diff --git a/CHANGELOG.md b/CHANGELOG.md index c673eeba0ad..34ac65be1eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- *(reborn)* add the explicit offline `plan` / `apply` / `resume` / `verify` / + `status` v1 migration workflow with sealed manifests, deterministic replay, + target quarantine, structural durable-store verification, and a same-version + companion in Docker and paired source builds. Native installers do not yet + package the companion pair. + ### Fixed - *(slack)* resolve known DM conversation IDs through an exact Slack lookup before encoding mentions, avoiding wrong-target posts when conversation lists are long or display names are ambiguous. diff --git a/Cargo.lock b/Cargo.lock index c74c72de7b9..5ab32fe9a8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4833,6 +4833,7 @@ dependencies = [ "ironclaw_reborn_config", "ironclaw_reborn_traces", "ironclaw_reborn_webui_ingress", + "libsql", "reqwest 0.12.28", "secrecy", "serde", @@ -5005,6 +5006,7 @@ dependencies = [ "chrono", "clap", "deadpool-postgres", + "futures", "ironclaw", "ironclaw_common", "ironclaw_extensions", @@ -5013,7 +5015,9 @@ dependencies = [ "ironclaw_host_runtime", "ironclaw_memory", "ironclaw_memory_native", + "ironclaw_projects", "ironclaw_reborn_composition", + "ironclaw_reborn_config", "ironclaw_reborn_identity", "ironclaw_secrets", "ironclaw_threads", @@ -5022,6 +5026,7 @@ dependencies = [ "secrecy", "serde", "serde_json", + "sha2 0.10.9", "tempfile", "thiserror 2.0.18", "tokio", diff --git a/Dockerfile.reborn b/Dockerfile.reborn index 09eb7a0ca6e..b40de6c1c7b 100644 --- a/Dockerfile.reborn +++ b/Dockerfile.reborn @@ -4,7 +4,9 @@ # docker build -f Dockerfile.reborn -t ironclaw-reborn:latest . # # Run locally: -# docker run --rm --env-file .env.reborn -p 127.0.0.1:3000:3000 ironclaw-reborn:latest +# docker run --rm --env-file .env.reborn \ +# -e IRONCLAW_REBORN_SERVE_HOST=0.0.0.0 \ +# -p 127.0.0.1:3000:3000 ironclaw-reborn:latest # # Railway: # Set Dockerfile path to Dockerfile.reborn and IRONCLAW_REBORN_SERVE_HOST=0.0.0.0. @@ -33,15 +35,18 @@ WORKDIR /app FROM chef AS planner COPY Cargo.toml Cargo.lock ./ +COPY build.rs build.rs +COPY src/ src/ COPY crates/ crates/ COPY tools/ironclaw_stress/ tools/ironclaw_stress/ COPY skills/ skills/ COPY tests/ tests/ COPY wit/ wit/ +COPY profiles/ profiles/ +COPY prompts/ prompts/ +COPY channels-src/telegram/telegram.capabilities.json channels-src/telegram/telegram.capabilities.json +COPY channels-src/discord/discord.capabilities.json channels-src/discord/discord.capabilities.json COPY providers.json providers.json -RUN mkdir -p src \ - && printf 'fn main() {}\n' > src/main.rs \ - && printf '\n' > src/lib.rs RUN cargo chef prepare --recipe-path recipe.json @@ -59,21 +64,29 @@ RUN cargo chef cook \ --profile dist \ --package ironclaw_reborn_cli \ --features webui-v2-beta,slack-v2-host-beta,libsql,postgres,inmemory-turn-state \ + --recipe-path recipe.json \ + && cargo chef cook \ + --profile dist \ + --package ironclaw_reborn_migration \ + --features libsql,postgres \ --recipe-path recipe.json FROM deps AS builder COPY Cargo.toml Cargo.lock ./ +COPY build.rs build.rs +COPY src/ src/ COPY crates/ crates/ COPY tools/ironclaw_stress/ tools/ironclaw_stress/ COPY migrations/ migrations/ COPY skills/ skills/ COPY tests/ tests/ COPY wit/ wit/ +COPY profiles/ profiles/ +COPY prompts/ prompts/ +COPY channels-src/telegram/telegram.capabilities.json channels-src/telegram/telegram.capabilities.json +COPY channels-src/discord/discord.capabilities.json channels-src/discord/discord.capabilities.json COPY providers.json providers.json -RUN mkdir -p src \ - && printf 'fn main() {}\n' > src/main.rs \ - && printf '\n' > src/lib.rs WORKDIR /app/crates/ironclaw_webui_v2/frontend RUN pnpm install --frozen-lockfile @@ -83,7 +96,12 @@ RUN cargo build \ --profile dist \ --package ironclaw_reborn_cli \ --features webui-v2-beta,slack-v2-host-beta,libsql,postgres,inmemory-turn-state \ - --bin ironclaw-reborn + --bin ironclaw-reborn \ + && cargo build \ + --profile dist \ + --package ironclaw_reborn_migration \ + --features libsql,postgres \ + --bin ironclaw-reborn-migration FROM debian:bookworm-slim AS runtime @@ -95,6 +113,7 @@ RUN apt-get -o Acquire::Retries=3 update \ && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/dist/ironclaw-reborn /usr/local/bin/ironclaw-reborn +COPY --from=builder /app/target/dist/ironclaw-reborn-migration /usr/local/bin/ironclaw-reborn-migration COPY docker/reborn/config.toml /opt/ironclaw/reborn/config.toml COPY docker/reborn/config.hosted-single-tenant.toml /opt/ironclaw/reborn/config.hosted-single-tenant.toml COPY docker/reborn/config.hosted-single-tenant-volume.toml /opt/ironclaw/reborn/config.hosted-single-tenant-volume.toml diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index ae4a4c7ac75..4b7b61fe082 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -712,17 +712,41 @@ Trace Commons issuer/TenantCtx note: the server-side `zmanian/tracedao-server` s | Gmail pub/sub | ✅ | ❌ | P3 | | | Inferred follow-up commitments | ✅ | ❌ | P3 | Heartbeat-delivered reminders; opt-in batched extraction | -**State migration (v1/engine-v2 → Reborn):** `crates/ironclaw_reborn_migration` -converts persisted automations. Cron routines and cron missions convert to -Reborn `TriggerRecord`s (mission threads land under `ThreadScope.mission_id`). +**State migration (v1/engine-v2 → Reborn):** `ironclaw-reborn migrate v1` +provides an explicit `plan → apply/resume → verify → status` workflow through a +same-version companion in the Docker image and paired source builds. Native +installers do not yet package the pair. Planning uses a read-only source adapter +and does not open the target; apply requires a stopped-source snapshot, a fresh +target, and the sealed source fingerprint. The versioned manifest inventories +known v1 tables/home artifacts from an explicit sealed source-home path and +labels each category as imported, converted, +archive-only, reset, re-auth/reinstall, or unsupported. `archive-only` currently +means the source category and count remain visible in the manifest; it does not +export or retain the source payload. The companion is resolved beside the +primary executable rather than from `PATH`, and database URLs/master keys remain +environment-only. See +`docs/reborn/v1-migration.md` for cutover and rollback. +Both target backends persist an atomic, run-bound migration claim; PostgreSQL +stores it in the shared database so all replicas block activation until +verification succeeds. + +The current conversion layer maps cron routines and cron missions to Reborn +`TriggerRecord`s (mission threads land under `ThreadScope.mission_id`). Because Reborn's `TriggerSourceKind` is `Schedule`-only, **event / system-event / webhook / manual routines and non-cron mission cadences have no `TriggerRecord` -target** and are recorded in the migration manifest rather than converted — even +target** and are recorded in the apply/resume migration report rather than converted — even where the runtime supports the *behavior* via hooks/`event_emit`, the durable automation row does not carry over. Guardrails, notify config, run counters, `routine_runs` history (no public run-history insert), and mission-only fields (focus/approach/success-criteria) likewise have no target. See the crate's -CLAUDE.md for the full mapping + gap catalog. +CLAUDE.md, manifest inventory, and retained apply/resume report for the full +mapping + gap catalog. Current verification checks structural counts for users, +projects, threads, messages, triggers, memory documents, secrets, and identity +records in production durable tables/paths; other domains do not receive an +independent readback, and this does not constitute a full production +cold-boot/readback test. Intermediate `applied` or `verifying` +states remain quarantined, and operators must complete a production canary +before accepting cutover. ### Owner: _Unassigned_ diff --git a/README.md b/README.md index 2b8b6afe6da..ce3ee812ebf 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ ## IronClaw Reborn Quick Start -IronClaw Reborn is the standalone runtime on the `reborn-integration` branch. -It uses the separate `ironclaw-reborn` binary from the +IronClaw Reborn is the standalone runtime in this workspace. It uses the +separate `ironclaw-reborn` binary from the `ironclaw_reborn_cli` package and a separate Reborn state root. It does not use the legacy `ironclaw` state directory as its config root. @@ -62,24 +62,61 @@ cargo build -p ironclaw_reborn_cli --bin ironclaw-reborn ./target/debug/ironclaw-reborn --help ``` +To use `migrate`, build the same-version companion into the same target +directory too. Compile the primary CLI with the target backend it must inspect +after migration (libSQL shown; use `--features postgres` for PostgreSQL): + +```bash +cargo build -p ironclaw_reborn_cli --features libsql +cargo build -p ironclaw_reborn_migration +./target/debug/ironclaw-reborn migrate v1 --help +``` + The default Reborn home is `$HOME/.ironclaw/reborn`. Override it with an absolute path when you want isolated state: ```bash -export IRONCLAW_REBORN_HOME="$PWD/.reborn-home" +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw-reborn-demo" cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- config path ``` `config path` and `doctor` are safe diagnostics; they report the resolved home, -profile, `config.toml`, `providers.json`, and `v1_state: not-used`. -They do not create Reborn state or seed config files. +profile, `config.toml`, `providers.json`, and `v1_state: not-used`. `doctor` +also reports `v1_migration_state`, including detected sources and any local or +durable target quarantine. They do not create Reborn state or seed config +files. + +### Migrate an existing v1 installation + +The Reborn Docker image includes a same-version migration companion. Source +builds must build both executables into the same target directory. Native +`cargo-dist` installers do not yet package the pair. Use the companion through +the primary binary; normal `run`, `repl`, `serve`, extension lifecycle, and +container startup never import v1 automatically. Runtime and extension +activation refuse quarantined migration states until verification succeeds: + +```bash +ironclaw-reborn migrate v1 plan \ + --source-libsql /backups/ironclaw-v1.db \ + --source-home /srv/ironclaw-v1 \ + --manifest /secure/migration-v1.json + +ironclaw-reborn migrate v1 status \ + --manifest /secure/migration-v1.json +``` + +Final apply requires v1 to be stopped and a consistent source snapshot. API +tokens and incompatible credentials require re-authentication; unsupported +executables are never enabled as placeholders. Review the complete backup, +apply, verification, and rollback procedure in +[`docs/reborn/v1-migration.md`](docs/reborn/v1-migration.md) before cutover. ### Configure the model route The CLI-native way to configure Reborn's default model route is: ```bash -export IRONCLAW_REBORN_HOME="$PWD/.reborn-home" +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw-reborn-demo" cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- models set-provider openai --model gpt-5-mini ``` @@ -179,7 +216,7 @@ seeded config does not include `[llm.default]`, so env-only model selection continues to work: ```bash -export IRONCLAW_REBORN_HOME="$PWD/.reborn-env-only" +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw-reborn-env-only" export LLM_BACKEND=openai export OPENAI_API_KEY="sk-..." cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- run --message "hello" @@ -245,7 +282,7 @@ env-bearer token and a user id at startup. It also needs the model route from the earlier section, including that provider's credential env var: ```bash -export IRONCLAW_REBORN_HOME="$PWD/.reborn-home" +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw-reborn-demo" export OPENAI_API_KEY="sk-..." # or the required env var for your configured provider export IRONCLAW_REBORN_WEBUI_TOKEN="$(openssl rand -hex 32)" export IRONCLAW_REBORN_WEBUI_USER_ID="reborn-cli" @@ -343,7 +380,7 @@ Slack support is compiled behind the `slack-v2-host-beta` Cargo feature. That feature includes `webui-v2-beta`, so Slack runs on the same `serve` command: ```bash -export IRONCLAW_REBORN_HOME="$PWD/.reborn-home" +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw-reborn-demo" export OPENAI_API_KEY="sk-..." # or the required env var for your configured provider export IRONCLAW_REBORN_WEBUI_TOKEN="$(openssl rand -hex 32)" export IRONCLAW_REBORN_WEBUI_USER_ID="reborn-cli" diff --git a/crates/AGENTS.md b/crates/AGENTS.md index 6e2eec4ea3b..dbc8825fcaf 100644 --- a/crates/AGENTS.md +++ b/crates/AGENTS.md @@ -139,6 +139,7 @@ Boundary rule: if you need an upstream crate in a low-level crate, stop and chec | `ironclaw_first_party_extensions` | `ironclaw_first_party_extensions/AGENTS.md`, `Cargo.toml` | Concrete first-party userland extension implementations and deterministic tool behavior behind scoped handles. | Host runtime composition, loop-facing ports, ambient runtime authority, dispatcher/network/secrets handles. | | `ironclaw_first_party_extension_ports` | `ironclaw_first_party_extension_ports/AGENTS.md`, `Cargo.toml` | Loop-facing adapters for first-party extensions: skill activation/context/execution ports over loop-host and turn-run contracts. | Concrete tool behavior, host runtime composition, product workflow, raw host authority. | | `ironclaw_reborn_cli` | `ironclaw_reborn_cli/AGENTS.md` | Standalone Reborn CLI, command files, CLI context, shell completions, doctor/home/profile commands. | V1 runtime imports, root `ironclaw` deps, side effects in pure commands. | +| `ironclaw_reborn_migration` | `ironclaw_reborn_migration/CLAUDE.md` | Same-release offline v1 migration companion: read-only source adapters, sealed manifests, deterministic converters, target quarantine, and structural verification. | Linking legacy source code into the normal Reborn runtime or bypassing the explicit plan/apply/resume/verify lifecycle. | | `ironclaw_product_adapters` | `ironclaw_product_adapters/AGENTS.md`, `ironclaw_product_adapters/CLAUDE.md` | Product-adapter contracts: adapter trait, auth, egress, identity, workflow, external/projection/inbound, redaction, fakes. | Host runtime internals or specific WASM runner implementation. | | `ironclaw_product_adapter_registry` | `ironclaw_product_adapter_registry/AGENTS.md`, `ironclaw_product_adapter_registry/CLAUDE.md` | ProductAdapter host-api projection and installation registry. | Adapter execution or product workflow orchestration. | | `ironclaw_product_workflow` | `ironclaw_product_workflow/AGENTS.md`, `ironclaw_product_workflow/CLAUDE.md` | Product-facing workflow facade: inbound turns, bindings, ledger, workflow/errors, Reborn service bridges, and feature-gated durable ledger adapters. | Low-level runtime lane internals, direct provider-specific transports, or durable ledger access outside the `IdempotencyLedger` port. | @@ -174,7 +175,7 @@ Boundary rule: if you need an upstream crate in a low-level crate, stop and chec - Reborn runtime execution: lane crate (`scripts`, `mcp`, `wasm`) first; `dispatcher` for routing; `host_runtime` for secrets/network/resources/redaction; `processes` for background lifecycle; `ironclaw_wasm_limiter` only for shared limiter mechanics. Use `ironclaw_engine` only for existing v1 engine maintenance. - Reborn turns/agent loop: `ironclaw_turns` for turn coordination; `ironclaw_agent_loop` for strategy/planner/executor contracts; `ironclaw_loop_host` for host support ports. Use `ironclaw_engine` only for existing v1 CodeAct/thread runtime maintenance. - Product adapter flow: `ironclaw_product_adapters` contracts -> `ironclaw_product_adapter_registry` installation/projection -> `ironclaw_product_workflow` orchestration -> concrete adapter crate. -- Reborn binary/composition: `ironclaw_reborn_config` for boot config; `ironclaw_reborn_composition` for production wiring; `ironclaw_reborn_cli` for commands; `ironclaw_runner` for standalone adapters/driver registry; `ironclaw_reborn_webui_ingress` for host-owned WebChat v2 listener lifecycle. +- Reborn binary/composition: `ironclaw_reborn_config` for boot config; `ironclaw_reborn_composition` for production wiring; `ironclaw_reborn_cli` for commands; `ironclaw_reborn_migration` for the offline v1 companion; `ironclaw_runner` for standalone adapters/driver registry; `ironclaw_reborn_webui_ingress` for host-owned WebChat v2 listener lifecycle. - Model/provider behavior: `ironclaw_llm`; do not leak provider auth/cache/retry concerns into engine or product workflow. - UI presentation: `ironclaw_webui_v2` and `ironclaw_webui_v2_static` for Reborn WebChat v2; `ironclaw_tui` and `ironclaw_gateway` only for existing v1 UI maintenance. Backend API/web channel code remains under root `src/` unless the surface is the Reborn WebChat v2 route crate. diff --git a/crates/README.md b/crates/README.md index ff58126f801..28ac9263fea 100644 --- a/crates/README.md +++ b/crates/README.md @@ -81,6 +81,7 @@ A good rule of thumb: if a change adds new authority or persistence, put it in t | `ironclaw_reborn_composition` | `ironclaw_reborn_composition` | Wiring layer that assembles Reborn services into the host runtime. Composition-only; no policy or persistence logic of its own. | | `ironclaw_reborn_config` | `ironclaw_reborn_config` | Reborn boot-config boundary: typed configuration, profiles, and validation consumed before services start. | | `ironclaw_reborn_cli` | `ironclaw_reborn_cli` | Reborn-first CLI surface (command modules, completion, shell entry points). Calls into composition; does not own host policy. | +| `ironclaw_reborn_migration` | `ironclaw_reborn_migration` | Same-release offline companion for manifest-driven v1 planning, conversion, quarantine, resume, and structural verification. It is invoked through `ironclaw-reborn migrate v1`, not linked into the normal runtime. | | `ironclaw_reborn_webui_ingress` | `ironclaw_reborn_webui_ingress` | Host-owned listener binding, authenticator implementations, and serve loop for the Reborn WebChat v2 HTTP gateway. | | `ironclaw_reborn_openai_compat` | `ironclaw_reborn_openai_compat` | OpenAI-compatible Chat/Responses DTOs, route descriptors, sanitized errors, fail-closed route fragment, and feature-gated durable ref/idempotency storage. | | `ironclaw_llm` | `ironclaw_llm` | LLM provider routing and abstraction used by Reborn product surfaces and the agent loop. | diff --git a/crates/ironclaw_memory/src/lib.rs b/crates/ironclaw_memory/src/lib.rs index 4145b803e9c..3545c64237e 100644 --- a/crates/ironclaw_memory/src/lib.rs +++ b/crates/ironclaw_memory/src/lib.rs @@ -39,9 +39,9 @@ pub use service::{ MEMORY_DISABLED_CONTEXT_ALIASES, MemoryContextProfileId, MemoryInvocation, MemoryProfileSetStatus, MemoryService, MemoryServiceContextRequest, MemoryServiceContextSnippet, MemoryServiceError, MemoryServiceErrorKind, - MemoryServiceProfileSetRequest, MemoryServiceProfileSetResponse, MemoryServiceReadRequest, - MemoryServiceReadResponse, MemoryServiceSearchRequest, MemoryServiceSearchResponse, - MemoryServiceSearchResult, MemoryServiceTreeRequest, MemoryServiceTreeResponse, - MemoryServiceWriteRequest, MemoryServiceWriteResponse, MemoryWriteStatus, - memory_context_disabled, + MemoryServiceMetadataResponse, MemoryServiceProfileSetRequest, MemoryServiceProfileSetResponse, + MemoryServiceReadRequest, MemoryServiceReadResponse, MemoryServiceSearchRequest, + MemoryServiceSearchResponse, MemoryServiceSearchResult, MemoryServiceTreeRequest, + MemoryServiceTreeResponse, MemoryServiceWriteRequest, MemoryServiceWriteResponse, + MemoryWriteStatus, memory_context_disabled, }; diff --git a/crates/ironclaw_memory/src/service.rs b/crates/ironclaw_memory/src/service.rs index a6d3ffd6207..359a8bd2db8 100644 --- a/crates/ironclaw_memory/src/service.rs +++ b/crates/ironclaw_memory/src/service.rs @@ -192,6 +192,15 @@ pub struct MemoryServiceReadResponse { pub word_count: usize, } +/// Result of a metadata-only document read. +/// +/// `metadata` is `None` when the scoped document does not exist. +#[derive(Debug, Clone, PartialEq)] +pub struct MemoryServiceMetadataResponse { + /// Parsed document metadata without loading document content. + pub metadata: Option, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct MemoryServiceTreeRequest { pub path: String, @@ -444,6 +453,18 @@ pub trait MemoryService: Send + Sync { Err(MemoryServiceError::unavailable()) } + /// Read only a scoped document's metadata, returning `None` for a missing + /// document. Providers that do not implement this operation fail closed + /// with [`MemoryServiceErrorKind::Unavailable`]. + async fn read_metadata( + &self, + invocation: MemoryInvocation, + request: MemoryServiceReadRequest, + ) -> Result { + let _ = (invocation, request); + Err(MemoryServiceError::unavailable()) + } + async fn tree( &self, invocation: MemoryInvocation, diff --git a/crates/ironclaw_memory_native/CLAUDE.md b/crates/ironclaw_memory_native/CLAUDE.md index 86835cc66fd..4db0536b3ca 100644 --- a/crates/ironclaw_memory_native/CLAUDE.md +++ b/crates/ironclaw_memory_native/CLAUDE.md @@ -12,7 +12,10 @@ agnostic `ironclaw_memory` crate, which this crate depends on and re-exports. - Keep semantic search, chunking, embeddings, and versioning behind memory-owned repository/indexer abstractions; do not put them in `ironclaw_filesystem`. - Reborn memory is **native and isolated**. The current implementation is a single `FilesystemMemoryDocumentRepository` over `RootFilesystem` (see the last bullet); the **deferred** SQL-backed target is dedicated `reborn_memory_*` tables with explicit `tenant_id`, `user_id`, `agent_id`, `project_id` scope columns — not legacy `memory_documents`. Do not encode Reborn scope into legacy `memory_documents.user_id` and do not introduce a `WorkspaceMemoryAdapter` or any other bridge over `src/workspace::Workspace`. - `src/workspace/*` and `src/db/libsql/workspace.rs` are **reference material only**. Pure behavior, schema validation, FTS escaping, chunking, version-hash semantics, RRF/weighted hybrid search fusion, and `.config` inheritance tests may be ported, but `ironclaw_memory_native` must not depend on the main app crate or any product modules to do so. -- Legacy migration and coexistence of existing `memory_documents` rows are **explicitly deferred** to a later issue that defines the product mapping. Do not migrate or alias legacy rows from this crate. +- Legacy import is owned by the offline `ironclaw_reborn_migration` companion, + which writes supported documents through this crate's native service. Do not + read, migrate, or alias legacy rows directly from this crate, and do not add a + dependency on v1. - Every read/list/search/write/version/chunk operation must filter by the full `(tenant_id, user_id, agent_id, project_id)` tuple. Do not infer project scope from path prefixes. Document uniqueness must be `UNIQUE (tenant_id, user_id, agent_id, project_id, path)`. - Use the empty string as the DB-only absent sentinel for `agent_id` and `project_id` (safe because `MemoryDocumentScope` rejects empty supplied IDs). Do not store `_none` in the database — `_none` is the **virtual-path** sentinel only. - Capability declarations (`MemoryBackendCapabilities`) are enforcement inputs: unsupported file/search behavior must fail closed before backend side effects. diff --git a/crates/ironclaw_memory_native/src/backend.rs b/crates/ironclaw_memory_native/src/backend.rs index ddcf4aaa5ff..e6249aebd20 100644 --- a/crates/ironclaw_memory_native/src/backend.rs +++ b/crates/ironclaw_memory_native/src/backend.rs @@ -133,6 +133,23 @@ pub trait MemoryBackend: Send + Sync { )) } + /// Read stored metadata without loading document content. + /// + /// Returns `None` when the scoped document is absent. The default fails + /// closed because metadata reads are an optional backend operation. + async fn read_document_metadata( + &self, + context: &MemoryContext, + path: &MemoryDocumentPath, + ) -> Result, FilesystemError> { + let _ = path; + Err(memory_backend_unsupported( + context.scope(), + FilesystemOperation::ReadFile, + "memory backend does not support document metadata", + )) + } + async fn write_document( &self, context: &MemoryContext, @@ -432,6 +449,27 @@ where self.repository.read_document(path).await } + async fn read_document_metadata( + &self, + context: &MemoryContext, + path: &MemoryDocumentPath, + ) -> Result, FilesystemError> { + ensure_file_documents_supported( + context, + FilesystemOperation::ReadFile, + self.capabilities.file_documents, + )?; + if !self.capabilities.metadata { + return Err(memory_backend_unsupported( + context.scope(), + FilesystemOperation::ReadFile, + "memory backend does not support document metadata", + )); + } + ensure_path_matches_context(context, path, FilesystemOperation::ReadFile)?; + self.repository.read_document_metadata(path).await + } + async fn write_document( &self, context: &MemoryContext, @@ -1160,6 +1198,22 @@ mod tests { ); } + #[tokio::test] + async fn metadata_capability_rejects_direct_backend_metadata_reads() { + let backend = make_backend().with_capabilities( + MemoryBackendCapabilities::default() + .set_file_documents(true) + .set_metadata(false), + ); + let result = backend + .read_document_metadata(&alpha_context(), &alpha_path()) + .await; + assert!( + result.is_err(), + "metadata reads must fail closed when metadata is unsupported" + ); + } + #[tokio::test] async fn vector_request_fails_closed_when_vector_search_is_unsupported() { let backend = make_search_backend( diff --git a/crates/ironclaw_memory_native/src/lib.rs b/crates/ironclaw_memory_native/src/lib.rs index 5c1b037e516..cf276b63cdf 100644 --- a/crates/ironclaw_memory_native/src/lib.rs +++ b/crates/ironclaw_memory_native/src/lib.rs @@ -58,9 +58,9 @@ pub use search::{FusionStrategy, MemorySearchRequest, MemorySearchResult}; pub use service::{ MemoryContextProfileId, MemoryInvocation, MemoryProfileSetStatus, MemoryService, MemoryServiceContextRequest, MemoryServiceContextSnippet, MemoryServiceError, - MemoryServiceErrorKind, MemoryServiceProfileSetRequest, MemoryServiceProfileSetResponse, - MemoryServiceReadRequest, MemoryServiceReadResponse, MemoryServiceSearchRequest, - MemoryServiceSearchResponse, MemoryServiceSearchResult, MemoryServiceTreeRequest, - MemoryServiceTreeResponse, MemoryServiceWriteRequest, MemoryServiceWriteResponse, - MemoryWriteStatus, NativeMemoryService, + MemoryServiceErrorKind, MemoryServiceMetadataResponse, MemoryServiceProfileSetRequest, + MemoryServiceProfileSetResponse, MemoryServiceReadRequest, MemoryServiceReadResponse, + MemoryServiceSearchRequest, MemoryServiceSearchResponse, MemoryServiceSearchResult, + MemoryServiceTreeRequest, MemoryServiceTreeResponse, MemoryServiceWriteRequest, + MemoryServiceWriteResponse, MemoryWriteStatus, NativeMemoryService, }; diff --git a/crates/ironclaw_memory_native/src/service.rs b/crates/ironclaw_memory_native/src/service.rs index 9d7d879cef6..7b697ef0cf6 100644 --- a/crates/ironclaw_memory_native/src/service.rs +++ b/crates/ironclaw_memory_native/src/service.rs @@ -28,11 +28,11 @@ use serde_json::{Map, Value, json}; pub use ironclaw_memory::{ MemoryContextProfileId, MemoryInvocation, MemoryProfileSetStatus, MemoryService, MemoryServiceContextRequest, MemoryServiceContextSnippet, MemoryServiceError, - MemoryServiceErrorKind, MemoryServiceProfileSetRequest, MemoryServiceProfileSetResponse, - MemoryServiceReadRequest, MemoryServiceReadResponse, MemoryServiceSearchRequest, - MemoryServiceSearchResponse, MemoryServiceSearchResult, MemoryServiceTreeRequest, - MemoryServiceTreeResponse, MemoryServiceWriteRequest, MemoryServiceWriteResponse, - MemoryWriteStatus, memory_context_disabled, + MemoryServiceErrorKind, MemoryServiceMetadataResponse, MemoryServiceProfileSetRequest, + MemoryServiceProfileSetResponse, MemoryServiceReadRequest, MemoryServiceReadResponse, + MemoryServiceSearchRequest, MemoryServiceSearchResponse, MemoryServiceSearchResult, + MemoryServiceTreeRequest, MemoryServiceTreeResponse, MemoryServiceWriteRequest, + MemoryServiceWriteResponse, MemoryWriteStatus, memory_context_disabled, }; const MEMORY_PATH: &str = "MEMORY.md"; @@ -238,6 +238,23 @@ impl MemoryService for NativeMemoryService { }) } + async fn read_metadata( + &self, + invocation: MemoryInvocation, + request: MemoryServiceReadRequest, + ) -> Result { + reject_local_or_traversal_path(&request.path)?; + let (scope, context) = self.scoped_context(&invocation)?; + let path = document_path(&scope, &request.path)?; + let metadata = self + .backend + .read_document_metadata(&context, &path) + .await + .map_err(MemoryServiceError::operation_from)? + .map(|value| DocumentMetadata::from_value(&value)); + Ok(MemoryServiceMetadataResponse { metadata }) + } + async fn tree( &self, invocation: MemoryInvocation, diff --git a/crates/ironclaw_reborn_cli/AGENTS.md b/crates/ironclaw_reborn_cli/AGENTS.md index 4bb9d5a97b9..70308c635c1 100644 --- a/crates/ironclaw_reborn_cli/AGENTS.md +++ b/crates/ironclaw_reborn_cli/AGENTS.md @@ -15,6 +15,13 @@ This crate owns the standalone `ironclaw-reborn` command surface. Keep it small, - Keep commands side-effect free unless the command name and issue explicitly require mutation. - Use `IRONCLAW_REBORN_HOME` / `~/.ironclaw/reborn`; do not write current v1 state. - no v1 runtime imports: do not depend on root `ironclaw`, `src/agent`, channels, worker, DB, setup, service, sandbox, or `ironclaw_engine`. +- The only v1-state exception is migration discovery/launch: the CLI may detect + non-secret source evidence and invoke the verified same-directory + `ironclaw-reborn-migration` companion. Source reads stay in that companion's + read-only adapters; neither binary may write v1 state. +- Commands that assemble runtime or extension services must call the canonical + migration activation guard before composition so quarantined, unknown, or + invalid target state cannot be activated through a new command path. - Do not add workspace dependencies beyond `ironclaw_reborn_composition`, `ironclaw_reborn_config`, `ironclaw_reborn_traces`, and `ironclaw_reborn_webui_ingress` (host-owned WebUI serve lifecycle) without an architecture test update and explicit PR rationale. Provider registry/auth/model UX should enter through the Reborn composition provider-admin facade, not a separate CLI-only path. ## Adding a command @@ -24,7 +31,9 @@ This crate owns the standalone `ironclaw-reborn` command surface. Keep it small, 3. If the command needs boot config, resolve `RebornCliContext` in `commands::Command::execute` and pass it into the command handler. 4. If the command is pure, do not resolve `RebornCliContext` just to run it. 5. Add a binary smoke test in `tests/smoke.rs` that invokes `env!("CARGO_BIN_EXE_ironclaw-reborn")`. -6. If the command can touch state, assert it uses Reborn home only and does not create/read v1 DB/settings/secrets. +6. If the command can touch state, assert it uses Reborn home only and does not + create/read v1 DB/settings/secrets, except for the narrow migration launcher + boundary above. 7. Run: - `cargo test -p ironclaw_reborn_cli` - `cargo test -p ironclaw_architecture reborn` @@ -44,6 +53,17 @@ cargo install --path crates/ironclaw_reborn_cli --features webui-v2-beta cargo build -p ironclaw_reborn_cli --features webui-v2-beta --release ``` +These examples build only the primary WebUI binary; they do not provide the +required migration companion. For `migrate`, build both workspace packages so +the same-version executables land beside each other. The primary CLI must +include the target backend it will inspect after migration (libSQL shown; +substitute `postgres` when appropriate): + +```bash +cargo build -p ironclaw_reborn_cli --features libsql --release +cargo build -p ironclaw_reborn_migration --release +``` + The beta WebUI static crate runs the frontend bundler from Cargo build scripts, so any `webui-v2-beta` or `--all-features` build needs Node.js/npm available even though the generated `static/dist/` bundle is not committed. diff --git a/crates/ironclaw_reborn_cli/Cargo.toml b/crates/ironclaw_reborn_cli/Cargo.toml index 40037fd8e18..b60fe7b7f5d 100644 --- a/crates/ironclaw_reborn_cli/Cargo.toml +++ b/crates/ironclaw_reborn_cli/Cargo.toml @@ -13,8 +13,8 @@ publish = false [package.metadata.ironclaw] layer = "app" -# Keep disabled until #3483 resolves cargo-dist tag/versioning + Windows WiX -# packaging for the standalone Reborn binary. +# Keep disabled until native cargo-dist packaging can install the Reborn CLI +# and its required same-release migration companion into one directory. [package.metadata.dist] dist = false @@ -59,10 +59,13 @@ openai-compat-beta = [ # Enable production PostgreSQL storage composition for `ironclaw-reborn`. postgres = [ "ironclaw_reborn_composition/postgres", + "ironclaw_reborn_composition/migration-support", ] # Enable embedded libSQL storage composition for volume-backed hosted previews. libsql = [ "ironclaw_reborn_composition/libsql", + "ironclaw_reborn_composition/migration-support", + "dep:libsql", ] # Coordinate turn state in a single in-process `InMemoryTurnStateStore` authority # instead of the per-user `state.json` filesystem CAS store (the runtime-wedge @@ -86,6 +89,7 @@ ironclaw_reborn_config = { path = "../ironclaw_reborn_config", version = "0.1.0" ironclaw_reborn_traces = { path = "../ironclaw_reborn_traces", version = "0.1.0" } ironclaw_reborn_webui_ingress = { path = "../ironclaw_reborn_webui_ingress", version = "0.1.0", optional = true } reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] } +libsql = { version = "0.9", optional = true, default-features = false, features = ["core", "replication", "remote", "tls"] } secrecy = "0.10" serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/ironclaw_reborn_cli/src/commands/doctor.rs b/crates/ironclaw_reborn_cli/src/commands/doctor.rs index cd5dd8d065d..95fa2cf9bee 100644 --- a/crates/ironclaw_reborn_cli/src/commands/doctor.rs +++ b/crates/ironclaw_reborn_cli/src/commands/doctor.rs @@ -1,3 +1,4 @@ +use anyhow::Context as _; use clap::Args; use ironclaw_reborn_composition::{ RebornRuntimeComponentStatus, reborn_runtime_readiness_snapshot, @@ -55,6 +56,15 @@ fn build_doctor_dto(context: &RebornCliContext) -> DoctorDto { detail: report.profile().to_string(), }); + checks.push(DoctorCheck { + name: "v1_state".to_string(), + category: CheckCategory::Core, + outcome: CheckOutcome::Pass, + detail: report.v1_state().to_string(), + }); + + checks.push(migration_check(context)); + let config_path = context.boot_config().home().config_file_path(); checks.push(check_config_file(&config_path)); @@ -192,12 +202,21 @@ impl Renderable for DoctorDto { CheckOutcome::Fail => "\u{2718}", CheckOutcome::Skip => "-", }; - writeln!( - w, - " {icon} {:<28} {}", - terminal_safe_text(&check.name), - terminal_safe_text(&check.detail) - )?; + if check.name == "v1_migration_state" { + writeln!( + w, + " {icon} {}: {}", + terminal_safe_text(&check.name), + terminal_safe_text(&check.detail) + )?; + } else { + writeln!( + w, + " {icon} {:<28} {}", + terminal_safe_text(&check.name), + terminal_safe_text(&check.detail) + )?; + } } writeln!(w)?; writeln!( @@ -209,6 +228,89 @@ impl Renderable for DoctorDto { } } +fn migration_check(context: &RebornCliContext) -> DoctorCheck { + let detail = match migration_state(context) { + Ok(Some(detail)) => detail, + Ok(None) if context.v1_migration_source_candidate().is_some() => "available".to_string(), + Ok(None) => "not_detected".to_string(), + Err(error) => format!("invalid: {error}"), + }; + let outcome = match detail + .split_once(':') + .map_or(detail.as_str(), |(state, _)| state) + { + "invalid" | "applying" | "failed" | "applied" | "verifying" => CheckOutcome::Fail, + "not_detected" | "available" | "explicitly_skipped" | "planned" => CheckOutcome::Skip, + "verified" => CheckOutcome::Pass, + _ => CheckOutcome::Fail, + }; + DoctorCheck { + name: "v1_migration_state".to_string(), + category: CheckCategory::Core, + outcome, + detail, + } +} + +fn migration_state(context: &RebornCliContext) -> anyhow::Result> { + match crate::commands::migrate::read_activation_state_status(context) { + Ok(Some(status)) => return Ok(Some(status.as_str().to_string())), + Err(error) => { + return Err(error).context("failed to inspect target migration quarantine state"); + } + Ok(None) => {} + } + + let marker = crate::commands::onboard::onboarding_marker_path(context.boot_config().home()); + let body = match std::fs::read_to_string(&marker) { + Ok(body) => body, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| { + format!("failed to read onboarding marker at {}", marker.display()) + }); + } + }; + let document: serde_json::Value = serde_json::from_str(&body) + .with_context(|| format!("invalid onboarding marker at {}", marker.display()))?; + let recorded = document + .pointer("/v1_migration/state") + .and_then(serde_json::Value::as_str) + .map(ToOwned::to_owned); + let Some(recorded) = recorded else { + return Ok(None); + }; + if !matches!( + recorded.as_str(), + "planned" | "applying" | "failed" | "applied" | "verifying" | "verified" + ) { + return Ok(Some(recorded)); + } + let Some(manifest_path) = document + .pointer("/v1_migration/manifest") + .and_then(serde_json::Value::as_str) + else { + return Ok(Some(recorded)); + }; + Ok(status_from_document(std::path::Path::new(manifest_path))? + .map(|status| status.as_str().to_string()) + .or(Some(recorded))) +} + +fn status_from_document( + path: &std::path::Path, +) -> anyhow::Result> { + let body = std::fs::read_to_string(path) + .with_context(|| format!("failed to read migration manifest at {}", path.display()))?; + let document = serde_json::from_str::(&body) + .with_context(|| format!("invalid migration manifest at {}", path.display()))?; + document + .get("status") + .and_then(serde_json::Value::as_str) + .map(crate::commands::migrate::MigrationLifecycleStatus::parse) + .transpose() +} + #[cfg(test)] mod tests { use super::*; @@ -237,6 +339,110 @@ mod tests { ); } + #[test] + fn doctor_fails_quarantined_migration_states() { + let (_tmp, context) = RebornCliContext::test_context(); + let marker = context + .boot_config() + .home() + .path() + .join(crate::commands::migrate::MIGRATION_STATE_MARKER_FILE); + std::fs::create_dir_all(context.boot_config().home().path()).expect("create home"); + + for status in ["applying", "applied", "verifying"] { + std::fs::write( + &marker, + serde_json::json!({ + "schema_version": "ironclaw.reborn.migration-state/v1", + "migration_protocol_version": 1, + "release_version": env!("CARGO_PKG_VERSION"), + "status": status, + }) + .to_string(), + ) + .expect("write marker"); + let check = migration_check(&context); + assert_eq!(check.outcome, CheckOutcome::Fail, "status {status}"); + } + } + + #[test] + fn doctor_reports_corrupt_onboarding_migration_state() { + let (_tmp, context) = RebornCliContext::test_context(); + let marker = crate::commands::onboard::onboarding_marker_path(context.boot_config().home()); + std::fs::create_dir_all(context.boot_config().home().path()).expect("create home"); + std::fs::write(&marker, "not-json").expect("write corrupt marker"); + + let check = migration_check(&context); + + assert_eq!(check.outcome, CheckOutcome::Fail); + assert!(check.detail.contains("invalid"), "detail: {}", check.detail); + assert!( + check.detail.contains("onboarding"), + "detail: {}", + check.detail + ); + } + + #[test] + fn doctor_reports_missing_recorded_migration_manifest() { + let (_tmp, context) = RebornCliContext::test_context(); + let marker = crate::commands::onboard::onboarding_marker_path(context.boot_config().home()); + let missing_manifest = context + .boot_config() + .home() + .path() + .join("missing-manifest.json"); + std::fs::create_dir_all(context.boot_config().home().path()).expect("create home"); + std::fs::write( + &marker, + serde_json::json!({ + "v1_migration": { + "state": "planned", + "manifest": missing_manifest, + } + }) + .to_string(), + ) + .expect("write marker"); + + let check = migration_check(&context); + + assert_eq!(check.outcome, CheckOutcome::Fail); + assert!(check.detail.contains("invalid"), "detail: {}", check.detail); + assert!( + check.detail.contains("manifest"), + "detail: {}", + check.detail + ); + } + + #[test] + fn doctor_does_not_open_manifests_for_non_manifest_onboarding_states() { + for state in ["not_detected", "available", "explicitly_skipped"] { + let (_tmp, context) = RebornCliContext::test_context(); + let marker = + crate::commands::onboard::onboarding_marker_path(context.boot_config().home()); + std::fs::create_dir_all(context.boot_config().home().path()).expect("create home"); + std::fs::write( + &marker, + serde_json::json!({ + "v1_migration": { + "state": state, + "manifest": context.boot_config().home().path().join("missing.json"), + } + }) + .to_string(), + ) + .expect("write marker"); + + let check = migration_check(&context); + + assert_eq!(check.outcome, CheckOutcome::Skip, "state {state}"); + assert_eq!(check.detail, state); + } + } + #[test] fn doctor_config_file_absent_is_skip() { let check = check_config_file(std::path::Path::new("/nonexistent/config.toml")); diff --git a/crates/ironclaw_reborn_cli/src/commands/extension.rs b/crates/ironclaw_reborn_cli/src/commands/extension.rs index c249414a4bc..7ab72f0f764 100644 --- a/crates/ironclaw_reborn_cli/src/commands/extension.rs +++ b/crates/ironclaw_reborn_cli/src/commands/extension.rs @@ -52,6 +52,7 @@ struct ExtensionPackageCommand { impl ExtensionCommand { pub(crate) fn execute(self, context: RebornCliContext) -> anyhow::Result<()> { + crate::commands::migrate::ensure_activation_allowed(&context)?; crate::runtime::init_tracing(); let (command, json, label) = match self.command { ExtensionSubcommand::Search(command) => ( @@ -115,3 +116,42 @@ fn execute_lifecycle_command( .map_err(anyhow::Error::from) }) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn quarantined_target_rejects_extension_before_service_assembly() { + let (_tmp, context) = RebornCliContext::test_context(); + let marker = context + .boot_config() + .home() + .path() + .join(crate::commands::migrate::MIGRATION_STATE_MARKER_FILE); + std::fs::create_dir_all(context.boot_config().home().path()).expect("create home"); + std::fs::write( + marker, + serde_json::json!({ + "schema_version": "ironclaw.reborn.migration-state/v1", + "migration_protocol_version": 1, + "release_version": env!("CARGO_PKG_VERSION"), + "status": "applying", + }) + .to_string(), + ) + .expect("write marker"); + let command = ExtensionCommand { + confirm_host_access: false, + command: ExtensionSubcommand::Search(ExtensionSearchCommand { + query: None, + json: false, + }), + }; + + let error = command + .execute(context) + .expect_err("quarantine must reject extension lifecycle commands"); + assert!(error.to_string().contains("quarantined"), "{error:#}"); + } +} diff --git a/crates/ironclaw_reborn_cli/src/commands/migrate.rs b/crates/ironclaw_reborn_cli/src/commands/migrate.rs new file mode 100644 index 00000000000..e7310d74c4f --- /dev/null +++ b/crates/ironclaw_reborn_cli/src/commands/migrate.rs @@ -0,0 +1,879 @@ +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, ExitStatus, Stdio}; + +use anyhow::{Context, ensure}; +use clap::{ArgGroup, Args, Subcommand}; +use serde::Deserialize; + +use crate::context::RebornCliContext; +use crate::context::V1MigrationSourceCandidate; + +#[cfg(not(windows))] +const COMPANION_FILE_STEM: &str = "ironclaw-reborn-migration"; +const COMPANION_HANDSHAKE_SCHEMA: &str = "ironclaw.reborn.migration-companion/v1"; +const COMPANION_PROTOCOL_VERSION: u32 = 1; +const COMPANION_ERROR_FORMAT_ENV: &str = "IRONCLAW_REBORN_MIGRATION_ERROR_FORMAT"; +pub(crate) const MIGRATION_STATE_MARKER_FILE: &str = ".v1-migration-state.json"; +const MIGRATION_STATE_MARKER_SCHEMA: &str = "ironclaw.reborn.migration-state/v1"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MigrationLifecycleStatus { + Planned, + Applying, + Failed, + Applied, + Verifying, + Verified, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +struct MigrationStateRecord { + schema_version: String, + migration_protocol_version: u32, + run_id: String, + status: String, + profile: String, + target_backend: String, + target_locator_fingerprint: String, + tenant_id: String, + agent_id: String, +} + +impl MigrationLifecycleStatus { + pub(crate) fn parse(status: &str) -> anyhow::Result { + match status { + "planned" => Ok(Self::Planned), + "applying" => Ok(Self::Applying), + "failed" => Ok(Self::Failed), + "applied" => Ok(Self::Applied), + "verifying" => Ok(Self::Verifying), + "verified" => Ok(Self::Verified), + _ => anyhow::bail!( + "Reborn target is quarantined because v1 migration status `{status}` is unknown" + ), + } + } + + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Planned => "planned", + Self::Applying => "applying", + Self::Failed => "failed", + Self::Applied => "applied", + Self::Verifying => "verifying", + Self::Verified => "verified", + } + } + + const fn is_activation_safe(self) -> bool { + matches!(self, Self::Planned | Self::Verified) + } +} + +/// Migrate persisted state into Reborn. +#[derive(Debug, Args)] +pub(crate) struct MigrateCommand { + #[command(subcommand)] + target: MigrationTarget, +} + +#[derive(Debug, Subcommand)] +enum MigrationTarget { + /// Migrate an IronClaw v1 installation. + V1(V1MigrationCommand), +} + +#[derive(Debug, Args)] +struct V1MigrationCommand { + #[command(subcommand)] + operation: V1MigrationOperation, +} + +#[derive(Debug, Subcommand)] +enum V1MigrationOperation { + /// Inventory a v1 snapshot and write a reviewable migration manifest. + Plan(PlanArgs), + /// Apply a reviewed plan into a fresh staged Reborn target. + Apply(ApplyArgs), + /// Resume an interrupted apply using its migration manifest. + Resume(ResumeArgs), + /// Verify an applied target with structural durable-store readback. + Verify(VerifyArgs), + /// Show the current migration status recorded in a manifest. + Status(StatusArgs), +} + +#[derive(Debug, Args)] +struct SourceArgs { + /// WAL-consistent libSQL/SQLite snapshot to inventory. + #[arg(long, value_name = "SNAPSHOT", group = "source")] + source_libsql: Option, + + /// Read the PostgreSQL snapshot URL from MIGRATION_SOURCE_POSTGRES. + #[arg(long, group = "source")] + source_postgres: bool, + + /// v1 home whose persistent artifacts belong to this source snapshot. + #[arg(long, value_name = "PATH")] + source_home: Option, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct PlanArgs { + #[command(flatten)] + source: SourceArgs, + + /// Destination for the versioned migration manifest. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, + + /// Fail after writing for blockers or nonzero archive/re-auth/reinstall/unsupported data. + #[arg(long)] + strict: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct ApplyArgs { + #[command(flatten)] + source: SourceArgs, + + /// Reviewed migration plan to apply. + #[arg(long, value_name = "PATH")] + plan: PathBuf, + + /// Confirm that v1 is stopped and the plan's source is a consistent snapshot. + #[arg(long, required = true)] + confirm_v1_stopped: bool, + + /// Confirm that the selected source was created as a consistent snapshot. + #[arg(long, required = true)] + confirm_source_snapshot: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct ResumeArgs { + #[command(flatten)] + source: SourceArgs, + + /// Migration manifest to resume. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, + + /// Confirm that v1 remains stopped. + #[arg(long, required = true)] + confirm_v1_stopped: bool, + + /// Confirm that the selected source is the plan's consistent snapshot. + #[arg(long, required = true)] + confirm_source_snapshot: bool, +} + +#[derive(Debug, Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct VerifyArgs { + #[command(flatten)] + source: SourceArgs, + + /// Migration manifest to verify. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, +} + +#[derive(Debug, Args)] +struct StatusArgs { + /// Migration manifest to inspect. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, + + /// Emit the machine-readable status document. + #[arg(long)] + json: bool, +} + +impl MigrateCommand { + pub(crate) fn execute(self) -> anyhow::Result<()> { + let args = self.target.forward_args(); + launch_companion(args) + } +} + +pub(crate) fn plan_detected_v1( + source: V1MigrationSourceCandidate, + source_home: &Path, + manifest: &Path, +) -> anyhow::Result<()> { + let mut args = vec![OsString::from("v1"), OsString::from("plan")]; + match source { + V1MigrationSourceCandidate::LibSql(path) => { + push_path_option(&mut args, "--source-libsql", path) + } + V1MigrationSourceCandidate::PostgresEnvironment => { + args.push(OsString::from("--source-postgres")); + } + } + push_path_option(&mut args, "--source-home", source_home.to_path_buf()); + push_path_option(&mut args, "--manifest", manifest.to_path_buf()); + launch_companion(args) +} + +pub(crate) fn ensure_activation_allowed(context: &RebornCliContext) -> anyhow::Result<()> { + let status = read_activation_state_status(context)?; + let Some(status) = status else { + return Ok(()); + }; + activation_status_allowed(status) +} + +pub(crate) fn read_activation_state_status( + context: &RebornCliContext, +) -> anyhow::Result> { + let marker = context + .boot_config() + .home() + .path() + .join(MIGRATION_STATE_MARKER_FILE); + let local = read_local_target_state(&marker)?; + let shared = read_shared_target_state(context)?; + if local.is_none() && shared.is_none() { + return Ok(None); + } + let binding = current_target_binding(context)?; + if let Some(record) = local.as_ref() { + validate_state_binding(record, &binding)?; + } + if let Some(record) = shared.as_ref() { + validate_state_binding(record, &binding)?; + } + match (local, shared) { + (None, None) => Ok(None), + (Some(_), None) => anyhow::bail!( + "Reborn target is quarantined because its local v1 migration marker has no matching target-owned durable state" + ), + (Some(local), Some(shared)) => { + ensure!( + local == shared, + "Reborn target is quarantined because local and durable v1 migration state do not match" + ); + Ok(Some(MigrationLifecycleStatus::parse(&shared.status)?)) + } + (None, Some(shared)) => Ok(Some(MigrationLifecycleStatus::parse(&shared.status)?)), + } +} + +fn read_local_target_state(marker: &Path) -> anyhow::Result> { + let body = match fs::read_to_string(marker) { + Ok(body) => body, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to inspect the target-owned v1 migration state at {}", + marker.display() + ) + }); + } + }; + let document: MigrationStateRecord = serde_json::from_str(&body).with_context(|| { + format!( + "target-owned v1 migration state at {} is invalid; keep the target quarantined and inspect the migration manifest recorded in that marker", + marker.display() + ) + })?; + validate_state_header( + Some(&document.schema_version), + Some(u64::from(document.migration_protocol_version)), + )?; + MigrationLifecycleStatus::parse(&document.status)?; + Ok(Some(document)) +} + +fn validate_state_header(schema: Option<&str>, protocol: Option) -> anyhow::Result<()> { + ensure!( + schema == Some(MIGRATION_STATE_MARKER_SCHEMA), + "Reborn target is quarantined because its v1 migration state marker has an unknown schema" + ); + ensure!( + protocol == Some(u64::from(COMPANION_PROTOCOL_VERSION)), + "Reborn target is quarantined because its v1 migration state marker has an incompatible protocol" + ); + Ok(()) +} + +fn activation_status_allowed(status: MigrationLifecycleStatus) -> anyhow::Result<()> { + if status.is_activation_safe() { + Ok(()) + } else { + anyhow::bail!( + "Reborn target is quarantined because v1 migration status is `{}`; do not start live workers or ingress until migration verification records `verified`", + status.as_str() + ) + } +} + +#[derive(Debug)] +struct CurrentTargetBinding { + profile: String, + target_backend: &'static str, + target_locator_fingerprint: String, + tenant_id: String, + agent_id: String, +} + +#[cfg(any(feature = "postgres", feature = "libsql"))] +fn current_target_binding(context: &RebornCliContext) -> anyhow::Result { + use ironclaw_reborn_composition::RebornMigrationTargetStore; + + let target = + ironclaw_reborn_composition::resolve_reborn_migration_target(context.boot_config()) + .context("failed to resolve the Reborn target for migration quarantine inspection")?; + let (target_backend, target_locator_fingerprint) = match &target.store { + #[cfg(feature = "postgres")] + RebornMigrationTargetStore::Postgres { url } => ( + "postgres", + ironclaw_reborn_composition::migration_postgres_locator_fingerprint(url) + .context("failed to identify configured PostgreSQL migration target")?, + ), + #[cfg(feature = "libsql")] + RebornMigrationTargetStore::LibSql { path } => ( + "libsql", + ironclaw_reborn_composition::migration_libsql_locator_fingerprint(path), + ), + }; + Ok(CurrentTargetBinding { + profile: target.profile.as_str().to_string(), + target_backend, + target_locator_fingerprint, + tenant_id: target.tenant_id.to_string(), + agent_id: target.agent_id.to_string(), + }) +} + +#[cfg(not(any(feature = "postgres", feature = "libsql")))] +fn current_target_binding(_context: &RebornCliContext) -> anyhow::Result { + anyhow::bail!("migration target inspection requires a binary built with libsql or postgres") +} + +fn validate_state_binding( + record: &MigrationStateRecord, + binding: &CurrentTargetBinding, +) -> anyhow::Result<()> { + ensure!( + record.profile == binding.profile + && record.target_backend == binding.target_backend + && record.target_locator_fingerprint == binding.target_locator_fingerprint + && record.tenant_id == binding.tenant_id + && record.agent_id == binding.agent_id, + "Reborn target is quarantined because v1 migration state does not match the configured target profile or scope" + ); + Ok(()) +} + +#[cfg(any(feature = "postgres", feature = "libsql"))] +fn read_shared_target_state( + context: &RebornCliContext, +) -> anyhow::Result> { + use ironclaw_reborn_composition::RebornMigrationTargetStore; + + let target = + match ironclaw_reborn_composition::resolve_reborn_migration_target(context.boot_config()) { + Ok(target) => target, + Err(_) => return Ok(None), + }; + match target.store { + #[cfg(feature = "postgres")] + RebornMigrationTargetStore::Postgres { url } => crate::runtime::block_on_cli(async move { + let pool = ironclaw_reborn_composition::open_reborn_postgres_pool(url).context( + "failed to open the Reborn PostgreSQL target for migration quarantine inspection", + )?; + let client = pool + .get() + .await + .context("failed to inspect shared PostgreSQL migration quarantine state")?; + let relation: Option = client + .query_one("SELECT to_regclass('reborn_migration_state')::text", &[]) + .await + .context("failed to inspect shared PostgreSQL migration quarantine schema")? + .try_get(0) + .context("invalid shared PostgreSQL migration quarantine schema result")?; + if relation.is_none() { + return Ok::, anyhow::Error>(None); + } + let row = client + .query_opt( + "SELECT schema_version, migration_protocol_version, run_id, status, profile, \ + target_backend, target_locator_fingerprint, tenant_id, agent_id \ + FROM reborn_migration_state WHERE singleton = TRUE", + &[], + ) + .await + .context("failed to read shared PostgreSQL migration quarantine state")? + .context("shared PostgreSQL migration quarantine table has no singleton state")?; + let schema: String = row + .try_get(0) + .context("invalid shared PostgreSQL migration quarantine schema version")?; + let protocol: i64 = row + .try_get(1) + .context("invalid shared PostgreSQL migration quarantine protocol version")?; + let run_id: String = row + .try_get(2) + .context("invalid shared PostgreSQL migration run id")?; + let status: String = row + .try_get(3) + .context("invalid shared PostgreSQL migration quarantine status")?; + let profile: String = row.try_get(4).context("invalid shared migration profile")?; + let target_backend: String = row.try_get(5).context("invalid shared target backend")?; + let target_locator_fingerprint: String = row + .try_get(6) + .context("invalid shared target fingerprint")?; + let tenant_id: String = row.try_get(7).context("invalid shared tenant id")?; + let agent_id: String = row.try_get(8).context("invalid shared agent id")?; + let protocol = u64::try_from(protocol) + .context("shared PostgreSQL migration quarantine protocol version is negative")?; + validate_state_header(Some(&schema), Some(protocol))?; + MigrationLifecycleStatus::parse(&status)?; + Ok::, anyhow::Error>(Some(MigrationStateRecord { + schema_version: schema, + migration_protocol_version: u32::try_from(protocol) + .context("shared migration protocol version is too large")?, + run_id, + status, + profile, + target_backend, + target_locator_fingerprint, + tenant_id, + agent_id, + })) + }), + #[cfg(feature = "libsql")] + RebornMigrationTargetStore::LibSql { path } => read_libsql_target_state(path), + } +} + +#[cfg(not(any(feature = "postgres", feature = "libsql")))] +fn read_shared_target_state( + _context: &RebornCliContext, +) -> anyhow::Result> { + Ok(None) +} + +#[cfg(feature = "libsql")] +fn read_libsql_target_state(path: PathBuf) -> anyhow::Result> { + if !path.is_file() { + return Ok(None); + } + crate::runtime::block_on_cli(async move { + let database = libsql::Builder::new_local(&path) + .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY) + .build() + .await + .context("failed to open libSQL migration quarantine state")?; + let connection = database + .connect() + .context("failed to connect libSQL target")?; + let mut schema = connection + .query( + "SELECT 1 FROM sqlite_schema WHERE type = 'table' AND name = 'reborn_migration_state'", + (), + ) + .await + .context("failed to inspect libSQL migration quarantine schema")?; + if schema + .next() + .await + .context("failed to read libSQL schema")? + .is_none() + { + return Ok::, anyhow::Error>(None); + } + let mut rows = connection + .query( + "SELECT schema_version, migration_protocol_version, run_id, status, profile, + target_backend, target_locator_fingerprint, tenant_id, agent_id + FROM reborn_migration_state WHERE singleton = 1", + (), + ) + .await + .context("failed to read libSQL migration quarantine state")?; + let row = rows + .next() + .await + .context("failed to read libSQL migration state row")? + .context("libSQL migration quarantine table has no singleton state")?; + let record = MigrationStateRecord { + schema_version: row.get(0).context("invalid libSQL migration schema")?, + migration_protocol_version: u32::try_from( + row.get::(1) + .context("invalid libSQL migration protocol")?, + ) + .context("invalid libSQL migration protocol")?, + run_id: row.get(2).context("invalid libSQL migration run id")?, + status: row.get(3).context("invalid libSQL migration status")?, + profile: row.get(4).context("invalid libSQL migration profile")?, + target_backend: row.get(5).context("invalid libSQL target backend")?, + target_locator_fingerprint: row.get(6).context("invalid libSQL target fingerprint")?, + tenant_id: row.get(7).context("invalid libSQL tenant id")?, + agent_id: row.get(8).context("invalid libSQL agent id")?, + }; + validate_state_header( + Some(&record.schema_version), + Some(u64::from(record.migration_protocol_version)), + )?; + MigrationLifecycleStatus::parse(&record.status)?; + Ok::, anyhow::Error>(Some(record)) + }) +} + +fn launch_companion(args: Vec) -> anyhow::Result<()> { + let companion = resolve_companion( + &std::env::current_exe() + .context("failed to locate the running ironclaw-reborn executable")?, + )?; + verify_handshake(&companion)?; + + let status = Command::new(&companion) + .args(args) + .env(COMPANION_ERROR_FORMAT_ENV, "json") + .stdin(Stdio::inherit()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .status() + .with_context(|| { + format!( + "failed to start the Reborn migration companion at {}", + companion.display() + ) + })?; + + propagate_status(status) +} + +impl MigrationTarget { + fn forward_args(self) -> Vec { + match self { + Self::V1(command) => command.forward_args(), + } + } +} + +impl V1MigrationCommand { + fn forward_args(self) -> Vec { + let mut args = vec![OsString::from("v1")]; + match self.operation { + V1MigrationOperation::Plan(command) => { + args.push(OsString::from("plan")); + push_source_args(&mut args, command.source); + push_path_option(&mut args, "--manifest", command.manifest); + if command.strict { + args.push(OsString::from("--strict")); + } + } + V1MigrationOperation::Apply(command) => { + args.push(OsString::from("apply")); + push_source_args(&mut args, command.source); + push_path_option(&mut args, "--plan", command.plan); + if command.confirm_v1_stopped { + args.push(OsString::from("--confirm-v1-stopped")); + } + if command.confirm_source_snapshot { + args.push(OsString::from("--confirm-source-snapshot")); + } + } + V1MigrationOperation::Resume(command) => { + args.push(OsString::from("resume")); + push_source_args(&mut args, command.source); + push_path_option(&mut args, "--manifest", command.manifest); + if command.confirm_v1_stopped { + args.push(OsString::from("--confirm-v1-stopped")); + } + if command.confirm_source_snapshot { + args.push(OsString::from("--confirm-source-snapshot")); + } + } + V1MigrationOperation::Verify(command) => { + args.push(OsString::from("verify")); + push_source_args(&mut args, command.source); + push_path_option(&mut args, "--manifest", command.manifest); + } + V1MigrationOperation::Status(command) => { + args.push(OsString::from("status")); + push_path_option(&mut args, "--manifest", command.manifest); + if command.json { + args.push(OsString::from("--json")); + } + } + } + args + } +} + +fn push_source_args(args: &mut Vec, source: SourceArgs) { + if let Some(path) = source.source_libsql { + push_path_option(args, "--source-libsql", path); + } else if source.source_postgres { + args.push(OsString::from("--source-postgres")); + } + if let Some(path) = source.source_home { + push_path_option(args, "--source-home", path); + } +} + +fn push_path_option(args: &mut Vec, option: &'static str, value: PathBuf) { + args.push(OsString::from(option)); + args.push(value.into_os_string()); +} + +#[derive(Debug, Deserialize)] +struct CompanionHandshake { + schema_version: String, + protocol_version: u32, + release_version: String, +} + +fn companion_file_name() -> &'static OsStr { + #[cfg(windows)] + { + OsStr::new("ironclaw-reborn-migration.exe") + } + #[cfg(not(windows))] + { + OsStr::new(COMPANION_FILE_STEM) + } +} + +fn resolve_companion(current_exe: &Path) -> anyhow::Result { + let bin_dir = current_exe.parent().with_context(|| { + format!( + "cannot resolve the installation directory for {}", + current_exe.display() + ) + })?; + let companion = bin_dir.join(companion_file_name()); + let metadata = fs::symlink_metadata(&companion).with_context(|| { + format!( + "the Reborn migration companion is missing at {}; reinstall or upgrade Reborn with migration support", + companion.display() + ) + })?; + ensure!( + !metadata.file_type().is_symlink() && metadata.is_file(), + "refusing migration companion at {} because it is not a regular, non-symlink file", + companion.display() + ); + + verify_companion_permissions(current_exe, &companion, &metadata)?; + Ok(companion) +} + +#[cfg(unix)] +fn verify_companion_permissions( + current_exe: &Path, + companion: &Path, + companion_metadata: &fs::Metadata, +) -> anyhow::Result<()> { + use std::os::unix::fs::MetadataExt; + + let current_metadata = fs::metadata(current_exe).with_context(|| { + format!( + "failed to inspect the running Reborn executable at {}", + current_exe.display() + ) + })?; + ensure!( + current_metadata.uid() == companion_metadata.uid(), + "refusing migration companion at {} because it is not owned by the Reborn binary owner", + companion.display() + ); + ensure!( + companion_metadata.mode() & 0o022 == 0, + "refusing migration companion at {} because it is group- or world-writable", + companion.display() + ); + let install_dir = companion + .parent() + .context("migration companion has no installation directory")?; + let install_metadata = fs::metadata(install_dir).with_context(|| { + format!( + "failed to inspect the Reborn installation directory at {}", + install_dir.display() + ) + })?; + ensure!( + install_metadata.uid() == current_metadata.uid() && install_metadata.mode() & 0o022 == 0, + "refusing migration companion because its installation directory {} is writable by another user", + install_dir.display() + ); + Ok(()) +} + +#[cfg(not(unix))] +fn verify_companion_permissions( + _current_exe: &Path, + _companion: &Path, + _companion_metadata: &fs::Metadata, +) -> anyhow::Result<()> { + Ok(()) +} + +fn verify_handshake(companion: &Path) -> anyhow::Result<()> { + let output = Command::new(companion) + .arg("__handshake") + .stdin(Stdio::null()) + .stderr(Stdio::piped()) + .output() + .with_context(|| { + format!( + "failed to query the migration companion at {}", + companion.display() + ) + })?; + ensure!( + output.status.success(), + "migration companion handshake failed at {}: {}", + companion.display(), + String::from_utf8_lossy(&output.stderr).trim() + ); + let handshake: CompanionHandshake = serde_json::from_slice(&output.stdout) + .context("migration companion returned an invalid handshake document")?; + ensure!( + handshake.schema_version == COMPANION_HANDSHAKE_SCHEMA, + "migration companion protocol schema mismatch: expected {}, found {}", + COMPANION_HANDSHAKE_SCHEMA, + handshake.schema_version + ); + ensure!( + handshake.protocol_version == COMPANION_PROTOCOL_VERSION, + "migration companion protocol mismatch: expected {}, found {}", + COMPANION_PROTOCOL_VERSION, + handshake.protocol_version + ); + ensure!( + handshake.release_version == env!("CARGO_PKG_VERSION"), + "migration companion release mismatch: ironclaw-reborn is {}, companion is {}; install both executables from the same release", + env!("CARGO_PKG_VERSION"), + handshake.release_version + ); + Ok(()) +} + +fn propagate_status(status: ExitStatus) -> anyhow::Result<()> { + if status.success() { + return Ok(()); + } + + if let Some(code) = status.code() { + std::process::exit(code); + } + + anyhow::bail!("migration companion terminated without an exit code") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn marker(release_version: &str, status: &str, binding: &CurrentTargetBinding) -> String { + serde_json::json!({ + "schema_version": MIGRATION_STATE_MARKER_SCHEMA, + "migration_protocol_version": COMPANION_PROTOCOL_VERSION, + "release_version": release_version, + "run_id": "01JTESTMIGRATIONRUN0000000000", + "status": status, + "profile": binding.profile, + "target_backend": binding.target_backend, + "target_locator_fingerprint": binding.target_locator_fingerprint, + "tenant_id": binding.tenant_id, + "agent_id": binding.agent_id, + }) + .to_string() + } + + #[test] + fn plan_forwarding_never_contains_database_urls_or_keys() { + let command = V1MigrationCommand { + operation: V1MigrationOperation::Plan(PlanArgs { + source: SourceArgs { + source_libsql: None, + source_postgres: true, + source_home: Some(PathBuf::from("v1-home")), + }, + manifest: PathBuf::from("manifest.json"), + strict: true, + }), + }; + + assert_eq!( + command.forward_args(), + vec![ + "v1", + "plan", + "--source-postgres", + "--source-home", + "v1-home", + "--manifest", + "manifest.json", + "--strict", + ] + .into_iter() + .map(OsString::from) + .collect::>() + ); + } + + #[test] + #[cfg(feature = "libsql")] + fn verified_marker_without_matching_durable_state_stays_quarantined() { + let (_tmp, context) = RebornCliContext::test_context(); + let binding = current_target_binding(&context).expect("target binding"); + let path = context + .boot_config() + .home() + .path() + .join(MIGRATION_STATE_MARKER_FILE); + fs::create_dir_all(context.boot_config().home().path()).expect("create home"); + fs::write(&path, marker("previous-release", "verified", &binding)).expect("write marker"); + + let error = ensure_activation_allowed(&context) + .expect_err("a local marker alone must not authorize activation"); + assert!( + error + .to_string() + .contains("no matching target-owned durable state") + ); + } + + #[test] + fn state_binding_rejects_changed_scope() { + let binding = CurrentTargetBinding { + profile: "local-dev".to_string(), + target_backend: "libsql", + target_locator_fingerprint: "target-fingerprint".to_string(), + tenant_id: "tenant-a".to_string(), + agent_id: "agent-a".to_string(), + }; + let mut record: MigrationStateRecord = + serde_json::from_str(&marker("release", "verified", &binding)).expect("marker record"); + record.tenant_id = "tenant-b".to_string(); + assert!(validate_state_binding(&record, &binding).is_err()); + } +} diff --git a/crates/ironclaw_reborn_cli/src/commands/mod.rs b/crates/ironclaw_reborn_cli/src/commands/mod.rs index 1f5d92454b5..bd7dbe11f07 100644 --- a/crates/ironclaw_reborn_cli/src/commands/mod.rs +++ b/crates/ironclaw_reborn_cli/src/commands/mod.rs @@ -7,6 +7,7 @@ pub(crate) mod doctor; pub(crate) mod extension; pub(crate) mod hooks; pub(crate) mod logs; +pub(crate) mod migrate; pub(crate) mod models; pub(crate) mod onboard; pub(crate) mod profile; @@ -42,6 +43,8 @@ pub(crate) enum Command { Hooks(hooks::HooksCommand), /// Inspect Reborn logs. Logs(logs::LogsCommand), + /// Plan, apply, resume, verify, and inspect migrations into Reborn. + Migrate(migrate::MigrateCommand), /// Inspect Reborn model slots and route status. Models(models::ModelsCommand), /// Initialize the standalone Reborn home and first-run setup marker. @@ -82,20 +85,27 @@ impl Command { } Self::Hooks(command) => command.execute(), Self::Logs(command) => command.execute(), + Self::Migrate(command) => command.execute(), Self::Models(command) => command.execute(), Self::Onboard(command) => { command.execute(crate::context::RebornCliContext::resolve_from_env()?) } Self::Profile(command) => command.execute(), Self::Repl(command) => { - command.execute(crate::context::RebornCliContext::resolve_from_env()?) + let context = crate::context::RebornCliContext::resolve_from_env()?; + migrate::ensure_activation_allowed(&context)?; + command.execute(context) } Self::Run(command) => { - command.execute(crate::context::RebornCliContext::resolve_from_env()?) + let context = crate::context::RebornCliContext::resolve_from_env()?; + migrate::ensure_activation_allowed(&context)?; + command.execute(context) } #[cfg(feature = "webui-v2-beta")] Self::Serve(command) => { - command.execute(crate::context::RebornCliContext::resolve_from_env()?) + let context = crate::context::RebornCliContext::resolve_from_env()?; + migrate::ensure_activation_allowed(&context)?; + command.execute(context) } Self::Skills(command) => { command.execute(crate::context::RebornCliContext::resolve_from_env()?) diff --git a/crates/ironclaw_reborn_cli/src/commands/onboard.rs b/crates/ironclaw_reborn_cli/src/commands/onboard.rs index c75ea2b6eb6..429be36184c 100644 --- a/crates/ironclaw_reborn_cli/src/commands/onboard.rs +++ b/crates/ironclaw_reborn_cli/src/commands/onboard.rs @@ -4,11 +4,36 @@ use clap::Args; use ironclaw_reborn_config::RebornHome; use crate::commands::config::init::{ExistingConfigPolicy, write_default_config_files}; -use crate::context::RebornCliContext; +use crate::context::{RebornCliContext, V1MigrationSourceCandidate}; use crate::file_write::{FileWriteAction, write_atomic}; const ONBOARDING_MARKER_FILE: &str = ".onboard-completed.json"; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum OnboardingMigrationState { + ExplicitlySkipped, + Planned, + Available, + NotDetected, +} + +impl OnboardingMigrationState { + const fn as_str(self) -> &'static str { + match self { + Self::ExplicitlySkipped => "explicitly_skipped", + Self::Planned => "planned", + Self::Available => "available", + Self::NotDetected => "not_detected", + } + } +} + +impl std::fmt::Display for OnboardingMigrationState { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(self.as_str()) + } +} + /// Initialize the standalone Reborn home and first-run setup marker. #[derive(Debug, Args)] pub(crate) struct OnboardCommand { @@ -20,27 +45,75 @@ pub(crate) struct OnboardCommand { #[arg(long = "dry-run")] dry_run: bool, - /// Reserve the history-import step in the onboarding summary. - /// - /// History import is not wired in this slice; the flag makes the missing - /// step explicit without touching v1 setup/import state. - #[arg(long = "import-history")] + /// Inventory a detected v1 installation and write a migration plan. + #[arg(long = "migrate-v1", conflicts_with = "skip_v1_migration")] + migrate_v1: bool, + + /// Deprecated alias for --migrate-v1. + #[arg( + long = "import-history", + hide = true, + conflicts_with_all = ["migrate_v1", "skip_v1_migration"] + )] import_history: bool, + + /// Record that v1 migration was explicitly skipped during onboarding. + #[arg(long = "skip-v1-migration", conflicts_with = "migrate_v1")] + skip_v1_migration: bool, } impl OnboardCommand { pub(crate) fn execute(self, context: RebornCliContext) -> anyhow::Result<()> { let home = context.boot_config().home(); let marker_path = onboarding_marker_path(home); + let source = context.v1_migration_source_candidate(); + let source_detected = source.is_some(); + let migration_requested = self.migrate_v1 || self.import_history; + let manifest_path = default_migration_manifest_path(home); + + if self.import_history { + eprintln!("warning: --import-history is deprecated; use --migrate-v1"); + } if self.dry_run { - print_dry_run(home, &marker_path, self.force, self.import_history); + print_dry_run( + home, + &marker_path, + self.force, + migration_requested, + self.skip_v1_migration, + source.as_ref(), + &manifest_path, + ); return Ok(()); } let outcome = write_default_config_files(home, self.force, ExistingConfigPolicy::Preserve)?; - let marker_action = - write_onboarding_marker(home, &marker_path, self.force, self.import_history)?; + let migration_state = if self.skip_v1_migration { + OnboardingMigrationState::ExplicitlySkipped + } else if migration_requested { + let source = source.ok_or_else(|| { + anyhow::anyhow!( + "--migrate-v1 was requested, but no v1 source was detected; set MIGRATION_SOURCE_POSTGRES or place a stopped-source snapshot at $IRONCLAW_BASE_DIR/ironclaw.db, then run `ironclaw-reborn migrate v1 plan --help`" + ) + })?; + let source_home = context.v1_source_home().ok_or_else(|| { + anyhow::anyhow!("could not resolve the v1 source home for migration planning") + })?; + crate::commands::migrate::plan_detected_v1(source, &source_home, &manifest_path)?; + OnboardingMigrationState::Planned + } else if source_detected { + OnboardingMigrationState::Available + } else { + OnboardingMigrationState::NotDetected + }; + let marker_action = write_onboarding_marker( + home, + &marker_path, + self.force || migration_requested || self.skip_v1_migration, + migration_state, + &manifest_path, + )?; println!("IronClaw Reborn onboarding"); println!("reborn_home: {}", home.path().display()); @@ -53,6 +126,7 @@ impl OnboardCommand { marker_action ); println!("v1_state: not-used"); + println!("v1_migration_state: {migration_state}"); println!(); println!("completed:"); println!("- reborn home initialized"); @@ -64,10 +138,18 @@ impl OnboardCommand { println!( "- run `ironclaw-reborn models set-provider --model ` as needed" ); - if self.import_history { - println!("- history import requested but not wired yet"); - } else { - println!("- history import not requested"); + match migration_state { + OnboardingMigrationState::Available => println!( + "- v1 data detected; review `ironclaw-reborn migrate v1 plan --help` before cutover" + ), + OnboardingMigrationState::Planned => println!( + "- review the v1 migration plan at {} before stopping v1 and applying", + manifest_path.display() + ), + OnboardingMigrationState::ExplicitlySkipped => { + println!("- v1 migration explicitly skipped") + } + OnboardingMigrationState::NotDetected => println!("- no v1 installation detected"), } Ok(()) } @@ -77,7 +159,19 @@ pub(crate) fn onboarding_marker_path(home: &RebornHome) -> PathBuf { home.path().join(ONBOARDING_MARKER_FILE) } -fn print_dry_run(home: &RebornHome, marker_path: &Path, force: bool, import_history: bool) { +pub(crate) fn default_migration_manifest_path(home: &RebornHome) -> PathBuf { + home.path().join("v1-migration-manifest.json") +} + +fn print_dry_run( + home: &RebornHome, + marker_path: &Path, + force: bool, + migration_requested: bool, + migration_skipped: bool, + source: Option<&V1MigrationSourceCandidate>, + manifest_path: &Path, +) { println!("IronClaw Reborn onboarding dry run"); println!("reborn_home: {}", home.path().display()); println!("home_source: {}", home.source_label()); @@ -96,7 +190,28 @@ fn print_dry_run(home: &RebornHome, marker_path: &Path, force: bool, import_hist "would_write" }; println!("{marker_action}: {}", marker_path.display()); - println!("import_history_requested: {import_history}"); + println!("migrate_v1_requested: {migration_requested}"); + println!("skip_v1_migration_requested: {migration_skipped}"); + println!( + "v1_migration_state: {}", + if migration_skipped { + "explicitly_skipped" + } else if migration_requested && source.is_some() { + "would_plan" + } else if source.is_some() { + "available" + } else { + "not_detected" + } + ); + if migration_requested && source.is_some() { + println!( + "would_write_migration_manifest: {}", + manifest_path.display() + ); + } else if migration_requested { + println!("migration_plan_blocker: no v1 source detected"); + } println!("v1_state: not-used"); } @@ -104,13 +219,14 @@ fn write_onboarding_marker( home: &RebornHome, marker_path: &Path, force: bool, - import_history: bool, + migration_state: OnboardingMigrationState, + manifest_path: &Path, ) -> anyhow::Result { if marker_path.exists() && !force { return Ok(FileWriteAction::Preserved); } let body = serde_json::to_string_pretty(&serde_json::json!({ - "schema_version": "ironclaw.reborn.onboarding/v1", + "schema_version": "ironclaw.reborn.onboarding/v2", "completed_at": chrono::Utc::now().to_rfc3339(), "reborn_home": home.path(), "home_source": home.source_label(), @@ -121,8 +237,12 @@ fn write_onboarding_marker( "config_files", "completion_marker" ], - "steps_pending": pending_steps(import_history), - "v1_state": "not-used" + "steps_pending": pending_steps(migration_state), + "v1_state": "not-used", + "v1_migration": { + "state": migration_state.as_str(), + "manifest": manifest_path, + } }))?; write_atomic( marker_path, @@ -132,10 +252,13 @@ fn write_onboarding_marker( ) } -fn pending_steps(import_history: bool) -> Vec<&'static str> { +fn pending_steps(migration_state: OnboardingMigrationState) -> Vec<&'static str> { let mut steps = vec!["llm_credentials", "model_selection", "channel_setup"]; - if import_history { - steps.push("history_import"); + if matches!( + migration_state, + OnboardingMigrationState::Available | OnboardingMigrationState::Planned + ) { + steps.push("v1_migration"); } steps } diff --git a/crates/ironclaw_reborn_cli/src/context.rs b/crates/ironclaw_reborn_cli/src/context.rs index 21e86e838f4..3b20577ef68 100644 --- a/crates/ironclaw_reborn_cli/src/context.rs +++ b/crates/ironclaw_reborn_cli/src/context.rs @@ -1,4 +1,14 @@ use ironclaw_reborn_config::RebornBootConfig; +use std::path::PathBuf; + +/// Non-secret evidence that a v1 source may be available for migration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum V1MigrationSourceCandidate { + LibSql(PathBuf), + /// The URL remains in `MIGRATION_SOURCE_POSTGRES`; only its presence is + /// represented here so it cannot leak into CLI diagnostics or argv. + PostgresEnvironment, +} /// Per-invocation context shared by Reborn CLI commands. #[derive(Debug, Clone)] @@ -34,4 +44,24 @@ impl RebornCliContext { pub(crate) fn boot_config(&self) -> &RebornBootConfig { &self.boot_config } + + pub(crate) fn v1_migration_source_candidate(&self) -> Option { + if std::env::var_os("MIGRATION_SOURCE_POSTGRES").is_some_and(|value| !value.is_empty()) { + return Some(V1MigrationSourceCandidate::PostgresEnvironment); + } + + let v1_base = self.v1_source_home()?; + let database = v1_base.join("ironclaw.db"); + database + .is_file() + .then_some(V1MigrationSourceCandidate::LibSql(database)) + } + + pub(crate) fn v1_source_home(&self) -> Option { + let v1_base = match std::env::var_os("IRONCLAW_BASE_DIR") { + Some(path) if !path.is_empty() => PathBuf::from(path), + _ => PathBuf::from(std::env::var_os("HOME")?).join(".ironclaw"), + }; + Some(v1_base) + } } diff --git a/crates/ironclaw_reborn_cli/tests/migration_cli.rs b/crates/ironclaw_reborn_cli/tests/migration_cli.rs new file mode 100644 index 00000000000..01160e99c6a --- /dev/null +++ b/crates/ironclaw_reborn_cli/tests/migration_cli.rs @@ -0,0 +1,321 @@ +#![cfg(unix)] + +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +const HANDSHAKE_SCHEMA: &str = "ironclaw.reborn.migration-companion/v1"; + +fn source_reborn_bin() -> &'static str { + env!("CARGO_BIN_EXE_ironclaw-reborn") +} + +struct InstalledPair { + _temp: tempfile::TempDir, + reborn: PathBuf, + capture: PathBuf, +} + +impl InstalledPair { + fn with_companion() -> Self { + let temp = tempfile::tempdir().expect("tempdir"); + let bin_dir = temp.path().join("bin"); + std::fs::create_dir(&bin_dir).expect("bin dir"); + set_mode(&bin_dir, 0o755); + let reborn = bin_dir.join("ironclaw-reborn"); + std::fs::copy(source_reborn_bin(), &reborn).expect("copy Reborn CLI"); + set_mode(&reborn, 0o755); + + let companion = bin_dir.join("ironclaw-reborn-migration"); + std::fs::write( + &companion, + "#!/bin/sh\n\ + if [ \"${1:-}\" = \"__handshake\" ]; then\n\ + printf '%s\\n' \"$MIGRATION_TEST_HANDSHAKE\"\n\ + exit 0\n\ + fi\n\ + printf '%s\\n' \"$@\" > \"$MIGRATION_TEST_CAPTURE\"\n\ + exit \"$MIGRATION_TEST_EXIT_CODE\"\n", + ) + .expect("write companion"); + set_mode(&companion, 0o755); + + let capture = temp.path().join("args.txt"); + Self { + _temp: temp, + reborn, + capture, + } + } + + fn without_companion() -> Self { + let temp = tempfile::tempdir().expect("tempdir"); + let bin_dir = temp.path().join("bin"); + std::fs::create_dir(&bin_dir).expect("bin dir"); + set_mode(&bin_dir, 0o755); + let reborn = bin_dir.join("ironclaw-reborn"); + std::fs::copy(source_reborn_bin(), &reborn).expect("copy Reborn CLI"); + set_mode(&reborn, 0o755); + let capture = temp.path().join("args.txt"); + Self { + _temp: temp, + reborn, + capture, + } + } + + fn run(&self, version: &str, exit_code: i32, args: &[&str]) -> Output { + let mut command = Command::new(&self.reborn); + command + .args(args) + .current_dir(self._temp.path()) + .env_clear() + .env("HOME", self._temp.path().join("home")) + .env("MIGRATION_TEST_CAPTURE", &self.capture) + .env("MIGRATION_TEST_EXIT_CODE", exit_code.to_string()) + .env("MIGRATION_TEST_HANDSHAKE", handshake_document(version)); + command.output().expect("run copied Reborn CLI") + } +} + +fn set_mode(path: &Path, mode: u32) { + let mut permissions = std::fs::metadata(path).expect("metadata").permissions(); + permissions.set_mode(mode); + std::fs::set_permissions(path, permissions).expect("set permissions"); +} + +fn handshake_document(version: &str) -> String { + serde_json::json!({ + "schema_version": HANDSHAKE_SCHEMA, + "protocol_version": 1, + "release_version": version, + }) + .to_string() +} + +#[test] +fn migrate_v1_requires_an_explicit_operation() { + let pair = InstalledPair::without_companion(); + let output = pair.run(env!("CARGO_PKG_VERSION"), 0, &["migrate", "v1"]); + + assert_eq!(output.status.code(), Some(2)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("Usage:"), "stderr: {stderr}"); + assert!(stderr.contains(""), "stderr: {stderr}"); +} + +#[test] +fn migrate_verify_help_describes_structural_readback() { + let pair = InstalledPair::without_companion(); + let output = pair.run( + env!("CARGO_PKG_VERSION"), + 0, + &["migrate", "v1", "verify", "--help"], + ); + + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("structural durable-store readback"), + "stdout: {stdout}" + ); + assert!( + !stdout.contains("production Reborn services"), + "stdout: {stdout}" + ); +} + +#[test] +fn migrate_rejects_a_missing_sibling_without_searching_path() { + let pair = InstalledPair::without_companion(); + let other_bin = pair._temp.path().join("other-bin"); + std::fs::create_dir(&other_bin).expect("other bin"); + let path_companion = other_bin.join("ironclaw-reborn-migration"); + std::fs::write(&path_companion, "#!/bin/sh\nexit 0\n").expect("PATH companion"); + set_mode(&path_companion, 0o755); + + let output = Command::new(&pair.reborn) + .args(["migrate", "v1", "status", "--manifest", "manifest.json"]) + .current_dir(pair._temp.path()) + .env_clear() + .env("HOME", pair._temp.path().join("home")) + .env("PATH", &other_bin) + .output() + .expect("run Reborn CLI"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("companion is missing"), "stderr: {stderr}"); + assert!( + stderr.contains(pair.reborn.parent().unwrap().to_string_lossy().as_ref()), + "stderr: {stderr}" + ); +} + +#[test] +fn migrate_rejects_a_companion_from_another_release() { + let pair = InstalledPair::with_companion(); + let output = pair.run( + "999.0.0", + 0, + &["migrate", "v1", "status", "--manifest", "manifest.json"], + ); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!(stderr.contains("release mismatch"), "stderr: {stderr}"); + assert!(stderr.contains("999.0.0"), "stderr: {stderr}"); + assert!(!pair.capture.exists(), "operation must not be forwarded"); +} + +#[test] +fn migrate_rejects_a_companion_from_a_group_writable_install_directory() { + let pair = InstalledPair::with_companion(); + set_mode(pair.reborn.parent().unwrap(), 0o775); + let output = pair.run( + env!("CARGO_PKG_VERSION"), + 0, + &["migrate", "v1", "status", "--manifest", "manifest.json"], + ); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("installation directory") && stderr.contains("writable by another user"), + "stderr: {stderr}" + ); + assert!(!pair.capture.exists(), "operation must not be forwarded"); +} + +#[test] +fn migrate_forwards_only_redacted_plan_options_and_propagates_exit_status() { + let pair = InstalledPair::with_companion(); + let snapshot = pair._temp.path().join("v1-snapshot.db"); + let manifest = pair._temp.path().join("migration.json"); + let output = Command::new(&pair.reborn) + .args([ + "migrate", + "v1", + "plan", + "--source-libsql", + snapshot.to_str().unwrap(), + "--manifest", + manifest.to_str().unwrap(), + "--strict", + ]) + .current_dir(pair._temp.path()) + .env_clear() + .env("HOME", pair._temp.path().join("home")) + .env("MIGRATION_TEST_CAPTURE", &pair.capture) + .env("MIGRATION_TEST_EXIT_CODE", "23") + .env( + "MIGRATION_TEST_HANDSHAKE", + handshake_document(env!("CARGO_PKG_VERSION")), + ) + .env( + "MIGRATION_SOURCE_POSTGRES", + "postgres://user:secret@example.invalid/db", + ) + .env("MIGRATION_SOURCE_SECRET_MASTER_KEY", "source-secret") + .env("MIGRATION_TARGET_SECRET_MASTER_KEY", "target-secret") + .output() + .expect("run Reborn CLI"); + + assert_eq!(output.status.code(), Some(23)); + let forwarded = std::fs::read_to_string(&pair.capture).expect("captured args"); + assert_eq!( + forwarded.lines().collect::>(), + vec![ + "v1", + "plan", + "--source-libsql", + snapshot.to_str().unwrap(), + "--manifest", + manifest.to_str().unwrap(), + "--strict", + ] + ); + assert!(!forwarded.contains("postgres://")); + assert!(!forwarded.contains("source-secret")); + assert!(!forwarded.contains("target-secret")); +} + +#[test] +fn migrate_apply_requires_the_snapshot_confirmation_before_launch() { + let pair = InstalledPair::with_companion(); + let output = pair.run( + env!("CARGO_PKG_VERSION"), + 0, + &[ + "migrate", + "v1", + "apply", + "--source-libsql", + "snapshot.db", + "--plan", + "manifest.json", + "--confirm-v1-stopped", + ], + ); + + assert_eq!(output.status.code(), Some(2)); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("--confirm-source-snapshot"), + "stderr: {stderr}" + ); + assert!(!pair.capture.exists(), "operation must not be forwarded"); +} + +#[test] +fn onboard_migrate_v1_invokes_plan_only_after_an_explicit_flag() { + let pair = InstalledPair::with_companion(); + let v1_home = pair._temp.path().join("v1-home"); + let reborn_home = pair._temp.path().join("reborn-home"); + std::fs::create_dir(&v1_home).expect("v1 home"); + let source = v1_home.join("ironclaw.db"); + std::fs::write(&source, b"snapshot evidence").expect("source evidence"); + + let output = Command::new(&pair.reborn) + .args(["onboard", "--migrate-v1"]) + .current_dir(pair._temp.path()) + .env_clear() + .env("HOME", pair._temp.path().join("home")) + .env("IRONCLAW_BASE_DIR", &v1_home) + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .env("MIGRATION_TEST_CAPTURE", &pair.capture) + .env("MIGRATION_TEST_EXIT_CODE", "0") + .env( + "MIGRATION_TEST_HANDSHAKE", + handshake_document(env!("CARGO_PKG_VERSION")), + ) + .output() + .expect("run onboarding"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let forwarded = std::fs::read_to_string(&pair.capture).expect("captured args"); + assert_eq!( + forwarded.lines().collect::>(), + vec![ + "v1", + "plan", + "--source-libsql", + source.to_str().unwrap(), + "--source-home", + v1_home.to_str().unwrap(), + "--manifest", + reborn_home + .join("v1-migration-manifest.json") + .to_str() + .unwrap(), + ] + ); + let marker = std::fs::read_to_string(reborn_home.join(".onboard-completed.json")) + .expect("onboarding marker"); + let marker: serde_json::Value = serde_json::from_str(&marker).expect("marker JSON"); + assert_eq!(marker["v1_migration"]["state"], "planned"); +} diff --git a/crates/ironclaw_reborn_cli/tests/smoke.rs b/crates/ironclaw_reborn_cli/tests/smoke.rs index 68a7bd0deab..86be0379e0e 100644 --- a/crates/ironclaw_reborn_cli/tests/smoke.rs +++ b/crates/ironclaw_reborn_cli/tests/smoke.rs @@ -1,3 +1,5 @@ +// arch-exempt: large_file, split smoke coverage by command and deployment surface, plan #8513 + #[cfg(feature = "webui-v2-beta")] use std::io::BufRead; use std::{ @@ -151,6 +153,21 @@ fn dockerfile_reborn_builds_with_postgres_feature() { !dockerfile.contains("\nVOLUME "), "Railway's Dockerfile builder rejects Docker VOLUME instructions; configure Railway volumes outside the image: {dockerfile}" ); + assert!( + dockerfile.contains("--package ironclaw_reborn_migration") + && dockerfile.contains("--bin ironclaw-reborn-migration") + && dockerfile.contains( + "COPY --from=builder /app/target/dist/ironclaw-reborn-migration /usr/local/bin/ironclaw-reborn-migration" + ), + "Dockerfile.reborn must build and install the same-image migration companion: {dockerfile}" + ); + assert!( + dockerfile.matches("COPY src/ src/").count() == 2 + && dockerfile.matches("COPY build.rs build.rs").count() == 2 + && dockerfile.contains("channels-src/telegram/telegram.capabilities.json") + && dockerfile.contains("channels-src/discord/discord.capabilities.json"), + "Dockerfile.reborn must provide the legacy library sources and compile-time assets needed by the migration companion: {dockerfile}" + ); } #[test] @@ -267,6 +284,47 @@ fn docker_reborn_entrypoint_uses_railway_volume_mount_for_home() { assert!(stdout.contains("args=--help"), "stdout: {stdout}"); } +#[cfg(unix)] +#[test] +fn docker_reborn_entrypoint_does_not_seed_target_for_migration_commands() { + let temp = tempfile::tempdir().expect("tempdir"); + let bin_dir = temp.path().join("bin"); + fake_reborn_bin(&bin_dir); + let reborn_home = temp.path().join("missing-reborn-home"); + + for args in [ + &["migrate", "v1", "--help"][..], + &["migrate", "v1", "apply", "--help"][..], + &["migrate", "v1", "resume", "-h"][..], + &["migrate", "v1", "verify", "--help"][..], + ] { + let output = Command::new("/bin/sh") + .arg(workspace_root().join("docker/reborn/entrypoint.sh")) + .args(args) + .env_clear() + .env("PATH", fake_bin_path(&bin_dir)) + .env("HOME", temp.path().join("home")) + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .output() + .expect("entrypoint should run"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(&format!("args={}", args.join(" "))), + "stdout: {stdout}" + ); + assert!( + !reborn_home.exists(), + "migration help dispatch must not create target state" + ); + } +} + #[cfg(unix)] #[test] fn docker_reborn_entrypoint_rejects_ephemeral_railway_without_volume() { @@ -443,6 +501,7 @@ fn help_mentions_reborn_commands() { assert!(stdout.contains("extension"), "stdout: {stdout}"); assert!(stdout.contains("hooks"), "stdout: {stdout}"); assert!(stdout.contains("logs"), "stdout: {stdout}"); + assert!(stdout.contains("migrate"), "stdout: {stdout}"); assert!(stdout.contains("models"), "stdout: {stdout}"); assert!(stdout.contains("onboard"), "stdout: {stdout}"); assert!(stdout.contains("profile"), "stdout: {stdout}"); @@ -1928,6 +1987,50 @@ fn run_reports_runtime_readiness_snapshot_without_touching_v1_state() { ); } +#[test] +#[cfg(not(any(feature = "libsql", feature = "postgres")))] +fn run_fails_closed_for_a_non_default_applied_migration_manifest() { + let temp = tempfile::tempdir().expect("tempdir"); + let reborn_home = temp.path().join("reborn-home"); + std::fs::create_dir(&reborn_home).expect("Reborn home"); + let non_default_manifest = temp.path().join("operator-selected-manifest.json"); + std::fs::write(&non_default_manifest, r#"{"status":"applied"}"#).expect("migration manifest"); + std::fs::write( + reborn_home.join(".v1-migration-state.json"), + serde_json::json!({ + "schema_version": "ironclaw.reborn.migration-state/v1", + "migration_protocol_version": 1, + "release_version": env!("CARGO_PKG_VERSION"), + "run_id": "01KXE6YQKR3047DV18GG8DCWDG", + "status": "applied", + "profile": "local-dev", + "target_backend": "libsql", + "target_locator_fingerprint": "test-only-unresolved-target", + "tenant_id": "local", + "agent_id": "default", + "manifest": non_default_manifest, + }) + .to_string(), + ) + .expect("target-owned migration state"); + + let output = Command::new(reborn_bin()) + .args(["run", "--dry-run"]) + .env_clear() + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .output() + .expect("ironclaw-reborn run should fail closed"); + + assert!(!output.status.success()); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains( + "migration target inspection requires a binary built with libsql or postgres" + ), + "stderr: {stderr}" + ); +} + #[test] fn doctor_uses_reborn_home_override_without_touching_v1_state() { let temp = tempfile::tempdir().expect("tempdir"); @@ -1957,8 +2060,10 @@ fn doctor_uses_reborn_home_override_without_touching_v1_state() { assert!(stdout.contains("local-dev"), "stdout: {stdout}"); assert!(stdout.contains("text_only_driver"), "stdout: {stdout}"); assert!( - !stdout.contains("v1_state"), - "doctor output should not include v1_state" + stdout + .lines() + .any(|line| line.contains("v1_state") && line.contains("not-used")), + "stdout: {stdout}" ); assert!( !reborn_home.exists(), @@ -1966,6 +2071,63 @@ fn doctor_uses_reborn_home_override_without_touching_v1_state() { ); } +#[test] +#[cfg(not(any(feature = "libsql", feature = "postgres")))] +fn doctor_reports_unverifiable_migration_state_without_a_storage_backend() { + let temp = tempfile::tempdir().expect("tempdir"); + let reborn_home = temp.path().join("reborn-home"); + std::fs::create_dir(&reborn_home).expect("Reborn home"); + let manifest = reborn_home.join("v1-migration-manifest.json"); + std::fs::write(&manifest, r#"{"status":"applied"}"#).expect("manifest"); + std::fs::write( + reborn_home.join(".v1-migration-state.json"), + serde_json::json!({ + "schema_version": "ironclaw.reborn.migration-state/v1", + "migration_protocol_version": 1, + "release_version": env!("CARGO_PKG_VERSION"), + "run_id": "01KXE6YQKR3047DV18GG8DCWDG", + "status": "applied", + "profile": "local-dev", + "target_backend": "libsql", + "target_locator_fingerprint": "test-only-unresolved-target", + "tenant_id": "local", + "agent_id": "default", + "manifest": manifest, + }) + .to_string(), + ) + .expect("target-owned migration state"); + std::fs::write( + reborn_home.join(".onboard-completed.json"), + serde_json::json!({ + "v1_migration": { + "state": "planned", + "manifest": manifest, + } + }) + .to_string(), + ) + .expect("onboarding marker"); + + let output = Command::new(reborn_bin()) + .arg("doctor") + .env_clear() + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .output() + .expect("ironclaw-reborn doctor should run"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("v1_migration_state: invalid"), + "stdout: {stdout}" + ); +} + #[test] fn repl_help_mentions_composed_runtime() { let output = Command::new(reborn_bin()) @@ -2877,14 +3039,56 @@ fn onboard_bootstraps_reborn_home_without_touching_v1_state() { assert!(marker_path.exists(), "onboarding marker missing"); let marker_text = std::fs::read_to_string(marker_path).expect("read marker"); let marker: serde_json::Value = serde_json::from_str(&marker_text).expect("valid marker JSON"); - assert_eq!(marker["schema_version"], "ironclaw.reborn.onboarding/v1"); + assert_eq!(marker["schema_version"], "ironclaw.reborn.onboarding/v2"); assert_eq!(marker["v1_state"], "not-used"); + assert_eq!(marker["v1_migration"]["state"], "not_detected"); assert!( !v1_home.exists(), "onboard must not create or read explicit v1 state" ); } +#[test] +fn onboard_reports_detected_v1_without_starting_migration() { + let temp = tempfile::tempdir().expect("tempdir"); + let reborn_home = temp.path().join("reborn-home"); + let v1_home = temp.path().join("v1-home"); + std::fs::create_dir(&v1_home).expect("v1 home"); + std::fs::write(v1_home.join("ironclaw.db"), b"snapshot evidence") + .expect("v1 database evidence"); + + let output = Command::new(reborn_bin()) + .arg("onboard") + .env_clear() + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .env("IRONCLAW_BASE_DIR", &v1_home) + .output() + .expect("ironclaw-reborn onboard should run"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains("v1_migration_state: available"), + "stdout: {stdout}" + ); + assert!( + stdout.contains("ironclaw-reborn migrate v1 plan --help"), + "stdout: {stdout}" + ); + let marker_text = + std::fs::read_to_string(reborn_home.join(".onboard-completed.json")).expect("read marker"); + let marker: serde_json::Value = serde_json::from_str(&marker_text).expect("marker JSON"); + assert_eq!(marker["v1_migration"]["state"], "available"); + assert!( + !reborn_home.join("v1-migration-manifest.json").exists(), + "detection must not run plan automatically" + ); +} + #[test] fn onboard_dry_run_is_read_only() { let temp = tempfile::tempdir().expect("tempdir"); @@ -2908,7 +3112,7 @@ fn onboard_dry_run_is_read_only() { "stdout: {stdout}" ); assert!( - stdout.contains("import_history_requested: true"), + stdout.contains("migrate_v1_requested: true"), "stdout: {stdout}" ); assert!(!reborn_home.exists(), "dry-run must not create Reborn home"); @@ -2944,16 +3148,16 @@ fn onboard_dry_run_reports_existing_marker_as_preserved() { } #[test] -fn onboard_import_history_records_pending_step() { +fn onboard_can_record_an_explicit_v1_migration_skip() { let temp = tempfile::tempdir().expect("tempdir"); let reborn_home = temp.path().join("reborn-home"); let output = Command::new(reborn_bin()) - .args(["onboard", "--import-history"]) + .args(["onboard", "--skip-v1-migration"]) .env_clear() .env("IRONCLAW_REBORN_HOME", &reborn_home) .output() - .expect("ironclaw-reborn onboard --import-history should run"); + .expect("ironclaw-reborn onboard --skip-v1-migration should run"); assert!( output.status.success(), @@ -2963,13 +3167,9 @@ fn onboard_import_history_records_pending_step() { let marker_text = std::fs::read_to_string(reborn_home.join(".onboard-completed.json")).expect("read marker"); let marker: serde_json::Value = serde_json::from_str(&marker_text).expect("valid marker JSON"); - let pending = marker["steps_pending"] - .as_array() - .expect("pending steps array"); - assert!( - pending.iter().any(|step| step == "history_import"), - "marker should record history import as pending: {marker_text}" - ); + assert_eq!(marker["v1_migration"]["state"], "explicitly_skipped"); + let pending = marker["steps_pending"].as_array().expect("pending steps"); + assert!(!pending.iter().any(|step| step == "v1_migration")); } #[test] @@ -3059,7 +3259,7 @@ fn onboard_with_force_overwrites_existing_files_and_marker() { assert!(config_text.contains("api_version = \"ironclaw.runtime/v1\"")); assert!(providers_text.contains("\"id\": \"acme-openrouter\"")); let marker: serde_json::Value = serde_json::from_str(&marker_text).expect("valid marker JSON"); - assert_eq!(marker["schema_version"], "ironclaw.reborn.onboarding/v1"); + assert_eq!(marker["schema_version"], "ironclaw.reborn.onboarding/v2"); } #[test] diff --git a/crates/ironclaw_reborn_composition/AGENTS.md b/crates/ironclaw_reborn_composition/AGENTS.md index 2f4c5ab8ff7..179f8614e22 100644 --- a/crates/ironclaw_reborn_composition/AGENTS.md +++ b/crates/ironclaw_reborn_composition/AGENTS.md @@ -16,6 +16,7 @@ - Top-level factories that expose `HostRuntime`, `TurnCoordinator`, readiness, runtime/profile inputs, and LLM catalog wiring: `RebornServices`/`build_reborn_services` (`factory`), `RebornBuildInput`/`RebornBuildError`, and the feature-gated LLM catalog resolvers (`llm_admin::llm_catalog`). - The `RebornRuntime` conversation-level facade (`RebornRuntime`/`build_reborn_runtime`, `AssistantReply`, `ConversationId`, `RebornRuntimeError`) and its runtime inputs (`RebornRuntimeInput`/`RebornRuntimeIdentity`, `TurnRunnerSettings`/`PollSettings`, heartbeat/poll-interval defaults). - Product-live adapter wiring (`product_live_adapters`): `ProductLivePlannedRuntimeAdapters`, capability authority/IO/model-route settings, `capability_allowlist`, `visible_capability_request_for_run`; and the WebUI facade (`webui`). +- Feature-gated, non-activating migration target resolution (`migration_support`); the migration crate owns v1 reads and conversion. - Production and migration-dry-run profile validation for required handles (`profile`, `readiness`). ## Do Not Move In Here diff --git a/crates/ironclaw_reborn_composition/CLAUDE.md b/crates/ironclaw_reborn_composition/CLAUDE.md index 0aa2aaab46a..f52536e62bf 100644 --- a/crates/ironclaw_reborn_composition/CLAUDE.md +++ b/crates/ironclaw_reborn_composition/CLAUDE.md @@ -7,6 +7,9 @@ - Outbound state stores are composition-owned via `RebornLocalRuntimeServices`; do not construct `FilesystemOutboundStateStore` in consumer modules (lint-enforced via `clippy::disallowed-methods`). - Do not depend on the root `ironclaw` crate or `src/` modules. - Do not add legacy bridge modes here until an accepted migration contract exists. +- The feature-gated `migration_support` module only resolves the production + target locator, scope, and encryption configuration without activation. The + migration companion owns every v1 source adapter and conversion bridge. - Do not route live v1/product traffic here; callers must opt in through explicit Reborn adapters. - Production and migration-dry-run profiles must fail closed on local-only or missing required handles. - Product auth composition must use `ironclaw_auth` trait-shaped ports. Do not diff --git a/crates/ironclaw_reborn_composition/src/admin_user_directory.rs b/crates/ironclaw_reborn_composition/src/admin_user_directory.rs index 967ba8c8210..c9b3db950dc 100644 --- a/crates/ironclaw_reborn_composition/src/admin_user_directory.rs +++ b/crates/ironclaw_reborn_composition/src/admin_user_directory.rs @@ -301,9 +301,9 @@ fn map_identity_error(error: RebornIdentityError) -> AdminUserError { RebornIdentityError::Backend(_) => AdminUserError::Unavailable, // A persisted-id inconsistency or a channel-actor misuse is not // retryable and not the client's fault. - RebornIdentityError::InvalidUserId(_) | RebornIdentityError::ChannelActorNotMintable => { - AdminUserError::Internal - } + RebornIdentityError::InvalidUserId(_) + | RebornIdentityError::UserImportConflict(_) + | RebornIdentityError::ChannelActorNotMintable => AdminUserError::Internal, // Only `resolve_or_create` (the SSO login path) raises this; the admin // directory operations never resolve external identities, so reaching // it here is a backend inconsistency rather than the client's fault. diff --git a/crates/ironclaw_reborn_composition/src/factory.rs b/crates/ironclaw_reborn_composition/src/factory.rs index fba4f664bf4..f65e4e391b6 100644 --- a/crates/ironclaw_reborn_composition/src/factory.rs +++ b/crates/ironclaw_reborn_composition/src/factory.rs @@ -3425,7 +3425,7 @@ fn validate_resolved_master_key( } #[cfg(any(feature = "libsql", feature = "postgres"))] -fn resolve_local_dev_secret_master_key( +pub(crate) fn resolve_local_dev_secret_master_key( root: &Path, ) -> Result { // Fail closed on an explicitly-set-but-unusable master key: only an diff --git a/crates/ironclaw_reborn_composition/src/input.rs b/crates/ironclaw_reborn_composition/src/input.rs index b5127f7495f..ad5ae9fdd69 100644 --- a/crates/ironclaw_reborn_composition/src/input.rs +++ b/crates/ironclaw_reborn_composition/src/input.rs @@ -814,11 +814,64 @@ struct ResolvedPostgresStorage { process_local_resource_governor_singleton: bool, } +#[cfg(feature = "postgres")] +pub(crate) struct ResolvedPostgresTargetConfig { + pub(crate) url: ironclaw_secrets::SecretMaterial, + pub(crate) secret_master_key: ironclaw_secrets::SecretMaterial, +} + #[cfg(feature = "postgres")] fn resolve_postgres_storage_from_config_and_env( profile: RebornCompositionProfile, config_file: Option<&ironclaw_reborn_config::RebornConfigFile>, ) -> Result { + let ResolvedPostgresTargetConfig { + url: database_url, + secret_master_key, + } = resolve_postgres_target_config(profile, config_file)?; + let process_local_resource_governor_singleton = + require_postgres_resource_governor_singleton_env()?; + let storage = config_file + .and_then(|file| file.storage.as_ref()) + .ok_or_else(|| RebornBuildError::InvalidConfig { + reason: format!( + "profile={profile} requires [storage] backend = \"postgres\" with url_env naming \ + an environment variable such as {DEFAULT_REBORN_POSTGRES_URL_ENV}" + ), + })?; + let (pool_max_size, pool_max_size_source) = + resolve_postgres_pool_max_size(storage.pool_max_size)?; + tracing::debug!( + %profile, + pool_max_size, + pool_max_size_source, + "resolved Reborn PostgreSQL pool size" + ); + let tls_options = postgres_pool_tls_options_from_env()?; + let pool = ironclaw_reborn_event_store::open_postgres_pool_with_tls_options( + database_url.clone(), + pool_max_size, + tls_options, + )?; + + Ok(ResolvedPostgresStorage { + pool, + url: database_url, + tls_options, + secret_master_key, + process_local_resource_governor_singleton, + }) +} + +/// Resolve the production target locator and encryption key without opening a +/// pool or enforcing runtime-worker settings. Both runtime construction and +/// migration target discovery use this function so their storage selection +/// cannot drift. +#[cfg(feature = "postgres")] +fn resolve_postgres_target_config( + profile: RebornCompositionProfile, + config_file: Option<&ironclaw_reborn_config::RebornConfigFile>, +) -> Result { let storage = config_file .and_then(|file| file.storage.as_ref()) .ok_or_else(|| RebornBuildError::InvalidConfig { @@ -857,32 +910,24 @@ fn resolve_postgres_storage_from_config_and_env( "Reborn secret master key", "storage.secret_master_key_env", )?; - let process_local_resource_governor_singleton = - require_postgres_resource_governor_singleton_env()?; - let (pool_max_size, pool_max_size_source) = - resolve_postgres_pool_max_size(storage.pool_max_size)?; - tracing::debug!( - %profile, - pool_max_size, - pool_max_size_source, - "resolved Reborn PostgreSQL pool size" - ); - let tls_options = postgres_pool_tls_options_from_env()?; - let pool = ironclaw_reborn_event_store::open_postgres_pool_with_tls_options( - database_url.clone(), - pool_max_size, - tls_options, - )?; - Ok(ResolvedPostgresStorage { - pool, + Ok(ResolvedPostgresTargetConfig { url: database_url, - tls_options, secret_master_key, - process_local_resource_governor_singleton, }) } +/// Resolve the exact PostgreSQL URL and secret key used by production without +/// constructing a runtime or starting workers. The migration companion maps +/// this composition-owned result into its own target types. +#[cfg(all(feature = "migration-support", feature = "postgres"))] +pub(crate) fn resolve_postgres_migration_target( + profile: RebornCompositionProfile, + config_file: Option<&ironclaw_reborn_config::RebornConfigFile>, +) -> Result { + resolve_postgres_target_config(profile, config_file) +} + #[cfg(feature = "postgres")] fn resolve_production_runtime_policy( profile: RebornCompositionProfile, @@ -1034,7 +1079,8 @@ fn require_postgres_resource_governor_singleton_env() -> Result Result { +pub(crate) fn postgres_pool_tls_options_from_env() +-> Result { let ssl_mode_override = match std::env::var(DATABASE_SSLMODE_ENV) { Ok(value) if value.trim().is_empty() => None, Ok(value) => Some( diff --git a/crates/ironclaw_reborn_composition/src/lib.rs b/crates/ironclaw_reborn_composition/src/lib.rs index 416b02d11ba..bcc9570c3e8 100644 --- a/crates/ironclaw_reborn_composition/src/lib.rs +++ b/crates/ironclaw_reborn_composition/src/lib.rs @@ -41,6 +41,8 @@ mod local_dev_authorization; mod local_dev_capability_policy; mod local_dev_mounts; mod local_runtime_profile; +#[cfg(feature = "migration-support")] +mod migration_support; mod observability; mod outbound; mod product_auth; @@ -148,6 +150,24 @@ pub use local_runtime_profile::{ local_dev_runtime_policy, local_dev_yolo_runtime_policy, local_runtime_build_input, local_runtime_build_input_with_options, }; +#[cfg(all(feature = "migration-support", feature = "libsql"))] +pub use migration_support::migration_libsql_locator_fingerprint; +#[cfg(all(feature = "migration-support", feature = "postgres"))] +pub use migration_support::migration_postgres_locator_fingerprint; +/// Re-exported for the migration companion's local apply path so it can use +/// production key resolution without depending on composition internals. The +/// `reborn_crate_dependency_boundaries_hold` architecture test constrains the +/// companion to this non-activating facade. +#[cfg(all(feature = "migration-support", feature = "libsql"))] +pub use migration_support::resolve_local_migration_target_key; +/// Re-exported for the Reborn CLI and migration companion so target selection +/// follows production composition without either caller opening the runtime. +/// `composition_public_pub_use_surface_matches_snapshot` pins this intentional +/// facade surface. +#[cfg(feature = "migration-support")] +pub use migration_support::{ + RebornMigrationTargetConfig, RebornMigrationTargetStore, resolve_reborn_migration_target, +}; pub use observability::budget::build_default_budget_accountant; pub use observability::budget_events::{BudgetEventObserver, TracingBudgetEventObserver}; pub use observability::hooks::{ @@ -903,7 +923,18 @@ where pub fn open_reborn_postgres_pool( url: secrecy::SecretString, ) -> Result { - Ok(ironclaw_reborn_event_store::open_postgres_pool(url)?) + let tls_options = input::postgres_pool_tls_options_from_env().map_err(|error| { + RebornCompositionError::InvalidConfig { + reason: error.to_string(), + } + })?; + Ok( + ironclaw_reborn_event_store::open_postgres_pool_with_tls_options( + url, + ironclaw_reborn_event_store::DEFAULT_POSTGRES_POOL_MAX_SIZE, + tls_options, + )?, + ) } /// Open a PostgreSQL pool for Reborn production storage with an explicit diff --git a/crates/ironclaw_reborn_composition/src/migration_support.rs b/crates/ironclaw_reborn_composition/src/migration_support.rs new file mode 100644 index 00000000000..a3ad07382b8 --- /dev/null +++ b/crates/ironclaw_reborn_composition/src/migration_support.rs @@ -0,0 +1,449 @@ +//! Non-activating target configuration resolution for migration. +//! +//! This module deliberately returns configuration only. It does not open the +//! target filesystem, run schema migrations, construct workers, or start +//! ingress. PostgreSQL locator/key selection is shared with runtime +//! composition, and local target paths use the canonical profile layout. +//! Profile and identity precedence mirror the standalone CLI and are parity +//! tested here; they are not yet selected through one shared CLI/composition +//! entry point. The migration companion owns the legacy bridge and maps these +//! values into its target selector. + +#[cfg(feature = "libsql")] +use std::path::Path; +#[cfg(feature = "libsql")] +use std::path::PathBuf; + +use ironclaw_host_api::{AgentId, TenantId}; +use ironclaw_reborn_config::{ + REBORN_PROFILE_ENV, RebornBootConfig, RebornConfigFile, RebornProfile, +}; +use ironclaw_secrets::SecretMaterial; +#[cfg(feature = "postgres")] +use secrecy::ExposeSecret as _; + +use crate::{RebornBuildError, RebornRuntimeIdentity}; +#[cfg(feature = "postgres")] +use crate::{RebornCompositionProfile, input}; + +/// Production-selected durable target, expressed without a dependency on the +/// migration crate (which itself depends on composition). +#[derive(Clone)] +pub enum RebornMigrationTargetStore { + /// Canonical local/volume-backed target database path. + #[cfg(feature = "libsql")] + LibSql { + /// Database path selected by the production profile layout. + path: PathBuf, + }, + /// Secret-bearing production PostgreSQL locator. Callers must not log or + /// serialize it into migration artifacts. + #[cfg(feature = "postgres")] + Postgres { + /// Production target URL held in redacting secret material. + url: SecretMaterial, + }, +} + +/// Resolved migration target scope and encryption material. +#[derive(Clone)] +pub struct RebornMigrationTargetConfig { + /// Effective production profile. + pub profile: RebornProfile, + /// Config-only target selector; resolving it does not open the store. + pub store: RebornMigrationTargetStore, + /// Target tenant scope. + pub tenant_id: TenantId, + /// Target agent scope. + pub agent_id: AgentId, + /// PostgreSQL resolves its configured key without opening the database. + /// Local libSQL resolves/generates the cached production key only after + /// apply preconditions, when the migration target is opened. + pub target_master_key: Option, +} + +/// Resolve migration target configuration without activating the runtime. +/// +/// Storage locator/key rules are shared with runtime composition. Profile and +/// identity precedence intentionally mirror the standalone CLI until the CLI +/// routes both live boot and migration through a single resolver. +pub fn resolve_reborn_migration_target( + boot: &RebornBootConfig, +) -> Result { + let config_file = RebornConfigFile::load(&boot.home().config_file_path()).map_err(|error| { + RebornBuildError::InvalidConfig { + reason: format!("Reborn migration target config could not be loaded: {error}"), + } + })?; + let profile = effective_profile(boot, config_file.as_ref())?; + if config_file + .as_ref() + .and_then(|file| file.storage.as_ref()) + .is_some() + && matches!( + profile, + RebornProfile::LocalDev + | RebornProfile::LocalDevYolo + | RebornProfile::HostedSingleTenantVolume + ) + { + return Err(RebornBuildError::InvalidConfig { + reason: format!( + "config file [storage] is not wired for profile={profile}; migration target resolution refuses to ignore it" + ), + }); + } + let identity = runtime_identity(config_file.as_ref()); + let tenant_id = + TenantId::new(identity.tenant_id).map_err(|error| RebornBuildError::InvalidConfig { + reason: format!("invalid migration target tenant identity: {error}"), + })?; + let agent_id = + AgentId::new(identity.agent_id).map_err(|error| RebornBuildError::InvalidConfig { + reason: format!("invalid migration target agent identity: {error}"), + })?; + let (store, target_master_key) = match profile { + RebornProfile::LocalDev + | RebornProfile::LocalDevYolo + | RebornProfile::HostedSingleTenantVolume => { + #[cfg(feature = "libsql")] + { + let root = boot + .home() + .path() + .join(profile.local_runtime_storage_subdir()); + ( + RebornMigrationTargetStore::LibSql { + path: root.join(crate::factory::LOCAL_DEV_DB_FILENAME), + }, + None, + ) + } + #[cfg(not(feature = "libsql"))] + { + return Err(RebornBuildError::InvalidConfig { + reason: format!( + "profile={profile} migration target requires a binary built with libsql" + ), + }); + } + } + RebornProfile::HostedSingleTenant + | RebornProfile::Production + | RebornProfile::MigrationDryRun => { + #[cfg(feature = "postgres")] + { + let composition_profile = profile + .as_str() + .parse::() + .map_err(|error| RebornBuildError::InvalidConfig { + reason: format!("invalid migration composition profile: {error}"), + })?; + let target = input::resolve_postgres_migration_target( + composition_profile, + config_file.as_ref(), + )?; + ( + RebornMigrationTargetStore::Postgres { url: target.url }, + Some(target.secret_master_key), + ) + } + #[cfg(not(feature = "postgres"))] + { + return Err(RebornBuildError::InvalidConfig { + reason: format!( + "profile={profile} migration target requires a binary built with postgres" + ), + }); + } + } + }; + + Ok(RebornMigrationTargetConfig { + profile, + store, + tenant_id, + agent_id, + target_master_key, + }) +} + +/// Compute the redacted local target identity consumed by the migration +/// companion and CLI activation guard. The composition public-surface snapshot +/// pins this shared boundary so both callers compare the same locator. +#[cfg(feature = "libsql")] +pub fn migration_libsql_locator_fingerprint(path: &Path) -> String { + use std::path::Component; + + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from(".")) + .join(path) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + other => normalized.push(other.as_os_str()), + } + } + let mut ancestor = normalized.clone(); + let mut missing_suffix = Vec::new(); + while !ancestor.exists() { + let Some(name) = ancestor.file_name() else { + break; + }; + missing_suffix.push(name.to_os_string()); + if !ancestor.pop() { + break; + } + } + let mut resolved = ancestor.canonicalize().unwrap_or(ancestor); + for component in missing_suffix.into_iter().rev() { + resolved.push(component); + } + ironclaw_common::hashing::sha256_hex(resolved.as_os_str().as_encoded_bytes()) +} + +/// Compute the credential-free PostgreSQL target identity consumed by the +/// migration companion and CLI activation guard. +#[cfg(feature = "postgres")] +pub fn migration_postgres_locator_fingerprint( + locator: &SecretMaterial, +) -> Result { + use deadpool_postgres::tokio_postgres::config::Host; + + let config = locator + .expose_secret() + .parse::() + .map_err(|_| RebornBuildError::InvalidConfig { + reason: "PostgreSQL migration target locator is invalid (details redacted)".to_string(), + })?; + if config.get_options().is_some() { + return Err(RebornBuildError::InvalidConfig { + reason: "PostgreSQL connection options are not supported for migration target identity (details redacted)".to_string(), + }); + } + let mut material = Vec::new(); + append_locator_field(&mut material, b"schema", b"postgres-locator-v1"); + append_locator_field( + &mut material, + b"database", + config + .get_dbname() + .or_else(|| config.get_user()) + .unwrap_or_default() + .as_bytes(), + ); + for (index, host) in config.get_hosts().iter().enumerate() { + let label = format!("host-{index}"); + match host { + Host::Tcp(host) => { + append_locator_field(&mut material, label.as_bytes(), host.as_bytes()) + } + #[cfg(unix)] + Host::Unix(path) => append_locator_field( + &mut material, + label.as_bytes(), + path.as_os_str().as_encoded_bytes(), + ), + } + let port = config.get_ports().get(index).copied().unwrap_or(5432); + append_locator_field( + &mut material, + format!("port-{index}").as_bytes(), + port.to_string().as_bytes(), + ); + } + for (index, address) in config.get_hostaddrs().iter().enumerate() { + append_locator_field( + &mut material, + format!("hostaddr-{index}").as_bytes(), + address.to_string().as_bytes(), + ); + } + Ok(ironclaw_common::hashing::sha256_hex(&material)) +} + +#[cfg(feature = "postgres")] +fn append_locator_field(material: &mut Vec, label: &[u8], value: &[u8]) { + material.extend_from_slice(label.len().to_string().as_bytes()); + material.push(b':'); + material.extend_from_slice(label); + material.extend_from_slice(value.len().to_string().as_bytes()); + material.push(b':'); + material.extend_from_slice(value); +} + +/// Resolve or create the cached local-runtime master key. Call this only from +/// the apply path after all source/manifest preconditions have passed and the +/// target database parent directory exists. +#[cfg(feature = "libsql")] +pub fn resolve_local_migration_target_key( + target_database_path: &Path, +) -> Result { + let root = target_database_path + .parent() + .ok_or_else(|| RebornBuildError::InvalidConfig { + reason: "local migration target database has no parent directory".to_string(), + })?; + crate::factory::resolve_local_dev_secret_master_key(root) +} + +fn effective_profile( + boot: &RebornBootConfig, + config_file: Option<&RebornConfigFile>, +) -> Result { + if std::env::var_os(REBORN_PROFILE_ENV).is_some() { + return Ok(boot.profile()); + } + let Some(profile) = config_file + .and_then(|file| file.boot.as_ref()) + .and_then(|section| section.profile.as_deref()) + else { + return Ok(boot.profile()); + }; + profile + .parse::() + .map_err(|error| RebornBuildError::InvalidConfig { + reason: format!("config file [boot].profile `{profile}` is invalid: {error}"), + }) +} + +fn runtime_identity(config_file: Option<&RebornConfigFile>) -> RebornRuntimeIdentity { + let default = RebornRuntimeIdentity::reborn_cli(); + let Some(identity) = config_file.and_then(|file| file.identity.as_ref()) else { + return default; + }; + RebornRuntimeIdentity { + tenant_id: identity + .tenant + .clone() + .unwrap_or_else(|| default.tenant_id.clone()), + agent_id: identity + .default_agent + .clone() + .unwrap_or_else(|| default.agent_id.clone()), + source_binding_id: default.source_binding_id, + reply_target_binding_id: default.reply_target_binding_id, + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "libsql")] + use std::fs; + + #[cfg(feature = "libsql")] + use ironclaw_reborn_config::{RebornBootConfig, RebornHome, RebornProfile}; + + #[cfg(feature = "libsql")] + use super::{RebornMigrationTargetStore, resolve_reborn_migration_target}; + + #[cfg(feature = "libsql")] + #[test] + fn local_target_resolution_matches_runtime_layout_without_creating_state() { + let temp = tempfile::tempdir().unwrap(); + let home_path = temp.path().join("reborn-home"); + let home = RebornHome::resolve_from_env_parts( + Some(home_path.clone().into_os_string()), + None, + None, + ) + .unwrap(); + let boot = RebornBootConfig::new(home, RebornProfile::LocalDev); + + let resolved = resolve_reborn_migration_target(&boot).unwrap(); + + let path = match resolved.store { + RebornMigrationTargetStore::LibSql { path } => path, + #[cfg(feature = "postgres")] + RebornMigrationTargetStore::Postgres { .. } => { + panic!("local-dev migration target must be libSQL") + } + }; + assert_eq!(path, home_path.join("local-dev/reborn-local-dev.db")); + assert_eq!(resolved.tenant_id.as_str(), "reborn-cli"); + assert_eq!(resolved.agent_id.as_str(), "reborn-cli-agent"); + assert!(resolved.target_master_key.is_none()); + assert!( + !home_path.exists(), + "resolution must not create target state" + ); + } + + #[cfg(feature = "libsql")] + #[test] + fn config_profile_and_identity_select_the_same_local_target_scope_as_boot() { + let temp = tempfile::tempdir().unwrap(); + let home_path = temp.path().join("reborn-home"); + fs::create_dir_all(&home_path).unwrap(); + fs::write( + home_path.join("config.toml"), + r#" +[boot] +profile = "hosted-single-tenant-volume" + +[identity] +tenant = "migrated-tenant" +default_agent = "migrated-agent" +"#, + ) + .unwrap(); + let home = RebornHome::resolve_from_env_parts( + Some(home_path.clone().into_os_string()), + None, + None, + ) + .unwrap(); + let boot = RebornBootConfig::new(home, RebornProfile::LocalDev); + + let resolved = resolve_reborn_migration_target(&boot).unwrap(); + + let path = match resolved.store { + RebornMigrationTargetStore::LibSql { path } => path, + #[cfg(feature = "postgres")] + RebornMigrationTargetStore::Postgres { .. } => { + panic!("hosted volume migration target must be libSQL") + } + }; + assert_eq!( + path, + home_path.join("hosted-single-tenant-volume/reborn-local-dev.db") + ); + assert_eq!(resolved.profile, RebornProfile::HostedSingleTenantVolume); + assert_eq!(resolved.tenant_id.as_str(), "migrated-tenant"); + assert_eq!(resolved.agent_id.as_str(), "migrated-agent"); + } + + #[cfg(feature = "libsql")] + #[test] + fn local_profile_rejects_storage_config_instead_of_ignoring_it() { + let temp = tempfile::tempdir().unwrap(); + let home_path = temp.path().join("reborn-home"); + fs::create_dir_all(&home_path).unwrap(); + fs::write( + home_path.join("config.toml"), + r#" +[storage] +backend = "postgres" +url_env = "IGNORED_DATABASE_URL" +"#, + ) + .unwrap(); + let home = RebornHome::resolve_from_env_parts(Some(home_path.into_os_string()), None, None) + .unwrap(); + let boot = RebornBootConfig::new(home, RebornProfile::LocalDev); + + let error = resolve_reborn_migration_target(&boot) + .err() + .expect("storage config must fail closed for a local target"); + + assert!(error.to_string().contains("refuses to ignore it")); + } +} diff --git a/crates/ironclaw_reborn_identity/CONTRACT.md b/crates/ironclaw_reborn_identity/CONTRACT.md index 2ee64a6641a..ced9b19a7f7 100644 --- a/crates/ironclaw_reborn_identity/CONTRACT.md +++ b/crates/ironclaw_reborn_identity/CONTRACT.md @@ -7,8 +7,9 @@ any runtime state (conversation binding, thread ownership) is touched. Identity provisioning lives here, not in WebUI ingress and not in `ironclaw_conversations` (which consumes an already-resolved `UserId`). -This crate is **also the durable home of the minimal user profile** (email, -display name, timestamps), not only an identity→`UserId` map. Resolving an +This crate is **also the durable home of the canonical user profile** (identity, +status, role, provenance, timestamps, and metadata), not only an +identity→`UserId` map. Resolving an identity persists a `StoredUser` record keyed by `UserId`, so "what do we know about this user, and where is it stored" is answered *here* — there is no separate users table elsewhere in the Reborn stack. Any future enumeration or @@ -49,7 +50,7 @@ segment maps to the `_` sentinel). | Record | Path (opaque segments base64url-encoded) | Fields | |---|---|---| -| `StoredUser` — the canonical **user profile** | `…/users/{user_id}.json` | `email`, `display_name`, `created_at`, `updated_at` | +| `StoredUser` — the canonical **user profile** | `…/users/{user_id}.json` | `tenant_id`, `email`, `display_name`, `status`, `role`, `created_by`, `created_at`, `updated_at`, `last_login_at`, `metadata` | | `StoredExternalIdentity` — one bound external login | `…/external/{tenant}/{surface}/{provider}/{instance}/{subject}.json` | `user_id`, `email`, `email_verified`, `created_at` | | `StoredVerifiedEmailIndex` — cross-provider link | `…/verified-email/{tenant}/{lower_email}.json` | `user_id` | @@ -58,6 +59,8 @@ login upserts the profile); this is why a user's email and display name are durably captured on SSO login without any separate directory. The record fields are `pub(super)` — the on-disk JSON is an implementation detail, and upstream consumers read through the resolver surface below rather than the raw records. +Legacy records default fields added after the minimal profile shape; historical +migration writes the full canonical state through `import_migrated_user`. (Known gap: `adopt_migrated_identity` does **not** write `StoredUser` today — tracked as #5616.) @@ -96,6 +99,11 @@ stack; the boundary tests still allow no new edge). the same email mints a *separate* user; admin-created users are token/API users, not pre-linked SSO accounts. Linking them is a future `link_email` action via `adopt_migrated_identity`, deliberately out of scope here.) +- `import_migrated_user` — historical import with an authoritative supplied + `UserId` and full canonical user state. Writes only the `users/` record and + never a verified-email index. Absent creates; an exact replay succeeds; a + divergent existing row fails with `UserImportConflict` and is not + overwritten. Migration calls this before `adopt_migrated_identity`. - `update_profile` / `update_status` / `update_role` — partial mutations through the shared `ironclaw_filesystem::cas_update` helper (never a per-record mutex; `ironclaw_filesystem/CLAUDE.md` invariant 2). Each bumps `updated_at`. @@ -193,6 +201,8 @@ Filed from the de-slop review: - **#5614** — cross-process divergent-email logins can split a principal. - **#5615** — `bind()` has no OAuth-surface guard (defense-in-depth). - **#5616** — `adopt_migrated_identity` never writes `StoredUser` and reverses the - index/identity write order. + index/identity write order. The migration path must call + `import_migrated_user` first; folding both operations into one atomic port is + still unresolved. - **#5617** — the login seam is tested only with fakes on both sides. - **#5618** — decide the `ExternalIdentityKey` + `lookup`/`bind` public surface. diff --git a/crates/ironclaw_reborn_identity/src/filesystem_store/directory.rs b/crates/ironclaw_reborn_identity/src/filesystem_store/directory.rs index 39f9562d131..aeb6f6de4fc 100644 --- a/crates/ironclaw_reborn_identity/src/filesystem_store/directory.rs +++ b/crates/ironclaw_reborn_identity/src/filesystem_store/directory.rs @@ -74,6 +74,27 @@ fn status_from_stored(status: StoredUserStatus) -> RebornUserStatus { } } +fn to_stored_user(user: &RebornUser) -> StoredUser { + StoredUser { + email: user.email.clone(), + display_name: user.display_name.clone(), + created_at: user.created_at.clone(), + updated_at: user.updated_at.clone(), + status: status_to_stored(user.status), + role: role_to_stored(user.role), + created_by: user + .created_by + .as_ref() + .map(|created_by| created_by.as_str().to_string()), + last_login_at: user.last_login_at.clone(), + tenant_id: user + .tenant_id + .as_ref() + .map(|tenant_id| tenant_id.as_str().to_string()), + metadata: user.metadata.clone(), + } +} + /// Map a persisted row to the public domain type, validating the persisted /// `user_id` / `created_by` / `tenant_id` strings on the way out (a malformed /// persisted id is a backend inconsistency, surfaced rather than dropped). @@ -355,6 +376,49 @@ where to_reborn_user(new_user_id.as_str().to_string(), record) } + async fn import_migrated_user( + &self, + user: RebornUser, + ) -> Result { + let path = user_path(user.user_id.as_str())?; + let record = to_stored_user(&user); + + if let Some(existing) = self.read_record::(&path).await? { + return if existing == record { + Ok(user) + } else { + Err(RebornIdentityError::UserImportConflict( + user.user_id.as_str().to_string(), + )) + }; + } + + match self + .write_record(&path, &record, CasExpectation::Absent) + .await + { + Ok(()) => Ok(user), + Err(FilesystemError::VersionMismatch { .. }) => { + let existing = self + .read_record::(&path) + .await? + .ok_or_else(|| { + RebornIdentityError::Backend( + "migrated user record vanished during reconciliation".to_string(), + ) + })?; + if existing == record { + Ok(user) + } else { + Err(RebornIdentityError::UserImportConflict( + user.user_id.as_str().to_string(), + )) + } + } + Err(error) => Err(backend(error)), + } + } + async fn update_profile( &self, user_id: &UserId, diff --git a/crates/ironclaw_reborn_identity/src/filesystem_store/tests.rs b/crates/ironclaw_reborn_identity/src/filesystem_store/tests.rs index d79e8b0140e..8509b38299a 100644 --- a/crates/ironclaw_reborn_identity/src/filesystem_store/tests.rs +++ b/crates/ironclaw_reborn_identity/src/filesystem_store/tests.rs @@ -7,7 +7,7 @@ use super::*; use crate::{ - ExternalSubjectId, ProviderInstanceId, ProviderKind, RebornUserDirectory, + ExternalSubjectId, ProviderInstanceId, ProviderKind, RebornUser, RebornUserDirectory, RebornUserProfileUpdate, RebornUserRole, RebornUserStatus, SurfaceKind, }; use ironclaw_filesystem::InMemoryBackend; @@ -51,6 +51,24 @@ fn tenant(id: &str) -> TenantId { TenantId::new(id).expect("tenant") } +fn migrated_user(id: &str) -> RebornUser { + let mut metadata = std::collections::BTreeMap::new(); + metadata.insert("legacy_plan".to_string(), "enterprise".to_string()); + RebornUser { + user_id: UserId::new(id).expect("user"), + email: Some("legacy@example.com".to_string()), + display_name: Some("Legacy User".to_string()), + status: RebornUserStatus::Suspended, + role: RebornUserRole::Admin, + created_at: "2024-01-02T03:04:05Z".to_string(), + updated_at: "2025-02-03T04:05:06Z".to_string(), + created_by: Some(UserId::new("legacy-admin").expect("creator")), + last_login_at: Some("2025-03-04T05:06:07Z".to_string()), + tenant_id: Some(tenant("acme")), + metadata, + } +} + fn oauth( tenant: &TenantId, provider: &str, @@ -881,6 +899,87 @@ async fn create_then_list_and_get_roundtrip() { assert!(other.is_empty(), "users are tenant-scoped in enumeration"); } +#[tokio::test] +async fn import_migrated_user_preserves_supplied_state_and_exact_replay_is_idempotent() { + let store = store(); + let imported = migrated_user("legacy-user-42"); + + let first = store + .import_migrated_user(imported.clone()) + .await + .expect("first historical import"); + let replay = store + .import_migrated_user(imported.clone()) + .await + .expect("exact replay"); + + assert_eq!(first, imported); + assert_eq!(replay, imported); + assert_eq!( + store + .get_user(&imported.user_id) + .await + .expect("get imported user"), + Some(imported.clone()), + "every supplied historical field must round-trip unchanged" + ); + assert!( + store + .read_record::( + &verified_email_path("acme", "legacy@example.com").unwrap() + ) + .await + .expect("read verified-email index") + .is_none(), + "importing a user profile must not establish verified-email linking" + ); +} + +#[tokio::test] +async fn import_migrated_user_rejects_divergent_existing_state_without_overwrite() { + let store = store(); + let original = migrated_user("legacy-user-42"); + store + .import_migrated_user(original.clone()) + .await + .expect("seed imported user"); + + let mut divergent = original.clone(); + divergent.role = RebornUserRole::Owner; + divergent.updated_at = "2026-01-01T00:00:00Z".to_string(); + let error = store + .import_migrated_user(divergent) + .await + .expect_err("divergent replay must conflict"); + + assert!( + matches!(error, RebornIdentityError::UserImportConflict(ref id) if id == "legacy-user-42"), + "divergent state must surface a typed import conflict, got {error:?}" + ); + assert_eq!( + store + .get_user(&original.user_id) + .await + .expect("get original"), + Some(original), + "a conflict must never overwrite the existing canonical row" + ); +} + +#[tokio::test] +async fn concurrent_exact_user_imports_converge_across_store_instances() { + let (store_a, store_b) = store_pair(); + let imported = migrated_user("legacy-user-42"); + + let (result_a, result_b) = tokio::join!( + store_a.import_migrated_user(imported.clone()), + store_b.import_migrated_user(imported.clone()), + ); + + assert_eq!(result_a.expect("import a"), imported); + assert_eq!(result_b.expect("import b"), imported); +} + #[tokio::test] async fn list_users_paginates_by_user_id_cursor() { // Regression: the admin listing must return bounded pages and page through diff --git a/crates/ironclaw_reborn_identity/src/lib.rs b/crates/ironclaw_reborn_identity/src/lib.rs index 0f8e26b31ec..b3f213689c7 100644 --- a/crates/ironclaw_reborn_identity/src/lib.rs +++ b/crates/ironclaw_reborn_identity/src/lib.rs @@ -118,6 +118,11 @@ pub enum RebornIdentityError { /// from `Backend` so the product-workflow facade can map it to a 404. #[error("no user record for id: {0}")] UserNotFound(String), + /// A historical import targeted a user id that already contains different + /// canonical state. Migration must stop rather than overwrite live or + /// previously imported user data. + #[error("migrated user conflicts with existing record for id: {0}")] + UserImportConflict(String), /// `resolve_or_create` resolved an external identity to an existing user /// whose account is suspended. Distinct from `Backend` so the SSO host /// adapter can map it to a fail-closed 403 (login refused) instead of a diff --git a/crates/ironclaw_reborn_identity/src/user_directory.rs b/crates/ironclaw_reborn_identity/src/user_directory.rs index 3b3f3dbf567..1c1e0deb80a 100644 --- a/crates/ironclaw_reborn_identity/src/user_directory.rs +++ b/crates/ironclaw_reborn_identity/src/user_directory.rs @@ -117,6 +117,19 @@ pub trait RebornUserDirectory: Send + Sync { created_by: &UserId, ) -> Result; + /// Import a canonical user from a historical store without minting a new + /// id or changing any supplied profile/lifecycle fields. + /// + /// This writes only the user record. It deliberately does not create a + /// verified-email index; external identity adoption remains the sole + /// migration path that may establish verified-email linking. Replaying an + /// exact record succeeds, while an existing divergent record returns + /// [`RebornIdentityError::UserImportConflict`] without overwriting it. + async fn import_migrated_user( + &self, + user: RebornUser, + ) -> Result; + /// Apply a partial profile update. Errors with /// [`RebornIdentityError::UserNotFound`] if the user does not exist. async fn update_profile( diff --git a/crates/ironclaw_reborn_migration/CLAUDE.md b/crates/ironclaw_reborn_migration/CLAUDE.md index 45cf4b2bfbd..12cb84f2774 100644 --- a/crates/ironclaw_reborn_migration/CLAUDE.md +++ b/crates/ironclaw_reborn_migration/CLAUDE.md @@ -1,111 +1,126 @@ # ironclaw_reborn_migration -Standalone tool + library that converts **IronClaw v1 / engine-v2 persisted -state** into the **Reborn** state substrate. Ships as its own binary -(`ironclaw-reborn-migration`); the conversion engine is a library -(`run_migration`) so it can later be wired into `ironclaw-reborn` startup. - -- **Read side** = the root `ironclaw` crate (`ironclaw::db::connect_with_handles`) - — one v1 database (PostgreSQL **or** libSQL). Engine-v2 state is **not** a - separate DB: missions/projects/threads were persisted by the v2 bridge as JSON - blobs inside the v1 `memory_documents` table under `engine/…` / - `.system/engine/…` paths. Parsed via the serde mirrors in `v2_model.rs` (the - engine-v2 types were deleted; they survive only at git tag `old_engine_v2`). -- **Write side** = Reborn domain stores built directly over a `RootFilesystem` / - triggers DB in `target.rs`, without booting a `RebornRuntime`. Threads / - secrets / identity force a concrete filesystem type, so they are built inside - the backend match arm and stored as `#[async_trait]` trait objects. -- **Philosophy: nothing is silently dropped.** Infrastructure errors abort; a - value with no Reborn representation is recorded as a `LossyItem` on the - `MigrationReport` (the manifest), with the reason and the Reborn gap named. +Transitional v1/engine-v2 migration bridge for Reborn. It ships as the +same-version `ironclaw-reborn-migration` companion and is invoked by +`ironclaw-reborn migrate v1`; do not link the v1 root crate into the normal +Reborn CLI/runtime. +The public lifecycle is: + +```rust +plan_migration(&MigrationOptions) +preflight_apply_migration(&MigrationOptions, &MigrationManifest, &MigrationSecretInputs, ApplyAcknowledgements) +apply_migration(MigrationOptions, &MigrationManifest, MigrationSecretInputs, ApplyAcknowledgements) +preflight_resume_migration(&MigrationOptions, &MigrationManifest, &MigrationSecretInputs, ApplyAcknowledgements) +resume_migration(MigrationOptions, &MigrationManifest, MigrationSecretInputs, ApplyAcknowledgements) +verify_migration(&MigrationOptions, &MigrationManifest) ``` -cargo run -p ironclaw_reborn_migration -- \ - --source-libsql ~/.ironclaw/ironclaw.db \ - --target-libsql ./reborn-local-dev.db \ - --tenant-id default --agent-id default --dry-run -``` -## What converts, and where losses go - -| v1 / engine-v2 source | Reborn target | Status | -|---|---|---| -| `conversations` + `conversation_messages` | `SessionThreadRecord` + transcript (orig id preserved via `EnsureThreadRequest.thread_id`; per-message role/ts/id in `metadata_json.legacy_v1`) | **full** | -| routine `Trigger::Cron` | `TriggerRecord` (`TriggerSchedule::Cron`) via `TriggerRepository::upsert_trigger` | **full** | -| engine-v2 mission `Cadence::Cron` | `TriggerRecord`; `thread_history` → threads under `ThreadScope.mission_id` | **full** | -| `memory_documents` (non-engine) | `ironclaw_memory` documents (`MemoryService::write`) | **full** | -| `secrets` | decrypt via v1 `SecretsStore` → re-encrypt via Reborn `SecretStore::put` (needs `--secret-master-key`) | **full** | -| `user_identities` (OAuth) + `channel_identities` | `RebornIdentityResolver::adopt_migrated_identity` (`SurfaceKind::Oauth` / `ChannelActor`) | **full** | -| `wasm_tools` / `wasm_channels` installs | `ExtensionInstallation` (+ synthesized `capability_provider` manifest) via composition's `migration-support` seam; `tool_capabilities.allowed_secrets` → credential bindings | **full (manifest is a placeholder — see below)** | -| routine `Trigger::{Event,SystemEvent,Webhook,Manual}` | — (Reborn `TriggerSourceKind` = `Schedule` only) | **gap → report** | -| mission `Cadence::{OnEvent,OnSystemEvent,Webhook,Manual}` | — | **gap → report** | -| routine guardrails / notify / run counters; mission focus / approach / success-criteria / notify | — (no trigger field / no durable mission entity) | **gap → report** | -| `routine_runs` history | — (`TriggerRepository` has no public run-history insert) | **gap → report** | -| routine/mission `Failed` status | `TriggerState::Paused` | **degraded → report** | -| non-user/assistant transcript messages (system/tool) | retained in thread `metadata_json.legacy_v1`, not a standalone row | **degraded → report** | -| `settings` (key/value) | — (Reborn config is typed `config.toml`/`providers.json`/`LlmKeyStore`, no generic KV store) | **gap → report** | -| `memory_document_versions` | — (no per-doc version history in Reborn) | **gap → report** | -| `agent_jobs` / `job_actions` / `job_events` | — (Reborn has no general job store) | **gap → report** | -| `heartbeat_state` | — (re-establish as a scheduled trigger) | **gap → report** | -| extension manifest fidelity + WASM binary; tool capability config; channel→secret binding; `pairing_requests` | — | **degraded/gap → report** | - -`Domain` + `LossReason` on each `LossyItem` make the manifest greppable; the -acceptance test asserts the **exact** gap set so a regression that silently drops -a domain fails the build. - -## Notes on the "full" deferred converters - -- **Secrets** — needs `--secret-master-key` (used verbatim as the HKDF IKM, as - in v1). The v1 store is built from the raw `DatabaseHandles`; each secret is - listed, decrypted (`get_decrypted`), and re-encrypted through - `RebornTarget::secret_store` (`FilesystemSecretStore`). Expiry is preserved; a - secret that fails to decrypt (expired / wrong key) is a per-secret loss, not a - run abort. Without a key, secrets are skipped with a recorded loss. -- **Identities** — `user_identities` read via the `Database` trait, - `channel_identities` via raw SQL (no trait accessor). Adoption preserves the v1 - `UserId` and seeds the verified-email index. Idempotent (safe to re-run). -- **Extensions** — installed tools/channels become `ExtensionInstallation`s with - activation from the v1 `status` and credential bindings from - `tool_capabilities.allowed_secrets`. The synthesized manifest declares - `ironclaw.capability_provider/v1` + one `ask`-permission placeholder capability - (a non-first-party manifest must declare a host API or capability). **The v1 - capability contract and WASM binary are NOT carried over** — the manifest is a - migration placeholder, recorded as a `manifest_fidelity` loss per installation. - The store is opened through the composition `migration-support` seam - `extension_installation_store_for_migration` (mirrors composition's - `*_for_test` accessors; ships zero bytes without the feature). - -## Remaining follow-up — wire into `ironclaw-reborn` startup - -Call `run_migration` in `crates/ironclaw_reborn_cli/src/runtime/mod.rs` after the -storage root is resolved and before `build_reborn_runtime`, mirroring -`with_run_local_trigger_fire_access_checker`; or add a `Command::Migrate` -subcommand. The `run --dry-run` output already reserves a `v1_state:` line. -Deferred per the original PR scope. - -## Mount layout caveat - -`mounts.rs` reproduces the production alias→path layout (memory `/memory`; -threads/secrets tenant/user-scoped) because the canonical resolver is **private** -in `ironclaw_reborn_composition`. It MUST be reconciled with composition when the -startup wiring lands so the runtime reads back exactly what was migrated. The -acceptance test verifies round-trip through the **same** services the migration -writes with, pinning conversion correctness independently of that reconciliation -(end-to-end runtime-readback is the wiring follow-up). - -## Tests - -`tests/migration_roundtrip.rs` (`required-features = ["libsql"]`, Docker-free): -seeds a rich v1+engine-v2 fixture (conversations, every routine trigger variant, -cron + non-cron missions with a mission thread, memory docs, settings, a secret, -an OAuth + a channel identity, an installed WASM tool), runs the migration, and -asserts converted counts (including secrets/identities/extensions), the exact gap -set, triggers read back through the **public** `LibSqlTriggerRepository`, and -on-disk durability of thread / secret / extension-installation documents via a -fresh connection. A second case asserts `--dry-run` reports fully but writes -nothing. Add a Postgres variant with the `postgres_pool_or_skip()` -skip-if-no-Docker helper (see `crates/ironclaw_reborn_composition/tests/postgres_substrate.rs`). +`run_migration` is a deprecated compatibility wrapper. New code must use the +explicit lifecycle. + +## Safety contract + +- Source readers are narrow, read-only adapters. Never use the v1 runtime + connection constructor because it runs schema migrations. +- Planning must not open or create the Reborn target. It may write a manifest + only at the explicit operator path. +- Apply requires a stopped v1 source and a consistent snapshot acknowledgement. +- The manifest contains redacted locator fingerprints, not database URLs, + keys, tokens, or decrypted values. +- Source and target keys are independent. Source key input is used only for v1 + decryption; target key resolution follows production Reborn composition. A + non-empty source `secrets` table makes the source key an apply/resume preflight + requirement. +- The v1 home is an explicit sealed input independent of the database snapshot + location. Omitting it records an apply blocker because home coverage is unknown. +- Unknown or incompatible executable artifacts are never enabled. +- `verify` performs read-only target-data checks for users, projects, threads, + messages, triggers, memory documents, secrets, and identity records in the + production persistence tables before transitioning the manifest to + `verified`. It writes lifecycle/quarantine state, does not independently + read back other manifest domains, and does not boot a full Reborn runtime or + prove every product service can consume the records. `applying`, `failed`, + `applied`, and `verifying` targets remain quarantined. +- Apply/resume preflight finishes before the CLI persists `applying`. Apply + accepts only `planned`; resume accepts `applying`, `failed`, or `applied` for + the same sealed run. +- libSQL and PostgreSQL targets keep an atomic, run-bound lifecycle claim as + well as the local marker. The claim binds release/protocol, profile, backend, + locator fingerprint, tenant, and agent. PostgreSQL makes it visible to every + replica; startup requires local and durable records to agree when both exist. + +## Source and target ownership +`source.rs` owns supported v1 schema reads for PostgreSQL and libSQL. Engine-v2 +state is stored as JSON documents inside v1 persistence; `v2_model.rs` contains +the compatibility DTOs. + +Target profile, store, tenant, agent, and encryption configuration come from +`ironclaw_reborn_composition::resolve_reborn_migration_target`. Mount aliases +must use composition's production resolver. Do not reintroduce migration-only +target paths or identity defaults. + +The companion protocol is intentionally small: + +```text +ironclaw-reborn-migration __handshake +ironclaw-reborn-migration v1 plan|apply|resume|verify|status ``` -cargo test -p ironclaw_reborn_migration --features libsql --test migration_roundtrip + +PostgreSQL source URLs and source keys come from +`MIGRATION_SOURCE_POSTGRES` and `MIGRATION_SOURCE_SECRET_MASTER_KEY`. +Production target URL/key names come from Reborn `config.toml`. They must never +be accepted as raw CLI values. + +## Disposition and fidelity + +The versioned inventory is the source of truth. Every known v1 table and +persistent home artifact must be classified as imported, semantically +converted, archive-only, re-auth/reinstall, intentionally reset, +operator-skipped, unsupported, or derived/rebuilt. A new v1 category without a +registry entry is a blocker, not an implicit skip. + +Converters must claim deterministic slots atomically and use absent-or-exact or +compare-and-create semantics. Exact replay is a no-op; a divergent target +collision fails without overwriting. Exact duplicate engine documents with the +same durable id are deduplicated, while divergent source duplicates fail the +migration. Non-engine memory retains its optional source-agent scope. Supported +automations import paused when their owner is not active, their next-fire time +is missing, or a mission references a project that was not imported; the latter +also omits the invalid project scope. +Unsupported transcript payloads are retained in thread metadata where the +converter explicitly says so. `archive_only` operational categories currently +retain inventory counts/checksums and a disposition only; the source payload is +not copied into a Reborn archive. Do not describe those payloads as archived. + +The persisted manifest carries inventory and lifecycle checkpoints. Converter +`LossyItem` details exist only in the `MigrationReport` JSON emitted by apply or +resume, so operator guidance must tell users to retain that output securely and +must not claim `status --json` contains per-record losses. + +API-token hashes cannot become Reborn signed sessions. Unknown v1 WASM/channel +packages cannot become runnable Reborn installations. Both require an explicit +re-auth/reinstall disposition. + +`heartbeat_state` is unsupported: only actual source rows produce a loss, no +durable heartbeat row is written, and operators must recreate the cadence. + +## Validation + +Minimum focused gates: + +```bash +cargo test -p ironclaw_reborn_migration --no-default-features --features libsql +cargo test -p ironclaw_reborn_migration --no-default-features --features postgres +cargo clippy -p ironclaw_reborn_migration --all-targets --all-features -- -D warnings ``` + +Acceptance coverage must prove source immutability, target absence after plan, +manifest redaction, stopped-snapshot enforcement, idempotent resume, collision +handling, full inventory disposition, libSQL/PostgreSQL parity, and cold +structural readback. Full runtime/service readback remains additional work; a +converter-only roundtrip does not prove that the normal Reborn runtime can see +all migrated data. + +The operator runbook is `docs/reborn/v1-migration.md`. diff --git a/crates/ironclaw_reborn_migration/Cargo.toml b/crates/ironclaw_reborn_migration/Cargo.toml index 01ec5691768..8ad4e5313ae 100644 --- a/crates/ironclaw_reborn_migration/Cargo.toml +++ b/crates/ironclaw_reborn_migration/Cargo.toml @@ -13,8 +13,8 @@ publish = false [package.metadata.ironclaw] layer = "app" -# Keep out of cargo-dist release packaging; this is an operator/migration tool, -# not a shipped product binary (mirrors ironclaw_reborn_cli). +# Docker/source builds ship this beside `ironclaw-reborn`. Keep cargo-dist +# disabled until one native installer can guarantee the sibling pair. [package.metadata.dist] dist = false @@ -59,12 +59,14 @@ ironclaw_threads = { path = "../ironclaw_threads" } ironclaw_triggers = { path = "../ironclaw_triggers" } ironclaw_memory = { path = "../ironclaw_memory" } ironclaw_memory_native = { path = "../ironclaw_memory_native" } +ironclaw_projects = { path = "../ironclaw_projects" } ironclaw_secrets = { path = "../ironclaw_secrets" } ironclaw_extensions = { path = "../ironclaw_extensions" } ironclaw_host_runtime = { path = "../ironclaw_host_runtime" } ironclaw_reborn_composition = { path = "../ironclaw_reborn_composition", features = [ "migration-support", ] } +ironclaw_reborn_config = { path = "../ironclaw_reborn_config" } ironclaw_reborn_identity = { path = "../ironclaw_reborn_identity" } # ── shared ────────────────────────────────────────────────────────────────── @@ -72,9 +74,11 @@ anyhow = "1" async-trait = "0.1" chrono = { version = "0.4", features = ["serde"] } clap = { version = "4", features = ["derive", "env"] } +futures = "0.3" secrecy = { version = "0.10", features = ["serde"] } serde = { version = "1", features = ["derive"] } serde_json = "1" +sha2 = "0.10" thiserror = "2" tokio = { version = "1", features = ["macros", "rt-multi-thread", "fs"] } tracing = "0.1" @@ -97,3 +101,8 @@ tempfile = "3" name = "migration_roundtrip" path = "tests/migration_roundtrip.rs" required-features = ["libsql"] + +[[test]] +name = "project_migration" +path = "tests/project_migration.rs" +required-features = ["libsql"] diff --git a/crates/ironclaw_reborn_migration/src/convert/automations.rs b/crates/ironclaw_reborn_migration/src/convert/automations.rs index ea1ff5dfd21..aecdcdd61fe 100644 --- a/crates/ironclaw_reborn_migration/src/convert/automations.rs +++ b/crates/ironclaw_reborn_migration/src/convert/automations.rs @@ -21,10 +21,11 @@ //! scoped under `ThreadScope.mission_id` — the one place "mission" survives in //! Reborn (a scope dimension, not a durable entity). -use std::collections::HashMap; +use std::collections::{BTreeMap, BTreeSet}; use ironclaw::agent::routine::{Routine, RoutineAction, Trigger}; use ironclaw_host_api::ProjectId; +use ironclaw_reborn_identity::RebornUserStatus; use ironclaw_triggers::{TriggerRecord, TriggerSchedule, TriggerSourceKind, TriggerState}; use uuid::Uuid; @@ -34,6 +35,7 @@ use crate::options::MigrationOptions; use crate::report::{Domain, LossReason, MigrationReport}; use crate::source::V1Source; use crate::target::RebornTarget; +use crate::target::ids; use crate::v2_model::{self, EngineThread, Mission, MissionCadence, MissionStatus}; pub(crate) async fn run( @@ -41,9 +43,10 @@ pub(crate) async fn run( tgt: &mut RebornTarget, options: &MigrationOptions, report: &mut MigrationReport, + imported_project_ids: &BTreeSet, ) -> Result<(), MigrationError> { convert_routines(src, tgt, options, report).await?; - convert_missions(src, tgt, options, report).await?; + convert_missions(src, tgt, options, report, imported_project_ids).await?; Ok(()) } @@ -120,12 +123,25 @@ async fn convert_routine( let prompt = routine_prompt(&routine.action); // v1 routines have no terminal-failed status distinct from disabled; // consecutive_failures is recorded as a field loss below. - let state = if routine.enabled { + let mut state = if routine.enabled { TriggerState::Scheduled } else { TriggerState::Paused }; - let now = routine.next_fire_at.unwrap_or(routine.created_at); + let now = match routine.next_fire_at { + Some(next_fire_at) => next_fire_at, + None => { + report.record_loss( + Domain::Routine, + &source_id, + "next_fire_at", + LossReason::Degraded, + "routine had no next_fire_at; its deterministic created_at is retained and the routine is imported Paused", + ); + state = TriggerState::Paused; + routine.created_at + } + }; record_routine_field_losses(report, &source_id, &routine, is_cron); @@ -135,9 +151,24 @@ async fn convert_routine( else { return Ok(()); }; + pause_for_inactive_owner( + tgt, + report, + Domain::Routine, + &source_id, + &creator_user_id, + &mut state, + ) + .await?; + let migration_identity = ids::MigrationIdentity::from_report(report)?; let record = TriggerRecord { - trigger_id: ironclaw_triggers::TriggerId::new(), + trigger_id: migration_identity.trigger_id( + "routine", + &routine.id.to_string(), + &tgt.tenant_id, + &tgt.agent_id, + )?, tenant_id: tgt.tenant_id.clone(), creator_user_id, agent_id: Some(tgt.agent_id.clone()), @@ -160,15 +191,10 @@ async fn convert_routine( }; if !options.dry_run { - tgt.trigger_repo - .upsert_trigger(record) - .await - .map_err(|e| MigrationError::WriteTarget { - domain: format!("trigger for {source_id}"), - reason: e.to_string(), - })?; + tgt.compare_and_upsert_trigger(&source_id, record).await?; } report.stats.routines += 1; + report.stats.triggers += 1; Ok(()) } @@ -273,28 +299,33 @@ async fn convert_missions( tgt: &mut RebornTarget, options: &MigrationOptions, report: &mut MigrationReport, + imported_project_ids: &BTreeSet, ) -> Result<(), MigrationError> { let users = src.distinct_users().await?; + let mut engine_threads: BTreeMap = BTreeMap::new(); + let mut missions: BTreeMap = BTreeMap::new(); for user_id in &users { - let docs = - src.db - .list_documents(user_id, None) - .await - .map_err(|e| MigrationError::ReadSource { - domain: "memory_documents(engine)".into(), - reason: e.to_string(), - })?; - - // Index engine threads by id so mission thread_history can resolve them. - let mut engine_threads: HashMap = HashMap::new(); - let mut missions: Vec = Vec::new(); + let docs = src.all_memory_documents(user_id).await?; for doc in &docs { if !v2_model::is_engine_path(&doc.path) { continue; } if doc.path.ends_with("mission.json") { - match serde_json::from_str::(&doc.content) { - Ok(mission) => missions.push(mission), + match parse_engine_document::(&doc.content) { + Ok((representation, mission)) => { + let owner = if mission.user_id.is_empty() { + user_id.clone() + } else { + mission.user_id.clone() + }; + let indexed = IndexedMission { + source: engine_document_source(doc), + owner, + representation, + mission, + }; + insert_mission(&mut missions, indexed)?; + } Err(e) => report.record_loss( Domain::Mission, doc.path.clone(), @@ -304,9 +335,14 @@ async fn convert_missions( ), } } else if doc.path.contains("/threads/") && doc.path.ends_with(".json") { - match serde_json::from_str::(&doc.content) { - Ok(thread) => { - engine_threads.insert(thread.id, thread); + match parse_engine_document::(&doc.content) { + Ok((representation, thread)) => { + let indexed = IndexedEngineThread { + source: engine_document_source(doc), + representation, + thread, + }; + insert_engine_thread(&mut engine_threads, indexed)?; } Err(e) => report.record_loss( Domain::Mission, @@ -318,30 +354,33 @@ async fn convert_missions( } } } + } - // Threads referenced by a mission's `thread_history`; anything parsed but - // never referenced has no Reborn owner to migrate it under. - let referenced: std::collections::HashSet = missions - .iter() - .flat_map(|m| m.thread_history.iter().copied()) - .collect(); - - for mission in &missions { - convert_mission(tgt, options, report, user_id, mission, &engine_threads).await?; - } - - for id in engine_threads.keys() { - if !referenced.contains(id) { - report.record_loss( - Domain::Mission, - format!("thread:{id}"), - "*", - LossReason::NoTargetConcept, - "engine thread blob is not referenced by any mission thread_history; \ - there is no Reborn mission owner to migrate it under" - .to_string(), - ); - } + let referenced: BTreeSet = missions + .values() + .flat_map(|indexed| indexed.mission.thread_history.iter().copied()) + .collect(); + for indexed in missions.values() { + convert_mission( + tgt, + options, + report, + &indexed.owner, + &indexed.mission, + &engine_threads, + imported_project_ids, + ) + .await?; + } + for id in engine_threads.keys() { + if !referenced.contains(id) { + report.record_loss( + Domain::Mission, + format!("thread:{id}"), + "*", + LossReason::NoTargetConcept, + "engine thread blob is not referenced by any mission thread_history; there is no Reborn mission owner to migrate it under", + ); } } Ok(()) @@ -353,7 +392,8 @@ async fn convert_mission( report: &mut MigrationReport, user_id: &str, mission: &Mission, - engine_threads: &HashMap, + engine_threads: &BTreeMap, + imported_project_ids: &BTreeSet, ) -> Result<(), MigrationError> { let source_id = format!("mission:{}", mission.name); let owner = if mission.user_id.is_empty() { @@ -374,7 +414,7 @@ async fn convert_mission( let tz = timezone.clone().unwrap_or_else(|| "UTC".to_string()); match TriggerSchedule::cron_with_timezone(expression.clone(), tz) { Ok(schedule) => { - let state = match mission.status { + let mut state = match mission.status { MissionStatus::Active => TriggerState::Scheduled, MissionStatus::Paused => TriggerState::Paused, MissionStatus::Completed => TriggerState::Completed, @@ -395,13 +435,42 @@ async fn convert_mission( if let Some(creator_user_id) = report.valid_user_id(Domain::Mission, &source_id, "user_id", &owner) { - let next_run_at = mission_next_run_at(report, &source_id, mission); + let (next_run_at, synthesized_next_run) = + mission_next_run_at(report, &source_id, mission); + // A source without a durable next fire cannot safely be + // armed. Keep the deterministic historical timestamp, + // but require an operator to review and resume it. + if synthesized_next_run && state == TriggerState::Scheduled { + state = TriggerState::Paused; + } + pause_for_inactive_owner( + tgt, + report, + Domain::Mission, + &source_id, + &creator_user_id, + &mut state, + ) + .await?; + let project_id = resolve_mission_project( + report, + &source_id, + mission, + imported_project_ids, + &mut state, + )?; + let migration_identity = ids::MigrationIdentity::from_report(report)?; let record = TriggerRecord { - trigger_id: ironclaw_triggers::TriggerId::new(), + trigger_id: migration_identity.trigger_id( + "mission", + &mission.id.to_string(), + &tgt.tenant_id, + &tgt.agent_id, + )?, tenant_id: tgt.tenant_id.clone(), creator_user_id, agent_id: Some(tgt.agent_id.clone()), - project_id: Option::::None, + project_id, name: mission.name.clone(), source: TriggerSourceKind::Schedule, schedule, @@ -422,13 +491,9 @@ async fn convert_mission( created_at: mission.created_at, }; if !options.dry_run { - tgt.trigger_repo.upsert_trigger(record).await.map_err(|e| { - MigrationError::WriteTarget { - domain: format!("trigger for {source_id}"), - reason: e.to_string(), - } - })?; + tgt.compare_and_upsert_trigger(&source_id, record).await?; } + report.stats.triggers += 1; } } Err(e) => report.record_loss( @@ -453,9 +518,15 @@ async fn convert_mission( report.stats.missions += 1; - // Migrate the mission's threads under ThreadScope.mission_id. + let thread_project_id = mission + .project_id + .filter(|id| imported_project_ids.contains(id.to_string().as_str())); + let mut migrated_thread_ids = BTreeSet::new(); for tid in &mission.thread_history { - let Some(thread) = engine_threads.get(tid) else { + if !migrated_thread_ids.insert(*tid) { + continue; + } + let Some(indexed_thread) = engine_threads.get(tid) else { report.record_loss( Domain::Mission, &source_id, @@ -467,10 +538,12 @@ async fn convert_mission( ); continue; }; + let thread = &indexed_thread.thread; let import = ThreadImport { thread_id: thread.id, owner_user: owner.clone(), title: thread.title.clone().or_else(|| Some(mission.name.clone())), + project_id: thread_project_id, mission_id: Some(mission.id), provenance: serde_json::json!({ "source": "engine_v2_mission_thread", @@ -509,31 +582,173 @@ async fn convert_mission( Ok(()) } +struct IndexedMission { + source: String, + owner: String, + representation: serde_json::Value, + mission: Mission, +} + +struct IndexedEngineThread { + source: String, + representation: serde_json::Value, + thread: EngineThread, +} + +fn parse_engine_document( + content: &str, +) -> Result<(serde_json::Value, T), serde_json::Error> { + let representation: serde_json::Value = serde_json::from_str(content)?; + let parsed = serde_json::from_value(representation.clone())?; + Ok((representation, parsed)) +} + +fn engine_document_source(document: &ironclaw::workspace::MemoryDocument) -> String { + format!( + "user={} agent={} path={}", + document.user_id, + document + .agent_id + .map_or_else(|| "unscoped".to_string(), |id| id.to_string()), + document.path + ) +} + +fn insert_mission( + missions: &mut BTreeMap, + candidate: IndexedMission, +) -> Result<(), MigrationError> { + if let Some(existing) = missions.get(&candidate.mission.id) { + if existing.representation == candidate.representation && existing.owner == candidate.owner + { + return Ok(()); + } + return Err(divergent_engine_document( + "mission", + candidate.mission.id, + &existing.source, + &candidate.source, + )); + } + missions.insert(candidate.mission.id, candidate); + Ok(()) +} + +fn insert_engine_thread( + threads: &mut BTreeMap, + candidate: IndexedEngineThread, +) -> Result<(), MigrationError> { + if let Some(existing) = threads.get(&candidate.thread.id) { + if existing.representation == candidate.representation { + return Ok(()); + } + return Err(divergent_engine_document( + "thread", + candidate.thread.id, + &existing.source, + &candidate.source, + )); + } + threads.insert(candidate.thread.id, candidate); + Ok(()) +} + +fn divergent_engine_document( + kind: &str, + id: Uuid, + existing_source: &str, + candidate_source: &str, +) -> MigrationError { + MigrationError::ReadSource { + domain: format!("engine {kind} {id}"), + reason: format!( + "source documents {existing_source} and {candidate_source} contain divergent state for the same durable id" + ), + } +} + +async fn pause_for_inactive_owner( + tgt: &RebornTarget, + report: &mut MigrationReport, + domain: Domain, + source_id: &str, + creator_user_id: &ironclaw_host_api::UserId, + state: &mut TriggerState, +) -> Result<(), MigrationError> { + let owner = tgt + .user_directory(creator_user_id.clone()) + .get_user(creator_user_id) + .await + .map_err(|error| MigrationError::WriteTarget { + domain: format!("trigger owner {creator_user_id}"), + reason: error.to_string(), + })?; + if !matches!(owner, Some(user) if user.status == RebornUserStatus::Active) { + *state = TriggerState::Paused; + report.record_loss( + domain, + source_id, + "owner.status", + LossReason::Degraded, + "automation owner is not an active migrated user; trigger imported Paused", + ); + } + Ok(()) +} + +fn resolve_mission_project( + report: &mut MigrationReport, + source_id: &str, + mission: &Mission, + imported_project_ids: &BTreeSet, + state: &mut TriggerState, +) -> Result, MigrationError> { + let Some(source_project_id) = mission.project_id else { + return Ok(None); + }; + let project_id = ProjectId::new(source_project_id.to_string()).map_err(|error| { + MigrationError::InvalidInput(format!( + "mission {} has invalid project id: {error}", + mission.id + )) + })?; + if imported_project_ids.contains(project_id.as_str()) { + return Ok(Some(project_id)); + } + *state = TriggerState::Paused; + report.record_loss( + Domain::Mission, + source_id, + "project_id", + LossReason::Degraded, + "mission references a project that was not imported; project scope was omitted and the trigger imported Paused", + ); + Ok(None) +} + /// The trigger's `next_run_at` for a migrated mission. A mission with an -/// explicit `next_fire_at` uses it; otherwise the fallback is the **migration -/// time**, not `mission.created_at` — the latter can be the `epoch_fallback` -/// synthesized when a drifted blob omits `created_at`, which would produce a -/// `1970`-dated, immediately-due trigger. The synthesized fallback is recorded -/// as a `Degraded` loss so it is never silent. +/// explicit `next_fire_at` uses it; otherwise the deterministic source +/// `created_at` is retained and an active mission is forced to `Paused`. The +/// degraded fallback is recorded so an old timestamp can never silently arm an +/// immediately-due trigger. fn mission_next_run_at( report: &mut MigrationReport, source_id: &str, mission: &Mission, -) -> chrono::DateTime { +) -> (chrono::DateTime, bool) { match mission.next_fire_at { - Some(next_fire_at) => next_fire_at, + Some(next_fire_at) => (next_fire_at, false), None => { report.record_loss( Domain::Mission, source_id, "next_fire_at", LossReason::Degraded, - "mission had no next_fire_at; the trigger's next run was synthesized to the \ - migration time (mission created_at may be an epoch fallback, which would make \ - the trigger immediately due)" + "mission had no next_fire_at; its deterministic created_at is retained and an \ + active mission is imported Paused so migration cannot fire it unexpectedly" .to_string(), ); - chrono::Utc::now() + (mission.created_at, true) } } } @@ -563,3 +778,83 @@ fn engine_role(role: v2_model::MessageRole) -> ImportRole { v2_model::MessageRole::System | v2_model::MessageRole::ActionResult => ImportRole::Other, } } + +#[cfg(test)] +mod tests { + use super::{ + IndexedEngineThread, IndexedMission, insert_engine_thread, insert_mission, + parse_engine_document, + }; + use std::collections::BTreeMap; + + #[test] + fn exact_engine_documents_are_deduplicated_by_durable_id() { + let mission_json = serde_json::json!({ + "id": "11111111-1111-4111-8111-111111111111", + "user_id": "alice", + "name": "daily", + "cadence": "Manual" + }); + let content = mission_json.to_string(); + let (representation, mission) = + parse_engine_document::(&content).unwrap(); + let mut missions = BTreeMap::new(); + insert_mission( + &mut missions, + IndexedMission { + source: "first".to_string(), + owner: "alice".to_string(), + representation: representation.clone(), + mission: mission.clone(), + }, + ) + .unwrap(); + insert_mission( + &mut missions, + IndexedMission { + source: "second".to_string(), + owner: "alice".to_string(), + representation, + mission, + }, + ) + .unwrap(); + assert_eq!(missions.len(), 1); + } + + #[test] + fn divergent_engine_documents_are_rejected_by_durable_id() { + let first = serde_json::json!({ + "id": "22222222-2222-4222-8222-222222222222", + "title": "first" + }); + let second = serde_json::json!({ + "id": "22222222-2222-4222-8222-222222222222", + "title": "second" + }); + let (first_representation, first_thread) = + parse_engine_document::(&first.to_string()).unwrap(); + let (second_representation, second_thread) = + parse_engine_document::(&second.to_string()).unwrap(); + let mut threads = BTreeMap::new(); + insert_engine_thread( + &mut threads, + IndexedEngineThread { + source: "first".to_string(), + representation: first_representation, + thread: first_thread, + }, + ) + .unwrap(); + let error = insert_engine_thread( + &mut threads, + IndexedEngineThread { + source: "second".to_string(), + representation: second_representation, + thread: second_thread, + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("divergent state")); + } +} diff --git a/crates/ironclaw_reborn_migration/src/convert/extensions.rs b/crates/ironclaw_reborn_migration/src/convert/extensions.rs index c760639414c..f40fba88a7f 100644 --- a/crates/ironclaw_reborn_migration/src/convert/extensions.rs +++ b/crates/ironclaw_reborn_migration/src/convert/extensions.rs @@ -1,40 +1,20 @@ -//! Extensions/channels/tools converter -//! (v1 `wasm_tools` / `wasm_channels` / `tool_capabilities` → Reborn -//! `ExtensionInstallation`). +//! v1 extension inventory and safe disposition. //! -//! Installed v1 WASM tools/channels are accumulated by validated -//! `ExtensionId`, then each group becomes one canonical Reborn -//! `ExtensionInstallation` with `installation_id == extension_id`. The -//! synthesized `InstalledLocal` manifest declares the -//! `ironclaw.capability_provider/v1` host API plus one placeholder, -//! approval-gated capability (a non-first-party manifest must declare a host API -//! or capability, and rejects top-level `[[capabilities]]`). Activation maps -//! from the v1 `status` column, failing closed when grouped source states -//! disagree; a tool's `tool_capabilities.allowed_secrets` become merged -//! `ExtensionCredentialBinding`s pointing at the migrated secrets. The store -//! itself is built by composition's `migration-support` seam -//! (`RebornTarget::extension_store`). +//! A v1 WASM row is not a runnable Reborn extension package: the source row +//! does not carry a Reborn manifest, package assets, schemas, prompt docs, or a +//! catalog identity. Production restore resolves *every* installation through +//! the available-extension catalog, including disabled installations. Writing +//! a synthesized placeholder would therefore make a later cold boot fail. //! -//! Losses recorded (per installation): the manifest is a placeholder — the v1 -//! tool's real capability contract and WASM binary are NOT carried over; tool -//! capability config beyond credential linkage (http_allowlist, rate limits, -//! workspace prefixes); and channel credential linkage (v1 has no explicit -//! channel→secret join; the secret *values* still migrate via the secrets -//! converter). +//! Until a converter can resolve a v1 artifact to a real bundled package, this +//! converter records an explicit reinstall/re-auth requirement and writes no +//! manifest or installation state. Unknown executable artifacts are never +//! silently made live. -use std::collections::BTreeMap; use std::sync::Arc; use ironclaw::channels::wasm::{StoredWasmChannel, WasmChannelStore}; -use ironclaw::tools::wasm::{StoredWasmTool, ToolStatus, WasmToolStore}; -use ironclaw_extensions::{ - ExtensionActivationState, ExtensionCredentialBinding, ExtensionCredentialHandle, - ExtensionInstallation, ExtensionInstallationError, ExtensionInstallationId, - ExtensionManifestRecord, ExtensionManifestRef, HostApiContractRegistry, InstallationOwner, - MANIFEST_SCHEMA_VERSION, ManifestSource, canonicalize_installation_rows, -}; -use ironclaw_host_api::{ExtensionId, HostPortCatalog, SecretHandle, UserId}; -use ironclaw_host_runtime::{default_host_api_contract_registry, default_host_port_catalog}; +use ironclaw::tools::wasm::{StoredWasmTool, WasmToolStore}; use crate::error::MigrationError; use crate::options::MigrationOptions; @@ -44,25 +24,13 @@ use crate::target::RebornTarget; pub(crate) async fn run( src: &V1Source, - tgt: &mut RebornTarget, - options: &MigrationOptions, + _tgt: &mut RebornTarget, + _options: &MigrationOptions, report: &mut MigrationReport, ) -> Result<(), MigrationError> { - let catalog = default_host_port_catalog().map_err(|e| MigrationError::WriteTarget { - domain: "extension host-port catalog".into(), - reason: e.to_string(), - })?; - let registry = - default_host_api_contract_registry().map_err(|e| MigrationError::WriteTarget { - domain: "extension host-api contract registry".into(), - reason: e.to_string(), - })?; - let tool_store = build_tool_store(src); let channel_store = build_channel_store(src); - let mut candidates_by_extension: BTreeMap> = BTreeMap::new(); - // Installed tools/channels are keyed by user_id; enumerate from both tables. let mut users: std::collections::BTreeSet = src.distinct_users().await?.into_iter().collect(); users.extend(src.distinct_user_ids_in("wasm_tools", "user_id").await?); @@ -70,367 +38,91 @@ pub(crate) async fn run( for user in users { if let Some(store) = tool_store.as_ref() { - let tools = store + for tool in store .list(&user) .await - .map_err(|e| MigrationError::ReadSource { + .map_err(|error| MigrationError::ReadSource { domain: "wasm_tools".into(), - reason: e.to_string(), - })?; - for tool in tools { - let bindings = tool_credential_bindings(store.as_ref(), &tool, report).await?; - collect_installation( - report, - &mut candidates_by_extension, - InstallInput { - owner: &user, - raw_name: &tool.name, - version: &tool.version, - description: &tool.description, - active: tool.status == ToolStatus::Active, - updated_at: tool.updated_at, - bindings, - }, - ); + reason: error.to_string(), + })? + { + record_tool_disposition(store.as_ref(), &tool, report).await?; } } if let Some(store) = channel_store.as_ref() { - let channels = store + for channel in store .list(&user) .await - .map_err(|e| MigrationError::ReadSource { + .map_err(|error| MigrationError::ReadSource { domain: "wasm_channels".into(), - reason: e.to_string(), - })?; - for channel in channels { - collect_installation( - report, - &mut candidates_by_extension, - channel_input(&user, &channel), - ); - report.record_loss( - Domain::Extension, - format!("channel:{}", channel.name), - "credential_binding", - LossReason::NoTargetField, - "v1 has no explicit channel→secret join; the credential value still \ - migrates via the secrets converter, but the installation binding is \ - not auto-linked" - .to_string(), - ); + reason: error.to_string(), + })? + { + record_channel_disposition(&channel, report); } } } - - for candidates in candidates_by_extension.into_values() { - write_canonical_installation(tgt, options, report, &catalog, ®istry, candidates).await?; - } Ok(()) } -struct InstallInput<'a> { - /// v1 owner user id. Candidates are merged into one private installation - /// membership set after the source id has been validated. - owner: &'a str, - raw_name: &'a str, - version: &'a str, - description: &'a str, - active: bool, - updated_at: chrono::DateTime, - bindings: Vec, -} - -struct InstallCandidate { - extension_id: ExtensionId, - owner: UserId, - raw_name: String, - version: String, - description: String, - active: bool, - updated_at: chrono::DateTime, - bindings: Vec, -} - -fn channel_input<'a>(owner: &'a str, channel: &'a StoredWasmChannel) -> InstallInput<'a> { - InstallInput { - owner, - raw_name: &channel.name, - version: &channel.version, - description: &channel.description, - active: channel.status == "active", - updated_at: channel.updated_at, - bindings: Vec::new(), - } -} - -fn collect_installation( +async fn record_tool_disposition( + store: &dyn WasmToolStore, + tool: &StoredWasmTool, report: &mut MigrationReport, - candidates_by_extension: &mut BTreeMap>, - input: InstallInput<'_>, -) { - let source_id = format!("extension:{}:{}", input.owner, input.raw_name); - let ext_id_str = sanitize_extension_id(input.raw_name); - let extension_id = match ExtensionId::new(&ext_id_str) { - Ok(id) => id, - Err(e) => { - report.record_loss( - Domain::Extension, - &source_id, - "id", - LossReason::Unparseable, - format!("could not derive a valid Reborn extension id: {e}"), - ); - return; - } - }; - - // The synthesized manifest is a migration placeholder: v1 tools have no - // Reborn capability contract and the WASM binary is not carried over, so a - // single generic host-mediated capability stands in. Record that gap. +) -> Result<(), MigrationError> { + let source_id = format!("tool:{}", tool.name); report.record_loss( Domain::Extension, &source_id, - "manifest_fidelity", - LossReason::Degraded, - "v1 tool capability contract + WASM binary are not migrated; a placeholder \ - capability_provider manifest is synthesized so the installation record + \ - activation + credential bindings carry over" + "package", + LossReason::NoTargetConcept, + "v1 WASM has no catalog-backed Reborn package; no installation was written or enabled. \ + Reinstall a compatible Reborn extension after cutover" .to_string(), ); - // v1 installs were per-user; carry each validated owner into the eventual - // canonical private membership set under the #5459 P1 ownership model. - let owner = match UserId::new(input.owner) { - Ok(user_id) => user_id, - Err(e) => { - report.record_loss( - Domain::Extension, - &source_id, - "owner", - LossReason::Unparseable, - format!("invalid owner user id: {e}"), - ); - return; - } - }; - candidates_by_extension - .entry(extension_id.clone()) - .or_default() - .push(InstallCandidate { - extension_id, - owner, - raw_name: input.raw_name.to_string(), - version: input.version.to_string(), - description: input.description.to_string(), - active: input.active, - updated_at: input.updated_at, - bindings: input.bindings, - }); -} - -// arch-exempt: too_many_args, migration converter scope + catalog + registry + group, plan #5459 -#[allow(clippy::too_many_arguments)] -async fn write_canonical_installation( - tgt: &RebornTarget, - options: &MigrationOptions, - report: &mut MigrationReport, - catalog: &HostPortCatalog, - registry: &HostApiContractRegistry, - candidates: Vec, -) -> Result<(), MigrationError> { - let Some((first, rest)) = candidates.split_first() else { - return Ok(()); - }; - let source_id = format!("extension:{}", first.extension_id); - - if rest.iter().any(|candidate| { - candidate.raw_name != first.raw_name - || candidate.version != first.version - || candidate.description != first.description - }) { - report.record_loss( + match store.get_capabilities(tool.id).await { + Ok(Some(_)) => report.record_loss( Domain::Extension, - &source_id, - "canonicalization", - LossReason::Unparseable, - "source names or manifest metadata sanitize to the same ExtensionId but disagree; \ - the group was skipped and no partial canonical installation was written" + source_id, + "capabilities", + LossReason::NoTargetField, + "v1 HTTP/workspace/rate-limit capability policy and allowed-secret bindings are \ + archive-only; review and bind credentials again after reinstall" .to_string(), - ); - return Ok(()); - } - - let manifest_toml = build_manifest_toml( - first.extension_id.as_str(), - &first.raw_name, - &first.version, - &first.description, - ); - let manifest = match ExtensionManifestRecord::from_toml_with_contracts( - manifest_toml, - ManifestSource::InstalledLocal, - catalog, - None, - registry, - ) { - Ok(manifest) => manifest, - Err(e) => { - report.record_loss( - Domain::Extension, - &source_id, - "manifest", - LossReason::Unparseable, - format!("synthesized manifest did not validate: {e}"), - ); - return Ok(()); - } - }; - - // Build one typed row per source install, then let the generic extension - // reducer own canonical ids, ownership, activation, credentials, and - // timestamps. The source-derived id exists only as a deterministic health - // tie-break; the reducer replaces it with `extension_id`. - let installations = candidates - .into_iter() - .map(|candidate| { - let installation_id = ExtensionInstallationId::new(format!( - "{}:{}", - candidate.extension_id.as_str(), - candidate.owner.as_str() - ))?; - ExtensionInstallation::new( - installation_id, - candidate.extension_id.clone(), - if candidate.active { - ExtensionActivationState::Enabled - } else { - ExtensionActivationState::Disabled - }, - ExtensionManifestRef::new(candidate.extension_id.clone(), None), - candidate.bindings, - candidate.updated_at, - InstallationOwner::user(candidate.owner), - ) - }) - .collect::, ExtensionInstallationError>>(); - let installations = match installations { - Ok(installations) => installations, + ), + Ok(None) => {} Err(error) => { - report.record_loss( - Domain::Extension, - &source_id, - "installation", - LossReason::Unparseable, - format!( - "could not build source installation rows: {error}; the group was skipped and no partial canonical installation was written" - ), - ); - return Ok(()); - } - }; - let mut canonical = match canonicalize_installation_rows(installations) { - Ok(canonical) => canonical, - Err(ExtensionInstallationError::ConflictingCredentialBinding { handle, .. }) => { - report.record_loss( - Domain::Extension, - &source_id, - "credential_bindings", - LossReason::Unparseable, - format!( - "credential handle '{handle}' maps to conflicting secret handles; the group was skipped and no partial canonical installation was written" - ), - ); - return Ok(()); - } - Err(error) => { - report.record_loss( - Domain::Extension, - &source_id, - "canonicalization", - LossReason::Unparseable, - format!( - "could not reduce source installation rows: {error}; the group was skipped and no partial canonical installation was written" - ), - ); - return Ok(()); + return Err(MigrationError::ReadSource { + domain: "tool_capabilities".into(), + reason: error.to_string(), + }); } - }; - let Some(installation) = canonical.pop() else { - report.record_loss( - Domain::Extension, - &source_id, - "canonicalization", - LossReason::Unparseable, - "no canonical installation row was produced; the group was skipped and no partial canonical installation was written".to_string(), - ); - return Ok(()); - }; - - if !options.dry_run { - tgt.extension_store - .upsert_manifest_and_installation(manifest, installation) - .await - .map_err(|e| MigrationError::WriteTarget { - domain: format!("extension {source_id}"), - reason: e.to_string(), - })?; } - report.stats.extensions += 1; Ok(()) } -/// Build credential bindings from a tool's `allowed_secrets`, recording the -/// capability config that has no Reborn target. -async fn tool_credential_bindings( - store: &dyn WasmToolStore, - tool: &StoredWasmTool, - report: &mut MigrationReport, -) -> Result, MigrationError> { - // A read *error* is a real infrastructure failure and aborts the run; a - // legitimate "no capabilities row" (`Ok(None)`) just yields no bindings. - let capabilities = match store.get_capabilities(tool.id).await { - Ok(Some(capabilities)) => capabilities, - Ok(None) => return Ok(Vec::new()), - Err(e) => { - return Err(MigrationError::ReadSource { - domain: "tool_capabilities".into(), - reason: e.to_string(), - }); - } - }; +fn record_channel_disposition(channel: &StoredWasmChannel, report: &mut MigrationReport) { + let source_id = format!("channel:{}", channel.name); report.record_loss( Domain::Extension, - format!("tool:{}", tool.name), - "capabilities", + &source_id, + "package", + LossReason::NoTargetConcept, + "v1 WASM channel has no catalog-backed Reborn package; no installation was written or \ + enabled. Reinstall a compatible Reborn extension after cutover" + .to_string(), + ); + report.record_loss( + Domain::Extension, + source_id, + "credential_binding", LossReason::NoTargetField, - "tool http_allowlist / rate limits / workspace prefixes have no Reborn \ - installation field" + "v1 has no explicit channel-to-secret join; re-authenticate and bind credentials after \ + reinstall" .to_string(), ); - let mut bindings_by_handle = BTreeMap::new(); - for secret_name in capabilities.allowed_secrets { - match ( - ExtensionCredentialHandle::new(secret_name.clone()), - SecretHandle::new(&secret_name), - ) { - (Ok(handle), Ok(secret_handle)) => { - bindings_by_handle.entry(handle).or_insert(secret_handle); - } - // An unconvertible secret name is recorded, not dropped silently. - _ => report.record_loss( - Domain::Extension, - format!("tool:{}", tool.name), - "allowed_secret", - LossReason::Unparseable, - format!("secret name '{secret_name}' is not a valid Reborn credential binding"), - ), - } - } - Ok(bindings_by_handle - .into_iter() - .map(|(handle, secret_handle)| ExtensionCredentialBinding::new(handle, secret_handle)) - .collect()) } fn build_tool_store(src: &V1Source) -> Option> { @@ -448,7 +140,6 @@ fn build_tool_store(src: &V1Source) -> Option> { } None } - fn build_channel_store(src: &V1Source) -> Option> { #[cfg(feature = "libsql")] if let Some(db) = src.handles.libsql_db.as_ref() { @@ -464,90 +155,3 @@ fn build_channel_store(src: &V1Source) -> Option> { } None } - -/// Sanitize a v1 tool/channel name into a valid Reborn `ExtensionId` -/// (`validate_name_segment`: lowercase, starts alnum, `[a-z0-9._-]`, ≤128). -fn sanitize_extension_id(raw: &str) -> String { - let mut out = String::with_capacity(raw.len()); - for ch in raw.chars() { - let lower = ch.to_ascii_lowercase(); - if lower.is_ascii_alphanumeric() || matches!(lower, '_' | '-' | '.') { - out.push(lower); - } else { - out.push('_'); - } - } - // Must start with an alphanumeric. - if !out - .chars() - .next() - .is_some_and(|c| c.is_ascii_alphanumeric()) - { - out.insert(0, 'x'); - } - out.truncate(128); - if out.is_empty() { - out.push_str("ext"); - } - out -} - -fn build_manifest_toml(ext_id: &str, name: &str, version: &str, description: &str) -> String { - // A valid non-first-party manifest declares the capability_provider host API - // and at least one namespaced, host-mediated capability (empty manifests and - // top-level `[[capabilities]]` are both rejected). `ask` permission keeps the - // migrated tool approval-gated. - format!( - r#"schema_version = "{schema}" -id = "{ext_id}" -name = "{name}" -version = "{version}" -description = "{description}" -trust = "third_party" - -[runtime] -kind = "wasm" -module = "wasm/{ext_id}.wasm" - -[[host_api]] -id = "ironclaw.capability_provider/v1" -section = "capability_provider.tools" - -[[capability_provider.tools.capabilities]] -id = "{ext_id}.invoke" -description = "Migrated v1 tool capability (placeholder)." -default_permission = "ask" -visibility = "model" -input_schema_ref = "schemas/{ext_id}/invoke.input.v1.json" -output_schema_ref = "schemas/{ext_id}/invoke.output.v1.json" -prompt_doc_ref = "prompts/{ext_id}/invoke.md" -"#, - schema = MANIFEST_SCHEMA_VERSION, - name = toml_escape(name), - version = toml_escape(normalize_version(version)), - description = toml_escape(description), - ) -} - -fn normalize_version(version: &str) -> &str { - if version.trim().is_empty() { - "0.1.0" - } else { - version - } -} - -/// Escape a value for a TOML basic string: backslash + quote escaped, control -/// characters (incl. newlines) dropped so the synthesized manifest stays valid. -fn toml_escape(value: &str) -> String { - let mut out = String::with_capacity(value.len()); - for ch in value.chars() { - match ch { - '\\' => out.push_str("\\\\"), - '"' => out.push_str("\\\""), - c if c.is_control() => out.push(' '), - c => out.push(c), - } - } - out -} diff --git a/crates/ironclaw_reborn_migration/src/convert/heartbeat.rs b/crates/ironclaw_reborn_migration/src/convert/heartbeat.rs index bf4d92c004a..434c7e5b8f3 100644 --- a/crates/ironclaw_reborn_migration/src/convert/heartbeat.rs +++ b/crates/ironclaw_reborn_migration/src/convert/heartbeat.rs @@ -17,9 +17,7 @@ pub(crate) async fn run( _options: &MigrationOptions, report: &mut MigrationReport, ) -> Result<(), MigrationError> { - // heartbeat_state is keyed per (user_id, agent_id); enumerate distinct users - // and record the gap. No typed all-user read API exists for heartbeat state. - for user_id in src.distinct_users().await? { + for user_id in src.heartbeat_user_ids().await? { report.record_loss( Domain::Heartbeat, user_id, diff --git a/crates/ironclaw_reborn_migration/src/convert/memory.rs b/crates/ironclaw_reborn_migration/src/convert/memory.rs index 9f757ed5405..7105422876b 100644 --- a/crates/ironclaw_reborn_migration/src/convert/memory.rs +++ b/crates/ironclaw_reborn_migration/src/convert/memory.rs @@ -2,15 +2,20 @@ //! `ironclaw_memory` documents). //! //! Each non-engine v1 document is written through the memory service under the -//! migrated (tenant, user, agent) scope; content and path are preserved. -//! Engine-v2 documents (mission/project/runtime blobs) are skipped here — they -//! are consumed by the automations converter. Chunks/embeddings are derived +//! migrated tenant/user scope while preserving its optional source-agent scope; +//! content and path are preserved. +//! Engine-v2 documents (mission/project/runtime blobs) are skipped here: +//! supported missions and projects are handled by their owning converters, +//! while other runtime blobs remain unsupported. Chunks/embeddings are derived //! state the memory service recomputes on write, so they are not migrated (not //! a loss). Version history (`memory_document_versions`) has no Reborn target //! and is recorded as a loss. -use ironclaw_host_api::{CorrelationId, InvocationId, ResourceScope}; -use ironclaw_memory::{DocumentMetadata, MemoryInvocation, MemoryServiceWriteRequest}; +use ironclaw_host_api::{AgentId, CorrelationId, InvocationId, ResourceScope}; +use ironclaw_memory::{ + DocumentMetadata, MemoryInvocation, MemoryServiceErrorKind, MemoryServiceReadRequest, + MemoryServiceWriteRequest, +}; use crate::error::MigrationError; use crate::options::MigrationOptions; @@ -27,18 +32,20 @@ pub(crate) async fn run( ) -> Result<(), MigrationError> { let users = src.distinct_users().await?; for user_id in &users { - let docs = - src.db - .list_documents(user_id, None) - .await - .map_err(|e| MigrationError::ReadSource { - domain: "memory_documents".into(), - reason: e.to_string(), - })?; + let docs = src.all_memory_documents(user_id).await?; for doc in docs { if v2_model::is_engine_path(&doc.path) { - continue; // engine-v2 state — handled by the automations converter + if !has_engine_converter(&doc.path) { + report.record_loss( + Domain::Memory, + doc.path, + "*", + LossReason::NoTargetConcept, + "engine-v2 runtime document has no supported Reborn converter and was not migrated", + ); + } + continue; } // A malformed source user id is a per-item loss, not a run abort. @@ -56,7 +63,14 @@ pub(crate) async fn run( let scope = ResourceScope { tenant_id: tgt.tenant_id.clone(), user_id: user, - agent_id: Some(tgt.agent_id.clone()), + agent_id: doc + .agent_id + .map(|agent_id| AgentId::new(agent_id.to_string())) + .transpose() + .map_err(|error| MigrationError::WriteTarget { + domain: format!("memory document {}", doc.path), + reason: format!("preserve source agent scope: {error}"), + })?, project_id: None, mission_id: None, thread_id: None, @@ -67,6 +81,63 @@ pub(crate) async fn run( correlation_id: CorrelationId::new(), }; let metadata = DocumentMetadata::from_value(&doc.metadata); + match tgt + .memory_service + .read( + invocation.clone(), + MemoryServiceReadRequest { + path: doc.path.clone(), + }, + ) + .await + { + Ok(existing) if existing.content == doc.content => { + let existing_metadata = tgt + .memory_service + .read_metadata( + invocation.clone(), + MemoryServiceReadRequest { + path: doc.path.clone(), + }, + ) + .await + .map_err(|error| MigrationError::WriteTarget { + domain: format!("memory document {}", doc.path), + reason: format!("read deterministic target metadata: {error}"), + })? + .metadata; + if existing_metadata.as_ref() == Some(&metadata) + || (existing_metadata.is_none() && metadata == DocumentMetadata::default()) + { + report.stats.memory_documents += 1; + continue; + } + if existing_metadata.is_some() { + return Err(MigrationError::WriteTarget { + domain: format!("memory document {}", doc.path), + reason: "deterministic memory path already contains divergent metadata; refusing to overwrite" + .to_string(), + }); + } + } + Ok(_) => { + return Err(MigrationError::WriteTarget { + domain: format!("memory document {}", doc.path), + reason: "deterministic memory path already contains divergent content; refusing to overwrite" + .to_string(), + }); + } + // Native memory reports a missing document as an input error. + // The subsequent write remains the authoritative path + // validation, so malformed source paths still fail closed. + Err(error) if error.kind() == MemoryServiceErrorKind::Input => {} + Err(error) => { + return Err(MigrationError::WriteTarget { + domain: format!("memory document {}", doc.path), + reason: format!("read deterministic target slot: {error}"), + }); + } + } let request = MemoryServiceWriteRequest { target: doc.path.clone(), content: doc.content.clone(), @@ -100,3 +171,30 @@ pub(crate) async fn run( ); Ok(()) } + +fn has_engine_converter(path: &str) -> bool { + if path.ends_with("mission.json") || (path.contains("/threads/") && path.ends_with(".json")) { + return true; + } + let segments: Vec<_> = path.trim_matches('/').split('/').collect(); + matches!( + segments.as_slice(), + ["engine", "projects", slug, "project.json"] + | [".system", "engine", "projects", slug, "project.json"] + if !slug.is_empty() + ) +} + +#[cfg(test)] +mod tests { + use super::has_engine_converter; + + #[test] + fn only_engine_documents_owned_by_specialized_converters_are_recognized() { + assert!(has_engine_converter("engine/missions/daily/mission.json")); + assert!(has_engine_converter(".system/engine/threads/id.json")); + assert!(has_engine_converter("engine/projects/demo/project.json")); + assert!(!has_engine_converter("engine/runtime/checkpoint.json")); + assert!(!has_engine_converter("engine/projects/project.json")); + } +} diff --git a/crates/ironclaw_reborn_migration/src/convert/mod.rs b/crates/ironclaw_reborn_migration/src/convert/mod.rs index 968efafeadb..9f3e95aaf92 100644 --- a/crates/ironclaw_reborn_migration/src/convert/mod.rs +++ b/crates/ironclaw_reborn_migration/src/convert/mod.rs @@ -7,6 +7,8 @@ pub(crate) mod heartbeat; pub(crate) mod identities; pub(crate) mod jobs; pub(crate) mod memory; +pub(crate) mod projects; pub(crate) mod secrets; pub(crate) mod settings; pub(crate) mod threads; +pub(crate) mod users; diff --git a/crates/ironclaw_reborn_migration/src/convert/projects.rs b/crates/ironclaw_reborn_migration/src/convert/projects.rs new file mode 100644 index 00000000000..e463283d9d7 --- /dev/null +++ b/crates/ironclaw_reborn_migration/src/convert/projects.rs @@ -0,0 +1,214 @@ +//! Engine-v2 project converter. +//! +//! Project state lived in v1 `memory_documents` under one user-visible and two +//! system layouts. All deserialize through the compatibility DTO and converge +//! on the canonical Reborn [`ProjectRepository`]. A repeated source/project id +//! must be exact; divergent source or target state fails without overwriting it. + +use std::collections::{BTreeMap, BTreeSet}; + +use ironclaw_host_api::ProjectId; +use ironclaw_projects::{ProjectError, ProjectRecord, ProjectState}; +use serde_json::json; + +use crate::error::MigrationError; +use crate::options::MigrationOptions; +use crate::report::{Domain, LossReason, MigrationReport}; +use crate::source::V1Source; +use crate::target::RebornTarget; +use crate::v2_model; + +pub(crate) async fn run( + source: &V1Source, + target: &RebornTarget, + options: &MigrationOptions, + report: &mut MigrationReport, +) -> Result, MigrationError> { + let mut projects: BTreeMap = BTreeMap::new(); + + for document in source.project_documents().await? { + let Some(slug) = project_slug(&document.path) else { + continue; + }; + let source_id = format!("project:{}:{}", slug, document.path); + let project = match serde_json::from_str::(&document.content) { + Ok(project) => project, + Err(error) => { + report.record_loss( + Domain::Project, + source_id, + "*", + LossReason::Unparseable, + format!("engine-v2 project JSON could not be parsed: {error}"), + ); + continue; + } + }; + let owner_raw = if project.user_id.is_empty() { + document.user_id.as_str() + } else { + project.user_id.as_str() + }; + let Some(owner_user_id) = + report.valid_user_id(Domain::Project, &source_id, "user_id", owner_raw) + else { + continue; + }; + let project_id = + ProjectId::new(project.id.to_string()).map_err(|error| MigrationError::ReadSource { + domain: source_id.clone(), + reason: format!("engine-v2 project UUID is not a valid Reborn ProjectId: {error}"), + })?; + let updated_at = project.updated_at.unwrap_or(project.created_at); + let record = ProjectRecord { + project_id: project_id.clone(), + tenant_id: target.tenant_id.clone(), + owner_user_id, + name: project.name, + description: project.description, + icon: None, + color: None, + metadata: json!({ + "legacy_engine_v2": { + "goals": project.goals, + "metrics": project.metrics, + "metadata": project.metadata, + "workspace_path": project.workspace_path, + } + }), + state: ProjectState::Active, + created_at: project.created_at, + updated_at, + }; + if let Err(error) = record.validate() { + report.record_loss( + Domain::Project, + source_id, + "*", + LossReason::Unparseable, + format!("engine-v2 project cannot satisfy the Reborn project contract: {error}"), + ); + continue; + } + + let key = project_id.as_str().to_string(); + if let Some((existing_source, existing)) = projects.get(&key) { + if existing != &record { + return Err(MigrationError::WriteTarget { + domain: format!("project {key}"), + reason: format!( + "source documents {existing_source} and {} contain divergent state for the same project id", + document.path + ), + }); + } + continue; + } + projects.insert(key, (document.path, record)); + } + + let imported_project_ids = projects.keys().cloned().collect(); + for (source_id, record) in projects.into_values() { + if !options.dry_run { + compare_and_create(target, &source_id, record).await?; + } + report.stats.projects = report.stats.projects.saturating_add(1); + } + Ok(imported_project_ids) +} + +async fn compare_and_create( + target: &RebornTarget, + source_id: &str, + record: ProjectRecord, +) -> Result<(), MigrationError> { + if let Some(existing) = target + .project_repo + .get_project(&record.tenant_id, &record.project_id) + .await + .map_err(|error| project_write_error(source_id, "read deterministic target slot", error))? + { + return if existing == record { + Ok(()) + } else { + Err(project_conflict(source_id, &record.project_id)) + }; + } + + match target.project_repo.create_project(record.clone()).await { + Ok(()) => Ok(()), + Err(ProjectError::AlreadyExists) => { + let existing = target + .project_repo + .get_project(&record.tenant_id, &record.project_id) + .await + .map_err(|error| { + project_write_error(source_id, "reconcile concurrent create", error) + })?; + match existing { + Some(existing) if existing == record => Ok(()), + Some(_) => Err(project_conflict(source_id, &record.project_id)), + None => Err(MigrationError::WriteTarget { + domain: format!("project {source_id}"), + reason: "project vanished while reconciling a concurrent create".to_string(), + }), + } + } + Err(error) => Err(project_write_error(source_id, "create project", error)), + } +} + +fn project_conflict(source_id: &str, project_id: &ProjectId) -> MigrationError { + MigrationError::WriteTarget { + domain: format!("project {source_id}"), + reason: format!( + "project id {} already contains divergent state; refusing to overwrite", + project_id.as_str() + ), + } +} + +fn project_write_error(source_id: &str, operation: &str, error: ProjectError) -> MigrationError { + MigrationError::WriteTarget { + domain: format!("project {source_id}"), + reason: format!("{operation}: {error}"), + } +} + +fn project_slug(path: &str) -> Option<&str> { + let segments: Vec<_> = path.trim_matches('/').split('/').collect(); + match segments.as_slice() { + ["projects", slug, ".project.json"] if !slug.is_empty() => Some(slug), + [".system", "engine", "projects", slug, "project.json"] if !slug.is_empty() => Some(slug), + ["engine", "projects", slug, "project.json"] if !slug.is_empty() => Some(slug), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::project_slug; + + #[test] + fn recognizes_both_project_document_layouts() { + assert_eq!(project_slug("projects/alpha/.project.json"), Some("alpha")); + assert_eq!( + project_slug(".system/engine/projects/beta/project.json"), + Some("beta") + ); + assert_eq!( + project_slug("engine/projects/legacy/project.json"), + Some("legacy") + ); + } + + #[test] + fn rejects_mission_and_near_match_paths() { + assert_eq!( + project_slug(".system/engine/projects/beta/missions/x/mission.json"), + None + ); + assert_eq!(project_slug("projects/alpha/project.json"), None); + assert_eq!(project_slug("projects//.project.json"), None); + } +} diff --git a/crates/ironclaw_reborn_migration/src/convert/secrets.rs b/crates/ironclaw_reborn_migration/src/convert/secrets.rs index 256d597ae79..31a8ad52089 100644 --- a/crates/ironclaw_reborn_migration/src/convert/secrets.rs +++ b/crates/ironclaw_reborn_migration/src/convert/secrets.rs @@ -2,17 +2,17 @@ //! //! v1 and Reborn both use AES-256-GCM but bind ciphertext to *different* schemes, //! so migration must **decrypt** each v1 secret and **re-encrypt** through -//! Reborn's `SecretStore::put`. Decryption uses the v1 secrets store constructed -//! with the supplied master key (`--secret-master-key`); the same key builds the -//! Reborn crypto in `RebornTarget::secret_store`. Without a master key, secrets -//! are skipped with a recorded loss. A secret whose decrypt fails (e.g. expired, -//! or wrong key) is recorded per-secret and skipped rather than aborting the run. +//! Reborn's secret store. Decryption uses the v1 secrets store constructed +//! with the independently supplied source key; `RebornTarget::secret_store` is +//! already built with the separately resolved target key. A secret whose decrypt fails (e.g. +//! expired or wrong key) is recorded per-secret and skipped rather than aborting +//! the run. use std::sync::Arc; use ironclaw::secrets::{SecretsCrypto, create_secrets_store}; use ironclaw_host_api::{InvocationId, ResourceScope, SecretHandle}; -use ironclaw_secrets::SecretMaterial; +use ironclaw_secrets::{SecretMaterial, SecretPutOutcome}; use crate::error::MigrationError; use crate::options::MigrationOptions; @@ -26,16 +26,12 @@ pub(crate) async fn run( options: &MigrationOptions, report: &mut MigrationReport, ) -> Result<(), MigrationError> { - let Some(master_key) = options.secret_master_key.as_ref() else { - report.record_loss( - Domain::Secret, - "secrets", - "*", - LossReason::NoTargetField, - "no --secret-master-key supplied; v1 secrets cannot be decrypted and were skipped" - .to_string(), - ); + let users = src.distinct_user_ids_in("secrets", "user_id").await?; + if users.is_empty() { return Ok(()); + } + let Some(master_key) = options.secret_master_key.as_ref() else { + return Err(MigrationError::MissingSecretKey); }; let Some(secret_store) = tgt.secret_store.clone() else { report.record_loss( @@ -65,7 +61,6 @@ pub(crate) async fn run( }; // v1 `list`/`get_decrypted` are per-user; enumerate users from the raw table. - let users = src.distinct_user_ids_in("secrets", "user_id").await?; for user_id in users { let refs = v1_store .list(&user_id) @@ -177,13 +172,24 @@ async fn migrate_one( invocation_id: InvocationId::new(), }; let material = SecretMaterial::from(decrypted.expose().to_string()); - secret_store - .put(scope, handle, material, expires_at) + let outcome = secret_store + .put_if_absent_or_matches(scope, handle, material, expires_at) .await .map_err(|e| MigrationError::WriteTarget { domain: format!("secret {user_id}:{name}"), reason: e.to_string(), })?; + if outcome == SecretPutOutcome::Divergent { + return Err(secret_collision(user_id, name)); + } report.stats.secrets += 1; Ok(()) } + +fn secret_collision(user_id: &str, name: &str) -> MigrationError { + MigrationError::WriteTarget { + domain: format!("secret {user_id}:{name}"), + reason: "deterministic secret slot already contains divergent state; refusing to overwrite" + .to_string(), + } +} diff --git a/crates/ironclaw_reborn_migration/src/convert/settings.rs b/crates/ironclaw_reborn_migration/src/convert/settings.rs index 58b277380e7..5a145072b95 100644 --- a/crates/ironclaw_reborn_migration/src/convert/settings.rs +++ b/crates/ironclaw_reborn_migration/src/convert/settings.rs @@ -22,19 +22,13 @@ pub(crate) async fn run( ) -> Result<(), MigrationError> { let users = src.distinct_users().await?; for user_id in &users { - let settings = - src.db - .get_all_settings(user_id) - .await - .map_err(|e| MigrationError::ReadSource { - domain: "settings".into(), - reason: e.to_string(), - })?; - for key in settings.keys() { + let settings = src.settings(user_id).await?; + for setting in settings { + let key = setting.key; report.record_loss( Domain::Setting, format!("{user_id}:{key}"), - key.clone(), + key, LossReason::NoTargetConcept, "Reborn config is a typed config.toml / providers.json / LlmKeyStore; \ there is no generic key/value settings store to migrate into. \ diff --git a/crates/ironclaw_reborn_migration/src/convert/threads.rs b/crates/ironclaw_reborn_migration/src/convert/threads.rs index 16ef869f151..82b812a83e1 100644 --- a/crates/ironclaw_reborn_migration/src/convert/threads.rs +++ b/crates/ironclaw_reborn_migration/src/convert/threads.rs @@ -5,17 +5,18 @@ //! become transcript messages in order through `SessionThreadService`. Because //! the append APIs assign their own timestamps and carry no per-message //! metadata, the original per-message `(role, created_at, id)` provenance is -//! preserved losslessly in the thread's `metadata_json` under a `legacy_v1` -//! key — content, ordering, and role all survive. +//! preserved in the thread's `metadata_json` under a `legacy_v1` key. Roles +//! without a first-class append port retain their content there as an archive; +//! user and assistant content lives in first-class transcript rows. //! //! The reusable [`write_thread`] helper is also used by the automations //! converter to migrate engine-v2 mission threads. use chrono::{DateTime, Utc}; -use ironclaw_host_api::{MissionId, ThreadId, UserId}; +use ironclaw_host_api::{MissionId, ProjectId, ThreadId, UserId}; use ironclaw_threads::{ AcceptInboundMessageRequest, AppendFinalizedAssistantMessageRequest, EnsureThreadRequest, - MessageContent, ThreadScope, + LoadContextMessagesRequest, MessageContent, MessageKind, MessageStatus, ThreadScope, }; use serde_json::json; use uuid::Uuid; @@ -25,6 +26,7 @@ use crate::options::MigrationOptions; use crate::report::{Domain, LossReason, MigrationReport}; use crate::source::V1Source; use crate::target::RebornTarget; +use crate::target::ids; /// Normalized role of a source transcript message. #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -60,6 +62,7 @@ pub(crate) struct ThreadImport { pub(crate) thread_id: Uuid, pub(crate) owner_user: String, pub(crate) title: Option, + pub(crate) project_id: Option, pub(crate) mission_id: Option, /// Provenance stored on the thread (channel, timestamps, source kind…). pub(crate) provenance: serde_json::Value, @@ -97,6 +100,7 @@ pub(crate) async fn run( thread_id: conv.id, owner_user: user_id.clone(), title: conv.title.clone(), + project_id: None, mission_id: None, provenance: json!({ "channel": conv.channel, @@ -169,11 +173,21 @@ pub(crate) async fn write_thread( }, None => None, }; + let project_id = match import.project_id { + Some(id) => match ProjectId::new(id.to_string()) { + Ok(project_id) => Some(project_id), + Err(e) => { + record_thread_id_loss(report, &import.thread_id, "project_id", e.to_string()); + return Ok(()); + } + }, + None => None, + }; let scope = ThreadScope { tenant_id: tgt.tenant_id.clone(), agent_id: tgt.agent_id.clone(), - project_id: None, + project_id, owner_user_id: Some(owner_user), mission_id, }; @@ -184,48 +198,103 @@ pub(crate) async fn write_thread( .unwrap_or_default(); let metadata_json = build_metadata_json(&import)?; + let migration_identity = ids::MigrationIdentity::from_report(report)?; - tgt.thread_service + let persisted_thread = tgt + .thread_service .ensure_thread(EnsureThreadRequest { scope: scope.clone(), thread_id: Some(thread_id.clone()), created_by_actor_id: actor_id.clone(), title: import.title.clone(), - metadata_json: Some(metadata_json), + metadata_json: Some(metadata_json.clone()), }) .await .map_err(|e| write_err("thread", &import.thread_id, e.to_string()))?; + if persisted_thread.created_by_actor_id != actor_id + || persisted_thread.title != import.title + || persisted_thread.metadata_json.as_deref() != Some(metadata_json.as_str()) + { + return Err(write_err( + "thread", + &import.thread_id, + "deterministic thread id already contains divergent source data; refusing to overwrite" + .to_string(), + )); + } - for message in import.messages { + let source_binding_id = migration_identity.thread_source_binding(import.thread_id); + for (message_index, message) in import.messages.into_iter().enumerate() { + let stable_message_key = migration_identity.message_key( + import.thread_id, + message_index, + message.orig_id.as_deref(), + ); match message.role { ImportRole::User => { - tgt.thread_service + let source_content = message.content; + let accepted = tgt + .thread_service .accept_inbound_message(AcceptInboundMessageRequest { scope: scope.clone(), thread_id: thread_id.clone(), actor_id: actor_id.clone(), - source_binding_id: None, + source_binding_id: Some(source_binding_id.clone()), reply_target_binding_id: None, - external_event_id: message.orig_id.clone(), - content: MessageContent::text(message.content), + external_event_id: Some(stable_message_key), + content: MessageContent::text(source_content.clone()), }) .await .map_err(|e| write_err("message", &import.thread_id, e.to_string()))?; + if accepted.idempotent_replay { + let replay = tgt + .thread_service + .load_context_messages(LoadContextMessagesRequest { + scope: scope.clone(), + thread_id: thread_id.clone(), + message_ids: vec![accepted.message_id], + }) + .await + .map_err(|e| write_err("message", &import.thread_id, e.to_string()))?; + let exact_replay = replay.messages.iter().any(|persisted| { + persisted.message_id == Some(accepted.message_id) + && persisted.kind == MessageKind::User + && persisted.content == source_content + }); + if !exact_replay { + return Err(write_err( + "message", + &import.thread_id, + "idempotency key already contains divergent user-message data; refusing to overwrite" + .to_string(), + )); + } + } report.stats.messages += 1; } ImportRole::Assistant => { - tgt.thread_service + let source_content = message.content; + let persisted = tgt + .thread_service .append_finalized_assistant_message(AppendFinalizedAssistantMessageRequest { scope: scope.clone(), thread_id: thread_id.clone(), - turn_run_id: message - .orig_id - .clone() - .unwrap_or_else(|| import.thread_id.to_string()), - content: MessageContent::text(message.content), + turn_run_id: stable_message_key, + content: MessageContent::text(source_content.clone()), }) .await .map_err(|e| write_err("message", &import.thread_id, e.to_string()))?; + if persisted.kind != MessageKind::Assistant + || persisted.status != MessageStatus::Finalized + || persisted.content.as_deref() != Some(source_content.as_str()) + { + return Err(write_err( + "message", + &import.thread_id, + "deterministic assistant-message key already contains divergent data; refusing to overwrite" + .to_string(), + )); + } report.stats.messages += 1; } ImportRole::Other => { @@ -250,6 +319,10 @@ fn build_metadata_json(import: &ThreadImport) -> Result "role": m.raw_role, "created_at": m.created_at.to_rfc3339(), "orig_id": m.orig_id, + // User/assistant content lives in first-class transcript rows. + // Roles without an append port are archived here so their + // payload is never silently reduced to metadata-only claims. + "archived_content": (m.role == ImportRole::Other).then_some(&m.content), }) }) .collect(); diff --git a/crates/ironclaw_reborn_migration/src/convert/users.rs b/crates/ironclaw_reborn_migration/src/convert/users.rs new file mode 100644 index 00000000000..96e201406a4 --- /dev/null +++ b/crates/ironclaw_reborn_migration/src/convert/users.rs @@ -0,0 +1,311 @@ +//! v1 canonical users -> Reborn canonical user directory. +//! +//! User ids and lifecycle timestamps are preserved. Reborn has no +//! `deactivated` state, so deactivated (and unknown) source statuses map to +//! `Suspended`, never `Active`. Unknown roles map to `Member`, never an admin +//! role. v1 API tokens contain only one-way hashes and are therefore reported +//! as requiring re-authentication; this converter never reads or writes token +//! hashes. + +use std::collections::{BTreeMap, BTreeSet}; + +use ironclaw::db::{ApiTokenRecord, UserRecord}; +use ironclaw_host_api::UserId; +use ironclaw_reborn_identity::{RebornUser, RebornUserRole, RebornUserStatus}; + +use crate::error::MigrationError; +use crate::options::MigrationOptions; +use crate::report::{Domain, LossReason, MigrationReport}; +use crate::source::V1Source; +use crate::target::RebornTarget; + +pub(crate) async fn run( + src: &V1Source, + tgt: &mut RebornTarget, + _options: &MigrationOptions, + report: &mut MigrationReport, +) -> Result<(), MigrationError> { + let (users, canonical_users_table_present) = match src.db.list_users(None).await { + Ok(users) => (users, true), + Err(error) if crate::source::is_missing_table_error(&error.to_string()) => { + (Vec::new(), false) + } + Err(error) => { + return Err(MigrationError::ReadSource { + domain: "users".to_string(), + reason: error.to_string(), + }); + } + }; + + let mut canonical_ids = BTreeSet::new(); + for source in users { + canonical_ids.insert(source.id.clone()); + report_api_tokens(src, &source, report).await?; + + let Some(user) = build_user(tgt, source, report) else { + continue; + }; + import_user(tgt, user, report).await?; + } + + // Import deterministic minimal users for ids that already own durable data + // so migrated records never point at a missing user. Only schemas without + // a canonical users table can safely treat those owners as active. + for raw_id in src.distinct_users().await? { + if canonical_ids.contains(&raw_id) { + continue; + } + let source_id = format!("data_owner:{raw_id}"); + let Some(user_id) = report.valid_user_id(Domain::User, &source_id, "user_id", &raw_id) + else { + continue; + }; + import_user( + tgt, + RebornUser { + user_id, + email: None, + display_name: Some(raw_id), + status: if canonical_users_table_present { + report.record_loss( + Domain::User, + &source_id, + "status", + LossReason::Degraded, + "durable-data owner is absent from the canonical users table; synthesized fail-closed as suspended", + ); + RebornUserStatus::Suspended + } else { + RebornUserStatus::Active + }, + role: RebornUserRole::Member, + created_at: "1970-01-01T00:00:00+00:00".to_string(), + updated_at: "1970-01-01T00:00:00+00:00".to_string(), + created_by: None, + last_login_at: None, + tenant_id: Some(tgt.tenant_id.clone()), + metadata: BTreeMap::from([ + ( + "migration.source".to_string(), + "ironclaw-v1-data-owner".to_string(), + ), + ("migration.synthesized".to_string(), "true".to_string()), + ]), + }, + report, + ) + .await?; + } + + Ok(()) +} + +async fn import_user( + tgt: &RebornTarget, + user: RebornUser, + report: &mut MigrationReport, +) -> Result<(), MigrationError> { + let directory = tgt.user_directory(user.user_id.clone()); + directory + .import_migrated_user(user.clone()) + .await + .map_err(|error| MigrationError::WriteTarget { + domain: format!("user {}", user.user_id), + reason: error.to_string(), + })?; + report.stats.users += 1; + Ok(()) +} + +fn build_user( + tgt: &RebornTarget, + source: UserRecord, + report: &mut MigrationReport, +) -> Option { + let source_id = format!("user:{}", source.id); + let user_id = report.valid_user_id(Domain::User, &source_id, "id", &source.id)?; + let status = map_status(&source.status, &source_id, report); + let role = map_role(&source.role, &source_id, report); + let created_by = source.created_by.as_deref().and_then(|raw| { + UserId::new(raw).map_or_else( + |error| { + report.record_loss( + Domain::User, + &source_id, + "created_by", + LossReason::Unparseable, + format!( + "source created_by is not a valid Reborn UserId and was omitted: {error}" + ), + ); + None + }, + Some, + ) + }); + let metadata = map_metadata(source.metadata, &source_id, report); + + Some(RebornUser { + user_id, + email: source.email, + display_name: Some(source.display_name), + status, + role, + created_at: source.created_at.to_rfc3339(), + updated_at: source.updated_at.to_rfc3339(), + created_by, + last_login_at: source.last_login_at.map(|at| at.to_rfc3339()), + tenant_id: Some(tgt.tenant_id.clone()), + metadata, + }) +} + +fn map_status(source: &str, source_id: &str, report: &mut MigrationReport) -> RebornUserStatus { + match source.trim().to_ascii_lowercase().as_str() { + "active" => RebornUserStatus::Active, + "suspended" => RebornUserStatus::Suspended, + "deactivated" => { + report.record_loss( + Domain::User, + source_id, + "status", + LossReason::Degraded, + "v1 deactivated has no Reborn equivalent; mapped fail-closed to suspended", + ); + RebornUserStatus::Suspended + } + _ => { + report.record_loss( + Domain::User, + source_id, + "status", + LossReason::Unparseable, + format!("unsupported v1 user status {source:?}; mapped fail-closed to suspended"), + ); + RebornUserStatus::Suspended + } + } +} + +fn map_role(source: &str, source_id: &str, report: &mut MigrationReport) -> RebornUserRole { + match source.trim().to_ascii_lowercase().as_str() { + "admin" => RebornUserRole::Admin, + "member" => RebornUserRole::Member, + _ => { + report.record_loss( + Domain::User, + source_id, + "role", + LossReason::Unparseable, + format!("unsupported v1 user role {source:?}; mapped fail-closed to member"), + ); + RebornUserRole::Member + } + } +} + +fn map_metadata( + source: serde_json::Value, + source_id: &str, + report: &mut MigrationReport, +) -> BTreeMap { + let serde_json::Value::Object(values) = source else { + report.record_loss( + Domain::User, + source_id, + "metadata", + LossReason::Unparseable, + "v1 user metadata was not a JSON object and was omitted", + ); + return BTreeMap::new(); + }; + + values + .into_iter() + .map(|(key, value)| match value { + serde_json::Value::String(value) => (key, value), + value => { + report.record_loss( + Domain::User, + source_id, + format!("metadata.{key}"), + LossReason::Degraded, + "non-string metadata value was preserved as compact JSON text", + ); + (key, value.to_string()) + } + }) + .collect() +} + +async fn report_api_tokens( + src: &V1Source, + user: &UserRecord, + report: &mut MigrationReport, +) -> Result<(), MigrationError> { + let tokens = match src.db.list_api_tokens(&user.id).await { + Ok(tokens) => tokens, + Err(error) if crate::source::is_missing_table_error(&error.to_string()) => Vec::new(), + Err(error) => { + return Err(MigrationError::ReadSource { + domain: "api_tokens".to_string(), + reason: error.to_string(), + }); + } + }; + for token in tokens { + record_token_reauth(token, report); + } + Ok(()) +} + +fn record_token_reauth(token: ApiTokenRecord, report: &mut MigrationReport) { + report.record_loss( + Domain::ApiToken, + format!("api_token:{}", token.id), + "*", + LossReason::NoTargetConcept, + "v1 API tokens store only one-way hashes and cannot be reused; issue a new Reborn credential after cutover", + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_lifecycle_values_map_fail_closed() { + let mut report = MigrationReport::new(false); + assert_eq!( + map_status("deactivated", "user:a", &mut report), + RebornUserStatus::Suspended + ); + assert_eq!( + map_status("future-state", "user:a", &mut report), + RebornUserStatus::Suspended + ); + assert_eq!( + map_role("owner", "user:a", &mut report), + RebornUserRole::Member + ); + assert_eq!(report.losses_in(Domain::User), 3); + } + + #[test] + fn metadata_preserves_strings_and_serializes_other_json_values() { + let mut report = MigrationReport::new(false); + let metadata = map_metadata( + serde_json::json!({"team": "infra", "quota": 3, "nested": {"a": true}}), + "user:a", + &mut report, + ); + assert_eq!(metadata.get("team").map(String::as_str), Some("infra")); + assert_eq!(metadata.get("quota").map(String::as_str), Some("3")); + assert_eq!( + metadata.get("nested").map(String::as_str), + Some("{\"a\":true}") + ); + assert_eq!(report.losses_in(Domain::User), 2); + } +} diff --git a/crates/ironclaw_reborn_migration/src/error.rs b/crates/ironclaw_reborn_migration/src/error.rs index 60f0d2893e0..ff8321060a5 100644 --- a/crates/ironclaw_reborn_migration/src/error.rs +++ b/crates/ironclaw_reborn_migration/src/error.rs @@ -8,6 +8,9 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum MigrationError { + #[error("migration preflight failed: {0}")] + Preflight(Box), + #[error("failed to open v1 source database: {0}")] OpenSource(String), @@ -35,3 +38,9 @@ pub enum MigrationError { #[error("i/o error: {0}")] Io(#[from] std::io::Error), } + +impl MigrationError { + pub fn is_preflight(&self) -> bool { + matches!(self, Self::Preflight(_)) + } +} diff --git a/crates/ironclaw_reborn_migration/src/inventory.rs b/crates/ironclaw_reborn_migration/src/inventory.rs new file mode 100644 index 00000000000..81fe1aac6fd --- /dev/null +++ b/crates/ironclaw_reborn_migration/src/inventory.rs @@ -0,0 +1,934 @@ +//! Complete registry of known v1 database and home-directory state. + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; + +use ironclaw_common::hashing::sha256_hex; + +use crate::manifest::{Disposition, InventoryEntry, InventorySourceKind}; +use crate::report::Domain; + +#[derive(Debug, Clone, Copy)] +struct DispositionRule { + name: &'static str, + domain: Domain, + disposition: Disposition, +} + +const TABLE_RULES: &[DispositionRule] = &[ + DispositionRule { + name: "_migrations", + domain: Domain::SchemaMetadata, + disposition: Disposition::IntentionallyReset, + }, + DispositionRule { + name: "refinery_schema_history", + domain: Domain::SchemaMetadata, + disposition: Disposition::IntentionallyReset, + }, + DispositionRule { + name: "conversations", + domain: Domain::Thread, + disposition: Disposition::Imported, + }, + DispositionRule { + name: "conversation_messages", + domain: Domain::Message, + disposition: Disposition::Imported, + }, + DispositionRule { + name: "agent_jobs", + domain: Domain::Job, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "job_actions", + domain: Domain::Job, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "job_events", + domain: Domain::Job, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "dynamic_tools", + domain: Domain::Extension, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "llm_calls", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "estimation_snapshots", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "repair_attempts", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "memory_documents", + domain: Domain::Memory, + disposition: Disposition::Imported, + }, + DispositionRule { + name: "memory_chunks", + domain: Domain::Memory, + disposition: Disposition::DerivedRebuilt, + }, + DispositionRule { + name: "memory_document_versions", + domain: Domain::Memory, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "heartbeat_state", + domain: Domain::Heartbeat, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "secrets", + domain: Domain::Secret, + disposition: Disposition::Imported, + }, + DispositionRule { + name: "wasm_tools", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "wasm_channels", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "tool_capabilities", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "leak_detection_patterns", + domain: Domain::SecurityAudit, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "tool_rate_limit_state", + domain: Domain::OperationalState, + disposition: Disposition::IntentionallyReset, + }, + DispositionRule { + name: "secret_usage_log", + domain: Domain::SecurityAudit, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "leak_detection_events", + domain: Domain::SecurityAudit, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "tool_failures", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "routines", + domain: Domain::Routine, + disposition: Disposition::SemanticallyConverted, + }, + DispositionRule { + name: "routine_runs", + domain: Domain::Routine, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "settings", + domain: Domain::Setting, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "users", + domain: Domain::User, + disposition: Disposition::Imported, + }, + DispositionRule { + name: "api_tokens", + domain: Domain::ApiToken, + disposition: Disposition::RequiresReauth, + }, + DispositionRule { + name: "user_identities", + domain: Domain::Identity, + disposition: Disposition::Imported, + }, + DispositionRule { + name: "channel_identities", + domain: Domain::Identity, + disposition: Disposition::SemanticallyConverted, + }, + DispositionRule { + name: "pairing_requests", + domain: Domain::Pairing, + disposition: Disposition::IntentionallyReset, + }, + DispositionRule { + name: "claude_code_events", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "root_filesystem_entries", + domain: Domain::WorkspaceFile, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "root_filesystem_events", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "root_filesystem_index_specs", + domain: Domain::OperationalState, + disposition: Disposition::DerivedRebuilt, + }, + DispositionRule { + name: "root_filesystem_sequences", + domain: Domain::OperationalState, + disposition: Disposition::IntentionallyReset, + }, + DispositionRule { + name: "hooks_predicate_invocations", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, + DispositionRule { + name: "hooks_predicate_values", + domain: Domain::OperationalState, + disposition: Disposition::ArchiveOnly, + }, +]; + +const FILE_RULES: &[DispositionRule] = &[ + DispositionRule { + name: ".env", + domain: Domain::Setting, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "settings.json", + domain: Domain::Setting, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "config.toml", + domain: Domain::Setting, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "providers.json", + domain: Domain::Provider, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "session.json", + domain: Domain::ApiToken, + disposition: Disposition::RequiresReauth, + }, + DispositionRule { + name: "mcp-servers.json", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "acp-agents.json", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "history", + domain: Domain::OperationalState, + disposition: Disposition::IntentionallyReset, + }, +]; + +const DIRECTORY_RULES: &[DispositionRule] = &[ + DispositionRule { + name: "profiles", + domain: Domain::Setting, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "skills", + domain: Domain::Skill, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "installed_skills", + domain: Domain::Skill, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "tools", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "channels", + domain: Domain::Extension, + disposition: Disposition::RequiresReinstall, + }, + DispositionRule { + name: "projects", + domain: Domain::Project, + disposition: Disposition::Unsupported, + }, + DispositionRule { + name: "logs", + domain: Domain::OperationalState, + disposition: Disposition::IntentionallyReset, + }, +]; + +#[derive(Debug, Clone)] +pub(crate) struct RawTableInventory { + pub name: String, + pub count: u64, + pub checksum: String, +} + +pub(crate) fn build_table_inventory(raw: Vec) -> Vec { + let mut by_name: BTreeMap<_, _> = raw + .into_iter() + .map(|item| (item.name.clone(), item)) + .collect(); + let mut out = Vec::with_capacity(TABLE_RULES.len() + by_name.len()); + for rule in TABLE_RULES { + let raw = by_name.remove(rule.name); + out.push(InventoryEntry { + source_kind: InventorySourceKind::Table, + source_name: rule.name.to_string(), + domain: rule.domain, + disposition: rule.disposition, + count: raw.as_ref().map_or(0, |item| item.count), + checksum: raw.map_or_else( + || sha256_hex(format!("missing:{}", rule.name).as_bytes()), + |item| item.checksum, + ), + blocker: None, + warning: None, + }); + } + for (_, raw) in by_name { + if let Some(rule) = dynamic_table_rule(&raw.name) { + out.push(InventoryEntry { + source_kind: InventorySourceKind::Table, + source_name: raw.name, + domain: rule.domain, + disposition: rule.disposition, + count: raw.count, + checksum: raw.checksum, + blocker: None, + warning: None, + }); + continue; + } + out.push(InventoryEntry { + source_kind: InventorySourceKind::Table, + source_name: raw.name, + domain: Domain::Unknown, + disposition: Disposition::UnsupportedUnknown, + count: raw.count, + checksum: raw.checksum, + blocker: Some( + "unknown v1 table requires an explicit migration disposition".to_string(), + ), + warning: None, + }); + } + out +} + +fn dynamic_table_rule(name: &str) -> Option { + if name == "memory_chunks_fts" + || matches!( + name.strip_prefix("memory_chunks_fts_"), + Some("config" | "content" | "data" | "docsize" | "idx") + ) + { + return Some(DispositionRule { + name: "memory_chunks_fts_shadow", + domain: Domain::Memory, + disposition: Disposition::DerivedRebuilt, + }); + } + // PostgreSQL installations can have one physical audit partition per + // month. They share the parent table's archive-only disposition, while an + // arbitrary unknown table remains a hard blocker. + if name + .strip_prefix("secret_usage_log_y") + .is_some_and(valid_audit_partition_suffix) + { + return Some(DispositionRule { + name: "secret_usage_log_partition", + domain: Domain::SecurityAudit, + disposition: Disposition::ArchiveOnly, + }); + } + None +} + +fn valid_audit_partition_suffix(suffix: &str) -> bool { + let bytes = suffix.as_bytes(); + let [year_a, year_b, year_c, year_d, b'm', month_a, month_b] = bytes else { + return false; + }; + if ![year_a, year_b, year_c, year_d, month_a, month_b] + .into_iter() + .all(u8::is_ascii_digit) + { + return false; + } + let month = (*month_a - b'0') * 10 + (*month_b - b'0'); + (1..=12).contains(&month) +} + +pub(crate) fn build_home_inventory( + home: Option<&Path>, + source_db: Option<&Path>, + target_db: Option<&Path>, +) -> Vec { + let mut out = Vec::new(); + let Some(home) = home else { return out }; + let mut excluded_paths = BTreeSet::new(); + for path in [source_db, target_db].into_iter().flatten() { + for suffix in ["", "-wal", "-shm"] { + let mut candidate = path.as_os_str().to_os_string(); + candidate.push(suffix); + excluded_paths.insert(normalized_path(Path::new(&candidate))); + } + } + if let Some(target_db) = target_db + && let Some(parent) = target_db.parent() + { + excluded_paths.insert(normalized_path( + &parent.join(".reborn-local-dev-secrets-master-key"), + )); + } + + for rule in FILE_RULES { + out.push(home_entry( + home.join(rule.name), + InventorySourceKind::HomeFile, + *rule, + &excluded_paths, + )); + } + for rule in DIRECTORY_RULES { + out.push(home_entry( + home.join(rule.name), + InventorySourceKind::HomeDirectory, + *rule, + &excluded_paths, + )); + } + + let known: BTreeSet<&str> = FILE_RULES + .iter() + .chain(DIRECTORY_RULES) + .map(|rule| rule.name) + .collect(); + match std::fs::read_dir(home) { + Ok(entries) => { + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(error) => { + out.push(home_inventory_error("home_directory_entry", &error)); + continue; + } + }; + let name = entry.file_name().to_string_lossy().into_owned(); + if known.contains(name.as_str()) + || excluded_paths.contains(&normalized_path(&entry.path())) + { + continue; + } + let kind = match entry.file_type() { + Ok(kind) if kind.is_dir() => InventorySourceKind::HomeDirectory, + Ok(_) => InventorySourceKind::HomeFile, + Err(error) => { + out.push(home_inventory_error("home_entry_type", &error)); + continue; + } + }; + let normalized_entry = normalized_path(&entry.path()); + if matches!(kind, InventorySourceKind::HomeDirectory) + && excluded_paths + .iter() + .any(|excluded| excluded.starts_with(&normalized_entry)) + && matches!( + path_has_nonexcluded_content(&entry.path(), &excluded_paths), + Ok(false) + ) + { + continue; + } + let (count, count_error) = + match count_path_entries_excluding(&entry.path(), &excluded_paths) { + Ok(count) => (count, None), + Err(error) => (0, Some(error)), + }; + let (checksum, checksum_error) = + match checksum_path_excluding(&entry.path(), &excluded_paths) { + Ok(checksum) => (checksum, None), + Err(error) => ( + sha256_hex(format!("unreadable-home:{name}:{error}").as_bytes()), + Some(error), + ), + }; + let blocker = count_error.or(checksum_error).map(|error| { + format!("v1 home artifact could not be inventoried completely: {error}") + }); + out.push(InventoryEntry { + source_kind: kind, + source_name: name.clone(), + domain: Domain::Unknown, + disposition: Disposition::UnsupportedUnknown, + count, + checksum, + blocker: blocker.or_else(|| { + Some( + "unknown v1 home artifact requires an explicit migration disposition" + .to_string(), + ) + }), + warning: None, + }); + } + } + Err(error) => out.push(home_inventory_error("home_directory_enumeration", &error)), + } + out +} + +fn home_entry( + path: PathBuf, + source_kind: InventorySourceKind, + rule: DispositionRule, + excluded_paths: &BTreeSet, +) -> InventoryEntry { + if excluded_paths.contains(&normalized_path(&path)) { + return InventoryEntry { + source_kind, + source_name: rule.name.to_string(), + domain: rule.domain, + disposition: rule.disposition, + count: 0, + checksum: sha256_hex(b"excluded-database-path"), + blocker: None, + warning: None, + }; + } + match path.try_exists() { + Ok(false) => { + return InventoryEntry { + source_kind, + source_name: rule.name.to_string(), + domain: rule.domain, + disposition: rule.disposition, + count: 0, + checksum: sha256_hex(b"missing"), + blocker: None, + warning: None, + }; + } + Ok(true) => {} + Err(error) => { + return InventoryEntry { + source_kind, + source_name: rule.name.to_string(), + domain: rule.domain, + disposition: rule.disposition, + count: 0, + checksum: sha256_hex(format!("unreadable:{}:{error}", rule.name).as_bytes()), + blocker: Some(format!( + "known v1 home artifact could not be inventoried completely: {error}" + )), + warning: None, + }; + } + } + if matches!( + path_has_nonexcluded_content(&path, excluded_paths), + Ok(false) + ) { + return InventoryEntry { + source_kind, + source_name: rule.name.to_string(), + domain: rule.domain, + disposition: rule.disposition, + count: 0, + checksum: sha256_hex(b"missing"), + blocker: None, + warning: None, + }; + } + let (count, count_error) = match count_path_entries_excluding(&path, excluded_paths) { + Ok(count) => (count, None), + Err(error) => (0, Some(error)), + }; + let (checksum, checksum_error) = match checksum_path_excluding(&path, excluded_paths) { + Ok(checksum) => (checksum, None), + Err(error) => ( + sha256_hex(format!("unreadable:{}:{error}", rule.name).as_bytes()), + Some(error), + ), + }; + InventoryEntry { + source_kind, + source_name: rule.name.to_string(), + domain: rule.domain, + disposition: rule.disposition, + count, + checksum, + blocker: count_error.or(checksum_error).map(|error| { + format!("known v1 home artifact could not be inventoried completely: {error}") + }), + warning: None, + } +} + +fn home_inventory_error(source_name: &str, error: &std::io::Error) -> InventoryEntry { + InventoryEntry { + source_kind: InventorySourceKind::HomeDirectory, + source_name: source_name.to_string(), + domain: Domain::Unknown, + disposition: Disposition::UnsupportedUnknown, + count: 0, + checksum: sha256_hex(format!("unreadable-home:{source_name}:{error}").as_bytes()), + blocker: Some(format!( + "v1 home directory could not be inventoried completely: {error}" + )), + warning: None, + } +} + +#[cfg(test)] +fn checksum_path(path: &Path) -> std::io::Result { + checksum_path_excluding(path, &BTreeSet::new()) +} + +fn checksum_path_excluding( + path: &Path, + excluded_paths: &BTreeSet, +) -> std::io::Result { + let metadata = std::fs::symlink_metadata(path)?; + let mut state = Fnv1a64::new(); + if metadata.file_type().is_symlink() { + // A symlink destination may itself contain a username, credential, or + // other operator-private path. Inventory needs only the artifact's + // shape, not a digest derived from its destination. + state.update(b"symlink\0"); + } else if metadata.is_file() { + // Never hash file contents here. Known home files include `.env`, + // provider configuration, and session state; even a one-way digest of + // their plaintext would violate the redacted manifest contract. + state.update(b"file\0"); + } else if metadata.is_dir() { + state.update(b"directory\0"); + let mut entries = std::fs::read_dir(path)?.collect::, _>>()?; + entries.sort_by_key(|entry| entry.file_name()); + for entry in entries { + if excluded_paths.contains(&normalized_path(&entry.path())) { + continue; + } + state.update(entry.file_name().as_encoded_bytes()); + state.update(b"\0"); + state.update(checksum_path_excluding(&entry.path(), excluded_paths)?.as_bytes()); + state.update(b"\0"); + } + } else { + state.update(b"other\0"); + } + let normalized = normalized_path(path); + if !metadata.is_dir() + || !excluded_paths + .iter() + .any(|excluded| excluded.starts_with(&normalized)) + { + update_metadata_shape(&mut state, &metadata); + } + Ok(format!("metadata-fnv1a64:{:016x}", state.finish())) +} + +fn update_metadata_shape(state: &mut Fnv1a64, metadata: &std::fs::Metadata) { + state.update(&metadata.len().to_le_bytes()); + if let Ok(modified) = metadata.modified() + && let Ok(since_epoch) = modified.duration_since(std::time::UNIX_EPOCH) + { + state.update(&since_epoch.as_secs().to_le_bytes()); + state.update(&since_epoch.subsec_nanos().to_le_bytes()); + } +} + +struct Fnv1a64(u64); + +impl Fnv1a64 { + const fn new() -> Self { + Self(0xcbf29ce484222325) + } + + fn update(&mut self, bytes: &[u8]) { + for byte in bytes { + self.0 ^= u64::from(*byte); + self.0 = self.0.wrapping_mul(0x100000001b3); + } + } + + const fn finish(self) -> u64 { + self.0 + } +} + +#[cfg(test)] +fn count_path_entries(path: &Path) -> std::io::Result { + count_path_entries_excluding(path, &BTreeSet::new()) +} + +fn count_path_entries_excluding( + path: &Path, + excluded_paths: &BTreeSet, +) -> std::io::Result { + let metadata = std::fs::symlink_metadata(path)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Ok(1); + } + let mut count = 1_u64; + for entry in std::fs::read_dir(path)? { + let entry = entry?; + if excluded_paths.contains(&normalized_path(&entry.path())) { + continue; + } + count = count.saturating_add(count_path_entries_excluding(&entry.path(), excluded_paths)?); + } + Ok(count) +} + +fn normalized_path(path: &Path) -> PathBuf { + crate::canonicalish(path) +} + +fn path_has_nonexcluded_content( + path: &Path, + excluded_paths: &BTreeSet, +) -> std::io::Result { + if excluded_paths.contains(&normalized_path(path)) { + return Ok(false); + } + let metadata = std::fs::symlink_metadata(path)?; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Ok(true); + } + for entry in std::fs::read_dir(path)? { + if path_has_nonexcluded_content(&entry?.path(), excluded_paths)? { + return Ok(true); + } + } + Ok(false) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_has_no_duplicate_source_names() { + let mut names = BTreeSet::new(); + for rule in TABLE_RULES { + assert!( + names.insert(rule.name), + "duplicate table rule: {}", + rule.name + ); + } + } + + #[test] + fn audit_partition_names_require_exact_year_and_valid_month() { + for valid in ["2024m01", "9999m12"] { + assert!(valid_audit_partition_suffix(valid), "{valid}"); + } + for invalid in [ + "ABCDm12", "2024m00", "2024m13", "2024x01", "2024m1", "20240m1", + ] { + assert!(!valid_audit_partition_suffix(invalid), "{invalid}"); + } + } + + #[test] + fn heartbeat_state_is_reported_as_unsupported() { + let inventory = build_table_inventory(vec![RawTableInventory { + name: "heartbeat_state".to_string(), + count: 1, + checksum: "checksum".to_string(), + }]); + let entry = inventory + .iter() + .find(|entry| entry.source_name == "heartbeat_state") + .expect("heartbeat inventory entry"); + assert_eq!(entry.disposition, Disposition::Unsupported); + } + + #[test] + fn home_checksum_is_not_derived_from_secret_file_contents() { + let directory = tempfile::tempdir().expect("tempdir"); + let secret = directory.path().join(".env"); + let canary = b"OPENAI_API_KEY=sk-secret-canary"; + std::fs::write(&secret, canary).expect("write canary"); + + let checksum = checksum_path(&secret).expect("metadata checksum"); + assert!(checksum.starts_with("metadata-fnv1a64:")); + + let mut content_digest = Fnv1a64::new(); + content_digest.update(canary); + assert_ne!( + checksum, + format!("fnv1a64:{:016x}", content_digest.finish()), + "manifest checksums must never be hashes of secret-bearing file contents" + ); + } + + #[test] + fn unconverted_known_sources_are_never_labeled_semantically_converted() { + for name in [ + "settings", + "root_filesystem_entries", + ".env", + "settings.json", + "config.toml", + "providers.json", + "profiles", + "skills", + "projects", + ] { + let rule = TABLE_RULES + .iter() + .chain(FILE_RULES) + .chain(DIRECTORY_RULES) + .find(|rule| rule.name == name) + .expect("known source rule"); + assert_eq!(rule.disposition, Disposition::Unsupported, "{name}"); + } + } + + #[test] + fn unreadable_home_enumeration_is_a_blocker() { + let directory = tempfile::tempdir().expect("tempdir"); + let not_a_directory = directory.path().join("home-file"); + std::fs::write(¬_a_directory, b"not a directory").expect("write file"); + + let inventory = build_home_inventory(Some(¬_a_directory), None, None); + let failure = inventory + .iter() + .find(|entry| entry.source_name == "home_directory_enumeration") + .expect("enumeration blocker"); + assert_eq!(failure.disposition, Disposition::UnsupportedUnknown); + assert!(failure.blocker.is_some()); + } + + #[test] + fn vanished_inventory_path_is_an_error() { + let directory = tempfile::tempdir().expect("tempdir"); + let vanished = directory.path().join("vanished"); + let error = count_path_entries(&vanished).expect_err("missing path must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::NotFound); + } + + #[test] + fn database_exclusion_is_exact_and_preserves_nested_siblings() { + let directory = tempfile::tempdir().expect("tempdir"); + let home = directory.path().join("home"); + let nested = home.join("custom"); + std::fs::create_dir_all(&nested).expect("create nested directory"); + let source = nested.join("source.db"); + std::fs::write(&source, b"database").expect("write source"); + std::fs::write(nested.join("notes.txt"), b"notes").expect("write sibling"); + std::fs::write(home.join("source.db"), b"unrelated").expect("write same-name artifact"); + + let inventory = build_home_inventory(Some(&home), Some(&source), None); + let nested_entry = inventory + .iter() + .find(|entry| entry.source_name == "custom") + .expect("nested directory remains inventoried"); + assert_eq!(nested_entry.count, 2, "directory plus non-database sibling"); + assert!( + inventory + .iter() + .any(|entry| entry.source_name == "source.db"), + "same basename at a different path must remain visible" + ); + } + + #[test] + fn target_only_directories_do_not_change_source_home_inventory() { + let directory = tempfile::tempdir().expect("tempdir"); + let home = directory.path().join("home"); + let target = home.join("reborn").join("reborn.db"); + std::fs::create_dir_all(&home).expect("create home"); + let before = build_home_inventory(Some(&home), None, Some(&target)); + + std::fs::create_dir_all(target.parent().expect("target parent")) + .expect("create target parent"); + std::fs::write(&target, b"target").expect("write target"); + std::fs::write( + target + .parent() + .expect("target parent") + .join(".reborn-local-dev-secrets-master-key"), + b"key", + ) + .expect("write target key"); + let after = build_home_inventory(Some(&home), None, Some(&target)); + + assert_eq!(after, before); + } + + #[test] + fn target_tree_under_known_directory_does_not_change_inventory() { + let directory = tempfile::tempdir().expect("tempdir"); + let home = directory.path().join("home"); + let target = home.join("projects").join("reborn").join("reborn.db"); + std::fs::create_dir_all(&home).expect("create home"); + let before = build_home_inventory(Some(&home), None, Some(&target)); + + std::fs::create_dir_all(target.parent().expect("target parent")) + .expect("create target parent"); + std::fs::write(&target, b"target").expect("write target"); + let after = build_home_inventory(Some(&home), None, Some(&target)); + + assert_eq!(after, before); + } + + #[test] + fn target_tree_exclusion_preserves_known_directory_siblings() { + let directory = tempfile::tempdir().expect("tempdir"); + let home = directory.path().join("home"); + let projects = home.join("projects"); + let target = projects.join("reborn.db"); + std::fs::create_dir_all(&projects).expect("create projects"); + std::fs::write(projects.join("legacy.json"), b"legacy").expect("write legacy sibling"); + let before = build_home_inventory(Some(&home), None, Some(&target)); + + std::fs::create_dir_all(target.parent().expect("target parent")) + .expect("create target parent"); + std::fs::write(&target, b"target").expect("write target"); + let after = build_home_inventory(Some(&home), None, Some(&target)); + + assert_eq!(after, before); + } +} diff --git a/crates/ironclaw_reborn_migration/src/lib.rs b/crates/ironclaw_reborn_migration/src/lib.rs index 709cb6da9d9..f844252f310 100644 --- a/crates/ironclaw_reborn_migration/src/lib.rs +++ b/crates/ironclaw_reborn_migration/src/lib.rs @@ -1,19 +1,14 @@ //! v1 / engine-v2 → Reborn state migration. //! -//! Reads a legacy IronClaw v1 database (PostgreSQL or libSQL) — which is also -//! where engine-v2 state lives, as JSON blobs in `memory_documents` — and -//! writes the equivalent Reborn state into the `RootFilesystem` KV substrate -//! plus the triggers database. Threads and automations (routines + engine-v2 -//! missions) convert without loss; anything that has no Reborn representation -//! today is recorded in a [`MigrationReport`] rather than silently dropped. -//! -//! The crate is a library (this module) plus a thin binary (`src/main.rs`) so -//! the conversion engine can later be reused inside `ironclaw-reborn` startup. +//! The public lifecycle is plan → apply/resume → verify. Planning opens only a +//! non-migrating source reader and never constructs a Reborn target writer. pub mod error; +pub mod manifest; pub mod options; pub mod report; +mod inventory; mod mounts; mod source; mod target; @@ -27,31 +22,813 @@ pub use extension_ownership::{ run_extension_ownership_migration, }; +use std::collections::BTreeMap; +use std::path::{Component, Path, PathBuf}; + +use chrono::Utc; +use ironclaw_common::hashing::sha256_hex; +#[cfg(feature = "postgres")] +use secrecy::ExposeSecret as _; + pub use error::MigrationError; -pub use options::{MigrationOptions, SourceDb, TargetStore}; +pub use manifest::{ + Disposition, DomainCheckpoint, InventoryEntry, InventorySourceKind, MANIFEST_SCHEMA_VERSION, + MIGRATION_PROTOCOL_VERSION, MigrationManifest, MigrationStatus, RedactedStoreDescriptor, + ResolvedScope, SourceFingerprint, StoreBackend, +}; +pub use options::{ + ApplyAcknowledgements, MigrationOptions, MigrationSecretInputs, SourceDb, TargetStore, +}; pub use report::{Domain, LossReason, LossyItem, MigrationReport, MigrationStats}; -/// Run a full migration: open the v1 source and Reborn target, convert every -/// in-scope domain, and return the outcome report. +/// Compare a currently resolved target with a manifest without serializing or +/// returning the target locator. Intended for `status`/`doctor` read paths. +pub fn manifest_target_matches(target: &TargetStore, manifest: &MigrationManifest) -> bool { + let Ok(current) = target_descriptor(target) else { + return false; + }; + current.backend == manifest.target.backend + && current.locator_fingerprint == manifest.target.locator_fingerprint +} + +/// Inspect a v1 snapshot and build a redacted, versioned migration plan. /// -/// Infrastructure failures (cannot open a store, cannot write a record) abort -/// with a [`MigrationError`]. Per-item representation gaps do not abort — they -/// are accumulated as [`LossyItem`]s on the returned report. -pub async fn run_migration(options: MigrationOptions) -> Result { - let mut report = MigrationReport::new(options.dry_run); +/// This function does not open the target or create target directories/files. +pub async fn plan_migration( + options: &MigrationOptions, +) -> Result { + validate_distinct_stores(&options.source, &options.target)?; + let source = source::V1Source::open(&options.source).await?; + let source_schema_version = source.schema_version().await?; + let inventory = collect_inventory(options, &source).await?; + let project_document_count = source.project_documents().await?.len() as u64; + // Fingerprint after every planning read has completed. libSQL may update + // connection-local WAL bookkeeping while opening a snapshot; apply runs + // the same read sequence before comparing the sealed fingerprint. + let source_fingerprint = source.fingerprint(&options.source).await?; + let source_inventory_checksum = inventory_checksum(&inventory)?; + + let mut domains = BTreeMap::new(); + for item in &inventory { + let checkpoint = domains + .entry(item.domain) + .or_insert_with(DomainCheckpoint::default); + checkpoint.planned = checkpoint.planned.saturating_add(item.count); + if let Some(blocker) = &item.blocker { + checkpoint.blockers.push(blocker.clone()); + } + if let Some(warning) = &item.warning { + checkpoint.warnings.push(warning.clone()); + } + } + let project_checkpoint = domains.entry(Domain::Project).or_default(); + project_checkpoint.planned = project_checkpoint + .planned + .saturating_add(project_document_count); + + let target = target_descriptor(&options.target)?; + // Local existence is observable without opening a target. PostgreSQL + // emptiness is deliberately deferred until apply so planning never makes + // a target connection. + let target_empty = match &options.target { + TargetStore::LibSql { path } => Some(!path.exists()), + TargetStore::Postgres { .. } => None, + }; + let mut manifest = MigrationManifest { + manifest_schema_version: MANIFEST_SCHEMA_VERSION, + migration_protocol_version: MIGRATION_PROTOCOL_VERSION, + tool_version: env!("CARGO_PKG_VERSION").to_string(), + release_version: env!("CARGO_PKG_VERSION").to_string(), + run_id: uuid::Uuid::new_v4(), + status: MigrationStatus::Planned, + source: source_descriptor(&options.source)?, + target, + source_schema_version, + source_fingerprint, + source_inventory_checksum, + plan_hash: String::new(), + scope: ResolvedScope { + profile: options.profile.clone(), + tenant_id: options.tenant_id.to_string(), + agent_id: options.agent_id.to_string(), + source_home_fingerprint: options + .source_home + .as_deref() + .map(canonicalish) + .as_deref() + .map(locator_hash), + user_mapping: BTreeMap::new(), + target_empty, + }, + inventory, + domains, + operator_acknowledgements: Vec::new(), + created_at: Utc::now(), + updated_at: Utc::now(), + }; + manifest.seal()?; + Ok(manifest) +} + +/// Apply a previously sealed plan after validating its source snapshot. +/// +/// Source and target keys are deliberately resolved independently. The target +/// is not opened until every acknowledgement and source fingerprint check has +/// passed. +pub async fn apply_migration( + options: MigrationOptions, + manifest: &MigrationManifest, + secrets: MigrationSecretInputs, + acknowledgements: ApplyAcknowledgements, +) -> Result { + apply_migration_inner(options, manifest, secrets, acknowledgements, false).await +} + +/// Validate a new apply without opening or claiming the target. +/// +/// This checks the stopped/snapshot acknowledgements, sealed source and target +/// identity, source secret-key requirements, manifest state, and target +/// freshness. Callers may persist `Applying` only after this succeeds. +pub async fn preflight_apply_migration( + options: &MigrationOptions, + manifest: &MigrationManifest, + secrets: &MigrationSecretInputs, + acknowledgements: ApplyAcknowledgements, +) -> Result<(), MigrationError> { + validate_apply_preconditions(options, manifest, secrets, acknowledgements, false) + .await + .map_err(|error| MigrationError::Preflight(Box::new(error))) +} + +/// Validate a resumable apply without opening or claiming the target. +/// +/// This accepts only resumable lifecycle states for the same sealed run and +/// performs the same source, key, acknowledgement, and binding checks as apply +/// without requiring the run-owned target to be empty. +pub async fn preflight_resume_migration( + options: &MigrationOptions, + manifest: &MigrationManifest, + secrets: &MigrationSecretInputs, + acknowledgements: ApplyAcknowledgements, +) -> Result<(), MigrationError> { + validate_apply_preconditions(options, manifest, secrets, acknowledgements, true) + .await + .map_err(|error| MigrationError::Preflight(Box::new(error))) +} - let src = source::V1Source::open(&options.source).await?; - let mut tgt = target::RebornTarget::open(&options).await?; +async fn apply_migration_inner( + options: MigrationOptions, + manifest: &MigrationManifest, + secrets: MigrationSecretInputs, + acknowledgements: ApplyAcknowledgements, + is_resume: bool, +) -> Result { + if is_resume { + preflight_resume_migration(&options, manifest, &secrets, acknowledgements).await?; + } else { + preflight_apply_migration(&options, manifest, &secrets, acknowledgements).await?; + } - convert::threads::run(&src, &mut tgt, &options, &mut report).await?; - convert::automations::run(&src, &mut tgt, &options, &mut report).await?; - convert::memory::run(&src, &mut tgt, &options, &mut report).await?; - convert::jobs::run(&src, &mut tgt, &options, &mut report).await?; - convert::secrets::run(&src, &mut tgt, &options, &mut report).await?; - convert::extensions::run(&src, &mut tgt, &options, &mut report).await?; - convert::identities::run(&src, &mut tgt, &options, &mut report).await?; - convert::heartbeat::run(&src, &mut tgt, &options, &mut report).await?; - convert::settings::run(&src, &mut tgt, &options, &mut report).await?; + let source_options = MigrationOptions { + secret_master_key: secrets.source_master_key, + dry_run: false, + ..options.clone() + }; + let target_options = MigrationOptions { + secret_master_key: secrets.target_master_key, + dry_run: false, + ..options + }; + let mut applied_manifest = if manifest.status == MigrationStatus::Applying { + manifest.clone() + } else { + manifest.transition(MigrationStatus::Applying)? + }; + applied_manifest.operator_acknowledgements = vec![ + "source_is_stopped".to_string(), + "source_is_snapshot".to_string(), + ]; + applied_manifest.seal()?; + target::write_shared_migration_state(&target_options.target, &applied_manifest).await?; + + // Re-open only after fingerprint validation, still without running v1 + // migrations. The converter surface exposes reads only. + let source = source::V1Source::open(&source_options.source).await?; + let mut target = target::RebornTarget::open(&target_options).await?; + let mut report = MigrationReport::new(false); + // Converter ledger keys may read the sealed installation fingerprint from + // the in-progress report without threading another context parameter + // through every domain converter. + report.manifest = Some(applied_manifest.clone()); + + let result = run_converters(&source, &mut target, &source_options, &mut report).await; + match result { + Ok(()) => { + record_applied_checkpoints(&mut applied_manifest, &report.stats); + applied_manifest.status = MigrationStatus::Applied; + } + Err(error) => { + applied_manifest.status = MigrationStatus::Failed; + applied_manifest.updated_at = Utc::now(); + applied_manifest.seal()?; + target::write_shared_migration_state(&target_options.target, &applied_manifest).await?; + report.manifest = Some(applied_manifest); + return Err(error); + } + } + applied_manifest.updated_at = Utc::now(); + applied_manifest.seal()?; + target::write_shared_migration_state(&target_options.target, &applied_manifest).await?; + report.manifest = Some(applied_manifest); Ok(report) } + +/// Resume uses the same deterministic compare-and-apply path as apply. Exact +/// replay is a no-op and divergent target state fails without overwriting it. +pub async fn resume_migration( + options: MigrationOptions, + manifest: &MigrationManifest, + secrets: MigrationSecretInputs, + acknowledgements: ApplyAcknowledgements, +) -> Result { + apply_migration_inner(options, manifest, secrets, acknowledgements, true).await +} + +/// Validate the sealed manifest and source snapshot, then structurally read +/// supported state from the production durable stores before marking the run +/// verified. The data readback is read-only, but lifecycle/quarantine state is +/// updated; this does not boot the production runtime. +pub async fn verify_migration( + options: &MigrationOptions, + manifest: &MigrationManifest, +) -> Result { + validate_manifest_source(options, manifest).await?; + match manifest.status { + MigrationStatus::Applied | MigrationStatus::Verifying | MigrationStatus::Verified => { + let mut verifying = if manifest.status != MigrationStatus::Verifying { + manifest.transition(MigrationStatus::Verifying)? + } else { + manifest.clone() + }; + target::write_shared_migration_state(&options.target, &verifying).await?; + let verification = async { + let readback = target::readback(&options.target, &options.tenant_id).await?; + verify_readback(&verifying, &readback)?; + Ok::<(), MigrationError>(()) + } + .await; + if let Err(error) = verification { + let failed = verifying.transition(MigrationStatus::Failed)?; + target::write_shared_migration_state(&options.target, &failed).await?; + return Err(error); + } + for checkpoint in verifying.domains.values_mut() { + checkpoint.verified = checkpoint.applied; + } + let verified = verifying.transition(MigrationStatus::Verified)?; + target::write_shared_migration_state(&options.target, &verified).await?; + Ok(verified) + } + status => Err(MigrationError::InvalidInput(format!( + "verify requires an applied or verifying manifest, got {status:?}" + ))), + } +} + +/// Temporary compatibility entry point. +/// +/// Dry-run callers now receive the non-writing plan manifest. Apply callers +/// retain the old one-shot behavior while the standalone CLI moves to the +/// explicit lifecycle API; the wrapper treats the one-shot invocation as the +/// legacy offline-snapshot acknowledgement. +pub async fn run_migration(options: MigrationOptions) -> Result { + let manifest = plan_migration(&options).await?; + if options.dry_run { + let mut report = MigrationReport::new(true); + report.manifest = Some(manifest); + return Ok(report); + } + let secrets = MigrationSecretInputs::from_legacy(&options); + apply_migration( + options, + &manifest, + secrets, + ApplyAcknowledgements::offline_snapshot(), + ) + .await +} + +async fn run_converters( + source: &source::V1Source, + target: &mut target::RebornTarget, + options: &MigrationOptions, + report: &mut MigrationReport, +) -> Result<(), MigrationError> { + convert::users::run(source, target, options, report).await?; + let imported_project_ids = convert::projects::run(source, target, options, report).await?; + convert::threads::run(source, target, options, report).await?; + convert::automations::run(source, target, options, report, &imported_project_ids).await?; + convert::memory::run(source, target, options, report).await?; + convert::jobs::run(source, target, options, report).await?; + convert::secrets::run(source, target, options, report).await?; + convert::extensions::run(source, target, options, report).await?; + convert::identities::run(source, target, options, report).await?; + convert::heartbeat::run(source, target, options, report).await?; + convert::settings::run(source, target, options, report).await?; + Ok(()) +} + +async fn validate_apply_preconditions( + options: &MigrationOptions, + manifest: &MigrationManifest, + secrets: &MigrationSecretInputs, + acknowledgements: ApplyAcknowledgements, + is_resume: bool, +) -> Result<(), MigrationError> { + if !acknowledgements.source_is_stopped || !acknowledgements.source_is_snapshot { + return Err(MigrationError::InvalidInput( + "apply requires both a stopped v1 source acknowledgement and a consistent snapshot acknowledgement" + .to_string(), + )); + } + if manifest.inventory.iter().any(|item| item.blocker.is_some()) { + return Err(MigrationError::InvalidInput( + "migration plan contains unresolved inventory blockers".to_string(), + )); + } + if secrets.source_master_key.is_none() + && manifest.inventory.iter().any(|item| { + item.source_kind == crate::manifest::InventorySourceKind::Table + && item.source_name == "secrets" + && item.count > 0 + }) + { + return Err(MigrationError::MissingSecretKey); + } + if !is_resume { + if manifest.status != MigrationStatus::Planned { + return Err(MigrationError::InvalidInput( + "apply requires a planned manifest; use resume for an existing run".to_string(), + )); + } + if manifest.scope.target_empty == Some(false) { + return Err(MigrationError::InvalidInput( + "apply requires a target that was empty at planning time".to_string(), + )); + } + if !target::target_is_empty(&options.target).await? { + return Err(MigrationError::InvalidInput( + "Reborn target is not empty; refusing to overwrite it".to_string(), + )); + } + } else if !matches!( + manifest.status, + MigrationStatus::Applying | MigrationStatus::Failed | MigrationStatus::Applied + ) { + return Err(MigrationError::InvalidInput( + "resume requires an applying, failed, or applied manifest".to_string(), + )); + } + validate_manifest_source(options, manifest).await +} + +fn record_applied_checkpoints(manifest: &mut MigrationManifest, stats: &MigrationStats) { + let applied = [ + (Domain::User, stats.users), + (Domain::Project, stats.projects), + (Domain::Thread, stats.threads), + (Domain::Message, stats.messages), + (Domain::Routine, stats.routines), + (Domain::Mission, stats.missions), + (Domain::Trigger, stats.triggers), + (Domain::Memory, stats.memory_documents), + (Domain::Secret, stats.secrets), + (Domain::Extension, stats.extensions), + (Domain::Identity, stats.identities), + (Domain::Heartbeat, stats.heartbeats), + ]; + let completed_at = Utc::now(); + for (domain, count) in applied { + let checkpoint = manifest.domains.entry(domain).or_default(); + checkpoint.applied = count as u64; + checkpoint.completed_at = Some(completed_at); + } +} + +fn verify_readback( + manifest: &MigrationManifest, + readback: &target::TargetReadback, +) -> Result<(), MigrationError> { + let expected = |domain: Domain| { + manifest + .domains + .get(&domain) + .map_or(0, |checkpoint| checkpoint.applied) + }; + let checks = [ + ("users", expected(Domain::User), readback.users), + ("projects", expected(Domain::Project), readback.projects), + ("threads", expected(Domain::Thread), readback.threads), + ("messages", expected(Domain::Message), readback.messages), + ("triggers", expected(Domain::Trigger), readback.triggers), + ( + "memory documents", + expected(Domain::Memory), + readback.memory_documents, + ), + ("secrets", expected(Domain::Secret), readback.secrets), + ]; + for (domain, expected, actual) in checks { + if actual != expected { + return Err(MigrationError::InvalidInput(format!( + "target verification failed for {domain}: expected {expected}, found {actual}" + ))); + } + } + let expected_identities = expected(Domain::Identity); + if readback.identity_records < expected_identities { + return Err(MigrationError::InvalidInput(format!( + "target verification failed for identities: expected at least {expected_identities} durable records, found {}", + readback.identity_records + ))); + } + Ok(()) +} + +async fn validate_manifest_source( + options: &MigrationOptions, + manifest: &MigrationManifest, +) -> Result<(), MigrationError> { + validate_distinct_stores(&options.source, &options.target)?; + manifest.validate_plan_hash()?; + if manifest.manifest_schema_version != MANIFEST_SCHEMA_VERSION + || manifest.migration_protocol_version != MIGRATION_PROTOCOL_VERSION + { + return Err(MigrationError::InvalidInput(format!( + "unsupported migration manifest/protocol version {}/{}", + manifest.manifest_schema_version, manifest.migration_protocol_version + ))); + } + let current_target = target_descriptor(&options.target)?; + let current_source_home_fingerprint = options + .source_home + .as_deref() + .map(canonicalish) + .as_deref() + .map(locator_hash); + if source_descriptor(&options.source)? != manifest.source + || current_target.backend != manifest.target.backend + || current_target.locator_fingerprint != manifest.target.locator_fingerprint + || options.profile != manifest.scope.profile + || options.tenant_id.to_string() != manifest.scope.tenant_id + || options.agent_id.to_string() != manifest.scope.agent_id + || current_source_home_fingerprint != manifest.scope.source_home_fingerprint + || !manifest.scope.user_mapping.is_empty() + { + return Err(MigrationError::InvalidInput( + "migration inputs do not match the sealed plan".to_string(), + )); + } + let source = source::V1Source::open(&options.source).await?; + #[cfg(feature = "postgres")] + if let (Some(source_pool), TargetStore::Postgres { url }) = + (source.handles.pg_pool.as_ref(), &options.target) + && !target::postgres_stores_are_distinct(source_pool, url).await? + { + return Err(MigrationError::InvalidInput( + "v1 source and Reborn target resolve to the same PostgreSQL database".to_string(), + )); + } + let current_inventory = collect_inventory(options, &source).await?; + if inventory_checksum(¤t_inventory)? != manifest.source_inventory_checksum { + return Err(MigrationError::InvalidInput( + "v1 source inventory changed after planning; create a new snapshot and plan" + .to_string(), + )); + } + let current = source.fingerprint(&options.source).await?; + if current != manifest.source_fingerprint { + return Err(MigrationError::InvalidInput( + "v1 source fingerprint changed after planning; create a new snapshot and plan" + .to_string(), + )); + } + Ok(()) +} + +async fn collect_inventory( + options: &MigrationOptions, + source: &source::V1Source, +) -> Result, MigrationError> { + let raw_tables = source.table_inventory().await?; + let source_path = match &options.source { + SourceDb::LibSql { path } => Some(path.as_path()), + SourceDb::Postgres { .. } => None, + }; + let target_path = match &options.target { + TargetStore::LibSql { path } => Some(path.as_path()), + TargetStore::Postgres { .. } => None, + }; + let mut inventory = inventory::build_table_inventory(raw_tables); + if options.source_home.is_none() { + inventory.push(InventoryEntry { + source_kind: InventorySourceKind::HomeDirectory, + source_name: "v1_home".to_string(), + domain: Domain::Setting, + disposition: Disposition::Unsupported, + count: 1, + checksum: sha256_hex(b"v1-home-not-specified"), + blocker: Some( + "v1 home was not specified; complete persistent-home inventory cannot be proven" + .to_string(), + ), + warning: None, + }); + } + inventory.extend(inventory::build_home_inventory( + options.source_home.as_deref(), + source_path, + target_path, + )); + Ok(inventory) +} + +fn inventory_checksum(inventory: &[InventoryEntry]) -> Result { + Ok(sha256_hex(&serde_json::to_vec(inventory)?)) +} + +fn source_descriptor(source: &SourceDb) -> Result { + Ok(match source { + SourceDb::LibSql { path } => RedactedStoreDescriptor { + backend: StoreBackend::Libsql, + locator_fingerprint: locator_hash(&canonicalish(path)), + exists: Some(path.exists()), + }, + SourceDb::Postgres { url } => RedactedStoreDescriptor { + backend: StoreBackend::Postgres, + locator_fingerprint: postgres_locator_fingerprint(url)?, + exists: None, + }, + }) +} + +fn target_descriptor(target: &TargetStore) -> Result { + Ok(match target { + TargetStore::LibSql { path } => RedactedStoreDescriptor { + backend: StoreBackend::Libsql, + locator_fingerprint: target_libsql_locator_fingerprint(path), + exists: Some(path.exists()), + }, + TargetStore::Postgres { url } => RedactedStoreDescriptor { + backend: StoreBackend::Postgres, + locator_fingerprint: target_postgres_locator_fingerprint(url)?, + exists: None, + }, + }) +} + +#[cfg(feature = "libsql")] +fn target_libsql_locator_fingerprint(path: &Path) -> String { + ironclaw_reborn_composition::migration_libsql_locator_fingerprint(path) +} + +#[cfg(not(feature = "libsql"))] +fn target_libsql_locator_fingerprint(path: &Path) -> String { + locator_hash(&canonicalish(path)) +} + +#[cfg(feature = "postgres")] +fn target_postgres_locator_fingerprint( + locator: &secrecy::SecretString, +) -> Result { + ironclaw_reborn_composition::migration_postgres_locator_fingerprint(locator) + .map_err(|error| MigrationError::InvalidInput(error.to_string())) +} + +#[cfg(not(feature = "postgres"))] +fn target_postgres_locator_fingerprint( + _locator: &secrecy::SecretString, +) -> Result { + Err(MigrationError::InvalidInput( + "PostgreSQL support is not compiled into this migrator".to_string(), + )) +} + +fn locator_hash(locator: &Path) -> String { + sha256_hex(locator.as_os_str().as_encoded_bytes()) +} + +#[cfg(feature = "postgres")] +fn postgres_locator_fingerprint(locator: &secrecy::SecretString) -> Result { + use tokio_postgres::config::Host; + + let config = locator + .expose_secret() + .parse::() + .map_err(|_| { + MigrationError::InvalidInput( + "PostgreSQL locator is invalid (connection details redacted)".to_string(), + ) + })?; + if config.get_options().is_some() { + return Err(MigrationError::InvalidInput( + "PostgreSQL connection options are not supported for migration locator identity; remove `options` and re-plan (connection details redacted)" + .to_string(), + )); + } + let mut material = Vec::new(); + append_locator_field(&mut material, b"schema", b"postgres-locator-v1"); + append_locator_field( + &mut material, + b"database", + config + .get_dbname() + .or_else(|| config.get_user()) + .unwrap_or_default() + .as_bytes(), + ); + for (index, host) in config.get_hosts().iter().enumerate() { + let label = format!("host-{index}"); + match host { + Host::Tcp(host) => { + append_locator_field(&mut material, label.as_bytes(), host.as_bytes()) + } + #[cfg(unix)] + Host::Unix(path) => append_locator_field( + &mut material, + label.as_bytes(), + path.as_os_str().as_encoded_bytes(), + ), + } + let port = config.get_ports().get(index).copied().unwrap_or(5432); + append_locator_field( + &mut material, + format!("port-{index}").as_bytes(), + port.to_string().as_bytes(), + ); + } + for (index, address) in config.get_hostaddrs().iter().enumerate() { + append_locator_field( + &mut material, + format!("hostaddr-{index}").as_bytes(), + address.to_string().as_bytes(), + ); + } + Ok(sha256_hex(&material)) +} + +#[cfg(not(feature = "postgres"))] +fn postgres_locator_fingerprint( + _locator: &secrecy::SecretString, +) -> Result { + Err(MigrationError::InvalidInput( + "PostgreSQL support is not compiled into this migrator".to_string(), + )) +} + +#[cfg(feature = "postgres")] +fn append_locator_field(material: &mut Vec, label: &[u8], value: &[u8]) { + material.extend_from_slice(label.len().to_string().as_bytes()); + material.push(b':'); + material.extend_from_slice(label); + material.extend_from_slice(value.len().to_string().as_bytes()); + material.push(b':'); + material.extend_from_slice(value); +} + +fn validate_distinct_stores(source: &SourceDb, target: &TargetStore) -> Result<(), MigrationError> { + let same = match (source, target) { + (SourceDb::LibSql { path: source }, TargetStore::LibSql { path: target }) => { + canonicalish(source) == canonicalish(target) + } + (SourceDb::Postgres { url: source }, TargetStore::Postgres { url: target }) => { + postgres_locator_fingerprint(source)? == postgres_locator_fingerprint(target)? + } + _ => false, + }; + if same { + return Err(MigrationError::InvalidInput( + "v1 source and Reborn target must be different stores".to_string(), + )); + } + Ok(()) +} + +fn canonicalish(path: &Path) -> PathBuf { + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir() + .unwrap_or_else(|_| PathBuf::from(".")) + .join(path) + }; + let mut normalized = PathBuf::new(); + for component in absolute.components() { + match component { + Component::CurDir => {} + Component::ParentDir => { + normalized.pop(); + } + other => normalized.push(other.as_os_str()), + } + } + + // Resolve symlinks and platform aliases (for example `/var` vs + // `/private/var` on macOS) without requiring the final target to exist. + // Hashing the lexical path before creation and the canonical path after + // creation would otherwise make a valid resume look like target drift. + let mut ancestor = normalized.clone(); + let mut missing_suffix = Vec::new(); + while !ancestor.exists() { + let Some(name) = ancestor.file_name() else { + break; + }; + missing_suffix.push(name.to_os_string()); + if !ancestor.pop() { + break; + } + } + let mut resolved = ancestor.canonicalize().unwrap_or(ancestor); + for component in missing_suffix.into_iter().rev() { + resolved.push(component); + } + resolved +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "postgres")] + use secrecy::SecretString; + + #[cfg(feature = "postgres")] + use super::{SourceDb, TargetStore, postgres_locator_fingerprint, validate_distinct_stores}; + + #[cfg(feature = "postgres")] + #[test] + fn postgres_locator_fingerprint_excludes_password_but_binds_database() { + let first = SecretString::from( + "postgresql://migration:password-one@database.example:5433/ironclaw", + ); + let rotated = SecretString::from( + "postgresql://migration:password-two@database.example:5433/ironclaw", + ); + let rotated_user = + SecretString::from("postgresql://new-user:password-two@database.example:5433/ironclaw"); + let other_database = + SecretString::from("postgresql://migration:password-one@database.example:5433/other"); + + let first_fingerprint = postgres_locator_fingerprint(&first).expect("first fingerprint"); + assert_eq!( + first_fingerprint, + postgres_locator_fingerprint(&rotated).expect("rotated fingerprint") + ); + assert_eq!( + first_fingerprint, + postgres_locator_fingerprint(&rotated_user).expect("rotated user fingerprint") + ); + assert_ne!( + first_fingerprint, + postgres_locator_fingerprint(&other_database).expect("other database fingerprint") + ); + assert!(!first_fingerprint.contains("password-one")); + assert!(!first_fingerprint.contains("database.example")); + } + + #[cfg(feature = "postgres")] + #[test] + fn postgres_store_identity_rejects_credential_only_differences() { + let source = SourceDb::Postgres { + url: SecretString::from( + "postgresql://source-user:source-password@database.example/ironclaw", + ), + }; + let target = TargetStore::Postgres { + url: SecretString::from( + "postgresql://target-user:target-password@database.example/ironclaw", + ), + }; + + let error = validate_distinct_stores(&source, &target) + .expect_err("different credentials must not disguise the same database"); + assert!(error.to_string().contains("must be different stores")); + assert!(!error.to_string().contains("password")); + assert!(!error.to_string().contains("database.example")); + } + + #[cfg(feature = "postgres")] + #[test] + fn postgres_locator_options_fail_closed_without_echoing_values() { + let locator = SecretString::from( + "postgresql://migration:password@database.example/ironclaw?options=-c%20secret.option%3Dcanary", + ); + + let error = postgres_locator_fingerprint(&locator) + .expect_err("connection options must not enter manifest hashes"); + let rendered = error.to_string(); + assert!(rendered.contains("options")); + assert!(!rendered.contains("canary")); + assert!(!rendered.contains("password")); + assert!(!rendered.contains("database.example")); + } +} diff --git a/crates/ironclaw_reborn_migration/src/main.rs b/crates/ironclaw_reborn_migration/src/main.rs index cce15bafd4e..89b0d2bac1b 100644 --- a/crates/ironclaw_reborn_migration/src/main.rs +++ b/crates/ironclaw_reborn_migration/src/main.rs @@ -1,110 +1,207 @@ -//! `ironclaw-reborn-migration` — convert IronClaw v1 / engine-v2 state into the -//! Reborn state substrate. -//! -//! Thin CLI wrapper over [`ironclaw_reborn_migration::run_migration`]. The -//! conversion engine lives in the library so it can later be reused inside -//! `ironclaw-reborn` startup (documented follow-up). - -use std::path::PathBuf; +//! Same-release migration companion for `ironclaw-reborn migrate v1`. + +use std::fs::OpenOptions; +use std::io::Write as _; +use std::path::{Path, PathBuf}; use std::process::ExitCode; -use clap::Parser; -use ironclaw_host_api::{AgentId, TenantId}; +use anyhow::{Context as _, ensure}; +use clap::{ArgGroup, Args, Parser, Subcommand}; +use ironclaw_reborn_composition::{RebornMigrationTargetStore, resolve_reborn_migration_target}; +use ironclaw_reborn_config::RebornBootConfig; use ironclaw_reborn_migration::{ - MigrationOptions, MigrationReport, SourceDb, TargetStore, run_migration, + ApplyAcknowledgements, Disposition, MIGRATION_PROTOCOL_VERSION, MigrationManifest, + MigrationOptions, MigrationSecretInputs, MigrationStatus, SourceDb, TargetStore, + apply_migration, manifest_target_matches, plan_migration, preflight_apply_migration, + preflight_resume_migration, resume_migration, verify_migration, }; use secrecy::SecretString; +use serde::Serialize; + +const HANDSHAKE_SCHEMA: &str = "ironclaw.reborn.migration-companion/v1"; +const SOURCE_POSTGRES_ENV: &str = "MIGRATION_SOURCE_POSTGRES"; +const SOURCE_MASTER_KEY_ENV: &str = "MIGRATION_SOURCE_SECRET_MASTER_KEY"; +const ERROR_FORMAT_ENV: &str = "IRONCLAW_REBORN_MIGRATION_ERROR_FORMAT"; +const TARGET_STATE_FILE: &str = ".v1-migration-state.json"; +const TARGET_STATE_SCHEMA: &str = "ironclaw.reborn.migration-state/v1"; -/// Migrate IronClaw v1 / engine-v2 persisted state into Reborn state. -/// -/// Deliberately no `Debug` derive: this struct holds the secrets master key and -/// PostgreSQL connection URLs (which embed `user:password@host`) as plain -/// strings, so a stray `{cli:?}` must not be able to leak them. `clap::Parser` -/// does not require `Debug`. #[derive(Parser)] #[command(name = "ironclaw-reborn-migration", version, about)] struct Cli { - /// v1 source: path to a libSQL/SQLite database file. - #[arg( - long, - conflicts_with = "source_postgres", - env = "MIGRATION_SOURCE_LIBSQL" - )] + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Machine-readable compatibility handshake used by ironclaw-reborn. + #[command(name = "__handshake", hide = true)] + Handshake, + /// Migrate an IronClaw v1 installation. + V1(V1Command), +} + +#[derive(Args)] +struct V1Command { + #[command(subcommand)] + operation: V1Operation, +} + +#[derive(Subcommand)] +enum V1Operation { + /// Inventory a v1 source without opening or creating the Reborn target. + Plan(PlanArgs), + /// Apply a reviewed plan to a fresh staged Reborn target. + Apply(ApplyArgs), + /// Resume an interrupted apply using the same stopped-source snapshot. + Resume(ResumeArgs), + /// Verify an applied target without activating workers or ingress. + Verify(VerifyArgs), + /// Inspect a migration manifest without opening either database. + Status(StatusArgs), +} + +#[derive(Args)] +struct SourceArgs { + /// WAL-consistent libSQL/SQLite snapshot. + #[arg(long, value_name = "SNAPSHOT", group = "source")] source_libsql: Option, - /// v1 source: PostgreSQL connection URL. - #[arg( - long, - conflicts_with = "source_libsql", - env = "MIGRATION_SOURCE_POSTGRES" - )] - source_postgres: Option, - - /// Reborn target: path to the libSQL store to write (created if absent). - #[arg( - long, - conflicts_with = "target_postgres", - env = "MIGRATION_TARGET_LIBSQL" - )] - target_libsql: Option, - - /// Reborn target: PostgreSQL connection URL. - #[arg( - long, - conflicts_with = "target_libsql", - env = "MIGRATION_TARGET_POSTGRES" - )] - target_postgres: Option, - - /// Reborn tenant all migrated state is written under. - #[arg(long, default_value = "default")] - tenant_id: String, - - /// Reborn agent migrated threads/triggers/memory are scoped to. - #[arg(long, default_value = "default")] - agent_id: String, - - /// Secrets master key (needed only to migrate secrets). Prefer the env var. - #[arg(long, env = "MIGRATION_SECRET_MASTER_KEY")] - secret_master_key: Option, - - /// Report what would be migrated without writing to the Reborn store. + /// Read the PostgreSQL snapshot URL from MIGRATION_SOURCE_POSTGRES. + #[arg(long, group = "source")] + source_postgres: bool, + + /// v1 home containing persistent files outside the database snapshot. + #[arg(long, value_name = "PATH")] + source_home: Option, +} + +#[derive(Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct PlanArgs { + #[command(flatten)] + source: SourceArgs, + + /// Destination for the versioned migration manifest. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, + + /// Fail after writing for blockers or nonzero archive/re-auth/reinstall/unsupported data. #[arg(long)] - dry_run: bool, + strict: bool, +} + +#[derive(Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct ApplyArgs { + #[command(flatten)] + source: SourceArgs, + + /// Reviewed migration plan to apply and update. + #[arg(long, value_name = "PATH")] + plan: PathBuf, + + /// Confirm that the v1 process and every other source writer are stopped. + #[arg(long, required = true)] + confirm_v1_stopped: bool, + + /// Confirm that the selected source is a consistent operator-created snapshot. + #[arg(long, required = true)] + confirm_source_snapshot: bool, +} + +#[derive(Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct ResumeArgs { + #[command(flatten)] + source: SourceArgs, - /// Write the JSON report to this path (otherwise printed to stdout). + /// Migration manifest to resume and update. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, + + /// Confirm that the v1 process and every other source writer remain stopped. + #[arg(long, required = true)] + confirm_v1_stopped: bool, + + /// Confirm that the selected source is the plan's consistent snapshot. + #[arg(long, required = true)] + confirm_source_snapshot: bool, +} + +#[derive(Args)] +#[command(group( + ArgGroup::new("source") + .required(true) + .multiple(false) +))] +struct VerifyArgs { + #[command(flatten)] + source: SourceArgs, + + /// Applied migration manifest to verify and update. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, +} + +#[derive(Args)] +struct StatusArgs { + /// Migration manifest to inspect. + #[arg(long, value_name = "PATH")] + manifest: PathBuf, + + /// Print the complete redacted manifest JSON. #[arg(long)] - report: Option, -} - -impl Cli { - fn into_options(self) -> anyhow::Result<(MigrationOptions, Option)> { - let source = match (self.source_libsql, self.source_postgres) { - (Some(path), None) => SourceDb::LibSql { path }, - (None, Some(url)) => SourceDb::Postgres { - url: SecretString::from(url), - }, - _ => anyhow::bail!("exactly one of --source-libsql / --source-postgres is required"), - }; - let target = match (self.target_libsql, self.target_postgres) { - (Some(path), None) => TargetStore::LibSql { path }, - (None, Some(url)) => TargetStore::Postgres { - url: SecretString::from(url), - }, - _ => anyhow::bail!("exactly one of --target-libsql / --target-postgres is required"), - }; - let tenant_id = TenantId::new(self.tenant_id)?; - let agent_id = AgentId::new(self.agent_id)?; - let options = MigrationOptions { - source, - target, - tenant_id, - agent_id, - secret_master_key: self.secret_master_key.map(SecretString::from), - dry_run: self.dry_run, - }; - Ok((options, self.report)) - } + json: bool, +} + +#[derive(Serialize)] +struct Handshake<'a> { + schema_version: &'a str, + protocol_version: u32, + release_version: &'a str, +} + +struct ResolvedRun { + options: MigrationOptions, + target_master_key: Option, + target_state_path: PathBuf, +} + +struct ResolvedTarget { + store: TargetStore, + tenant_id: ironclaw_host_api::TenantId, + agent_id: ironclaw_host_api::AgentId, + master_key: Option, + profile: String, + state_path: PathBuf, +} + +#[derive(Serialize)] +struct TargetMigrationState<'a> { + schema_version: &'static str, + migration_protocol_version: u32, + release_version: &'static str, + run_id: String, + status: &'static str, + profile: &'a str, + target_backend: &'static str, + target_locator_fingerprint: &'a str, + tenant_id: &'a str, + agent_id: &'a str, + manifest: &'a Path, } #[tokio::main] @@ -117,40 +214,413 @@ async fn main() -> ExitCode { .with_writer(std::io::stderr) .init(); - match run().await { + match run(Cli::parse()).await { Ok(()) => ExitCode::SUCCESS, Err(error) => { - eprintln!("migration failed: {error:#}"); + if std::env::var(ERROR_FORMAT_ENV).as_deref() == Ok("json") { + eprintln!( + "{}", + serde_json::json!({ + "schema_version": "ironclaw.reborn.migration-error/v1", + "code": "migration_failed", + "message": error.to_string(), + }) + ); + } else { + eprintln!("migration failed: {error:#}"); + } ExitCode::FAILURE } } } -async fn run() -> anyhow::Result<()> { - let cli = Cli::parse(); - let (options, report_path) = cli.into_options()?; - let dry_run = options.dry_run; +async fn run(cli: Cli) -> anyhow::Result<()> { + match cli.command { + Command::Handshake => emit_handshake(), + Command::V1(command) => run_v1(command.operation).await, + } +} - let report = run_migration(options).await?; - emit_report(&report, report_path.as_deref()).await?; +fn emit_handshake() -> anyhow::Result<()> { + println!( + "{}", + serde_json::to_string(&Handshake { + schema_version: HANDSHAKE_SCHEMA, + protocol_version: MIGRATION_PROTOCOL_VERSION, + release_version: env!("CARGO_PKG_VERSION"), + })? + ); + Ok(()) +} + +async fn run_v1(operation: V1Operation) -> anyhow::Result<()> { + match operation { + V1Operation::Plan(command) => { + let run = resolve_run(command.source)?; + let manifest = plan_migration(&run.options).await?; + manifest.write_atomic(&command.manifest, false)?; + println!("{}", manifest.to_json()?); - let losses = report.lossy.len(); - if dry_run { - eprintln!("dry run complete: {} lossy item(s) reported", losses); - } else { - eprintln!("migration complete: {} lossy item(s) reported", losses); + if command.strict && manifest_has_strict_loss(&manifest) { + anyhow::bail!( + "strict migration planning found unsupported data or blockers; review {}", + command.manifest.display() + ); + } + eprintln!( + "migration plan written: {} ({} inventory categories)", + command.manifest.display(), + manifest.inventory.len() + ); + Ok(()) + } + V1Operation::Apply(command) => { + let run = resolve_run(command.source)?; + let manifest = read_manifest(&command.plan)?; + let secrets = migration_secrets(run.target_master_key.clone())?; + let acknowledgements = ApplyAcknowledgements { + source_is_stopped: command.confirm_v1_stopped, + source_is_snapshot: command.confirm_source_snapshot, + }; + preflight_apply_migration(&run.options, &manifest, &secrets, acknowledgements).await?; + let applying = manifest.transition(MigrationStatus::Applying)?; + write_target_state(&run.target_state_path, &applying, &command.plan)?; + applying.write_atomic(&command.plan, true)?; + let result = apply_migration(run.options, &manifest, secrets, acknowledgements).await; + let report = match result { + Ok(report) => report, + Err(error) => { + if error.is_preflight() { + write_target_state(&run.target_state_path, &manifest, &command.plan)?; + manifest.write_atomic(&command.plan, true)?; + } else { + let failed = applying.transition(MigrationStatus::Failed)?; + write_target_state(&run.target_state_path, &failed, &command.plan)?; + failed.write_atomic(&command.plan, true)?; + } + return Err(error.into()); + } + }; + persist_report_manifest(&report, &command.plan)?; + write_target_state( + &run.target_state_path, + report + .manifest + .as_ref() + .context("migration apply completed without an updated manifest")?, + &command.plan, + )?; + println!("{}", report.to_json()?); + Ok(()) + } + V1Operation::Resume(command) => { + let run = resolve_run(command.source)?; + let manifest = read_manifest(&command.manifest)?; + let secrets = migration_secrets(run.target_master_key.clone())?; + let acknowledgements = ApplyAcknowledgements { + source_is_stopped: command.confirm_v1_stopped, + source_is_snapshot: command.confirm_source_snapshot, + }; + preflight_resume_migration(&run.options, &manifest, &secrets, acknowledgements).await?; + let applying = if manifest.status == MigrationStatus::Applying { + manifest.clone() + } else { + manifest.transition(MigrationStatus::Applying)? + }; + write_target_state(&run.target_state_path, &applying, &command.manifest)?; + applying.write_atomic(&command.manifest, true)?; + let result = resume_migration(run.options, &manifest, secrets, acknowledgements).await; + let report = match result { + Ok(report) => report, + Err(error) => { + let failed = applying.transition(MigrationStatus::Failed)?; + write_target_state(&run.target_state_path, &failed, &command.manifest)?; + failed.write_atomic(&command.manifest, true)?; + return Err(error.into()); + } + }; + persist_report_manifest(&report, &command.manifest)?; + write_target_state( + &run.target_state_path, + report + .manifest + .as_ref() + .context("migration resume completed without an updated manifest")?, + &command.manifest, + )?; + println!("{}", report.to_json()?); + Ok(()) + } + V1Operation::Verify(command) => { + let run = resolve_run(command.source)?; + let manifest = read_manifest(&command.manifest)?; + let verifying = if manifest.status == MigrationStatus::Verifying { + manifest.clone() + } else { + manifest.transition(MigrationStatus::Verifying)? + }; + write_target_state(&run.target_state_path, &verifying, &command.manifest)?; + verifying.write_atomic(&command.manifest, true)?; + let verified = match verify_migration(&run.options, &manifest).await { + Ok(verified) => verified, + Err(error) => { + if verifying.status == MigrationStatus::Verifying { + let failed = verifying.transition(MigrationStatus::Failed)?; + write_target_state(&run.target_state_path, &failed, &command.manifest)?; + failed.write_atomic(&command.manifest, true)?; + } + return Err(error.into()); + } + }; + write_target_state(&run.target_state_path, &verified, &command.manifest)?; + verified.write_atomic(&command.manifest, true)?; + println!("{}", verified.to_json()?); + ensure!( + verified.status == MigrationStatus::Verified, + "verification did not reach the verified state; the target remains quarantined and must not be started" + ); + Ok(()) + } + V1Operation::Status(command) => { + let manifest = read_manifest(&command.manifest)?; + let target = resolve_target()?; + let target_matches = manifest_target_matches(&target.store, &manifest); + if command.json { + println!( + "{}", + serde_json::to_string_pretty(&serde_json::json!({ + "target_fingerprint_match": target_matches, + "manifest": manifest, + }))? + ); + } else { + println!("IronClaw Reborn v1 migration"); + println!("status: {}", status_label(manifest.status)); + println!("run_id: {}", manifest.run_id); + println!("profile: {}", manifest.scope.profile); + println!("tenant_id: {}", manifest.scope.tenant_id); + println!("agent_id: {}", manifest.scope.agent_id); + println!("inventory_categories: {}", manifest.inventory.len()); + println!("target_fingerprint_match: {target_matches}"); + println!("manifest: {}", command.manifest.display()); + } + Ok(()) + } } - Ok(()) } -async fn emit_report( - report: &MigrationReport, - path: Option<&std::path::Path>, +fn resolve_run(source: SourceArgs) -> anyhow::Result { + let source_home = source.source_home.clone(); + let source = resolve_source(source)?; + let target = resolve_target()?; + Ok(ResolvedRun { + options: MigrationOptions { + source, + source_home, + target: target.store, + profile: target.profile.clone(), + tenant_id: target.tenant_id, + agent_id: target.agent_id, + secret_master_key: None, + dry_run: false, + }, + target_master_key: target.master_key, + target_state_path: target.state_path, + }) +} + +fn resolve_target() -> anyhow::Result { + let boot = RebornBootConfig::resolve_from_env() + .context("failed to resolve the Reborn migration target configuration")?; + let state_path = boot.home().path().join(TARGET_STATE_FILE); + let target = resolve_reborn_migration_target(&boot) + .context("failed to resolve the production Reborn migration target")?; + let store = match target.store { + #[cfg(feature = "libsql")] + RebornMigrationTargetStore::LibSql { path } => TargetStore::LibSql { path }, + #[cfg(feature = "postgres")] + RebornMigrationTargetStore::Postgres { url } => TargetStore::Postgres { url }, + }; + Ok(ResolvedTarget { + store, + tenant_id: target.tenant_id, + agent_id: target.agent_id, + master_key: target.target_master_key, + profile: target.profile.as_str().to_owned(), + state_path, + }) +} + +fn write_target_state( + path: &Path, + manifest: &MigrationManifest, + manifest_path: &Path, ) -> anyhow::Result<()> { - let json = report.to_json()?; - match path { - Some(path) => tokio::fs::write(path, json).await?, - None => println!("{json}"), + let parent = path + .parent() + .context("target migration state has no parent directory")?; + std::fs::create_dir_all(parent).with_context(|| { + format!( + "failed to create target migration state directory {}", + parent.display() + ) + })?; + let absolute_manifest = std::path::absolute(manifest_path).with_context(|| { + format!( + "failed to resolve migration manifest path {}", + manifest_path.display() + ) + })?; + let document = serde_json::to_vec_pretty(&TargetMigrationState { + schema_version: TARGET_STATE_SCHEMA, + migration_protocol_version: MIGRATION_PROTOCOL_VERSION, + release_version: env!("CARGO_PKG_VERSION"), + run_id: manifest.run_id.to_string(), + status: status_label(manifest.status), + profile: &manifest.scope.profile, + target_backend: match manifest.target.backend { + ironclaw_reborn_migration::StoreBackend::Libsql => "libsql", + ironclaw_reborn_migration::StoreBackend::Postgres => "postgres", + }, + target_locator_fingerprint: &manifest.target.locator_fingerprint, + tenant_id: &manifest.scope.tenant_id, + agent_id: &manifest.scope.agent_id, + manifest: &absolute_manifest, + })?; + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(TARGET_STATE_FILE); + let temporary = path.with_file_name(format!(".{file_name}.{}.tmp", uuid::Uuid::new_v4())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut file = options.open(&temporary).with_context(|| { + format!( + "failed to create temporary target migration state {}", + temporary.display() + ) + })?; + if let Err(error) = file + .write_all(&document) + .and_then(|()| file.write_all(b"\n")) + .and_then(|()| file.sync_all()) + { + let _ = std::fs::remove_file(&temporary); + return Err(error).context("failed to persist target migration state"); + } + drop(file); + if let Err(error) = std::fs::rename(&temporary, path) { + let _ = std::fs::remove_file(&temporary); + return Err(error).with_context(|| { + format!( + "failed to install target migration state at {}", + path.display() + ) + }); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) +} + +fn resolve_source(source: SourceArgs) -> anyhow::Result { + match (source.source_libsql, source.source_postgres) { + (Some(path), false) => Ok(SourceDb::LibSql { path }), + (None, true) => Ok(SourceDb::Postgres { + url: required_secret_env(SOURCE_POSTGRES_ENV)?, + }), + _ => anyhow::bail!("exactly one v1 source selector is required"), + } +} + +fn migration_secrets( + target_master_key: Option, +) -> anyhow::Result { + Ok(MigrationSecretInputs { + source_master_key: optional_secret_env(SOURCE_MASTER_KEY_ENV)?, + target_master_key, + }) +} + +fn required_secret_env(name: &'static str) -> anyhow::Result { + optional_secret_env(name)? + .with_context(|| format!("required environment variable {name} is not set")) +} + +fn optional_secret_env(name: &'static str) -> anyhow::Result> { + match std::env::var(name) { + Ok(value) => { + ensure!( + !value.trim().is_empty(), + "environment variable {name} is empty" + ); + Ok(Some(SecretString::from(value))) + } + Err(std::env::VarError::NotPresent) => Ok(None), + Err(std::env::VarError::NotUnicode(_)) => { + anyhow::bail!("environment variable {name} is not valid UTF-8") + } } +} + +fn read_manifest(path: &Path) -> anyhow::Result { + let body = std::fs::read_to_string(path) + .with_context(|| format!("failed to read migration manifest at {}", path.display()))?; + let manifest: MigrationManifest = + serde_json::from_str(&body).context("migration manifest is not valid versioned JSON")?; + manifest.validate_plan_hash()?; + ensure!( + manifest.migration_protocol_version == MIGRATION_PROTOCOL_VERSION, + "migration manifest protocol {} is not supported by this companion (expected {})", + manifest.migration_protocol_version, + MIGRATION_PROTOCOL_VERSION + ); + Ok(manifest) +} + +fn persist_report_manifest( + report: &ironclaw_reborn_migration::MigrationReport, + path: &Path, +) -> anyhow::Result<()> { + let manifest = report + .manifest + .as_ref() + .context("migration lifecycle completed without an updated manifest")?; + manifest.write_atomic(path, true)?; Ok(()) } + +fn manifest_has_strict_loss(manifest: &MigrationManifest) -> bool { + manifest.inventory.iter().any(|entry| { + entry.blocker.is_some() + || (entry.count > 0 + && matches!( + entry.disposition, + Disposition::ArchiveOnly + | Disposition::RequiresReauth + | Disposition::RequiresReinstall + | Disposition::Unsupported + | Disposition::UnsupportedUnknown + )) + }) +} + +const fn status_label(status: MigrationStatus) -> &'static str { + match status { + MigrationStatus::Planned => "planned", + MigrationStatus::Applying => "applying", + MigrationStatus::Failed => "failed", + MigrationStatus::Applied => "applied", + MigrationStatus::Verifying => "verifying", + MigrationStatus::Verified => "verified", + } +} diff --git a/crates/ironclaw_reborn_migration/src/manifest.rs b/crates/ironclaw_reborn_migration/src/manifest.rs new file mode 100644 index 00000000000..da7a44b8330 --- /dev/null +++ b/crates/ironclaw_reborn_migration/src/manifest.rs @@ -0,0 +1,317 @@ +//! Versioned, redacted migration plan contract. + +use std::collections::BTreeMap; +use std::fs::OpenOptions; +use std::io::Write as _; +use std::path::Path; + +use chrono::{DateTime, Utc}; +use ironclaw_common::hashing::sha256_hex; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::error::MigrationError; +use crate::report::Domain; + +/// Current serialized manifest schema version. +pub const MANIFEST_SCHEMA_VERSION: u32 = 3; +/// Companion lifecycle protocol version required by the primary CLI. +pub const MIGRATION_PROTOCOL_VERSION: u32 = 1; + +/// Persisted lifecycle state. Every state between `Applying` and `Verified` is +/// quarantined from live runtime activation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MigrationStatus { + /// Source inventory is sealed and no target writes have begun. + Planned, + /// Apply or resume may be writing target records. + Applying, + /// Apply or verification failed and requires operator action. + Failed, + /// Conversion completed but structural verification has not. + Applied, + /// Structural durable-store verification is in progress. + Verifying, + /// Structural verification completed successfully. + Verified, +} + +/// Durable-store family named by a redacted descriptor. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum StoreBackend { + /// Embedded libSQL/SQLite store. + Libsql, + /// PostgreSQL store. + Postgres, +} + +/// A store locator safe for reports and logs. +/// +/// `locator_fingerprint` is a one-way hash of a local path or credential-free +/// PostgreSQL locator components. It is sufficient for equality checks without +/// serializing credentials, usernames, hosts, or home-directory paths. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RedactedStoreDescriptor { + /// Store family. + pub backend: StoreBackend, + /// One-way, credential-free locator identity. + pub locator_fingerprint: String, + /// Local existence observation; `None` when planning cannot inspect it + /// without opening a remote target. + pub exists: Option, +} + +/// Versioned digest that seals the source snapshot contents. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct SourceFingerprint { + /// Fingerprint algorithm identifier. + pub algorithm: String, + /// Hex-encoded digest. + pub value: String, +} + +/// Planned handling for one known source category. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Disposition { + /// Copied into the equivalent Reborn concept. + Imported, + /// Converted into a different Reborn concept. + SemanticallyConverted, + /// Counted and checksummed only; payload is not copied. + ArchiveOnly, + /// Operator must issue new credentials after cutover. + RequiresReauth, + /// Operator must reinstall the artifact after cutover. + RequiresReinstall, + /// Transient state deliberately starts clean. + IntentionallyReset, + /// Excluded by an explicit operator-selected scope. + SkippedByOperator, + /// Known source data has no safe representation in this release. + Unsupported, + /// Unrecognized source data blocks apply. + UnsupportedUnknown, + /// Derived state is recomputed by Reborn. + DerivedRebuilt, +} + +/// Physical source category represented by an inventory entry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum InventorySourceKind { + /// Database table. + Table, + /// Persistent v1-home file. + HomeFile, + /// Persistent v1-home directory. + HomeDirectory, +} + +/// Counted and classified source category in a sealed plan. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct InventoryEntry { + /// Physical category kind. + pub source_kind: InventorySourceKind, + /// Stable table or home-artifact name. + pub source_name: String, + /// Logical conversion domain. + pub domain: Domain, + /// Required handling in this release. + pub disposition: Disposition, + /// Number of source records or path entries observed. + pub count: u64, + /// Digest of non-secret inventory metadata. Never a digest of plaintext + /// secret material or bearer tokens. + pub checksum: String, + /// Apply-preventing issue discovered during inventory. + pub blocker: Option, + /// Reviewable non-blocking caveat. + pub warning: Option, +} + +/// Lifecycle accounting for one logical domain. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct DomainCheckpoint { + /// Source records planned for the domain. + pub planned: u64, + /// Records converted successfully. + pub applied: u64, + /// Records covered by structural readback. + pub verified: u64, + /// Divergent records that prevented overwrite. + pub conflicts: u64, + /// Apply-preventing domain issues. + pub blockers: Vec, + /// Non-blocking domain caveats. + pub warnings: Vec, + /// Completion time for the latest successful domain phase. + pub completed_at: Option>, +} + +/// Production profile and authority scope sealed into the plan. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ResolvedScope { + /// Effective Reborn profile. + pub profile: String, + /// Target tenant identity. + pub tenant_id: String, + /// Target agent identity. + pub agent_id: String, + /// One-way identity of the explicit v1 home, when supplied. + pub source_home_fingerprint: Option, + /// Deterministic source-to-target user identity mapping. + pub user_mapping: BTreeMap, + /// Planning-time local target emptiness; remote targets use `None`. + pub target_empty: Option, +} + +/// Versioned, redacted plan and lifecycle authority for one migration run. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MigrationManifest { + /// Serialized manifest schema version. + pub manifest_schema_version: u32, + /// Companion protocol version. + pub migration_protocol_version: u32, + /// Migrator build version. + pub tool_version: String, + /// Release version that must match the primary CLI. + pub release_version: String, + /// Unique migration-run identity. + pub run_id: Uuid, + /// Current lifecycle state. + pub status: MigrationStatus, + /// Redacted source locator. + pub source: RedactedStoreDescriptor, + /// Redacted target locator. + pub target: RedactedStoreDescriptor, + /// v1 schema version, when the source records one. + pub source_schema_version: Option, + /// Source snapshot content seal. + pub source_fingerprint: SourceFingerprint, + /// Digest over the classified inventory. + pub source_inventory_checksum: String, + /// Seal over the complete manifest with this field cleared. + pub plan_hash: String, + /// Production scope and identity mapping. + pub scope: ResolvedScope, + /// Complete classified source inventory. + pub inventory: Vec, + /// Per-domain lifecycle accounting. + pub domains: BTreeMap, + /// Offline assertions recorded at apply time. + pub operator_acknowledgements: Vec, + /// Plan creation time. + pub created_at: DateTime, + /// Latest lifecycle update time. + pub updated_at: DateTime, +} + +impl MigrationManifest { + pub(crate) fn seal(&mut self) -> Result<(), MigrationError> { + self.plan_hash.clear(); + self.plan_hash = sha256_hex(&serde_json::to_vec(self)?); + Ok(()) + } + + /// Validate that no sealed manifest field changed since the last seal. + pub fn validate_plan_hash(&self) -> Result<(), MigrationError> { + let expected = self.plan_hash.clone(); + let mut candidate = self.clone(); + candidate.seal()?; + if candidate.plan_hash != expected { + return Err(MigrationError::InvalidInput( + "migration manifest plan hash does not match its contents".to_string(), + )); + } + Ok(()) + } + + /// Return a resealed lifecycle copy after checking the state transition. + /// Callers can atomically persist `Applying` before target writes and + /// `Failed` on an error path, so a crash never leaves only an in-memory + /// partial report. + pub fn transition(&self, status: MigrationStatus) -> Result { + let valid = matches!( + (self.status, status), + (MigrationStatus::Planned, MigrationStatus::Applying) + | (MigrationStatus::Applying, MigrationStatus::Failed) + | (MigrationStatus::Applying, MigrationStatus::Applied) + | (MigrationStatus::Failed, MigrationStatus::Applying) + | (MigrationStatus::Applied, MigrationStatus::Applying) + | (MigrationStatus::Applied, MigrationStatus::Verifying) + | (MigrationStatus::Verified, MigrationStatus::Verifying) + | (MigrationStatus::Verifying, MigrationStatus::Failed) + | (MigrationStatus::Verifying, MigrationStatus::Verified) + ); + if !valid { + return Err(MigrationError::InvalidInput(format!( + "invalid migration status transition {:?} -> {:?}", + self.status, status + ))); + } + let mut next = self.clone(); + next.status = status; + next.updated_at = Utc::now(); + next.seal()?; + Ok(next) + } + + /// Serialize the redacted manifest as pretty JSON. + pub fn to_json(&self) -> serde_json::Result { + serde_json::to_string_pretty(self) + } + + /// Persist a manifest via a same-directory temporary file. + /// + /// The default is no-clobber. On Unix, both the temporary and final file + /// are owner-readable/writable only. `hard_link` provides an atomic + /// create-if-absent operation without a check-then-rename race. + pub fn write_atomic(&self, path: &Path, overwrite: bool) -> Result<(), MigrationError> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let file_name = path + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("migration-manifest.json"); + let tmp_path = path.with_file_name(format!(".{file_name}.{}.tmp", Uuid::new_v4())); + + let mut open = OpenOptions::new(); + open.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + open.mode(0o600); + } + let mut file = open.open(&tmp_path)?; + let json = self.to_json()?; + file.write_all(json.as_bytes())?; + file.sync_all()?; + drop(file); + + let result = if overwrite { + std::fs::rename(&tmp_path, path) + } else { + std::fs::hard_link(&tmp_path, path).and_then(|()| std::fs::remove_file(&tmp_path)) + }; + if result.is_err() { + let _ = std::fs::remove_file(&tmp_path); + } + result?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))?; + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + std::fs::File::open(parent)?.sync_all()?; + } + Ok(()) + } +} diff --git a/crates/ironclaw_reborn_migration/src/mounts.rs b/crates/ironclaw_reborn_migration/src/mounts.rs index d2622e0ea65..597a3bcd3d8 100644 --- a/crates/ironclaw_reborn_migration/src/mounts.rs +++ b/crates/ironclaw_reborn_migration/src/mounts.rs @@ -1,75 +1,36 @@ -//! Canonical per-domain mount views. +//! Production-owned mount resolution for migration writers. //! -//! Each Reborn domain service resolves records through a `ScopedFilesystem` -//! whose resolver maps a [`ResourceScope`] to a [`MountView`] — alias → a -//! tenant/user-scoped virtual path. These builders mirror the production -//! layout (`ironclaw_reborn_composition::local_dev_mounts` for memory; the -//! tenant/user `/threads` + `/secrets` shape the runtime resolves through). -//! -//! FOLLOW-UP: the production mount resolver lives (private) in -//! `ironclaw_reborn_composition`. Until a shared `pub` accessor exists, this -//! module reproduces that layout; it MUST be reconciled with composition when -//! the migration step is wired into `ironclaw-reborn` startup so the runtime -//! reads back exactly what was migrated. The acceptance test verifies -//! round-trip through these same services, which pins conversion correctness -//! independently of that reconciliation. +//! Migration must never reproduce Reborn's virtual-path layout. Every scoped +//! writer delegates to composition's canonical resolver so a cold production +//! runtime reopens the exact tenant/user paths written here. -use ironclaw_host_api::{ - HostApiError, MountAlias, MountGrant, MountPermissions, MountView, ResourceScope, - SYSTEM_RESERVED_ID, VirtualPath, -}; +use ironclaw_host_api::{HostApiError, MountView, ResourceScope}; -fn grant(alias: &str, target: String) -> Result { - Ok(MountGrant::new( - MountAlias::new(alias)?, - VirtualPath::new(target)?, - MountPermissions::read_write_list_delete(), - )) +pub(crate) fn production_mount_view(scope: &ResourceScope) -> Result { + ironclaw_reborn_composition::invocation_mount_view(scope) } -/// Map a scope segment to its on-disk path form, mirroring production's -/// `ironclaw_reborn_composition::invocation_mount_view`: the system sentinel -/// ([`SYSTEM_RESERVED_ID`]) carries control bytes and must render as -/// `__system__` (a valid path segment) so system-scoped service operations -/// (e.g. `FilesystemSessionThreadService` idempotency lookups under -/// `ResourceScope::system()`) resolve to the same paths the runtime reads back. -fn scope_segment(value: &str) -> &str { - if value == SYSTEM_RESERVED_ID { - "__system__" - } else { - value - } -} +#[cfg(test)] +mod tests { + use ironclaw_host_api::{AgentId, InvocationId, ResourceScope, TenantId, UserId}; -/// `/threads` → `/tenants//users//threads`. Sub-scope (agent, project, -/// mission) is path-encoded by `FilesystemSessionThreadService` inside the alias. -pub(crate) fn threads_mount_view(scope: &ResourceScope) -> Result { - MountView::new(vec![grant( - "/threads", - format!( - "/tenants/{}/users/{}/threads", - scope_segment(scope.tenant_id.as_str()), - scope_segment(scope.user_id.as_str()) - ), - )?]) -} + use super::production_mount_view; -/// `/secrets` → `/tenants//users//secrets`. `FilesystemSecretStore` -/// path-encodes agent/project inside the alias. -pub(crate) fn secrets_mount_view(scope: &ResourceScope) -> Result { - MountView::new(vec![grant( - "/secrets", - format!( - "/tenants/{}/users/{}/secrets", - scope_segment(scope.tenant_id.as_str()), - scope_segment(scope.user_id.as_str()) - ), - )?]) -} + #[test] + fn migration_mount_view_is_the_production_mount_view() { + let scope = ResourceScope { + tenant_id: TenantId::new("tenant-a").unwrap(), + user_id: UserId::new("user-a").unwrap(), + agent_id: Some(AgentId::new("agent-a").unwrap()), + project_id: None, + mission_id: None, + thread_id: None, + invocation_id: InvocationId::new(), + }; -/// Identity records live under the store's fixed `/tenant-shared/reborn-identity` -/// root (partitioned by tenant inside the record path), so the mount exposes the -/// `/tenant-shared` alias — matching the identity crate's own store wiring. -pub(crate) fn identity_mount_view(_scope: &ResourceScope) -> Result { - MountView::new(vec![grant("/tenant-shared", "/tenant-shared".to_string())?]) + assert_eq!( + production_mount_view(&scope).unwrap(), + ironclaw_reborn_composition::invocation_mount_view(&scope).unwrap() + ); + } } diff --git a/crates/ironclaw_reborn_migration/src/options.rs b/crates/ironclaw_reborn_migration/src/options.rs index 929d7ab4515..7a61a7be94b 100644 --- a/crates/ironclaw_reborn_migration/src/options.rs +++ b/crates/ironclaw_reborn_migration/src/options.rs @@ -12,21 +12,78 @@ use secrecy::SecretString; pub struct MigrationOptions { /// Backend + connection details for the v1 source database. pub source: SourceDb, + /// Explicit v1 home whose persistent artifacts must be inventoried. + /// When absent, planning records a blocker rather than guessing from the + /// database snapshot location. + pub source_home: Option, /// Where to write Reborn state. pub target: TargetStore, + /// Effective production Reborn profile selected by composition. + pub profile: String, /// Reborn tenant that all migrated state belongs to. pub tenant_id: TenantId, /// Reborn agent that migrated threads/triggers/memory are scoped to. pub agent_id: AgentId, - /// Secrets master key (v1 ciphertext re-encrypts under Reborn's ported - /// AES-256-GCM scheme). Required only when migrating secrets. + /// Legacy single-key compatibility input used only by `run_migration`. + /// New lifecycle callers must use [`MigrationSecretInputs`] so v1 + /// decryption and Reborn encryption resolve independently. pub secret_master_key: Option, - /// Report only; write nothing to the Reborn store. + /// Legacy compatibility flag used only by [`crate::run_migration`]. + /// Explicit lifecycle callers use [`crate::plan_migration`] for the + /// non-writing phase. pub dry_run: bool, } -/// v1 source database selector. Mirrors `ironclaw::config::DatabaseConfig` -/// enough to open a read connection via `ironclaw::db::connect_with_handles`. +/// Secret material used by the two sides of an apply operation. +/// +/// v1 ciphertext and Reborn ciphertext do not have to use the same master key. +/// Keeping the values in a separate, non-`Debug` input also makes it harder for +/// a lifecycle request or serialized manifest to accidentally disclose them. +#[derive(Clone, Default)] +pub struct MigrationSecretInputs { + /// Key used only while decrypting values read from the v1 snapshot. + pub source_master_key: Option, + /// Key used only while encrypting values written to Reborn. + pub target_master_key: Option, +} + +impl MigrationSecretInputs { + /// Build the split input from the legacy single-key option. + /// + /// This exists only for the temporary [`crate::run_migration`] + /// compatibility wrapper. New callers should resolve both sides + /// independently and construct this type directly. + pub fn from_legacy(options: &MigrationOptions) -> Self { + Self { + source_master_key: options.secret_master_key.clone(), + target_master_key: options.secret_master_key.clone(), + } + } +} + +/// Operator assertions required before the first-release offline apply path. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ApplyAcknowledgements { + /// The source v1 process and every other writer have been stopped. + pub source_is_stopped: bool, + /// The selected source is an operator-created, consistent snapshot. + pub source_is_snapshot: bool, +} + +impl ApplyAcknowledgements { + /// Assert both offline-apply preconditions for an operator-confirmed + /// stopped, consistent source snapshot. + pub const fn offline_snapshot() -> Self { + Self { + source_is_stopped: true, + source_is_snapshot: true, + } + } +} + +/// v1 source selector consumed by the crate's narrow read-only adapter. +/// +/// Source opening must not use runtime constructors that run schema migrations. #[derive(Debug, Clone)] pub enum SourceDb { /// libSQL/SQLite file on disk. @@ -37,9 +94,9 @@ pub enum SourceDb { Postgres { url: SecretString }, } -/// Where Reborn state is written. The `RootFilesystem` KV substrate (threads, -/// memory, secrets, extensions, identity) and the triggers DB share the same -/// underlying backend handle. +/// Where Reborn state is written. The `RootFilesystem` KV substrate (users, +/// projects, threads, memory, secrets, identity) and the triggers DB share the +/// same underlying backend handle. #[derive(Debug, Clone)] pub enum TargetStore { /// Local libSQL file (the `reborn-local-dev.db` shape). diff --git a/crates/ironclaw_reborn_migration/src/report.rs b/crates/ironclaw_reborn_migration/src/report.rs index 7845f981fa2..c65b2e6b76e 100644 --- a/crates/ironclaw_reborn_migration/src/report.rs +++ b/crates/ironclaw_reborn_migration/src/report.rs @@ -1,31 +1,43 @@ //! Migration outcome accounting. //! -//! Two shapes: [`MigrationStats`] counts what was converted per domain, and -//! [`LossyItem`] records every source field/entity that could **not** be -//! represented in Reborn. Together they form the [`MigrationReport`], which is -//! JSON-serializable so an operator (or a follow-up in-process migration step) -//! can inspect exactly what carried over and what was dropped. Nothing is ever -//! silently lost: a value that has no Reborn home lands here as a `LossyItem`. +//! [`MigrationStats`] and [`LossyItem`] retain converter-level accounting while +//! the attached versioned [`crate::manifest::MigrationManifest`] carries the +//! complete source inventory, lifecycle status, and checkpoints. Both are +//! JSON-serializable; values without a Reborn representation must appear in one +//! of these explicit accounting surfaces rather than being silently dropped. use ironclaw_host_api::UserId; use serde::{Deserialize, Serialize}; /// The domain a converted item or a loss belongs to. Keeps report entries /// grouped and greppable rather than free-text. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum Domain { + User, + ApiToken, Thread, Message, + ConversationBinding, + Project, Routine, Mission, + Trigger, Job, Memory, Secret, + SecurityAudit, Extension, + Skill, Identity, + Pairing, Heartbeat, Setting, + Provider, + WorkspaceFile, + OperationalState, + SchemaMetadata, + Unknown, } /// Why a source value did not fully carry over into Reborn. @@ -61,10 +73,18 @@ pub struct LossyItem { /// Per-domain counts of successfully converted source entities. #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct MigrationStats { + /// Canonical or synthesized users imported. + pub users: usize, + /// Threads imported. pub threads: usize, + /// Messages imported. pub messages: usize, + /// Engine-v2 projects imported into the project repository. + pub projects: usize, pub routines: usize, pub missions: usize, + /// Durable Reborn trigger records created from supported routines/missions. + pub triggers: usize, pub trigger_runs: usize, pub jobs: usize, pub memory_documents: usize, @@ -81,8 +101,14 @@ pub struct MigrationStats { pub struct MigrationReport { /// True when the run was a dry run (nothing written to the Reborn store). pub dry_run: bool, + /// Successful conversion counts. pub stats: MigrationStats, + /// Per-record losses emitted only in the apply/resume report. pub lossy: Vec, + /// The lifecycle manifest that produced this report. Older callers may + /// omit it while the compatibility wrapper is being retired. + #[serde(skip_serializing_if = "Option::is_none")] + pub manifest: Option, } impl MigrationReport { @@ -148,7 +174,7 @@ impl MigrationReport { .count() } - /// Pretty JSON for `--report ` / stdout. + /// Pretty JSON emitted by successful apply/resume operations. pub fn to_json(&self) -> serde_json::Result { serde_json::to_string_pretty(self) } diff --git a/crates/ironclaw_reborn_migration/src/source.rs b/crates/ironclaw_reborn_migration/src/source.rs index 13bb21f6049..408b44d8732 100644 --- a/crates/ironclaw_reborn_migration/src/source.rs +++ b/crates/ironclaw_reborn_migration/src/source.rs @@ -5,21 +5,28 @@ //! v1 stores (secrets, wasm tools, identities) need. Engine-v2 mission/project //! state is not a separate connection — it lives as JSON blobs inside the //! `memory_documents` table and is read through the same `Database` handle -//! (see [`crate::convert::automations`] and [`crate::v2_model`]). +//! (see [`crate::convert::automations`], [`crate::convert::projects`], and +//! [`crate::v2_model`]). use std::sync::Arc; +use sha2::{Digest as _, Sha256}; + +#[cfg(feature = "postgres")] use ironclaw::config::{DatabaseBackend, DatabaseConfig, SslMode}; -use ironclaw::db::{Database, DatabaseHandles, connect_with_handles}; +use ironclaw::db::{Database, DatabaseHandles}; +#[cfg(feature = "postgres")] use secrecy::SecretString; use crate::error::MigrationError; +use crate::inventory::RawTableInventory; +use crate::manifest::SourceFingerprint; use crate::options::SourceDb; -/// A live, migrations-applied handle to the v1 source database. +/// A read-only-by-contract handle to a v1 source snapshot. /// -/// Crate-internal: the only public entry point is [`crate::run_migration`], and -/// this handle is consumed exclusively by the in-crate converters (mirrors the +/// Crate-internal: the public lifecycle functions construct this handle, and it +/// is consumed exclusively by the in-crate planners/converters (mirrors the /// symmetric `RebornTarget` visibility). pub(crate) struct V1Source { pub(crate) db: Arc, @@ -28,36 +35,512 @@ pub(crate) struct V1Source { pub(crate) handles: DatabaseHandles, } +pub(crate) struct ProjectDocument { + pub(crate) user_id: String, + pub(crate) path: String, + pub(crate) content: String, +} + /// Tables a v1 user_id can appear in. Queried independently so a DB missing one /// (e.g. a minimal libSQL install without `settings`) still discovers users /// from the others. -const USER_ID_TABLES: [&str; 4] = ["conversations", "routines", "memory_documents", "settings"]; +const USER_ID_TABLES: &[(&str, &str)] = &[ + ("conversations", "user_id"), + ("agent_jobs", "user_id"), + ("memory_documents", "user_id"), + ("heartbeat_state", "user_id"), + ("secrets", "user_id"), + ("wasm_tools", "user_id"), + ("wasm_channels", "user_id"), + ("tool_rate_limit_state", "user_id"), + ("secret_usage_log", "user_id"), + ("leak_detection_events", "user_id"), + ("routines", "user_id"), + ("settings", "user_id"), + ("api_tokens", "user_id"), + ("user_identities", "user_id"), + ("channel_identities", "owner_id"), + ("pairing_requests", "owner_id"), +]; impl V1Source { pub(crate) async fn open(source: &SourceDb) -> Result { - let config = source_to_config(source); - let (db, handles) = connect_with_handles(&config) - .await - .map_err(|e| MigrationError::OpenSource(e.to_string()))?; + if let SourceDb::LibSql { path } = source { + let metadata = std::fs::metadata(path).map_err(|error| { + MigrationError::OpenSource(format!( + "snapshot {} must already exist and be readable: {error}", + path.display() + )) + })?; + if !metadata.is_file() { + return Err(MigrationError::OpenSource(format!( + "snapshot {} is not a regular file", + path.display() + ))); + } + } + // This constructor intentionally skips every v1 schema migration. The + // migration tool reads historical schemas; it must never upgrade the + // operator's source as a side effect of inspection. + let (db, handles) = match source { + SourceDb::LibSql { path: source_path } => { + #[cfg(feature = "libsql")] + { + let backend = + ironclaw::db::libsql::LibSqlBackend::new_local_read_only(source_path) + .await + .map_err(source_open_error)?; + let handles = handles_with_libsql(backend.shared_db()); + (Arc::new(backend) as Arc, handles) + } + #[cfg(not(feature = "libsql"))] + { + let _ = source_path; + return Err(MigrationError::OpenSource( + "libSQL support is not compiled into this migrator".to_string(), + )); + } + } + SourceDb::Postgres { .. } => { + #[cfg(feature = "postgres")] + { + let config = source_to_config(source)?; + let backend = ironclaw::db::postgres::PgBackend::new(&config) + .await + .map_err(|_| { + MigrationError::OpenSource( + "PostgreSQL source connection failed (connection details redacted)" + .to_string(), + ) + })?; + let handles = handles_with_postgres(backend.pool()); + (Arc::new(backend) as Arc, handles) + } + #[cfg(not(feature = "postgres"))] + { + return Err(MigrationError::OpenSource( + "PostgreSQL support is not compiled into this migrator".to_string(), + )); + } + } + }; + + #[cfg(feature = "libsql")] + if let Some(database) = handles.libsql_db.as_ref() { + let connection = database + .connect() + .map_err(|error| MigrationError::OpenSource(error.to_string()))?; + connection + .execute("PRAGMA query_only = ON", ()) + .await + .map_err(|error| MigrationError::OpenSource(error.to_string()))?; + } Ok(Self { db, handles }) } + pub(crate) async fn fingerprint( + &self, + source: &SourceDb, + ) -> Result { + match source { + SourceDb::LibSql { path } => fingerprint_local_snapshot(path).await, + SourceDb::Postgres { .. } => { + let tables = self.table_inventory().await?; + let mut material = String::from("ironclaw-v1-postgres-v2\n"); + for table in tables { + material.push_str(&table.name); + material.push(':'); + material.push_str(&table.count.to_string()); + material.push(':'); + material.push_str(&table.checksum); + material.push('\n'); + } + Ok(SourceFingerprint { + algorithm: "sha256-table-content-v2".to_string(), + value: ironclaw_common::hashing::sha256_hex(material.as_bytes()), + }) + } + } + } + + pub(crate) async fn schema_version(&self) -> Result, MigrationError> { + #[cfg(feature = "libsql")] + if let Some(database) = self.handles.libsql_db.as_ref() { + let connection = database.connect().map_err(source_open_error)?; + connection + .execute("PRAGMA query_only = ON", ()) + .await + .map_err(source_open_error)?; + let mut rows = match connection + .query("SELECT MAX(version) FROM _migrations", ()) + .await + { + Ok(rows) => rows, + Err(error) if is_missing_table_error(&error.to_string()) => return Ok(None), + Err(error) => return Err(source_read_error("schema", error)), + }; + let Some(row) = rows + .next() + .await + .map_err(|error| source_read_error("schema", error))? + else { + return Ok(None); + }; + return match row.get::>(0) { + Ok(version) => Ok(version.map(|value| value.to_string())), + Err(error) => Err(source_read_error("schema", error)), + }; + } + #[cfg(feature = "postgres")] + if let Some(pool) = self.handles.pg_pool.as_ref() { + let client = pool.get().await.map_err(source_open_error)?; + let row = match client + .query_opt( + "SELECT MAX(version)::text FROM refinery_schema_history", + &[], + ) + .await + { + Ok(row) => row, + Err(error) if is_missing_table_error(&error.to_string()) => return Ok(None), + Err(error) => return Err(source_read_error("schema", error)), + }; + return match row { + Some(row) => row + .try_get::<_, Option>(0) + .map_err(|error| source_read_error("schema", error)), + None => Ok(None), + }; + } + Ok(None) + } + + pub(crate) async fn table_inventory(&self) -> Result, MigrationError> { + #[cfg(feature = "libsql")] + if let Some(database) = self.handles.libsql_db.as_ref() { + let connection = database.connect().map_err(source_open_error)?; + connection + .execute("PRAGMA query_only = ON", ()) + .await + .map_err(source_open_error)?; + let mut rows = connection + .query( + "SELECT name FROM sqlite_schema WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + (), + ) + .await + .map_err(|error| source_read_error("inventory", error))?; + let mut names = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| source_read_error("inventory", error))? + { + names.push( + row.get::(0) + .map_err(|error| source_read_error("inventory", error))?, + ); + } + let mut inventory = Vec::with_capacity(names.len()); + for name in names { + let sql = format!("SELECT COUNT(*) FROM {}", quote_identifier(&name)); + let mut rows = connection + .query(&sql, ()) + .await + .map_err(|error| source_read_error(&name, error))?; + let row = rows + .next() + .await + .map_err(|error| source_read_error(&name, error))? + .ok_or_else(|| MigrationError::ReadSource { + domain: name.clone(), + reason: "COUNT(*) returned no row".to_string(), + })?; + let count = row + .get::(0) + .map_err(|error| source_read_error(&name, error))? + .try_into() + .map_err(|_| MigrationError::ReadSource { + domain: name.clone(), + reason: "negative row count".to_string(), + })?; + inventory.push(RawTableInventory { + name: name.clone(), + count, + checksum: ironclaw_common::hashing::sha256_hex( + format!("libsql-table-v1:{name}:{count}").as_bytes(), + ), + }); + } + return Ok(inventory); + } + #[cfg(feature = "postgres")] + if let Some(pool) = self.handles.pg_pool.as_ref() { + use futures::{TryStreamExt as _, pin_mut}; + + let client = pool.get().await.map_err(source_open_error)?; + let rows = client + .query( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type = 'BASE TABLE' ORDER BY table_name", + &[], + ) + .await + .map_err(|error| source_read_error("inventory", error))?; + let mut inventory = Vec::with_capacity(rows.len()); + for row in rows { + let name: String = row + .try_get(0) + .map_err(|error| source_read_error("inventory", error))?; + let sql = format!( + "SELECT to_jsonb(source_row)::text FROM {} AS source_row \ + ORDER BY to_jsonb(source_row)::text", + quote_identifier(&name), + ); + let rows = client + .query_raw( + &sql, + std::iter::empty::<&(dyn tokio_postgres::types::ToSql + Sync)>(), + ) + .await + .map_err(|error| source_read_error(&name, error))?; + pin_mut!(rows); + let mut count = 0_u64; + let mut checksum = PostgresTableChecksum::new(&name); + while let Some(row) = rows + .try_next() + .await + .map_err(|error| source_read_error(&name, error))? + { + let encoded = row + .try_get::<_, String>(0) + .map_err(|error| source_read_error(&name, error))?; + checksum.update(&encoded); + count = count.saturating_add(1); + } + inventory.push(RawTableInventory { + name: name.clone(), + count, + checksum: checksum.finish(), + }); + } + return Ok(inventory); + } + Ok(Vec::new()) + } + /// Discover every distinct v1 `user_id` present in the source. v1 single-user /// installs (especially libSQL) may have no `users` table, so users are /// discovered from the data rows themselves, tolerating any table that does /// not exist. pub(crate) async fn distinct_users(&self) -> Result, MigrationError> { let mut users = std::collections::BTreeSet::new(); - for table in USER_ID_TABLES { - for uid in self.distinct_user_ids_in(table, "user_id").await? { + for (table, column) in USER_ID_TABLES { + for uid in self.distinct_user_ids_in(table, column).await? { if !uid.is_empty() { users.insert(uid); } } } + for uid in self.distinct_user_ids_in("users", "id").await? { + if !uid.is_empty() { + users.insert(uid); + } + } Ok(users.into_iter().collect()) } + pub(crate) async fn heartbeat_user_ids(&self) -> Result, MigrationError> { + self.distinct_user_ids_in("heartbeat_state", "user_id") + .await + } + + #[allow(dead_code, reason = "staged historical-user converter read port")] + pub(crate) async fn users(&self) -> Result, MigrationError> { + self.db.list_users(None).await.or_else(|error| { + if is_missing_table_error(&error.to_string()) { + Ok(Vec::new()) + } else { + Err(source_read_error("users", error)) + } + }) + } + + #[allow(dead_code, reason = "staged typed-settings converter read port")] + pub(crate) async fn settings( + &self, + user_id: &str, + ) -> Result, MigrationError> { + self.db.list_settings(user_id).await.or_else(|error| { + if is_missing_table_error(&error.to_string()) { + Ok(Vec::new()) + } else { + Err(source_read_error("settings", error)) + } + }) + } + + #[allow(dead_code, reason = "staged projects and memory converter read port")] + pub(crate) async fn memory_documents( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, MigrationError> { + self.db + .list_documents(user_id, agent_id) + .await + .map_err(|error| source_read_error("memory_documents", error)) + } + + pub(crate) async fn all_memory_documents( + &self, + user_id: &str, + ) -> Result, MigrationError> { + let mut agent_ids = self.memory_document_agent_ids(user_id).await?; + let mut documents = self.memory_documents(user_id, None).await?; + for agent_id in agent_ids.drain(..) { + documents.extend(self.memory_documents(user_id, Some(agent_id)).await?); + } + Ok(documents) + } + + async fn memory_document_agent_ids( + &self, + user_id: &str, + ) -> Result, MigrationError> { + let read_err = + |error: &dyn std::fmt::Display| source_read_error("memory_documents.agent_id", error); + #[cfg(feature = "libsql")] + if let Some(database) = self.handles.libsql_db.as_ref() { + let connection = database.connect().map_err(|error| read_err(&error))?; + let mut rows = match connection + .query( + "SELECT DISTINCT agent_id FROM memory_documents \ + WHERE user_id = ?1 AND agent_id IS NOT NULL ORDER BY agent_id", + [user_id], + ) + .await + { + Ok(rows) => rows, + Err(error) if is_missing_table_error(&error.to_string()) => return Ok(Vec::new()), + Err(error) => return Err(read_err(&error)), + }; + let mut agent_ids = Vec::new(); + while let Some(row) = rows.next().await.map_err(|error| read_err(&error))? { + let raw = row.get::(0).map_err(|error| read_err(&error))?; + agent_ids.push(raw.parse().map_err(|error| read_err(&error))?); + } + return Ok(agent_ids); + } + #[cfg(feature = "postgres")] + if let Some(pool) = self.handles.pg_pool.as_ref() { + let client = pool.get().await.map_err(|error| read_err(&error))?; + let rows = match client + .query( + "SELECT DISTINCT agent_id FROM memory_documents \ + WHERE user_id = $1 AND agent_id IS NOT NULL ORDER BY agent_id", + &[&user_id], + ) + .await + { + Ok(rows) => rows, + Err(error) if is_missing_table_error(&error.to_string()) => return Ok(Vec::new()), + Err(error) => return Err(read_err(&error)), + }; + return rows + .iter() + .map(|row| row.try_get(0).map_err(|error| read_err(&error))) + .collect(); + } + Ok(Vec::new()) + } + + /// Read every engine-v2 project document regardless of its optional + /// `agent_id`. The v1 `list_documents(user, None)` API means + /// `agent_id IS NULL`, not "all agents", so project discovery needs this + /// narrow raw read to avoid silently omitting agent-scoped metadata. + pub(crate) async fn project_documents(&self) -> Result, MigrationError> { + #[cfg(feature = "libsql")] + if let Some(database) = self.handles.libsql_db.as_ref() { + let connection = database.connect().map_err(source_open_error)?; + connection + .execute("PRAGMA query_only = ON", ()) + .await + .map_err(source_open_error)?; + let mut rows = match connection + .query( + "SELECT user_id, path, content FROM memory_documents + WHERE path LIKE 'projects/%/.project.json' + OR path LIKE '.system/engine/projects/%/project.json' + OR path LIKE 'engine/projects/%/project.json' + ORDER BY path, user_id, COALESCE(agent_id, '')", + (), + ) + .await + { + Ok(rows) => rows, + Err(error) if is_missing_table_error(&error.to_string()) => return Ok(Vec::new()), + Err(error) => return Err(source_read_error("projects", error)), + }; + let mut documents = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| source_read_error("projects", error))? + { + documents.push(ProjectDocument { + user_id: row + .get(0) + .map_err(|error| source_read_error("projects", error))?, + path: row + .get(1) + .map_err(|error| source_read_error("projects", error))?, + content: row + .get(2) + .map_err(|error| source_read_error("projects", error))?, + }); + } + return Ok(documents); + } + + #[cfg(feature = "postgres")] + if let Some(pool) = self.handles.pg_pool.as_ref() { + let client = pool.get().await.map_err(source_open_error)?; + let rows = match client + .query( + "SELECT user_id, path, content FROM memory_documents + WHERE path LIKE 'projects/%/.project.json' + OR path LIKE '.system/engine/projects/%/project.json' + OR path LIKE 'engine/projects/%/project.json' + ORDER BY path, user_id, COALESCE(agent_id::text, '')", + &[], + ) + .await + { + Ok(rows) => rows, + Err(error) if is_missing_table_error(&error.to_string()) => return Ok(Vec::new()), + Err(error) => return Err(source_read_error("projects", error)), + }; + return rows + .into_iter() + .map(|row| { + Ok(ProjectDocument { + user_id: row + .try_get(0) + .map_err(|error| source_read_error("projects", error))?, + path: row + .try_get(1) + .map_err(|error| source_read_error("projects", error))?, + content: row + .try_get(2) + .map_err(|error| source_read_error("projects", error))?, + }) + }) + .collect(); + } + + Ok(Vec::new()) + } + /// `SELECT DISTINCT FROM ` against the raw handle. `column` /// is the user-id column, which is `user_id` on data tables but `id` on the /// `users` table. @@ -79,7 +562,7 @@ impl V1Source { domain: table.to_string(), reason: e.to_string(), }; - let sql = format!("SELECT DISTINCT {column} FROM {table}"); + let sql = format!("SELECT DISTINCT {column} FROM {table} WHERE {column} IS NOT NULL"); #[cfg(feature = "libsql")] if let Some(db) = self.handles.libsql_db.as_ref() { let conn = db.connect().map_err(|e| read_err(&e))?; @@ -125,9 +608,10 @@ pub(crate) fn is_missing_table_error(message: &str) -> bool { || (lower.contains("relation") && lower.contains("does not exist")) } -fn source_to_config(source: &SourceDb) -> DatabaseConfig { +#[cfg(feature = "postgres")] +fn source_to_config(source: &SourceDb) -> Result { match source { - SourceDb::LibSql { path } => DatabaseConfig { + SourceDb::LibSql { path } => Ok(DatabaseConfig { backend: DatabaseBackend::LibSql, // libSQL backend ignores `url`; the resolver uses this sentinel too. url: SecretString::from("unused://libsql"), @@ -136,15 +620,393 @@ fn source_to_config(source: &SourceDb) -> DatabaseConfig { libsql_path: Some(path.clone()), libsql_url: None, libsql_auth_token: None, - }, - SourceDb::Postgres { url } => DatabaseConfig { - backend: DatabaseBackend::Postgres, - url: url.clone(), - pool_size: 4, - ssl_mode: SslMode::default(), - libsql_path: None, - libsql_url: None, - libsql_auth_token: None, - }, + }), + SourceDb::Postgres { url } => { + use secrecy::ExposeSecret as _; + + let parsed = url + .expose_secret() + .parse::() + .map_err(|_| { + MigrationError::OpenSource( + "invalid PostgreSQL source connection URL (details redacted)".to_string(), + ) + })?; + let remote = !is_local_postgres_config(&parsed); + let ssl_mode = match parsed.get_ssl_mode() { + tokio_postgres::config::SslMode::Disable if remote => { + return Err(MigrationError::OpenSource( + "remote PostgreSQL source requires TLS; sslmode=disable is rejected" + .to_string(), + )); + } + tokio_postgres::config::SslMode::Disable => SslMode::Disable, + _ if remote => SslMode::Require, + _ => SslMode::Prefer, + }; + Ok(DatabaseConfig { + backend: DatabaseBackend::Postgres, + url: SecretString::from(postgres_read_only_locator( + url.expose_secret(), + parsed.get_options(), + )), + pool_size: 4, + ssl_mode, + libsql_path: None, + libsql_url: None, + libsql_auth_token: None, + }) + } + } +} + +#[cfg(feature = "postgres")] +fn postgres_read_only_locator(locator: &str, existing_options: Option<&str>) -> String { + let options = match existing_options { + Some(existing) if !existing.is_empty() => { + format!("{existing} -c default_transaction_read_only=on") + } + _ => "-c default_transaction_read_only=on".to_string(), + }; + if locator.starts_with("postgres://") || locator.starts_with("postgresql://") { + let (base, query) = locator.split_once('?').unwrap_or((locator, "")); + let mut parameters = query + .split('&') + .filter(|parameter| !parameter.is_empty() && !parameter.starts_with("options=")) + .map(str::to_string) + .collect::>(); + parameters.push(format!("options={}", percent_encode_query_value(&options))); + format!("{base}?{}", parameters.join("&")) + } else { + let escaped = options.replace('\\', "\\\\").replace('\'', "\\'"); + format!("{locator} options='{escaped}'") + } +} + +#[cfg(feature = "postgres")] +fn percent_encode_query_value(value: &str) -> String { + let mut encoded = String::with_capacity(value.len()); + for byte in value.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + encoded.push(char::from(byte)); + } else { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + encoded.push('%'); + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + } + encoded +} + +#[cfg(feature = "postgres")] +struct PostgresTableChecksum(Sha256); + +#[cfg(feature = "postgres")] +impl PostgresTableChecksum { + fn new(name: &str) -> Self { + let mut hash = Sha256::new(); + hash.update(b"ironclaw-v1-postgres-table-content-v2\0"); + hash.update((name.len() as u64).to_le_bytes()); + hash.update(name.as_bytes()); + Self(hash) + } + + fn update(&mut self, row: &str) { + self.0.update((row.len() as u64).to_le_bytes()); + self.0.update(row.as_bytes()); + } + + fn finish(self) -> String { + format!("{:x}", self.0.finalize()) + } +} + +#[cfg(all(test, feature = "postgres"))] +fn postgres_table_content_checksum(name: &str, rows: &[String]) -> String { + let mut hash = PostgresTableChecksum::new(name); + for row in rows { + hash.update(row); + } + hash.finish() +} + +#[cfg(feature = "postgres")] +fn is_local_postgres_config(config: &tokio_postgres::Config) -> bool { + use tokio_postgres::config::Host; + + let hosts = config.get_hosts(); + let hostaddrs = config.get_hostaddrs(); + if hosts.is_empty() && hostaddrs.is_empty() { + return true; + } + for host in hosts { + match host { + #[cfg(unix)] + Host::Unix(_) => continue, + Host::Tcp(name) => { + if !matches!( + name.as_str(), + "localhost" | "127.0.0.1" | "::1" | "[::1]" | "0.0.0.0" + ) { + return false; + } + } + } + } + for address in hostaddrs { + if !address.is_loopback() && !address.is_unspecified() { + return false; + } + } + true +} + +fn quote_identifier(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} + +#[cfg(all(feature = "libsql", feature = "postgres"))] +fn handles_with_libsql(db: Arc) -> DatabaseHandles { + DatabaseHandles { + libsql_db: Some(db), + pg_pool: None, + } +} + +#[cfg(all(feature = "libsql", not(feature = "postgres")))] +fn handles_with_libsql(db: Arc) -> DatabaseHandles { + DatabaseHandles { + libsql_db: Some(db), + } +} + +#[cfg(all(feature = "postgres", feature = "libsql"))] +fn handles_with_postgres(pool: deadpool_postgres::Pool) -> DatabaseHandles { + DatabaseHandles { + pg_pool: Some(pool), + libsql_db: None, + } +} + +#[cfg(all(feature = "postgres", not(feature = "libsql")))] +fn handles_with_postgres(pool: deadpool_postgres::Pool) -> DatabaseHandles { + DatabaseHandles { + pg_pool: Some(pool), + } +} + +fn source_open_error(error: impl std::fmt::Display) -> MigrationError { + MigrationError::OpenSource(error.to_string()) +} + +fn source_read_error(domain: &str, error: impl std::fmt::Display) -> MigrationError { + MigrationError::ReadSource { + domain: domain.to_string(), + reason: error.to_string(), + } +} + +async fn fingerprint_local_snapshot( + path: &std::path::Path, +) -> Result { + let path = path.to_path_buf(); + tokio::task::spawn_blocking(move || { + let mut hash = Sha256::new(); + hash.update(b"ironclaw-v1-libsql-content-set-v1\0"); + for (role, candidate) in [ + (b"database".as_slice(), path.clone()), + ( + b"wal".as_slice(), + std::path::PathBuf::from(format!("{}-wal", path.display())), + ), + ] { + hash.update(role); + hash.update(b"\0"); + let mut file = match std::fs::File::open(&candidate) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + hash.update(b"missing\0"); + continue; + } + Err(error) => return Err(error.into()), + }; + hash.update(b"present\0"); + let length = file.metadata()?.len(); + hash.update(length.to_le_bytes()); + std::io::copy(&mut file, &mut DigestWriter(&mut hash))?; + } + Ok(SourceFingerprint { + algorithm: "sha256-file-content-set-v1".to_string(), + value: format!("{:x}", hash.finalize()), + }) + }) + .await + .map_err(|error| MigrationError::OpenSource(format!("snapshot fingerprint task: {error}")))? +} + +struct DigestWriter<'a>(&'a mut Sha256); + +impl std::io::Write for DigestWriter<'_> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + self.0.update(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[cfg(test)] +mod tests { + #[cfg(feature = "postgres")] + use secrecy::{ExposeSecret as _, SecretString}; + + use super::fingerprint_local_snapshot; + #[cfg(feature = "postgres")] + use super::postgres_table_content_checksum; + #[cfg(feature = "postgres")] + use super::source_to_config; + #[cfg(any(feature = "libsql", feature = "postgres"))] + use crate::options::SourceDb; + + #[cfg(feature = "postgres")] + #[test] + fn remote_postgres_source_rejects_disabled_tls() { + let source = SourceDb::Postgres { + url: SecretString::from( + "postgresql://user:password@database.example/ironclaw?sslmode=disable", + ), + }; + let error = source_to_config(&source).expect_err("remote plaintext source must fail"); + let rendered = error.to_string(); + assert!(rendered.contains("requires TLS")); + assert!(!rendered.contains("password")); + assert!(!rendered.contains("database.example")); + } + + #[cfg(feature = "postgres")] + #[test] + fn local_postgres_source_can_explicitly_disable_tls() { + let source = SourceDb::Postgres { + url: SecretString::from( + "postgresql://user:password@localhost/ironclaw?sslmode=disable", + ), + }; + let config = source_to_config(&source).expect("local plaintext source"); + assert_eq!(config.ssl_mode, ironclaw::config::SslMode::Disable); + let parsed = config + .url + .expose_secret() + .parse::() + .expect("parse read-only source URL"); + assert_eq!( + parsed.get_options(), + Some("-c default_transaction_read_only=on") + ); + } + + #[cfg(feature = "postgres")] + #[test] + fn postgres_source_overrides_write_capable_session_options() { + let source = SourceDb::Postgres { + url: SecretString::from( + "postgresql://user:password@localhost/ironclaw?sslmode=disable&options=-c%20default_transaction_read_only%3Doff", + ), + }; + let config = source_to_config(&source).expect("read-only source config"); + let parsed = config + .url + .expose_secret() + .parse::() + .expect("parse read-only source URL"); + assert_eq!( + parsed.get_options(), + Some("-c default_transaction_read_only=off -c default_transaction_read_only=on") + ); + } + + #[cfg(feature = "postgres")] + #[test] + fn postgres_table_checksum_is_bound_to_row_contents() { + let before = postgres_table_content_checksum( + "users", + &[r#"{"id": "alice", "status": "active"}"#.to_string()], + ); + let after = postgres_table_content_checksum( + "users", + &[r#"{"id": "alice", "status": "suspended"}"#.to_string()], + ); + + assert_ne!(before, after); + } + + #[tokio::test] + async fn local_snapshot_fingerprint_is_bound_to_file_contents() { + let directory = tempfile::tempdir().expect("tempdir"); + let snapshot = directory.path().join("snapshot.db"); + std::fs::write(&snapshot, b"same-length-a").expect("write snapshot"); + let original_modified = std::fs::metadata(&snapshot) + .expect("snapshot metadata") + .modified() + .expect("modified time"); + + let before = fingerprint_local_snapshot(&snapshot) + .await + .expect("initial fingerprint"); + std::fs::write(&snapshot, b"same-length-b").expect("replace snapshot"); + std::fs::File::options() + .write(true) + .open(&snapshot) + .expect("open snapshot") + .set_times(std::fs::FileTimes::new().set_modified(original_modified)) + .expect("restore modified time"); + let after = fingerprint_local_snapshot(&snapshot) + .await + .expect("replacement fingerprint"); + + assert_eq!(before.algorithm, "sha256-file-content-set-v1"); + assert_ne!(before.value, after.value); + } + + #[cfg(feature = "libsql")] + #[tokio::test] + async fn user_discovery_includes_satellite_store_owners() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join("source.db"); + let database = libsql::Builder::new_local(&path) + .build() + .await + .expect("build source"); + let connection = database.connect().expect("connect source"); + connection + .execute_batch( + "CREATE TABLE secrets (user_id TEXT NOT NULL);\ + CREATE TABLE wasm_tools (user_id TEXT NOT NULL);\ + CREATE TABLE wasm_channels (user_id TEXT NOT NULL);\ + CREATE TABLE channel_identities (owner_id TEXT NOT NULL);\ + INSERT INTO secrets VALUES ('secret-owner');\ + INSERT INTO wasm_tools VALUES ('tool-owner');\ + INSERT INTO wasm_channels VALUES ('channel-owner');\ + INSERT INTO channel_identities VALUES ('identity-owner');", + ) + .await + .expect("seed owners"); + drop(connection); + drop(database); + + let source = super::V1Source::open(&SourceDb::LibSql { path }) + .await + .expect("open source"); + assert_eq!( + source.distinct_users().await.expect("discover users"), + vec![ + "channel-owner", + "identity-owner", + "secret-owner", + "tool-owner" + ] + ); } } diff --git a/crates/ironclaw_reborn_migration/src/target.rs b/crates/ironclaw_reborn_migration/src/target.rs index b3199224984..481ea152503 100644 --- a/crates/ironclaw_reborn_migration/src/target.rs +++ b/crates/ironclaw_reborn_migration/src/target.rs @@ -5,8 +5,9 @@ //! the converters. Threads / secrets / identity force a concrete filesystem //! type, so they are constructed inside the backend match arm where `F` is //! known, then stored as `#[async_trait]` trait objects so the converters stay -//! backend-agnostic. All state is written under one (tenant, agent) scope from -//! [`MigrationOptions`]; each v1 `user_id` becomes the per-record Reborn `UserId`. +//! backend-agnostic. Target identity comes from [`MigrationOptions`]; each v1 +//! `user_id` becomes the per-record Reborn `UserId`, and memory retains its +//! optional v1 agent scope. use std::sync::Arc; @@ -15,6 +16,7 @@ use ironclaw_filesystem::{RootFilesystem, ScopedFilesystem}; use ironclaw_host_api::{AgentId, ProjectId, TenantId, UserId}; use ironclaw_memory::MemoryService; use ironclaw_memory_native::NativeMemoryService; +use ironclaw_projects::{FilesystemProjectRepository, ProjectRepository}; use ironclaw_reborn_identity::{ FilesystemRebornIdentityStore, RebornIdentityResolver, RebornUserDirectory, }; @@ -27,6 +29,662 @@ use crate::error::MigrationError; use crate::mounts; use crate::options::{MigrationOptions, TargetStore}; +#[path = "target_ids.rs"] +pub(crate) mod ids; + +#[derive(Debug, Default, PartialEq, Eq)] +pub(crate) struct TargetReadback { + pub(crate) users: u64, + pub(crate) threads: u64, + pub(crate) messages: u64, + pub(crate) projects: u64, + pub(crate) triggers: u64, + pub(crate) memory_documents: u64, + pub(crate) secrets: u64, + pub(crate) identity_records: u64, +} + +/// Inspect whether a target contains live Reborn state without applying schema +/// migrations or creating any target object. +pub(crate) async fn target_is_empty(target: &TargetStore) -> Result { + match target { + TargetStore::LibSql { path } => Ok(!path.exists()), + #[cfg(feature = "postgres")] + TargetStore::Postgres { url } => { + let pool = open_postgres_pool(url)?; + let client = pool.get().await.map_err(|error| { + MigrationError::OpenTarget(format!( + "PostgreSQL target emptiness probe failed (details redacted): {}", + error + )) + })?; + let relations = client + .query( + "SELECT format('%I.%I', namespace.nspname, relation.relname) + FROM pg_catalog.pg_class relation + JOIN pg_catalog.pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE relation.relkind IN ('r', 'p') + AND namespace.nspname NOT LIKE 'pg\\_%' ESCAPE '\\' + AND namespace.nspname <> 'information_schema'", + &[], + ) + .await + .map_err(|error| { + MigrationError::OpenTarget(format!( + "PostgreSQL target schema inventory failed: {error}" + )) + })?; + for relation in relations { + let table: String = relation + .try_get(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let sql = format!("SELECT EXISTS (SELECT 1 FROM {table} LIMIT 1)"); + let populated: bool = client + .query_one(&sql, &[]) + .await + .map_err(|error| { + MigrationError::OpenTarget(format!( + "PostgreSQL target data probe failed for {table}: {error}" + )) + })? + .try_get(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + if populated { + return Ok(false); + } + } + Ok(true) + } + #[cfg(not(feature = "postgres"))] + TargetStore::Postgres { .. } => Err(MigrationError::OpenTarget( + "binary built without the postgres feature".to_string(), + )), + } +} + +pub(crate) async fn write_shared_migration_state( + target: &TargetStore, + manifest: &crate::manifest::MigrationManifest, +) -> Result<(), MigrationError> { + match target { + #[cfg(feature = "libsql")] + TargetStore::LibSql { path } => { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let database = libsql::Builder::new_local(path) + .build() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let connection = database + .connect() + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + connection + .execute_batch( + "PRAGMA busy_timeout = 5000; + CREATE TABLE IF NOT EXISTS reborn_migration_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + schema_version TEXT NOT NULL, + migration_protocol_version INTEGER NOT NULL, + release_version TEXT NOT NULL, + run_id TEXT NOT NULL, + status TEXT NOT NULL, + profile TEXT NOT NULL, + target_backend TEXT NOT NULL, + target_locator_fingerprint TEXT NOT NULL, + tenant_id TEXT NOT NULL, + agent_id TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + BEGIN IMMEDIATE;", + ) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let result = write_libsql_shared_state(&connection, manifest).await; + match result { + Ok(()) => connection + .execute("COMMIT", ()) + .await + .map(|_| ()) + .map_err(|error| MigrationError::OpenTarget(error.to_string())), + Err(error) => { + let _ = connection.execute("ROLLBACK", ()).await; + Err(error) + } + } + } + #[cfg(not(feature = "libsql"))] + TargetStore::LibSql { .. } => Err(MigrationError::OpenTarget( + "binary built without the libsql feature".to_string(), + )), + #[cfg(feature = "postgres")] + TargetStore::Postgres { url } => { + let pool = open_postgres_pool(url)?; + let mut client = pool.get().await.map_err(postgres_identity_error)?; + client + .batch_execute( + "CREATE TABLE IF NOT EXISTS reborn_migration_state ( + singleton BOOLEAN PRIMARY KEY DEFAULT TRUE CHECK (singleton), + schema_version TEXT NOT NULL, + migration_protocol_version BIGINT NOT NULL, + release_version TEXT NOT NULL, + run_id TEXT NOT NULL, + status TEXT NOT NULL, + profile TEXT NOT NULL, + target_backend TEXT, + target_locator_fingerprint TEXT, + tenant_id TEXT, + agent_id TEXT, + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + ALTER TABLE reborn_migration_state ADD COLUMN IF NOT EXISTS target_backend TEXT; + ALTER TABLE reborn_migration_state ADD COLUMN IF NOT EXISTS target_locator_fingerprint TEXT; + ALTER TABLE reborn_migration_state ADD COLUMN IF NOT EXISTS tenant_id TEXT; + ALTER TABLE reborn_migration_state ADD COLUMN IF NOT EXISTS agent_id TEXT;", + ) + .await + .map_err(postgres_identity_error)?; + let transaction = client + .transaction() + .await + .map_err(postgres_identity_error)?; + let schema_version = "ironclaw.reborn.migration-state/v1"; + let protocol_version = i64::from(crate::manifest::MIGRATION_PROTOCOL_VERSION); + let run_id = manifest.run_id.to_string(); + let status = migration_status_label(manifest.status); + let existing = transaction + .query_opt( + "SELECT schema_version, migration_protocol_version, release_version, + run_id, status, profile, target_backend, + target_locator_fingerprint, tenant_id, agent_id + FROM reborn_migration_state + WHERE singleton = TRUE + FOR UPDATE", + &[], + ) + .await + .map_err(postgres_identity_error)?; + if let Some(existing) = existing { + let existing_schema: String = + existing.try_get(0).map_err(postgres_identity_error)?; + let existing_protocol: i64 = + existing.try_get(1).map_err(postgres_identity_error)?; + let existing_release: String = + existing.try_get(2).map_err(postgres_identity_error)?; + let existing_run_id: String = + existing.try_get(3).map_err(postgres_identity_error)?; + let existing_status: String = + existing.try_get(4).map_err(postgres_identity_error)?; + let existing_profile: String = + existing.try_get(5).map_err(postgres_identity_error)?; + let existing_backend: Option = + existing.try_get(6).map_err(postgres_identity_error)?; + let existing_fingerprint: Option = + existing.try_get(7).map_err(postgres_identity_error)?; + let existing_tenant: Option = + existing.try_get(8).map_err(postgres_identity_error)?; + let existing_agent: Option = + existing.try_get(9).map_err(postgres_identity_error)?; + if existing_schema != schema_version + || existing_protocol != protocol_version + || existing_release != manifest.release_version + || existing_run_id != run_id + || existing_profile != manifest.scope.profile + || existing_backend.as_deref() + != Some(store_backend_label(manifest.target.backend)) + || existing_fingerprint.as_deref() + != Some(manifest.target.locator_fingerprint.as_str()) + || existing_tenant.as_deref() != Some(manifest.scope.tenant_id.as_str()) + || existing_agent.as_deref() != Some(manifest.scope.agent_id.as_str()) + { + return Err(MigrationError::OpenTarget( + "PostgreSQL target is claimed by a different migration run or protocol" + .to_string(), + )); + } + if !shared_state_transition_allowed(&existing_status, manifest.status) { + return Err(MigrationError::OpenTarget(format!( + "invalid shared migration state transition {existing_status} -> {status}" + ))); + } + transaction + .execute( + "UPDATE reborn_migration_state + SET status = $1, updated_at = NOW() + WHERE singleton = TRUE AND run_id = $2", + &[&status, &run_id], + ) + .await + .map_err(postgres_identity_error)?; + } else { + if manifest.status != crate::manifest::MigrationStatus::Applying { + return Err(MigrationError::OpenTarget( + "PostgreSQL target has no active migration claim".to_string(), + )); + } + let target_backend = store_backend_label(manifest.target.backend); + let parameters: [&(dyn tokio_postgres::types::ToSql + Sync); 10] = [ + &schema_version, + &protocol_version, + &manifest.release_version, + &run_id, + &status, + &manifest.scope.profile, + &target_backend, + &manifest.target.locator_fingerprint, + &manifest.scope.tenant_id, + &manifest.scope.agent_id, + ]; + transaction + .execute( + "INSERT INTO reborn_migration_state ( + singleton, schema_version, migration_protocol_version, + release_version, run_id, status, profile, target_backend, + target_locator_fingerprint, tenant_id, agent_id, updated_at + ) VALUES (TRUE, $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, NOW())", + ¶meters, + ) + .await + .map_err(postgres_identity_error)?; + } + transaction + .commit() + .await + .map_err(postgres_identity_error)?; + Ok(()) + } + #[cfg(not(feature = "postgres"))] + TargetStore::Postgres { .. } => Err(MigrationError::OpenTarget( + "binary built without the postgres feature".to_string(), + )), + } +} + +#[cfg(feature = "libsql")] +async fn write_libsql_shared_state( + connection: &libsql::Connection, + manifest: &crate::manifest::MigrationManifest, +) -> Result<(), MigrationError> { + let mut rows = connection + .query( + "SELECT schema_version, migration_protocol_version, release_version, + run_id, status, profile, target_backend, + target_locator_fingerprint, tenant_id, agent_id + FROM reborn_migration_state WHERE singleton = 1", + (), + ) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing = rows + .next() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let status = migration_status_label(manifest.status); + let run_id = manifest.run_id.to_string(); + if let Some(existing) = existing { + let existing_schema = existing + .get::(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_protocol = existing + .get::(1) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_release = existing + .get::(2) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_run_id = existing + .get::(3) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_status = existing + .get::(4) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_profile = existing + .get::(5) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_backend = existing + .get::(6) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_fingerprint = existing + .get::(7) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_tenant = existing + .get::(8) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let existing_agent = existing + .get::(9) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + if existing_schema != "ironclaw.reborn.migration-state/v1" + || existing_protocol != i64::from(crate::manifest::MIGRATION_PROTOCOL_VERSION) + || existing_release != manifest.release_version + || existing_run_id != run_id + || existing_profile != manifest.scope.profile + || existing_backend != store_backend_label(manifest.target.backend) + || existing_fingerprint != manifest.target.locator_fingerprint + || existing_tenant != manifest.scope.tenant_id + || existing_agent != manifest.scope.agent_id + { + return Err(MigrationError::OpenTarget( + "libSQL target is claimed by a different migration run or protocol".to_string(), + )); + } + if !shared_state_transition_allowed(&existing_status, manifest.status) { + return Err(MigrationError::OpenTarget(format!( + "invalid shared migration state transition {existing_status} -> {status}" + ))); + } + connection + .execute( + "UPDATE reborn_migration_state + SET status = ?1, updated_at = ?2 + WHERE singleton = 1 AND run_id = ?3", + libsql::params![status, chrono::Utc::now().to_rfc3339(), run_id], + ) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + } else { + if manifest.status != crate::manifest::MigrationStatus::Applying { + return Err(MigrationError::OpenTarget( + "libSQL target has no active migration claim".to_string(), + )); + } + connection + .execute( + "INSERT INTO reborn_migration_state ( + singleton, schema_version, migration_protocol_version, + release_version, run_id, status, profile, target_backend, + target_locator_fingerprint, tenant_id, agent_id, updated_at + ) VALUES (1, ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + libsql::params![ + "ironclaw.reborn.migration-state/v1", + i64::from(crate::manifest::MIGRATION_PROTOCOL_VERSION), + manifest.release_version.clone(), + run_id, + status, + manifest.scope.profile.clone(), + store_backend_label(manifest.target.backend), + manifest.target.locator_fingerprint.clone(), + manifest.scope.tenant_id.clone(), + manifest.scope.agent_id.clone(), + chrono::Utc::now().to_rfc3339(), + ], + ) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + } + Ok(()) +} + +const fn store_backend_label(backend: crate::manifest::StoreBackend) -> &'static str { + match backend { + crate::manifest::StoreBackend::Libsql => "libsql", + crate::manifest::StoreBackend::Postgres => "postgres", + } +} + +const fn migration_status_label(status: crate::manifest::MigrationStatus) -> &'static str { + use crate::manifest::MigrationStatus; + match status { + MigrationStatus::Planned => "planned", + MigrationStatus::Applying => "applying", + MigrationStatus::Failed => "failed", + MigrationStatus::Applied => "applied", + MigrationStatus::Verifying => "verifying", + MigrationStatus::Verified => "verified", + } +} + +fn shared_state_transition_allowed(current: &str, next: crate::manifest::MigrationStatus) -> bool { + use crate::manifest::MigrationStatus; + + current == migration_status_label(next) + || matches!( + (current, next), + ( + "applying", + MigrationStatus::Failed | MigrationStatus::Applied + ) | ("failed", MigrationStatus::Applying) + | ( + "applied", + MigrationStatus::Applying | MigrationStatus::Verifying + ) + | ( + "verifying", + MigrationStatus::Failed | MigrationStatus::Verified + ) + | ("verified", MigrationStatus::Verifying) + ) +} + +/// Read migrated state through the same durable tables used by production, +/// without running migrations or starting workers/ingress. +pub(crate) async fn readback( + target: &TargetStore, + tenant_id: &TenantId, +) -> Result { + let tenant = tenant_id.as_str(); + let tenant_pattern = escape_like_component(tenant); + let thread_pattern = format!("/tenants/{tenant_pattern}/users/%/threads/%/thread.json"); + let message_pattern = format!("/tenants/{tenant_pattern}/users/%/threads/%/messages/%.json"); + let append_pattern = format!("/tenants/{tenant_pattern}/users/%/threads/%/message_appends"); + let memory_pattern = format!("/memory/tenants/{tenant_pattern}/%"); + let secret_pattern = format!("/tenants/{tenant_pattern}/users/%/secrets/%/secrets/%.json"); + let identity_pattern = format!("/tenants/{tenant_pattern}/shared/reborn-identity/external/%"); + let user_pattern = format!("/tenants/{tenant_pattern}/shared/reborn-identity/users/%.json"); + let project_pattern = + format!("/tenants/{tenant_pattern}/shared/reborn-projects/%/records/%.json"); + + match target { + #[cfg(feature = "libsql")] + TargetStore::LibSql { path } => { + if !path.is_file() { + return Err(MigrationError::OpenTarget( + "Reborn target does not exist for verification".to_string(), + )); + } + let database = libsql::Builder::new_local(path) + .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY) + .build() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let connection = database + .connect() + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + connection + .execute("PRAGMA query_only = ON", ()) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + Ok(TargetReadback { + users: count_libsql(&connection, "root_filesystem_entries", &user_pattern).await?, + threads: count_libsql(&connection, "root_filesystem_entries", &thread_pattern) + .await?, + messages: count_libsql(&connection, "root_filesystem_entries", &message_pattern) + .await? + + count_libsql(&connection, "root_filesystem_events", &append_pattern).await?, + projects: count_libsql(&connection, "root_filesystem_entries", &project_pattern) + .await?, + triggers: count_libsql_tenant(&connection, "trigger_records", tenant).await?, + memory_documents: count_libsql_files(&connection, &memory_pattern).await?, + secrets: count_libsql(&connection, "root_filesystem_entries", &secret_pattern) + .await?, + identity_records: count_libsql( + &connection, + "root_filesystem_entries", + &identity_pattern, + ) + .await?, + }) + } + #[cfg(not(feature = "libsql"))] + TargetStore::LibSql { .. } => Err(MigrationError::OpenTarget( + "binary built without the libsql feature".to_string(), + )), + #[cfg(feature = "postgres")] + TargetStore::Postgres { url } => { + let pool = open_postgres_pool(url)?; + let client = pool + .get() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + Ok(TargetReadback { + users: count_postgres(&client, "root_filesystem_entries", &user_pattern).await?, + threads: count_postgres(&client, "root_filesystem_entries", &thread_pattern) + .await?, + messages: count_postgres(&client, "root_filesystem_entries", &message_pattern) + .await? + + count_postgres(&client, "root_filesystem_events", &append_pattern).await?, + projects: count_postgres(&client, "root_filesystem_entries", &project_pattern) + .await?, + triggers: count_postgres_tenant(&client, "trigger_records", tenant).await?, + memory_documents: count_postgres_files(&client, &memory_pattern).await?, + secrets: count_postgres(&client, "root_filesystem_entries", &secret_pattern) + .await?, + identity_records: count_postgres( + &client, + "root_filesystem_entries", + &identity_pattern, + ) + .await?, + }) + } + #[cfg(not(feature = "postgres"))] + TargetStore::Postgres { .. } => Err(MigrationError::OpenTarget( + "binary built without the postgres feature".to_string(), + )), + } +} + +fn escape_like_component(value: &str) -> String { + let mut escaped = String::with_capacity(value.len()); + for character in value.chars() { + if matches!(character, '^' | '%' | '_') { + escaped.push('^'); + } + escaped.push(character); + } + escaped +} + +#[cfg(feature = "libsql")] +async fn count_libsql( + connection: &libsql::Connection, + table: &str, + pattern: &str, +) -> Result { + let sql = format!("SELECT COUNT(*) FROM {table} WHERE path LIKE ?1 ESCAPE '^'"); + let mut rows = connection + .query(&sql, [pattern]) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let count = rows + .next() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))? + .ok_or_else(|| MigrationError::OpenTarget("verification count returned no row".into()))? + .get::(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + u64::try_from(count).map_err(|_| MigrationError::OpenTarget("negative target count".into())) +} + +#[cfg(feature = "libsql")] +async fn count_libsql_tenant( + connection: &libsql::Connection, + table: &str, + tenant: &str, +) -> Result { + let sql = format!("SELECT COUNT(*) FROM {table} WHERE tenant_id = ?1"); + let mut rows = connection + .query(&sql, [tenant]) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let count = rows + .next() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))? + .ok_or_else(|| MigrationError::OpenTarget("verification count returned no row".into()))? + .get::(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + u64::try_from(count).map_err(|_| MigrationError::OpenTarget("negative target count".into())) +} + +#[cfg(feature = "libsql")] +async fn count_libsql_files( + connection: &libsql::Connection, + pattern: &str, +) -> Result { + let mut rows = connection + .query( + "SELECT COUNT(*) FROM root_filesystem_entries + WHERE path LIKE ?1 ESCAPE '^' AND is_dir = 0 + AND path NOT LIKE '%.meta' + AND path NOT LIKE '%.versions/%' + AND path NOT LIKE '%.chunks/%'", + [pattern], + ) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + let count = rows + .next() + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))? + .ok_or_else(|| MigrationError::OpenTarget("verification count returned no row".into()))? + .get::(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + u64::try_from(count).map_err(|_| MigrationError::OpenTarget("negative target count".into())) +} + +#[cfg(feature = "postgres")] +async fn count_postgres( + client: &deadpool_postgres::Client, + table: &str, + pattern: &str, +) -> Result { + let sql = format!("SELECT COUNT(*)::bigint FROM {table} WHERE path LIKE $1 ESCAPE '^'"); + let count: i64 = client + .query_one(&sql, &[&pattern]) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))? + .try_get(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + u64::try_from(count).map_err(|_| MigrationError::OpenTarget("negative target count".into())) +} + +#[cfg(feature = "postgres")] +async fn count_postgres_tenant( + client: &deadpool_postgres::Client, + table: &str, + tenant: &str, +) -> Result { + let sql = format!("SELECT COUNT(*)::bigint FROM {table} WHERE tenant_id = $1"); + let count: i64 = client + .query_one(&sql, &[&tenant]) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))? + .try_get(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + u64::try_from(count).map_err(|_| MigrationError::OpenTarget("negative target count".into())) +} + +#[cfg(feature = "postgres")] +async fn count_postgres_files( + client: &deadpool_postgres::Client, + pattern: &str, +) -> Result { + let count: i64 = client + .query_one( + "SELECT COUNT(*)::bigint FROM root_filesystem_entries + WHERE path LIKE $1 ESCAPE '^' AND is_dir = FALSE + AND path NOT LIKE '%.meta' + AND path NOT LIKE '%.versions/%' + AND path NOT LIKE '%.chunks/%'", + &[&pattern], + ) + .await + .map_err(|error| MigrationError::OpenTarget(error.to_string()))? + .try_get(0) + .map_err(|error| MigrationError::OpenTarget(error.to_string()))?; + u64::try_from(count).map_err(|_| MigrationError::OpenTarget("negative target count".into())) +} + /// The concrete Reborn backend the migration writes into. Both the KV substrate /// and the triggers DB share this one handle. pub(crate) enum Backend { @@ -55,9 +713,9 @@ pub(crate) struct RebornTarget { pub(crate) agent_id: AgentId, pub(crate) thread_service: Arc, pub(crate) memory_service: Arc, + pub(crate) project_repo: Arc, pub(crate) trigger_repo: Arc, - pub(crate) extension_store: Arc, - /// Present only when a secrets master key was supplied. + /// Present when composition resolved the production target key. pub(crate) secret_store: Option>, } @@ -93,10 +751,12 @@ impl ExtensionOwnershipTarget { })?; let user_directory = match &backend { #[cfg(feature = "libsql")] - Backend::LibSql { root, .. } => build_user_directory(root.clone(), tenant_id.clone())?, + Backend::LibSql { root, .. } => { + build_extension_ownership_user_directory(root.clone(), tenant_id.clone())? + } #[cfg(feature = "postgres")] Backend::Postgres { root, .. } => { - build_user_directory(root.clone(), tenant_id.clone())? + build_extension_ownership_user_directory(root.clone(), tenant_id.clone())? } }; @@ -110,43 +770,46 @@ impl ExtensionOwnershipTarget { impl RebornTarget { pub(crate) async fn open(options: &MigrationOptions) -> Result { - let crypto = match &options.secret_master_key { + // Open the target only after apply preconditions (the lifecycle caller + // enforces that ordering), then resolve the exact local-runtime key. + // Local key generation writes beside the DB, so it must never happen in + // plan mode. PostgreSQL keys arrive already resolved by composition. + let backend = open_backend(&options.target).await?; + let target_master_key = match (&options.secret_master_key, &options.target) { + (Some(key), _) => Some(key.clone()), + #[cfg(feature = "libsql")] + (None, TargetStore::LibSql { path }) => Some( + ironclaw_reborn_composition::resolve_local_migration_target_key(path).map_err( + |error| MigrationError::OpenTarget(format!("secrets master key: {error}")), + )?, + ), + _ => None, + }; + let crypto = match &target_master_key { Some(key) => Some(Arc::new(build_crypto(key)?)), None => None, }; - let backend = open_backend(&options.target).await?; - let (thread_service, memory_service, secret_store) = match &backend { + let (thread_service, memory_service, project_repo, secret_store) = match &backend { #[cfg(feature = "libsql")] - Backend::LibSql { root, .. } => build_kv_services(root.clone(), crypto.clone()), + Backend::LibSql { root, .. } => { + build_kv_services(root.clone(), crypto.clone(), options.agent_id.clone())? + } #[cfg(feature = "postgres")] - Backend::Postgres { root, .. } => build_kv_services(root.clone(), crypto.clone()), + Backend::Postgres { root, .. } => { + build_kv_services(root.clone(), crypto.clone(), options.agent_id.clone())? + } }; let trigger_repo = build_trigger_repo(&backend).await?; - // Extension installation store is owned by composition; the migration - // seam builds it over our root filesystem at the default state path. - let root_dyn: Arc = match &backend { - #[cfg(feature = "libsql")] - Backend::LibSql { root, .. } => root.clone(), - #[cfg(feature = "postgres")] - Backend::Postgres { root, .. } => root.clone(), - }; - let extension_store = - ironclaw_reborn_composition::extension_installation_store_for_migration(root_dyn, None) - .await - .map_err(|e| { - MigrationError::OpenTarget(format!("extension installation store: {e}")) - })?; - Ok(Self { backend, tenant_id: options.tenant_id.clone(), agent_id: options.agent_id.clone(), thread_service, memory_service, + project_repo, trigger_repo, - extension_store, secret_store, }) } @@ -169,6 +832,66 @@ impl RebornTarget { } } } + + /// Build the canonical user-directory port over the production identity + /// mount. The supplied caller id only fills the store's per-user scope; + /// canonical user records themselves live in the tenant-shared directory. + pub(crate) fn user_directory(&self, caller_id: UserId) -> Arc { + let tenant = self.tenant_id.clone(); + let agent = self.agent_id.clone(); + match &self.backend { + #[cfg(feature = "libsql")] + Backend::LibSql { root, .. } => { + build_user_directory(root.clone(), tenant, caller_id, agent) + } + #[cfg(feature = "postgres")] + Backend::Postgres { root, .. } => { + build_user_directory(root.clone(), tenant, caller_id, agent) + } + } + } + + /// Create a trigger only when the deterministic target slot is empty. + /// Replays accept an exact match; a different record at the same stable id + /// is a migration conflict and must never be overwritten by `upsert`. + pub(crate) async fn compare_and_upsert_trigger( + &self, + source_id: &str, + record: ironclaw_triggers::TriggerRecord, + ) -> Result<(), MigrationError> { + if self + .trigger_repo + .insert_trigger_if_absent(record.clone()) + .await + .map_err(|error| MigrationError::WriteTarget { + domain: format!("trigger for {source_id}"), + reason: format!("claim deterministic target slot: {error}"), + })? + { + return Ok(()); + } + match self + .trigger_repo + .get_trigger(record.tenant_id.clone(), record.trigger_id) + .await + .map_err(|error| MigrationError::WriteTarget { + domain: format!("trigger for {source_id}"), + reason: format!("reconcile deterministic target slot: {error}"), + })? { + Some(existing) if existing == record => Ok(()), + Some(_) => Err(MigrationError::WriteTarget { + domain: format!("trigger for {source_id}"), + reason: format!( + "deterministic trigger id {} already contains divergent state; refusing to overwrite", + record.trigger_id + ), + }), + None => Err(MigrationError::WriteTarget { + domain: format!("trigger for {source_id}"), + reason: "trigger vanished while reconciling an atomic insert conflict".to_string(), + }), + } + } } fn build_crypto(key: &SecretString) -> Result { @@ -180,37 +903,46 @@ fn build_crypto(key: &SecretString) -> Result { type KvServices = ( Arc, Arc, + Arc, Option>, ); /// Build the filesystem-backed KV services over one concrete backend, returning /// them as trait objects. -fn build_kv_services(root: Arc, crypto: Option>) -> KvServices +fn build_kv_services( + root: Arc, + crypto: Option>, + agent_id: AgentId, +) -> Result where F: RootFilesystem + 'static, { - let threads_scoped = Arc::new(ScopedFilesystem::new( + let scoped = Arc::new(ScopedFilesystem::new( root.clone(), - mounts::threads_mount_view, + mounts::production_mount_view, )); let thread_service: Arc = - Arc::new(FilesystemSessionThreadService::new(threads_scoped)); + Arc::new(FilesystemSessionThreadService::new(scoped.clone())); + + let migration_user = UserId::new("reborn-migration").map_err(|error| { + MigrationError::OpenTarget(format!("migration project repository caller: {error}")) + })?; + let project_repo: Arc = Arc::new(FilesystemProjectRepository::new( + scoped.clone(), + migration_user, + agent_id, + )); let root_dyn: Arc = root.clone(); let memory_service: Arc = Arc::new(NativeMemoryService::from_filesystem(root_dyn, None)); let secret_store: Option> = crypto.map(|crypto| { - let secrets_scoped = Arc::new(ScopedFilesystem::new( - root.clone(), - mounts::secrets_mount_view, - )); - let store: Arc = - Arc::new(FilesystemSecretStore::new(secrets_scoped, crypto)); + let store: Arc = Arc::new(FilesystemSecretStore::new(scoped, crypto)); store }); - (thread_service, memory_service, secret_store) + Ok((thread_service, memory_service, project_repo, secret_store)) } #[allow(dead_code)] // wired for the identity row-by-row follow-up @@ -223,7 +955,7 @@ fn build_identity_store( where F: RootFilesystem + 'static, { - let scoped = Arc::new(ScopedFilesystem::new(root, mounts::identity_mount_view)); + let scoped = Arc::new(ScopedFilesystem::new(root, mounts::production_mount_view)); let project_id: Option = None; let store: Arc = Arc::new(FilesystemRebornIdentityStore::new( scoped, tenant_id, user_id, agent_id, project_id, @@ -234,6 +966,21 @@ where fn build_user_directory( root: Arc, tenant_id: TenantId, + caller_id: UserId, + agent_id: AgentId, +) -> Arc +where + F: RootFilesystem + 'static, +{ + let scoped = Arc::new(ScopedFilesystem::new(root, mounts::production_mount_view)); + Arc::new(FilesystemRebornIdentityStore::new( + scoped, tenant_id, caller_id, agent_id, None, + )) +} + +fn build_extension_ownership_user_directory( + root: Arc, + tenant_id: TenantId, ) -> Result, MigrationError> where F: RootFilesystem + 'static, @@ -361,6 +1108,79 @@ fn open_postgres_pool( .map_err(|e| MigrationError::OpenTarget(e.to_string())) } +#[cfg(feature = "postgres")] +pub(crate) async fn postgres_stores_are_distinct( + source_pool: &deadpool_postgres::Pool, + target_url: &secrecy::SecretString, +) -> Result { + const LOCK_NAMESPACE: i64 = 0x4943_4d47; + + let source = source_pool.get().await.map_err(postgres_identity_error)?; + let target_pool = open_postgres_pool(target_url)?; + let target = target_pool.get().await.map_err(postgres_identity_error)?; + let source_database_oid: i64 = source + .query_one( + "SELECT oid::bigint FROM pg_database WHERE datname = current_database()", + &[], + ) + .await + .map_err(postgres_identity_error)? + .get(0); + let target_database_oid: i64 = target + .query_one( + "SELECT oid::bigint FROM pg_database WHERE datname = current_database()", + &[], + ) + .await + .map_err(postgres_identity_error)? + .get(0); + let source_lock_key = (LOCK_NAMESPACE << 32) | (source_database_oid & 0xffff_ffff); + let target_lock_key = (LOCK_NAMESPACE << 32) | (target_database_oid & 0xffff_ffff); + let source_locked: bool = source + .query_one("SELECT pg_try_advisory_lock($1)", &[&source_lock_key]) + .await + .map_err(postgres_identity_error)? + .get(0); + if !source_locked { + return Err(MigrationError::OpenSource( + "could not establish PostgreSQL source identity lock".to_string(), + )); + } + + let target_locked = target + .query_one("SELECT pg_try_advisory_lock($1)", &[&target_lock_key]) + .await + .map(|row| row.get::<_, bool>(0)); + let distinct = match target_locked { + Ok(locked) => { + if locked { + let _ = target + .query_one("SELECT pg_advisory_unlock($1)", &[&target_lock_key]) + .await; + } + locked + } + Err(error) => { + let _ = source + .query_one("SELECT pg_advisory_unlock($1)", &[&source_lock_key]) + .await; + return Err(postgres_identity_error(error)); + } + }; + source + .query_one("SELECT pg_advisory_unlock($1)", &[&source_lock_key]) + .await + .map_err(postgres_identity_error)?; + Ok(distinct) +} + +#[cfg(feature = "postgres")] +fn postgres_identity_error(error: impl std::fmt::Display) -> MigrationError { + MigrationError::OpenTarget(format!( + "PostgreSQL store identity probe failed (connection details redacted): {error}" + )) +} + /// True when the parsed Postgres `Config` targets only loopback hosts / Unix /// sockets. Anything else is treated as remote and must use TLS. Mirrors the /// event-store's `is_local_postgres_config`. @@ -395,3 +1215,153 @@ fn is_local_postgres_config(config: &tokio_postgres::Config) -> bool { } true } + +#[cfg(test)] +mod tests { + #[cfg(feature = "libsql")] + use std::collections::BTreeMap; + + use super::escape_like_component; + #[cfg(feature = "postgres")] + use super::shared_state_transition_allowed; + #[cfg(feature = "libsql")] + use super::write_shared_migration_state; + #[cfg(feature = "libsql")] + use crate::TargetStore; + use crate::manifest::MigrationStatus; + #[cfg(feature = "libsql")] + use crate::manifest::{ + MANIFEST_SCHEMA_VERSION, MIGRATION_PROTOCOL_VERSION, MigrationManifest, + RedactedStoreDescriptor, ResolvedScope, SourceFingerprint, StoreBackend, + }; + #[cfg(feature = "libsql")] + use chrono::Utc; + + #[test] + fn verification_like_component_escapes_wildcards_and_escape_character() { + assert_eq!(escape_like_component("tenant%_caret^"), "tenant^%^_caret^^"); + } + + #[cfg(feature = "libsql")] + fn applying_manifest(path: &std::path::Path) -> MigrationManifest { + let now = Utc::now(); + let mut manifest = MigrationManifest { + manifest_schema_version: MANIFEST_SCHEMA_VERSION, + migration_protocol_version: MIGRATION_PROTOCOL_VERSION, + tool_version: "test".to_string(), + release_version: "test".to_string(), + run_id: uuid::Uuid::new_v4(), + status: MigrationStatus::Applying, + source: RedactedStoreDescriptor { + backend: StoreBackend::Libsql, + locator_fingerprint: "source".to_string(), + exists: Some(true), + }, + target: RedactedStoreDescriptor { + backend: StoreBackend::Libsql, + locator_fingerprint: + ironclaw_reborn_composition::migration_libsql_locator_fingerprint(path), + exists: Some(false), + }, + source_schema_version: None, + source_fingerprint: SourceFingerprint { + algorithm: "test".to_string(), + value: "source".to_string(), + }, + source_inventory_checksum: "inventory".to_string(), + plan_hash: String::new(), + scope: ResolvedScope { + profile: "local-dev".to_string(), + tenant_id: "tenant".to_string(), + agent_id: "agent".to_string(), + source_home_fingerprint: None, + user_mapping: BTreeMap::new(), + target_empty: Some(true), + }, + inventory: Vec::new(), + domains: BTreeMap::new(), + operator_acknowledgements: Vec::new(), + created_at: now, + updated_at: now, + }; + manifest.seal().expect("seal manifest"); + manifest + } + + #[cfg(feature = "libsql")] + #[tokio::test] + async fn libsql_shared_state_claim_rejects_a_different_run() { + let directory = tempfile::tempdir().expect("tempdir"); + let path = directory.path().join("reborn.db"); + let target = TargetStore::LibSql { path: path.clone() }; + let first = applying_manifest(&path); + write_shared_migration_state(&target, &first) + .await + .expect("initial claim"); + write_shared_migration_state(&target, &first) + .await + .expect("exact replay"); + + let mut second = first.clone(); + second.run_id = uuid::Uuid::new_v4(); + second.seal().expect("seal second run"); + let error = write_shared_migration_state(&target, &second) + .await + .expect_err("different run must not replace the claim"); + assert!(error.to_string().contains("different migration run")); + } + + #[cfg(feature = "postgres")] + #[test] + fn shared_state_accepts_replay_and_manifest_lifecycle_transitions() { + assert!(shared_state_transition_allowed( + "applying", + MigrationStatus::Applying + )); + assert!(shared_state_transition_allowed( + "applying", + MigrationStatus::Applied + )); + assert!(shared_state_transition_allowed( + "failed", + MigrationStatus::Applying + )); + assert!(shared_state_transition_allowed( + "applied", + MigrationStatus::Verifying + )); + assert!(shared_state_transition_allowed( + "verifying", + MigrationStatus::Verified + )); + assert!(shared_state_transition_allowed( + "verified", + MigrationStatus::Verified + )); + assert!(shared_state_transition_allowed( + "verified", + MigrationStatus::Verifying + )); + } + + #[cfg(feature = "postgres")] + #[test] + fn shared_state_rejects_skipped_and_unknown_transitions() { + assert!(!shared_state_transition_allowed( + "applying", + MigrationStatus::Verified + )); + assert!(!shared_state_transition_allowed( + "failed", + MigrationStatus::Verified + )); + assert!(!shared_state_transition_allowed( + "verified", + MigrationStatus::Applying + )); + assert!(!shared_state_transition_allowed( + "unknown", + MigrationStatus::Applying + )); + } +} diff --git a/crates/ironclaw_reborn_migration/src/target_ids.rs b/crates/ironclaw_reborn_migration/src/target_ids.rs new file mode 100644 index 00000000000..99993245b32 --- /dev/null +++ b/crates/ironclaw_reborn_migration/src/target_ids.rs @@ -0,0 +1,185 @@ +//! Stable target identifiers for replay-safe migration writes. +//! +//! The namespace and seed format are versioned. Changing either would mint a +//! second copy of every migrated record, so a future incompatible format must +//! introduce a new manifest schema version. + +use ironclaw_host_api::{AgentId, TenantId}; +use ironclaw_triggers::TriggerId; +use uuid::Uuid; + +use crate::error::MigrationError; +use crate::report::MigrationReport; + +const MIGRATION_ID_SCHEMA: &str = "ironclaw-v1-to-reborn/v2"; +const MIGRATION_NAMESPACE: Uuid = Uuid::from_u128(0xd735d0a7_891d_4e57_ba2e_368f2a36a82c); + +#[derive(Debug, Clone)] +pub(crate) struct MigrationIdentity { + manifest_schema_version: u32, + source_fingerprint: String, +} + +impl MigrationIdentity { + pub(crate) fn from_report(report: &MigrationReport) -> Result { + let manifest = report.manifest.as_ref().ok_or_else(|| { + MigrationError::InvalidInput( + "converter execution requires a sealed migration manifest".to_string(), + ) + })?; + Ok(Self { + manifest_schema_version: manifest.manifest_schema_version, + source_fingerprint: manifest.source_fingerprint.value.clone(), + }) + } + + pub(crate) fn trigger_id( + &self, + domain: &str, + source_primary_id: &str, + tenant_id: &TenantId, + agent_id: &AgentId, + ) -> Result { + let uuid = self.scoped_uuid( + domain, + source_primary_id, + tenant_id.as_str(), + agent_id.as_str(), + ); + let ulid = ulid::Ulid::from(uuid.as_u128()); + TriggerId::parse(&ulid.to_string()).map_err(|error| { + MigrationError::InvalidInput(format!( + "could not derive deterministic {domain} target id: {error}" + )) + }) + } + + pub(crate) fn message_key( + &self, + thread_id: Uuid, + message_index: usize, + source_primary_id: Option<&str>, + ) -> String { + let source_primary_id = source_primary_id + .map(str::to_owned) + .unwrap_or_else(|| format!("{thread_id}\0{message_index}")); + self.scoped_uuid("message", &source_primary_id, "transcript", "transcript") + .to_string() + } + + pub(crate) fn thread_source_binding(&self, thread_id: Uuid) -> String { + let suffix = self.scoped_uuid("thread-binding", &thread_id.to_string(), "thread", "thread"); + format!("migration:v1:{suffix}") + } + + fn scoped_uuid( + &self, + domain: &str, + source_primary_id: &str, + tenant_id: &str, + agent_id: &str, + ) -> Uuid { + let manifest_schema_version = self.manifest_schema_version.to_string(); + let mut seed = Vec::new(); + for field in [ + MIGRATION_ID_SCHEMA, + manifest_schema_version.as_str(), + self.source_fingerprint.as_str(), + domain, + source_primary_id, + tenant_id, + agent_id, + ] { + seed.extend_from_slice(field.len().to_string().as_bytes()); + seed.push(b':'); + seed.extend_from_slice(field.as_bytes()); + } + Uuid::new_v5(&MIGRATION_NAMESPACE, &seed) + } + + #[cfg(test)] + fn for_test(source_fingerprint: &str) -> Self { + Self { + manifest_schema_version: 1, + source_fingerprint: source_fingerprint.to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use ironclaw_host_api::{AgentId, TenantId}; + use uuid::Uuid; + + use super::MigrationIdentity; + + #[test] + fn trigger_ids_are_stable_and_scope_sensitive() { + let tenant_a = TenantId::new("tenant-a").unwrap(); + let tenant_b = TenantId::new("tenant-b").unwrap(); + let agent = AgentId::new("agent-a").unwrap(); + let identity = MigrationIdentity::for_test("source-a"); + + let first = identity + .trigger_id("routine", "routine-1", &tenant_a, &agent) + .unwrap(); + let replay = identity + .trigger_id("routine", "routine-1", &tenant_a, &agent) + .unwrap(); + let other_scope = identity + .trigger_id("routine", "routine-1", &tenant_b, &agent) + .unwrap(); + + assert_eq!(first, replay); + assert_ne!(first, other_scope); + } + + #[test] + fn source_fingerprint_partitions_target_ids() { + let tenant = TenantId::new("tenant-a").unwrap(); + let agent = AgentId::new("agent-a").unwrap(); + let source_a = MigrationIdentity::for_test("source-a"); + let source_b = MigrationIdentity::for_test("source-b"); + + assert_ne!( + source_a + .trigger_id("routine", "routine-1", &tenant, &agent) + .unwrap(), + source_b + .trigger_id("routine", "routine-1", &tenant, &agent) + .unwrap() + ); + } + + #[test] + fn scoped_ids_are_injective_across_field_boundaries() { + let identity = MigrationIdentity::for_test("source-a"); + + let first = identity.scoped_uuid("routine\0legacy", "item", "tenant", "agent"); + let second = identity.scoped_uuid("routine", "legacy\0item", "tenant", "agent"); + + assert_ne!(first, second); + } + + #[test] + fn synthesized_message_keys_are_stable_and_order_sensitive() { + let thread = Uuid::parse_str("1cdfa15a-a8e7-4868-a25d-6fbde771d438").unwrap(); + let identity = MigrationIdentity::for_test("source-a"); + assert_eq!( + identity.message_key(thread, 2, None), + identity.message_key(thread, 2, None) + ); + assert_ne!( + identity.message_key(thread, 2, None), + identity.message_key(thread, 3, None) + ); + assert_eq!( + identity.message_key(thread, 2, Some("source-id")), + identity.message_key(thread, 9, Some("source-id")) + ); + assert_eq!( + identity.thread_source_binding(thread), + identity.thread_source_binding(thread) + ); + } +} diff --git a/crates/ironclaw_reborn_migration/src/v2_model.rs b/crates/ironclaw_reborn_migration/src/v2_model.rs index 3ec28a1a54e..89278e21521 100644 --- a/crates/ironclaw_reborn_migration/src/v2_model.rs +++ b/crates/ironclaw_reborn_migration/src/v2_model.rs @@ -127,8 +127,16 @@ pub(crate) struct Project { pub description: String, #[serde(default)] pub goals: Vec, + #[serde(default)] + pub metrics: Vec, + #[serde(default)] + pub metadata: serde_json::Value, + #[serde(default)] + pub workspace_path: Option, #[serde(default = "epoch_fallback")] pub created_at: DateTime, + #[serde(default)] + pub updated_at: Option>, } /// `ironclaw_engine::types::thread::Thread` (subset) — a mission's execution diff --git a/crates/ironclaw_reborn_migration/tests/companion_cli.rs b/crates/ironclaw_reborn_migration/tests/companion_cli.rs new file mode 100644 index 00000000000..7d8e09bec98 --- /dev/null +++ b/crates/ironclaw_reborn_migration/tests/companion_cli.rs @@ -0,0 +1,215 @@ +use std::process::Command; + +#[cfg(feature = "libsql")] +use std::path::Path; + +fn companion_bin() -> &'static str { + env!("CARGO_BIN_EXE_ironclaw-reborn-migration") +} + +#[test] +fn handshake_is_exact_and_machine_readable() { + let output = Command::new(companion_bin()) + .arg("__handshake") + .env_clear() + .output() + .expect("run migration companion"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + let handshake: serde_json::Value = + serde_json::from_slice(&output.stdout).expect("handshake JSON"); + assert_eq!( + handshake["schema_version"], + "ironclaw.reborn.migration-companion/v1" + ); + assert_eq!( + handshake["protocol_version"], + ironclaw_reborn_migration::MIGRATION_PROTOCOL_VERSION + ); + assert_eq!(handshake["release_version"], env!("CARGO_PKG_VERSION")); +} + +#[test] +fn lifecycle_help_never_accepts_raw_postgres_urls_or_keys() { + let output = Command::new(companion_bin()) + .args(["v1", "plan", "--help"]) + .env_clear() + .output() + .expect("run migration companion help"); + + assert!(output.status.success()); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!(stdout.contains("--source-postgres"), "stdout: {stdout}"); + assert!(stdout.contains("--source-home"), "stdout: {stdout}"); + assert!( + !stdout.contains("--source-postgres-url"), + "stdout: {stdout}" + ); + assert!(!stdout.contains("--target-postgres"), "stdout: {stdout}"); + assert!(!stdout.contains("--secret-master-key"), "stdout: {stdout}"); +} + +#[test] +fn companion_can_emit_a_machine_readable_error_envelope() { + let output = Command::new(companion_bin()) + .args(["v1", "status", "--manifest", "/missing/manifest.json"]) + .env_clear() + .env("IRONCLAW_REBORN_MIGRATION_ERROR_FORMAT", "json") + .output() + .expect("run migration companion"); + + assert!(!output.status.success()); + let error: serde_json::Value = + serde_json::from_slice(&output.stderr).expect("machine-readable error JSON"); + assert_eq!( + error["schema_version"], + "ironclaw.reborn.migration-error/v1" + ); + assert_eq!(error["code"], "migration_failed"); + assert!( + error["message"] + .as_str() + .is_some_and(|message| message.contains("failed to read migration manifest")), + "error: {error}" + ); +} + +#[cfg(feature = "libsql")] +async fn seed_empty_v1_source(path: &Path) { + let database = libsql::Builder::new_local(path) + .build() + .await + .expect("build source"); + let connection = database.connect().expect("connect source"); + connection + .execute_batch( + "CREATE TABLE settings ( + user_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (user_id, key) + );", + ) + .await + .expect("seed empty source"); +} + +#[cfg(feature = "libsql")] +#[tokio::test] +async fn strict_plan_ignores_empty_lossy_categories() { + let directory = tempfile::tempdir().expect("tempdir"); + let source_home = directory.path().join("v1-home"); + let reborn_home = directory.path().join("reborn-home"); + let source = source_home.join("ironclaw.db"); + let manifest = directory.path().join("migration.json"); + std::fs::create_dir_all(&source_home).expect("create source home"); + seed_empty_v1_source(&source).await; + + let output = Command::new(companion_bin()) + .args([ + "v1", + "plan", + "--source-libsql", + source.to_str().expect("UTF-8 source path"), + "--source-home", + source_home.to_str().expect("UTF-8 source home"), + "--manifest", + manifest.to_str().expect("UTF-8 manifest path"), + "--strict", + ]) + .env_clear() + .env("HOME", directory.path()) + .env("IRONCLAW_REBORN_HOME", reborn_home) + .env("IRONCLAW_REBORN_PROFILE", "local-dev") + .output() + .expect("run strict migration plan"); + + assert!( + output.status.success(), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert!( + manifest.exists(), + "strict plan should still write its manifest" + ); +} + +#[cfg(feature = "libsql")] +#[tokio::test] +async fn apply_preflight_failure_preserves_the_planned_manifest() { + let directory = tempfile::tempdir().expect("tempdir"); + let source_home = directory.path().join("v1-home"); + let reborn_home = directory.path().join("reborn-home"); + let source = source_home.join("ironclaw.db"); + let manifest_path = directory.path().join("migration.json"); + std::fs::create_dir_all(&source_home).expect("create source home"); + seed_empty_v1_source(&source).await; + + let plan = Command::new(companion_bin()) + .args([ + "v1", + "plan", + "--source-libsql", + source.to_str().expect("UTF-8 source path"), + "--source-home", + source_home.to_str().expect("UTF-8 source home"), + "--manifest", + manifest_path.to_str().expect("UTF-8 manifest path"), + ]) + .env_clear() + .env("HOME", directory.path()) + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .env("IRONCLAW_REBORN_PROFILE", "local-dev") + .output() + .expect("plan migration"); + assert!( + plan.status.success(), + "stderr: {}", + String::from_utf8_lossy(&plan.stderr) + ); + + let target = reborn_home.join("local-dev").join("reborn-local-dev.db"); + std::fs::create_dir_all(target.parent().expect("target parent")).expect("target parent"); + std::fs::write(&target, b"existing Reborn state").expect("existing target"); + + let apply = Command::new(companion_bin()) + .args([ + "v1", + "apply", + "--source-libsql", + source.to_str().expect("UTF-8 source path"), + "--source-home", + source_home.to_str().expect("UTF-8 source home"), + "--plan", + manifest_path.to_str().expect("UTF-8 manifest path"), + "--confirm-v1-stopped", + "--confirm-source-snapshot", + ]) + .env_clear() + .env("HOME", directory.path()) + .env("IRONCLAW_REBORN_HOME", &reborn_home) + .env("IRONCLAW_REBORN_PROFILE", "local-dev") + .output() + .expect("apply migration"); + assert!(!apply.status.success()); + + let manifest: serde_json::Value = serde_json::from_slice( + &std::fs::read(&manifest_path).expect("read preserved migration manifest"), + ) + .expect("migration manifest JSON"); + assert_eq!( + manifest["status"], + "planned", + "stderr: {}", + String::from_utf8_lossy(&apply.stderr) + ); + assert!( + !reborn_home.join(".v1-migration-state.json").exists(), + "preflight failure must not install a target quarantine marker" + ); +} diff --git a/crates/ironclaw_reborn_migration/tests/migration_roundtrip.rs b/crates/ironclaw_reborn_migration/tests/migration_roundtrip.rs index 8a0c0f7fce9..a7c863207eb 100644 --- a/crates/ironclaw_reborn_migration/tests/migration_roundtrip.rs +++ b/crates/ironclaw_reborn_migration/tests/migration_roundtrip.rs @@ -14,19 +14,28 @@ use ironclaw::agent::routine::{NotifyConfig, Routine, RoutineAction, RoutineGuar use ironclaw::config::{DatabaseBackend, DatabaseConfig, SslMode}; use ironclaw::db::{Database, DatabaseHandles, UserIdentityRecord, connect_with_handles}; use ironclaw::secrets::{CreateSecretParams, SecretsCrypto, create_secrets_store}; -use ironclaw::tools::wasm::{ - LibSqlWasmToolStore, StoreToolParams, ToolStatus, TrustLevel, WasmToolStore, -}; +use ironclaw::tools::wasm::{LibSqlWasmToolStore, StoreToolParams, TrustLevel, WasmToolStore}; use ironclaw_host_api::TenantId; -use ironclaw_reborn_migration::{Domain, MigrationOptions, SourceDb, TargetStore, run_migration}; -use ironclaw_triggers::{LibSqlTriggerRepository, TriggerRepository, TriggerSchedule}; +use ironclaw_reborn_migration::{ + ApplyAcknowledgements, Domain, MigrationOptions, MigrationSecretInputs, MigrationStatus, + SourceDb, TargetStore, apply_migration, plan_migration, resume_migration, run_migration, + verify_migration, +}; +use ironclaw_triggers::{ + LibSqlTriggerRepository, TriggerRepository, TriggerSchedule, TriggerState, +}; use secrecy::SecretString; use uuid::Uuid; const TENANT: &str = "acme"; const AGENT: &str = "assistant"; const USER: &str = "alice"; -const USER_BOB: &str = "bob"; +const SUSPENDED_USER: &str = "bob"; +const DEACTIVATED_USER: &str = "carol"; +const LEGACY_DATA_OWNER: &str = "legacy-owner"; +const SOURCE_AGENT: &str = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; +const OTHER_SOURCE_AGENT: &str = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; +const UNMIGRATED_PROJECT: &str = "cccccccc-cccc-4ccc-8ccc-cccccccccccc"; /// 64-char string ≥ 32 bytes (used verbatim as HKDF IKM by v1 + Reborn crypto). const MASTER_KEY: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; @@ -129,9 +138,13 @@ async fn seed_v1_fixture(dir: &std::path::Path) -> PathBuf { .expect("m5"); // ── routines: every trigger variant × both actions ── + let mut cron_without_next = routine("cron-light", cron("0 9 * * *"), lightweight(), true); + cron_without_next.next_fire_at = None; + let mut suspended_owner = routine("cron-fulljob", cron("0 18 * * MON-FRI"), full_job(), true); + suspended_owner.user_id = SUSPENDED_USER.to_string(); for r in [ - routine("cron-light", cron("0 9 * * *"), lightweight(), true), - routine("cron-fulljob", cron("0 18 * * MON-FRI"), full_job(), false), + cron_without_next, + suspended_owner, routine( "event-r", Trigger::Event { @@ -169,7 +182,7 @@ async fn seed_v1_fixture(dir: &std::path::Path) -> PathBuf { let mission_thread_id = Uuid::new_v4(); let cron_mission = serde_json::json!({ "id": Uuid::new_v4(), - "project_id": Uuid::new_v4(), + "project_id": UNMIGRATED_PROJECT, "user_id": USER, "name": "daily-digest", "goal": "compile a daily digest of important updates", @@ -189,6 +202,12 @@ async fn seed_v1_fixture(dir: &std::path::Path) -> PathBuf { &cron_mission, ) .await; + write_engine_doc( + db.as_ref(), + "engine/projects/p1/missions/daily-digest/mission.json", + &cron_mission, + ) + .await; let event_mission = serde_json::json!({ "id": Uuid::new_v4(), @@ -226,8 +245,28 @@ async fn seed_v1_fixture(dir: &std::path::Path) -> PathBuf { ) .await; - // ── a non-engine memory document ── - write_doc(db.as_ref(), "context/vision.md", "# Vision\nbe helpful").await; + // ── non-engine memory documents across distinct agent scopes ── + let scoped_doc = write_doc_for_agent( + db.as_ref(), + Some(Uuid::parse_str(SOURCE_AGENT).expect("source agent")), + "context/vision.md", + "# Vision\nbe helpful", + ) + .await; + db.update_document_metadata( + scoped_doc, + &serde_json::json!({"legacy_label": "source-scoped"}), + ) + .await + .expect("write source memory metadata"); + write_doc_for_agent(db.as_ref(), None, "context/vision.md", "# Vision\nunscoped").await; + write_doc_for_agent( + db.as_ref(), + Some(Uuid::parse_str(OTHER_SOURCE_AGENT).expect("other source agent")), + "context/vision.md", + "# Vision\nother agent", + ) + .await; // ── settings ── let mut settings = std::collections::HashMap::new(); @@ -236,6 +275,15 @@ async fn seed_v1_fixture(dir: &std::path::Path) -> PathBuf { db.set_all_settings(USER, &settings) .await .expect("seed settings"); + db.set_all_settings( + LEGACY_DATA_OWNER, + &std::collections::HashMap::from([( + "legacy_timezone".to_string(), + serde_json::json!("UTC"), + )]), + ) + .await + .expect("seed pre-users-table data owner"); seed_identities(db.as_ref(), &handles).await; seed_secret(&handles).await; @@ -256,27 +304,67 @@ async fn seed_identities(db: &dyn Database, handles: &DatabaseHandles) { .expect("connect"); // Raw users insert — `get_or_create_user` would additionally seed an // assistant thread (a real v1 behavior, but it would perturb the thread - // counts this test pins), so insert the rows directly. - for (user_id, email, display_name) in [ - (USER, "alice@example.com", "Alice"), - (USER_BOB, "bob@example.com", "Bob"), - ] { - conn.execute( - "INSERT INTO users (id, email, display_name, status, role, created_at, updated_at, metadata) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?7)", - ( - user_id.to_string(), - email.to_string(), - display_name.to_string(), - "active".to_string(), - "member".to_string(), - now.to_rfc3339(), - "{}".to_string(), - ), - ) - .await - .expect("seed user row"); - } + // counts this test pins), so insert the row directly. + conn.execute( + "INSERT INTO users (id, email, display_name, status, role, created_at, updated_at, metadata) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?7)", + ( + USER.to_string(), + "alice@example.com".to_string(), + "Alice".to_string(), + "active".to_string(), + "member".to_string(), + now.to_rfc3339(), + "{}".to_string(), + ), + ) + .await + .expect("seed user row"); + + // Case-insensitive historical values, lifecycle fields, creator relation, + // and object metadata must survive the canonical user import. + conn.execute( + "INSERT INTO users (id, email, display_name, status, role, created_at, updated_at, last_login_at, created_by, metadata) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?6, ?7, ?8)", + ( + SUSPENDED_USER.to_string(), + "bob@example.com".to_string(), + "Bob".to_string(), + "SUSPENDED".to_string(), + "ADMIN".to_string(), + now.to_rfc3339(), + USER.to_string(), + serde_json::json!({"team": "infra"}).to_string(), + ), + ) + .await + .expect("seed suspended admin user"); + conn.execute( + "INSERT INTO users (id, email, display_name, status, role, created_at, updated_at, created_by, metadata) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?7, ?8)", + ( + DEACTIVATED_USER.to_string(), + "carol@example.com".to_string(), + "Carol".to_string(), + "deactivated".to_string(), + "member".to_string(), + now.to_rfc3339(), + SUSPENDED_USER.to_string(), + serde_json::json!({"team": "operations", "quota": 3}).to_string(), + ), + ) + .await + .expect("seed deactivated user"); + + db.create_api_token( + USER, + "migration-fixture-token", + &[0xAB; 32], + "deadbeef", + None, + ) + .await + .expect("seed API token hash"); db.create_identity(&UserIdentityRecord { id: Uuid::new_v4(), @@ -321,105 +409,37 @@ async fn seed_secret(handles: &DatabaseHandles) { async fn seed_wasm_tool(handles: &DatabaseHandles) { let db = handles.libsql_db.clone().expect("libsql handle"); let store = LibSqlWasmToolStore::new(db.clone()); - // Same-named installs for two users exercise canonicalization. The - // disagreeing source states must produce one disabled canonical row. - for user_id in [USER, USER_BOB] { - let tool = store - .store(StoreToolParams { - user_id: user_id.to_string(), - name: "weather".to_string(), - version: "1.0.0".to_string(), - wit_version: "0.1.0".to_string(), - description: "Weather lookup".to_string(), - wasm_binary: b"\0asm-fake-binary".to_vec(), - parameters_schema: serde_json::json!({"type": "object"}), - source_url: None, - trust_level: TrustLevel::User, - }) - .await - .expect("seed wasm tool"); - store - .update_status( - user_id, - "weather", - if user_id == USER { - ToolStatus::Active - } else { - ToolStatus::Disabled - }, - ) - .await - .expect("seed tool status"); - - // Seed tool_capabilities with an allowed secret (no trait writer - // exists) so both source rows contribute the same credential binding. - let conn = db.connect().expect("connect"); - conn.execute( - "INSERT INTO tool_capabilities (id, wasm_tool_id, allowed_secrets) VALUES (?1, ?2, ?3)", - ( - Uuid::new_v4().to_string(), - tool.id.to_string(), - serde_json::json!(["openai_api_key"]).to_string(), - ), - ) - .await - .expect("seed tool capabilities"); - } -} - -async fn set_wasm_tool_status(path: &Path, user_id: &str, status: ToolStatus) { - let db = Arc::new( - libsql::Builder::new_local(path) - .build() - .await - .expect("open v1 fixture for status update"), - ); - let store = LibSqlWasmToolStore::new(db); - store - .update_status(user_id, "weather", status) - .await - .expect("update wasm tool status"); -} - -async fn set_wasm_tool_description(path: &Path, user_id: &str, description: &str) { - let db = libsql::Builder::new_local(path) - .build() - .await - .expect("open v1 fixture for metadata update"); - let conn = db.connect().expect("connect"); - let changed = conn - .execute( - "UPDATE wasm_tools SET description = ?1 WHERE user_id = ?2 AND name = ?3", - ( - description.to_string(), - user_id.to_string(), - "weather".to_string(), - ), - ) + // The source artifact is inventoried and reported for reinstall; it must + // not become a synthesized Reborn installation. + let tool = store + .store(StoreToolParams { + user_id: USER.to_string(), + name: "weather".to_string(), + version: "1.0.0".to_string(), + wit_version: "0.1.0".to_string(), + description: "Weather lookup".to_string(), + wasm_binary: b"\0asm-fake-binary".to_vec(), + parameters_schema: serde_json::json!({"type": "object"}), + source_url: None, + trust_level: TrustLevel::User, + }) .await - .expect("update wasm tool description"); - assert_eq!(changed, 1, "expected one source tool metadata row"); -} + .expect("seed wasm tool"); -async fn set_wasm_tool_allowed_secret(path: &Path, user_id: &str, secret: &str) { - let db = libsql::Builder::new_local(path) - .build() - .await - .expect("open v1 fixture for credential update"); + // Seed tool_capabilities with an allowed secret (no trait writer exists) so + // the migration records the unsupported capability and credential-binding + // policy explicitly. let conn = db.connect().expect("connect"); - let changed = conn - .execute( - "UPDATE tool_capabilities SET allowed_secrets = ?1 \ - WHERE wasm_tool_id = (SELECT id FROM wasm_tools WHERE user_id = ?2 AND name = ?3)", - ( - serde_json::json!([secret]).to_string(), - user_id.to_string(), - "weather".to_string(), - ), - ) - .await - .expect("update wasm tool credentials"); - assert_eq!(changed, 1, "expected one source tool capability row"); + conn.execute( + "INSERT INTO tool_capabilities (id, wasm_tool_id, allowed_secrets) VALUES (?1, ?2, ?3)", + ( + Uuid::new_v4().to_string(), + tool.id.to_string(), + serde_json::json!(["openai_api_key"]).to_string(), + ), + ) + .await + .expect("seed tool capabilities"); } fn cron(expr: &str) -> Trigger { @@ -434,19 +454,38 @@ async fn write_engine_doc(db: &dyn Database, path: &str, value: &serde_json::Val } async fn write_doc(db: &dyn Database, path: &str, content: &str) { + write_doc_for_agent( + db, + Some(Uuid::parse_str(SOURCE_AGENT).expect("source agent")), + path, + content, + ) + .await; +} + +async fn write_doc_for_agent( + db: &dyn Database, + agent_id: Option, + path: &str, + content: &str, +) -> Uuid { let doc = db - .get_or_create_document_by_path(USER, None, path) + .get_or_create_document_by_path(USER, agent_id, path) .await .expect("create doc"); db.update_document(doc.id, content) .await .expect("write doc content"); + doc.id } fn options(src: PathBuf, dst: PathBuf, dry_run: bool) -> MigrationOptions { + let source_home = src.parent().map(Path::to_path_buf); MigrationOptions { source: SourceDb::LibSql { path: src }, + source_home, target: TargetStore::LibSql { path: dst }, + profile: "test-migration".to_string(), tenant_id: TenantId::new(TENANT).unwrap(), agent_id: ironclaw_host_api::AgentId::new(AGENT).unwrap(), secret_master_key: Some(SecretString::from(MASTER_KEY)), @@ -473,9 +512,10 @@ async fn reborn_entry_count(path: &Path, like: &str) -> i64 { row.get::(0).expect("count") } -/// Read the `contents` blob of the first Reborn entry matching a LIKE pattern, -/// as UTF-8 — used to assert the shape of a written installation/thread doc. -async fn reborn_entry_content(path: &Path, like: &str) -> String { +/// Count append-log records. Finalized assistant messages are durably appended +/// to the production thread service's message log and indexed, rather than +/// always materialized as an entry row; both representations are first-class. +async fn reborn_event_count(path: &Path, like: &str) -> i64 { let db = libsql::Builder::new_local(path) .build() .await @@ -483,14 +523,83 @@ async fn reborn_entry_content(path: &Path, like: &str) -> String { let conn = db.connect().expect("connect"); let mut rows = conn .query( - "SELECT contents FROM root_filesystem_entries WHERE path LIKE ?1 LIMIT 1", + "SELECT count(*) FROM root_filesystem_events WHERE path LIKE ?1", [like], ) .await - .expect("query entry contents"); + .expect("query events"); let row = rows.next().await.expect("row").expect("some row"); - let blob = row.get::>(0).expect("contents blob"); - String::from_utf8(blob).expect("utf-8 contents") + row.get::(0).expect("count") +} + +async fn reborn_message_count(path: &Path) -> i64 { + reborn_entry_count(path, "/tenants/acme/users/alice/threads/%/messages/%.json").await + + reborn_event_count(path, "/tenants/acme/users/alice/threads/%/message_appends").await +} + +/// Whether any persisted entry under `like` contains a UTF-8 payload fragment. +async fn reborn_entry_contains(path: &Path, like: &str, needle: &str) -> bool { + let db = libsql::Builder::new_local(path) + .build() + .await + .expect("open reborn db"); + let conn = db.connect().expect("connect"); + let mut rows = conn + .query( + "SELECT contents FROM root_filesystem_entries WHERE path LIKE ?1", + [like], + ) + .await + .expect("query entry contents"); + while let Some(row) = rows.next().await.expect("row") { + let blob = row.get::>(0).expect("contents blob"); + if String::from_utf8_lossy(&blob).contains(needle) { + return true; + } + } + false +} + +async fn reborn_entry_json(path: &Path, entry_path: &str) -> serde_json::Value { + let db = libsql::Builder::new_local(path) + .build() + .await + .expect("open reborn db"); + let conn = db.connect().expect("connect"); + let mut rows = conn + .query( + "SELECT contents FROM root_filesystem_entries WHERE path = ?1", + [entry_path], + ) + .await + .expect("query entry contents"); + let row = rows.next().await.expect("row").expect("persisted entry"); + serde_json::from_slice(&row.get::>(0).expect("contents blob")).expect("entry JSON") +} + +async fn delete_reborn_entry(path: &Path, entry_path: &str) { + let db = libsql::Builder::new_local(path) + .build() + .await + .expect("open reborn db"); + let conn = db.connect().expect("connect"); + conn.execute( + "DELETE FROM root_filesystem_entries WHERE path = ?1", + [entry_path], + ) + .await + .expect("delete reborn entry"); +} + +fn reborn_user_entry_path(user_id: &str) -> String { + let encoded = match user_id { + USER => "YWxpY2U", + SUSPENDED_USER => "Ym9i", + DEACTIVATED_USER => "Y2Fyb2w", + LEGACY_DATA_OWNER => "bGVnYWN5LW93bmVy", + other => panic!("missing expected base64url fixture path for {other}"), + }; + format!("/tenants/{TENANT}/shared/reborn-identity/users/{encoded}.json") } async fn reborn_triggers(path: &Path) -> Vec { @@ -507,36 +616,57 @@ async fn reborn_triggers(path: &Path) -> Vec { .expect("list triggers") } -/// Count thread.json documents in the Reborn store via a fresh connection -/// (proves on-disk durability independent of the migration's live handles). -async fn reborn_thread_doc_count(path: &Path) -> i64 { +async fn overwrite_trigger_prompt(path: &Path, name: &str, prompt: &str) { let db = libsql::Builder::new_local(path) .build() .await .expect("open reborn db"); let conn = db.connect().expect("connect"); - let mut rows = conn - .query( - "SELECT count(*) FROM root_filesystem_entries WHERE path LIKE '%/thread.json'", - (), - ) - .await - .expect("query thread docs"); - let row = rows.next().await.expect("row").expect("some row"); - row.get::(0).expect("count") + conn.execute( + "UPDATE trigger_records SET prompt = ?1 WHERE tenant_id = ?2 AND name = ?3", + (prompt, TENANT, name), + ) + .await + .expect("mutate trigger collision fixture"); } #[tokio::test] async fn migrates_v1_and_engine_v2_state_without_loss() { let dir = tempfile::tempdir().unwrap(); let src = seed_v1_fixture(dir.path()).await; - let dst = dir.path().join("reborn.db"); + // Keep the Reborn home distinct from the v1 source home. Target-owned + // database/key artifacts must never become new source inventory on resume. + let dst = dir.path().join("reborn/reborn.db"); - let report = run_migration(options(src.clone(), dst.clone(), false)) + let migration_options = options(src.clone(), dst.clone(), false); + let manifest = plan_migration(&migration_options) .await - .expect("migration runs"); + .expect("migration plan"); + let blockers: Vec<_> = manifest + .inventory + .iter() + .filter(|item| item.blocker.is_some()) + .collect(); + assert!( + blockers.is_empty(), + "the known-source fixture must have an explicit disposition for every category: {blockers:#?}" + ); + let report = apply_migration( + migration_options, + &manifest, + MigrationSecretInputs { + source_master_key: Some(SecretString::from(MASTER_KEY)), + // Local Reborn composition resolves/persists its own production + // target key after apply preconditions. + target_master_key: None, + }, + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect("migration runs"); // ── converted counts ── + assert_eq!(report.stats.users, 4, "users: {:?}", report.stats); // 2 conversations + 1 mission thread. assert_eq!(report.stats.threads, 3, "threads: {:?}", report.stats); // 2 cron routines converted (event/sysevent/webhook/manual do not). @@ -546,7 +676,7 @@ async fn migrates_v1_and_engine_v2_state_without_loss() { // user+assistant messages: conv1 (2) + conv2 (2) + mission thread (2) = 6. assert_eq!(report.stats.messages, 6, "messages: {:?}", report.stats); assert_eq!( - report.stats.memory_documents, 1, + report.stats.memory_documents, 3, "memory: {:?}", report.stats ); @@ -555,33 +685,41 @@ async fn migrates_v1_and_engine_v2_state_without_loss() { // dropped (or newly-recovered) value fails the build. Counts are pinned to // the fixture above; see the inline breakdown per domain. ── let expected_losses = [ + // deactivated -> suspended, one non-string metadata value -> JSON text, + // and one orphan durable-data owner synthesized fail-closed as suspended. + (Domain::User, 3), + // Hash-only API tokens cannot be converted and require re-authentication. + (Domain::ApiToken, 1), // owner/thread/mission ids are all valid → no thread-identity losses. (Domain::Thread, 0), // conv1's single "system" transcript message (no first-class append path). (Domain::Message, 1), // 6 routines: each cron routine records 3 field losses // (action + guardrails/notify/counters + routine_runs); each non-cron - // routine records 1 trigger-source loss + 2 field losses. 6 × 3 = 18. - (Domain::Routine, 18), + // routine records 1 trigger-source loss + 2 field losses. The active + // cron without a next fire and the cron owned by a suspended user each + // add one fail-closed degradation. 6 × 3 + 2 = 20. + (Domain::Routine, 20), // daily-digest: mission_only_fields + status.failed + next_fire_at // (the fixture mission has no next_fire_at → synthesized) = 3; - // on-deploy: cadence.on_event (1). No orphan threads (blob referenced). - (Domain::Mission, 4), + // on-deploy: cadence.on_event (1). The missing referenced project is + // omitted and paused (1). No orphan threads (blob referenced). + (Domain::Mission, 5), // fixture seeds no jobs → the job converter records nothing. (Domain::Job, 0), // single unconditional memory_document_versions gap. (Domain::Memory, 1), // the seeded secret decrypts, re-encrypts, and carries no expiry → 0. (Domain::Secret, 0), - // the two source installs each record manifest_fidelity + capabilities. - // They merge into one canonical installation row. - (Domain::Extension, 4), + // the migrated wasm tool: manifest_fidelity + capabilities (2). + // Unknown v1 WASM is report-only: package reinstall + capabilities. + (Domain::Extension, 2), // unconditional pairing_requests gap (both identities adopt cleanly). (Domain::Identity, 1), - // unconditional heartbeat_state gap. - (Domain::Heartbeat, 1), - // one gap per seeded setting key (model, timezone). - (Domain::Setting, 2), + // fixture seeds no heartbeat state rows. + (Domain::Heartbeat, 0), + // one gap per seeded setting key (model, timezone, legacy_timezone). + (Domain::Setting, 3), ]; for (domain, expected) in expected_losses { assert_eq!( @@ -614,7 +752,7 @@ async fn migrates_v1_and_engine_v2_state_without_loss() { for (domain, field) in [ (Domain::Mission, "cadence.on_event"), (Domain::Mission, "status.failed"), - (Domain::Extension, "manifest_fidelity"), + (Domain::Extension, "package"), (Domain::Extension, "capabilities"), ] { assert!( @@ -631,51 +769,28 @@ async fn migrates_v1_and_engine_v2_state_without_loss() { assert_eq!(report.stats.secrets, 1, "secrets: {:?}", report.stats); // 1 OAuth identity + 1 channel identity adopted. assert_eq!(report.stats.identities, 2, "identities: {:?}", report.stats); - // 2 same-named user installs → 1 canonical ExtensionInstallation. - assert_eq!(report.stats.extensions, 1, "extensions: {:?}", report.stats); - - // Extension installation invariants: the on-disk installation record must - // carry the canonical id, both private owners, fail-closed Disabled - // activation, and the merged credential binding. - let installation_doc = - reborn_entry_content(&dst, "%/system/extensions/.installations/state.json").await; - let installation_state: serde_json::Value = - serde_json::from_str(&installation_doc).expect("installation state JSON"); - let installations = installation_state["installations"] - .as_array() - .expect("installation array"); - assert_eq!( - installations.len(), - 1, - "same-named source installs must canonicalize to one row" - ); - let installation = &installations[0]; - assert_eq!(installation["installation_id"], "weather"); - assert_eq!(installation["extension_id"], "weather"); - assert_eq!(installation["activation_state"], "disabled"); - assert_eq!(installation["owner"]["kind"], "users"); - assert_eq!( - installation["owner"]["user_ids"], - serde_json::json!([USER, USER_BOB]) - ); - assert_eq!( - installation["credential_bindings"] - .as_array() - .expect("credential binding array") - .len(), - 1, - "agreeing duplicate credential bindings must be merged" + // Unknown executable artifacts are accounted for but never installed. + assert_eq!(report.stats.extensions, 0, "extensions: {:?}", report.stats); + let report_json = report.to_json().expect("report JSON"); + assert!(!report_json.contains("deadbeef"), "token prefix leaked"); + assert!(!report_json.contains(&"ab".repeat(32)), "token hash leaked"); + assert!( + dst.parent() + .expect("target parent") + .join(".reborn-local-dev-secrets-master-key") + .is_file(), + "apply must persist the same local target key a production Reborn boot resolves" ); - assert!(installation_doc.contains("openai_api_key")); // On-disk durability of the deferred domains (fresh connection). assert!( reborn_entry_count(&dst, "%/secrets/%openai_api_key.json").await >= 1, "expected the migrated secret document on disk" ); - assert!( - reborn_entry_count(&dst, "%/system/extensions/.installations/state.json").await >= 1, - "expected the extension installation state document on disk" + assert_eq!( + reborn_entry_count(&dst, "%/system/extensions/.installations/state.json").await, + 0, + "an incompatible placeholder installation must never be persisted" ); // ── round-trip through the Reborn triggers repo ── @@ -686,160 +801,276 @@ async fn migrates_v1_and_engine_v2_state_without_loss() { assert!(names.contains(&"cron-light")); assert!(names.contains(&"cron-fulljob")); assert!(names.contains(&"daily-digest")); + let cron_light = triggers.iter().find(|t| t.name == "cron-light").unwrap(); + assert_eq!(cron_light.state, TriggerState::Paused); + let cron_fulljob = triggers.iter().find(|t| t.name == "cron-fulljob").unwrap(); + assert_eq!(cron_fulljob.state, TriggerState::Paused); let digest = triggers.iter().find(|t| t.name == "daily-digest").unwrap(); + assert_eq!(digest.state, TriggerState::Paused); + assert!(digest.project_id.is_none()); match &digest.schedule { TriggerSchedule::Cron { expression, .. } => assert_eq!(expression, "0 7 * * *"), other => panic!("expected cron schedule, got {other:?}"), } - // ── on-disk durability of threads (fresh connection) ── + // ── production-path durability of threads (fresh connection) ── + assert_eq!( + reborn_entry_count(&dst, "/tenants/acme/users/alice/threads/%/thread.json").await, + 3, + "migration must use composition's production tenant/user mount layout" + ); + assert_eq!( + reborn_message_count(&dst).await, + 6, + "expected exactly the supported transcript rows across production's row and append-log representations" + ); assert!( - reborn_thread_doc_count(&dst).await >= 3, - "expected >=3 persisted thread.json docs" + reborn_entry_count(&dst, "/tenants/acme/shared/reborn-identity/%").await >= 2, + "identity rows must use production's tenant-shared mount, not the legacy global path" + ); + assert_eq!( + reborn_entry_count(&dst, "/tenants/acme/shared/reborn-identity/users/%.json").await, + 4, + "all canonical and synthesized users must use the tenant-shared production path" + ); + let bob = reborn_entry_json(&dst, &reborn_user_entry_path(SUSPENDED_USER)).await; + assert_eq!(bob["status"], "suspended"); + assert_eq!(bob["role"], "admin"); + assert_eq!(bob["created_by"], USER); + assert_eq!(bob["tenant_id"], TENANT); + assert_eq!(bob["last_login_at"], "2024-01-02T03:04:05+00:00"); + assert_eq!(bob["metadata"]["team"], "infra"); + let carol = reborn_entry_json(&dst, &reborn_user_entry_path(DEACTIVATED_USER)).await; + assert_eq!(carol["status"], "suspended"); + assert_eq!(carol["role"], "member"); + assert_eq!(carol["created_by"], SUSPENDED_USER); + assert_eq!(carol["metadata"]["quota"], "3"); + let legacy = reborn_entry_json(&dst, &reborn_user_entry_path(LEGACY_DATA_OWNER)).await; + assert_eq!(legacy["status"], "suspended"); + assert_eq!(legacy["role"], "member"); + assert_eq!(legacy["created_at"], "1970-01-01T00:00:00+00:00"); + assert_eq!(legacy["metadata"]["migration.synthesized"], "true"); + assert_eq!( + reborn_entry_count(&dst, "/tenant-shared/%").await, + 0, + "migration must not write identity data to an unscoped global mount" + ); + assert!( + reborn_entry_contains(&dst, "%/thread.json", "session started").await, + "non-user/assistant content must be retained in the migration archive metadata" + ); + assert!( + !reborn_entry_contains(&dst, "%/thread.json", UNMIGRATED_PROJECT).await, + "mission threads must not retain references to projects that were not imported" ); - // ── idempotency: re-running the migration into the same target re-adopts - // identities (first-writer-wins) and upserts the extension installation by - // its deterministic id, so no duplicate installation doc is written. (Trigger - // ids are freshly minted per run, so triggers are intentionally not - // deduplicated — the tool is a one-shot converter.) ── - let report2 = run_migration(options(src, dst.clone(), false)) - .await - .expect("second migration run"); + let scoped_memory_path = format!( + "/memory/tenants/{TENANT}/users/{USER}/agents/{SOURCE_AGENT}/projects/_none/context/vision.md" + ); + let unscoped_memory_path = format!( + "/memory/tenants/{TENANT}/users/{USER}/agents/_none/projects/_none/context/vision.md" + ); + let other_scoped_memory_path = format!( + "/memory/tenants/{TENANT}/users/{USER}/agents/{OTHER_SOURCE_AGENT}/projects/_none/context/vision.md" + ); + assert!(reborn_entry_contains(&dst, &scoped_memory_path, "be helpful").await); + assert!(reborn_entry_contains(&dst, &unscoped_memory_path, "unscoped").await); + assert!(reborn_entry_contains(&dst, &other_scoped_memory_path, "other agent").await); + + // ── idempotency: resume replays the same sealed source identity and must + // compare-and-apply without duplicating triggers or transcript rows. ── + let applied_manifest = report.manifest.clone().expect("applied manifest"); + let applying_checkpoint = applied_manifest + .transition(MigrationStatus::Applying) + .expect("interrupted applying checkpoint"); + let scoped_metadata_path = format!("{scoped_memory_path}.meta"); + delete_reborn_entry(&dst, &scoped_metadata_path).await; + let replay_options = options(src.clone(), dst.clone(), false); + let report2 = resume_migration( + replay_options, + &applying_checkpoint, + MigrationSecretInputs { + source_master_key: Some(SecretString::from(MASTER_KEY)), + target_master_key: None, + }, + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect("migration resume"); assert_eq!( report2.stats.identities, 2, "re-run must re-adopt the same 2 identities" ); + assert_eq!(report2.stats.users, 4, "resume must replay exact users"); + assert_eq!( + report2.stats.memory_documents, 3, + "resume must replay every distinct memory scope" + ); + assert_eq!( + reborn_entry_json(&dst, &scoped_metadata_path).await["legacy_label"], + "source-scoped", + "resume must complete a body-only partial memory write" + ); + assert_eq!( + reborn_entry_count(&dst, "/tenants/acme/shared/reborn-identity/users/%.json").await, + 4, + "resume duplicated users" + ); assert_eq!( - report2.stats.extensions, 1, - "re-run must upsert the same installation, not duplicate" + report2.stats.extensions, 0, + "resume must not install incompatible extensions" ); assert_eq!( reborn_entry_count(&dst, "%/system/extensions/.installations/state.json").await, - 1, - "re-run must not write a second installation state document" + 0, + "resume must not write placeholder installation state" ); assert_eq!( - reborn_entry_content(&dst, "%/system/extensions/.installations/state.json").await, - installation_doc, - "re-run must preserve deterministic canonical extension state" + reborn_triggers(&dst).await.len(), + 3, + "resume duplicated triggers" + ); + assert_eq!( + reborn_message_count(&dst).await, + 6, + "resume duplicated transcript messages" + ); + assert_eq!( + reborn_entry_count(&dst, "%/secret-leases/%").await, + 0, + "secret comparison during resume must not create one-shot leases" ); -} -/// Caller-level migration coverage: run the public converter and reopen the -/// persisted extension state, proving that two agreeing active source rows -/// stay enabled after the shared canonical reducer merges their owners. -#[tokio::test] -async fn migrates_all_active_duplicate_users_as_enabled() { - let dir = tempfile::tempdir().unwrap(); - let src = seed_v1_fixture(dir.path()).await; - set_wasm_tool_status(&src, USER_BOB, ToolStatus::Active).await; - let dst = dir.path().join("reborn-all-active.db"); + let verified = verify_migration( + &options(src.clone(), dst.clone(), false), + report2.manifest.as_ref().expect("resume manifest"), + ) + .await + .expect("cold target verification"); + assert_eq!(verified.status, MigrationStatus::Verified); - let report = run_migration(options(src, dst.clone(), false)) - .await - .expect("migration runs"); - - assert_eq!(report.stats.extensions, 1); - let installation_doc = - reborn_entry_content(&dst, "%/system/extensions/.installations/state.json").await; - let state: serde_json::Value = serde_json::from_str(&installation_doc).unwrap(); - let installations = state["installations"].as_array().unwrap(); - assert_eq!(installations.len(), 1); - assert_eq!(installations[0]["activation_state"], "enabled"); - assert_eq!(installations[0]["owner"]["kind"], "users"); + // A stale applied manifest cannot reopen a verified target claim or + // overwrite operator/runtime state. + overwrite_trigger_prompt(&dst, "cron-light", "divergent target prompt").await; + let resume_manifest = report2.manifest.as_ref().expect("resume manifest"); + let collision = resume_migration( + options(src, dst.clone(), false), + resume_manifest, + MigrationSecretInputs { + source_master_key: Some(SecretString::from(MASTER_KEY)), + target_master_key: None, + }, + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("verified target claim must fail closed"); + assert!( + collision + .to_string() + .contains("invalid shared migration state transition verified -> applying"), + "unexpected collision error: {collision}" + ); assert_eq!( - installations[0]["owner"]["user_ids"], - serde_json::json!([USER, USER_BOB]) + reborn_triggers(&dst) + .await + .into_iter() + .find(|trigger| trigger.name == "cron-light") + .expect("cron-light trigger") + .prompt, + "divergent target prompt", + "migration must not overwrite a divergent deterministic slot" ); } -/// Caller-level migration coverage for incompatible source metadata: the -/// grouped source rows are reported and skipped before a manifest is written. #[tokio::test] -async fn migration_records_metadata_conflict_without_partial_extension() { +async fn apply_requires_source_key_when_inventory_contains_secrets() { let dir = tempfile::tempdir().unwrap(); let src = seed_v1_fixture(dir.path()).await; - set_wasm_tool_description(&src, USER_BOB, "Different weather metadata").await; - let dst = dir.path().join("reborn-metadata-conflict.db"); - - let report = run_migration(options(src, dst.clone(), false)) + let dst = dir.path().join("missing-key-target/reborn.db"); + let migration_options = options(src, dst.clone(), false); + let manifest = plan_migration(&migration_options) .await - .expect("migration runs"); - - assert_eq!(report.stats.extensions, 0); - assert_eq!(report.losses_in(Domain::Extension), 5); - assert!(report.lossy.iter().any(|loss| { - loss.domain == Domain::Extension - && loss.field == "canonicalization" - && loss.detail.contains("no partial canonical installation") - })); - assert_eq!( - reborn_entry_count(&dst, "%/system/extensions/.installations/state.json").await, - 0, - "metadata conflict must not write a partial canonical installation" + .expect("migration plan"); + + let error = apply_migration( + migration_options, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("source key must be required before applying secret rows"); + + assert!(error.to_string().contains("secrets master key required")); + assert!( + !dst.exists(), + "preflight failure must not create the target" ); } -/// Caller-level migration coverage for distinct v1 allowed secrets: because -/// v1 stores secret names rather than a separate credential-handle mapping, -/// distinct names are distinct target bindings and must merge safely. The -/// shared reducer's conflicting-handle fail-closed policy is covered at the -/// persisted-store seam, where legacy rows can contain that ambiguity. #[tokio::test] -async fn migration_merges_distinct_credential_bindings() { +async fn apply_tolerates_historical_schema_without_settings_table() { let dir = tempfile::tempdir().unwrap(); - let src = seed_v1_fixture(dir.path()).await; - set_wasm_tool_allowed_secret(&src, USER_BOB, "different_api_key").await; - let dst = dir.path().join("reborn-credential-conflict.db"); + let src = dir.path().join("without-settings.db"); + let (db, handles) = connect_with_handles(&libsql_config(&src)) + .await + .expect("create historical fixture"); + drop(db); + let connection = handles + .libsql_db + .as_ref() + .expect("libsql handle") + .connect() + .expect("connect"); + connection + .execute("DROP TABLE settings", ()) + .await + .expect("remove optional settings table"); + drop(connection); + drop(handles); - let report = run_migration(options(src, dst.clone(), false)) + let dst = dir.path().join("without-settings-target/reborn.db"); + let migration_options = options(src, dst.clone(), false); + let manifest = plan_migration(&migration_options) .await - .expect("migration runs"); - - assert_eq!(report.stats.extensions, 1); - assert_eq!(report.losses_in(Domain::Extension), 4); - let installation_doc = - reborn_entry_content(&dst, "%/system/extensions/.installations/state.json").await; - let state: serde_json::Value = serde_json::from_str(&installation_doc).unwrap(); - let bindings = state["installations"][0]["credential_bindings"] - .as_array() - .unwrap(); - assert_eq!(bindings.len(), 2); - assert!(bindings.iter().any(|binding| { - binding["credential_handle"] == "openai_api_key" - && binding["secret_handle"] == "openai_api_key" - })); - assert!(bindings.iter().any(|binding| { - binding["credential_handle"] == "different_api_key" - && binding["secret_handle"] == "different_api_key" - })); + .expect("migration plan"); + let report = apply_migration( + migration_options, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect("missing optional settings table must be treated as empty"); + + assert_eq!(report.losses_in(Domain::Setting), 0); + assert!(dst.exists()); } #[tokio::test] async fn dry_run_reports_without_writing() { let dir = tempfile::tempdir().unwrap(); let src = seed_v1_fixture(dir.path()).await; - let dst = dir.path().join("reborn-dry.db"); + let target_parent = dir.path().join("missing-target"); + let dst = target_parent.join("reborn-dry.db"); let report = run_migration(options(src, dst.clone(), true)) .await .expect("dry run"); - // Same counts as a real run … - assert_eq!(report.stats.threads, 3); - assert_eq!(report.stats.routines, 2); - assert_eq!(report.stats.missions, 2); + // Planning reports inventory through the manifest without invoking writers. + let manifest = report.manifest.expect("plan manifest"); + assert!( + manifest + .inventory + .iter() + .any(|item| item.source_name == "conversations" && item.count >= 2) + ); assert!(report.dry_run); - // … but nothing was written to the Reborn store. + // No read helper is invoked here: opening libSQL would itself create state. assert!( - reborn_triggers(&dst).await.is_empty(), - "dry run wrote triggers" - ); - assert_eq!( - reborn_thread_doc_count(&dst).await, - 0, - "dry run wrote thread docs" + !dst.exists() && !target_parent.exists(), + "planning created the target path or its parent" ); } diff --git a/crates/ironclaw_reborn_migration/tests/migration_safety.rs b/crates/ironclaw_reborn_migration/tests/migration_safety.rs new file mode 100644 index 00000000000..47a6ba0c49b --- /dev/null +++ b/crates/ironclaw_reborn_migration/tests/migration_safety.rs @@ -0,0 +1,429 @@ +#![cfg(feature = "libsql")] + +use std::path::{Path, PathBuf}; + +use ironclaw_host_api::{AgentId, TenantId}; +use ironclaw_reborn_migration::{ + ApplyAcknowledgements, Domain, MigrationOptions, MigrationSecretInputs, MigrationStatus, + SourceDb, TargetStore, apply_migration, manifest_target_matches, plan_migration, + verify_migration, +}; + +async fn seed_source(path: &Path) { + let database = libsql::Builder::new_local(path) + .build() + .await + .expect("build source"); + let connection = database.connect().expect("connect source"); + connection + .execute_batch( + "CREATE TABLE settings ( + user_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (user_id, key) + ); + INSERT INTO settings (user_id, key, value) + VALUES ('user-1', 'model', '\"gpt-test\"');", + ) + .await + .expect("seed source"); +} + +async fn source_contents(path: &Path) -> Vec<(String, String, String)> { + let database = libsql::Builder::new_local(path) + .build() + .await + .expect("open source"); + let connection = database.connect().expect("connect source"); + let mut rows = connection + .query( + "SELECT user_id, key, value FROM settings ORDER BY user_id, key", + (), + ) + .await + .expect("query source"); + let mut out = Vec::new(); + while let Some(row) = rows.next().await.expect("next row") { + out.push(( + row.get(0).expect("user id"), + row.get(1).expect("key"), + row.get(2).expect("value"), + )); + } + out +} + +fn options(source: PathBuf, target: PathBuf) -> MigrationOptions { + let source_home = source.parent().map(Path::to_path_buf); + MigrationOptions { + source: SourceDb::LibSql { path: source }, + source_home, + target: TargetStore::LibSql { path: target }, + profile: "test-migration".to_string(), + tenant_id: TenantId::new("migration-tenant").expect("tenant"), + agent_id: AgentId::new("migration-agent").expect("agent"), + secret_master_key: None, + dry_run: true, + } +} + +#[tokio::test] +async fn apply_rejects_scope_drift_before_creating_target() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("new-target").join("target.db"); + seed_source(&source).await; + let planned_options = options(source, target.clone()); + let manifest = plan_migration(&planned_options).await.expect("plan"); + + let mut changed = planned_options; + changed.profile = "different-profile".to_string(); + let error = apply_migration( + changed, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("scope drift must fail"); + + assert!(error.to_string().contains("sealed plan")); + assert!(!target.exists()); +} + +#[tokio::test] +async fn explicit_source_home_is_sealed_and_not_inferred_from_snapshot() { + let directory = tempfile::tempdir().expect("tempdir"); + let source_home = directory.path().join("v1-home"); + let backup_dir = directory.path().join("backups"); + std::fs::create_dir_all(&source_home).expect("source home"); + std::fs::create_dir_all(&backup_dir).expect("backup dir"); + std::fs::write(source_home.join("settings.json"), b"{}").expect("home artifact"); + let source = backup_dir.join("source.db"); + seed_source(&source).await; + let target = directory.path().join("target.db"); + let mut migration_options = options(source, target.clone()); + migration_options.source_home = Some(source_home.clone()); + + let manifest = plan_migration(&migration_options).await.expect("plan"); + assert_eq!( + manifest + .inventory + .iter() + .find(|entry| entry.source_name == "settings.json") + .expect("settings inventory") + .count, + 1 + ); + + migration_options.source_home = Some(backup_dir); + let error = apply_migration( + migration_options, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("source home drift must fail"); + assert!(error.to_string().contains("sealed plan")); + assert!(!target.exists()); +} + +#[tokio::test] +async fn missing_source_home_blocks_apply() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("target.db"); + seed_source(&source).await; + let mut migration_options = options(source, target.clone()); + migration_options.source_home = None; + + let manifest = plan_migration(&migration_options).await.expect("plan"); + assert!( + manifest + .inventory + .iter() + .any(|entry| entry.source_name == "v1_home" && entry.blocker.is_some()) + ); + let error = apply_migration( + migration_options, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("missing source home must block apply"); + assert!(error.to_string().contains("inventory blockers")); + assert!(!target.exists()); +} + +#[tokio::test] +async fn plan_is_source_read_only_and_does_not_create_target() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source-with-password-canary.db"); + let target = directory.path().join("new-reborn-home").join("target.db"); + seed_source(&source).await; + let before = source_contents(&source).await; + let bytes_before = std::fs::read(&source).expect("read source snapshot"); + + let manifest = plan_migration(&options(source.clone(), target.clone())) + .await + .expect("plan"); + + assert_eq!(manifest.status, MigrationStatus::Planned); + assert_eq!(source_contents(&source).await, before); + assert_eq!( + std::fs::read(&source).expect("read source after plan"), + bytes_before, + "planning modified source database bytes" + ); + assert!(!target.exists(), "planning created the target database"); + assert!( + !target.parent().expect("target parent").exists(), + "planning created the target home" + ); + let json = manifest.to_json().expect("manifest json"); + assert!(!json.contains("source-with-password-canary.db")); + assert!(!json.contains(&source.display().to_string())); + manifest.validate_plan_hash().expect("sealed plan"); + assert!(manifest_target_matches( + &TargetStore::LibSql { + path: target.clone() + }, + &manifest + )); + assert!(!manifest_target_matches( + &TargetStore::LibSql { + path: directory.path().join("different-target.db") + }, + &manifest + )); + std::fs::create_dir_all(target.parent().expect("target parent")).expect("create target parent"); + std::fs::File::create(&target).expect("create target"); + assert!( + manifest_target_matches(&TargetStore::LibSql { path: target }, &manifest), + "target locator fingerprint must be stable across missing to created" + ); +} + +#[tokio::test] +async fn plan_inventory_accounts_for_known_tables_and_home_artifacts() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("target.db"); + seed_source(&source).await; + + let manifest = plan_migration(&options(source, target)) + .await + .expect("plan"); + for table in [ + "conversations", + "conversation_messages", + "users", + "api_tokens", + "settings", + "memory_documents", + "memory_document_versions", + "routines", + "routine_runs", + "secrets", + "wasm_tools", + "wasm_channels", + "pairing_requests", + "claude_code_events", + "root_filesystem_entries", + ] { + assert!( + manifest + .inventory + .iter() + .any(|item| item.source_name == table), + "missing table disposition for {table}" + ); + } + for artifact in [ + ".env", + "settings.json", + "config.toml", + "providers.json", + "session.json", + "mcp-servers.json", + "acp-agents.json", + "profiles", + "skills", + "installed_skills", + "tools", + "channels", + "projects", + "history", + "logs", + ] { + assert!( + manifest + .inventory + .iter() + .any(|item| item.source_name == artifact), + "missing home disposition for {artifact}" + ); + } + assert_eq!( + manifest + .inventory + .iter() + .find(|item| item.source_name == "settings") + .expect("settings inventory") + .count, + 1 + ); + assert!(manifest.domains.contains_key(&Domain::Setting)); +} + +#[tokio::test] +async fn apply_rejects_changed_source_before_creating_target() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("new-target").join("target.db"); + seed_source(&source).await; + let options = options(source.clone(), target.clone()); + let manifest = plan_migration(&options).await.expect("plan"); + + let database = libsql::Builder::new_local(&source) + .build() + .await + .expect("open source"); + database + .connect() + .expect("connect source") + .execute( + "UPDATE settings SET value = '\"changed\"' WHERE key = 'model'", + (), + ) + .await + .expect("change source"); + + let error = apply_migration( + options, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("changed source must fail"); + assert!(error.to_string().contains("fingerprint changed")); + assert!(!target.exists()); +} + +#[tokio::test] +async fn same_source_and_target_is_rejected_without_writes() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + seed_source(&source).await; + let before = source_contents(&source).await; + + let error = plan_migration(&options(source.clone(), source.clone())) + .await + .expect_err("same store must fail"); + assert!(error.to_string().contains("must be different")); + assert_eq!(source_contents(&source).await, before); +} + +#[tokio::test] +async fn apply_requires_both_offline_snapshot_acknowledgements() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("new-target").join("target.db"); + seed_source(&source).await; + let options = options(source, target.clone()); + let manifest = plan_migration(&options).await.expect("plan"); + + let error = apply_migration( + options, + &manifest, + MigrationSecretInputs::default(), + ApplyAcknowledgements { + source_is_stopped: true, + source_is_snapshot: false, + }, + ) + .await + .expect_err("missing snapshot acknowledgement must fail"); + assert!(error.to_string().contains("consistent snapshot")); + assert!(!target.exists()); +} + +#[tokio::test] +async fn manifest_write_is_owner_only_and_no_clobber_by_default() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("target.db"); + seed_source(&source).await; + let manifest = plan_migration(&options(source, target)) + .await + .expect("plan"); + let manifest_path = directory.path().join("reports").join("plan.json"); + + manifest + .write_atomic(&manifest_path, false) + .expect("write manifest"); + assert!(manifest.write_atomic(&manifest_path, false).is_err()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let mode = std::fs::metadata(&manifest_path) + .expect("manifest metadata") + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + } +} + +#[tokio::test] +async fn verification_requires_an_active_target_claim() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("source.db"); + let target = directory.path().join("target.db"); + seed_source(&source).await; + let options = options(source, target); + let planned = plan_migration(&options).await.expect("plan"); + let applying = planned + .transition(MigrationStatus::Applying) + .expect("applying"); + let applied = applying + .transition(MigrationStatus::Applied) + .expect("applied"); + let error = verify_migration(&options, &applied) + .await + .expect_err("an unclaimed target cannot be verified"); + assert!( + error.to_string().contains("no active migration claim"), + "unexpected verification error: {error}" + ); + + let verifying = applied + .transition(MigrationStatus::Verifying) + .expect("persist verifying before readback"); + let resumed_error = verify_migration(&options, &verifying) + .await + .expect_err("resumed verification still requires an active target claim"); + assert!( + resumed_error + .to_string() + .contains("no active migration claim"), + "unexpected resumed verification error: {resumed_error}" + ); + + let verified = verifying + .transition(MigrationStatus::Verified) + .expect("verified"); + let reverify_error = verify_migration(&options, &verified) + .await + .expect_err("verified manifests still require an active target claim"); + assert!( + reverify_error + .to_string() + .contains("no active migration claim"), + "unexpected re-verification error: {reverify_error}" + ); +} diff --git a/crates/ironclaw_reborn_migration/tests/project_migration.rs b/crates/ironclaw_reborn_migration/tests/project_migration.rs new file mode 100644 index 00000000000..7e028409f6a --- /dev/null +++ b/crates/ironclaw_reborn_migration/tests/project_migration.rs @@ -0,0 +1,220 @@ +#![cfg(feature = "libsql")] + +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use ironclaw::config::{DatabaseBackend, DatabaseConfig, SslMode}; +use ironclaw::db::{Database, connect_with_handles}; +use ironclaw_filesystem::ScopedFilesystem; +use ironclaw_host_api::{AgentId, ProjectId, TenantId, UserId}; +use ironclaw_projects::{FilesystemProjectRepository, ProjectRepository}; +use ironclaw_reborn_migration::{ + ApplyAcknowledgements, Domain, MigrationOptions, MigrationSecretInputs, SourceDb, TargetStore, + apply_migration, plan_migration, resume_migration, +}; +use secrecy::SecretString; +use uuid::Uuid; + +const TENANT: &str = "acme"; +const AGENT: &str = "assistant"; +const USER: &str = "alice"; +const MASTER_KEY: &str = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + +fn libsql_config(path: &Path) -> DatabaseConfig { + DatabaseConfig { + backend: DatabaseBackend::LibSql, + url: SecretString::from("unused://libsql"), + pool_size: 2, + ssl_mode: SslMode::default(), + libsql_path: Some(path.to_path_buf()), + libsql_url: None, + libsql_auth_token: None, + } +} + +async fn seed_source(path: &Path, project_a: Uuid, project_b: Uuid) { + let (database, _handles) = connect_with_handles(&libsql_config(path)) + .await + .expect("open v1 fixture"); + write_project( + database.as_ref(), + "projects/alpha/.project.json", + project_a, + "Alpha", + "First project", + ("2024-01-02T03:04:05Z", "2024-02-03T04:05:06Z"), + None, + ) + .await; + write_project( + database.as_ref(), + ".system/engine/projects/beta/project.json", + project_b, + "Beta", + "Second project", + ("2023-01-02T03:04:05Z", "2023-02-03T04:05:06Z"), + Some(Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap()), + ) + .await; +} + +async fn write_project( + database: &dyn Database, + path: &str, + id: Uuid, + name: &str, + description: &str, + timestamps: (&str, &str), + agent_id: Option, +) { + let (created_at, updated_at) = timestamps; + let document = database + .get_or_create_document_by_path(USER, agent_id, path) + .await + .expect("create project document"); + let body = serde_json::json!({ + "id": id, + "user_id": USER, + "name": name, + "description": description, + "goals": [format!("ship {name}")], + "metrics": [{"name": "quality", "target": "high"}], + "metadata": {"legacy_label": name.to_ascii_lowercase()}, + "workspace_path": format!("/srv/projects/{}", name.to_ascii_lowercase()), + "created_at": created_at, + "updated_at": updated_at, + }); + database + .update_document(document.id, &body.to_string()) + .await + .expect("write project document"); +} + +fn options(source: PathBuf, target: PathBuf) -> MigrationOptions { + let source_home = source.parent().map(Path::to_path_buf); + MigrationOptions { + source: SourceDb::LibSql { path: source }, + source_home, + target: TargetStore::LibSql { path: target }, + profile: "test-migration".to_string(), + tenant_id: TenantId::new(TENANT).expect("tenant"), + agent_id: AgentId::new(AGENT).expect("agent"), + secret_master_key: Some(SecretString::from(MASTER_KEY)), + dry_run: false, + } +} + +fn migration_secrets() -> MigrationSecretInputs { + MigrationSecretInputs { + source_master_key: Some(SecretString::from(MASTER_KEY)), + target_master_key: None, + } +} + +async fn project_repository( + target: &Path, +) -> FilesystemProjectRepository { + let database = Arc::new( + libsql::Builder::new_local(target) + .build() + .await + .expect("open Reborn target"), + ); + let root = Arc::new(ironclaw_filesystem::LibSqlRootFilesystem::new(database)); + let scoped = Arc::new(ScopedFilesystem::new( + root, + ironclaw_reborn_composition::invocation_mount_view, + )); + FilesystemProjectRepository::new( + scoped, + UserId::new("project-readback").expect("readback user"), + AgentId::new(AGENT).expect("agent"), + ) +} + +#[tokio::test] +async fn migrates_both_project_layouts_and_replay_conflicts_fail_closed() { + let directory = tempfile::tempdir().expect("tempdir"); + let source = directory.path().join("v1.db"); + let target = directory.path().join("reborn/reborn.db"); + let project_a = Uuid::parse_str("11111111-1111-4111-8111-111111111111").unwrap(); + let project_b = Uuid::parse_str("22222222-2222-4222-8222-222222222222").unwrap(); + seed_source(&source, project_a, project_b).await; + + let migration_options = options(source.clone(), target.clone()); + let manifest = plan_migration(&migration_options).await.expect("plan"); + assert_eq!( + manifest + .domains + .get(&Domain::Project) + .expect("project checkpoint") + .planned, + 2, + "planning must count supported engine-v2 project documents" + ); + let report = apply_migration( + migration_options, + &manifest, + migration_secrets(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect("apply projects"); + assert_eq!(report.stats.projects, 2); + assert_eq!(report.losses_in(Domain::Project), 0); + + let repository = project_repository(&target).await; + let tenant = TenantId::new(TENANT).unwrap(); + let alpha = repository + .get_project(&tenant, &ProjectId::new(project_a.to_string()).unwrap()) + .await + .expect("read alpha") + .expect("alpha exists"); + assert_eq!(alpha.owner_user_id.as_str(), USER); + assert_eq!(alpha.name, "Alpha"); + assert_eq!(alpha.description, "First project"); + assert_eq!(alpha.created_at.to_rfc3339(), "2024-01-02T03:04:05+00:00"); + assert_eq!(alpha.updated_at.to_rfc3339(), "2024-02-03T04:05:06+00:00"); + assert_eq!(alpha.metadata["legacy_engine_v2"]["goals"][0], "ship Alpha"); + assert_eq!( + alpha.metadata["legacy_engine_v2"]["metadata"]["legacy_label"], + "alpha" + ); + assert!( + repository + .get_project(&tenant, &ProjectId::new(project_b.to_string()).unwrap()) + .await + .expect("read beta") + .is_some(), + "the .system/engine layout must also import" + ); + + let replay = resume_migration( + options(source.clone(), target.clone()), + report.manifest.as_ref().expect("applied manifest"), + migration_secrets(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect("exact replay"); + assert_eq!(replay.stats.projects, 2); + + let mut divergent = alpha; + divergent.description = "operator changed this project".to_string(); + repository + .update_project(divergent) + .await + .expect("mutate deterministic target slot"); + let error = resume_migration( + options(source, target), + replay.manifest.as_ref().expect("replay manifest"), + migration_secrets(), + ApplyAcknowledgements::offline_snapshot(), + ) + .await + .expect_err("divergent project must conflict"); + assert!( + error.to_string().contains("refusing to overwrite"), + "unexpected project conflict: {error}" + ); +} diff --git a/crates/ironclaw_secrets/src/filesystem_store.rs b/crates/ironclaw_secrets/src/filesystem_store.rs index f381caae947..982bb95f7c4 100644 --- a/crates/ironclaw_secrets/src/filesystem_store.rs +++ b/crates/ironclaw_secrets/src/filesystem_store.rs @@ -40,6 +40,8 @@ //! TODO(reborn/fs-secrets): once `EncryptedBackend` ships, replace the inline //! `encrypt`/`decrypt` calls with plaintext writes wrapped by the decorator. +// arch-exempt: large_file, filesystem secret-store decomposition is tracked, plan #4088 + use std::{collections::HashSet, sync::Arc}; use async_trait::async_trait; @@ -58,7 +60,7 @@ use crate::{ CredentialAccount, CredentialAccountId, CredentialAccountStatus, CredentialAccountStore, CredentialBrokerError, CredentialSession, CredentialSessionId, CredentialSessionStore, DEFAULT_SECRET_LEASE_TTL_SECONDS, SecretError, SecretLease, SecretLeaseId, SecretLeaseStatus, - SecretMaterial, SecretMetadata, SecretStore, SecretStoreError, SecretsCrypto, + SecretMaterial, SecretMetadata, SecretPutOutcome, SecretStore, SecretStoreError, SecretsCrypto, credential_account_aad, credential_session_aad, filesystem_secret_aad, }; @@ -85,7 +87,7 @@ const CREDENTIAL_SESSION_KIND: &str = "credential_session"; // nothing in this file ever writes plaintext secret material to the // filesystem. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] struct StoredSecret { scope: ResourceScope, handle: SecretHandle, @@ -267,15 +269,7 @@ where async fn write_secret(&self, secret: &StoredSecret) -> Result<(), SecretStoreError> { let path = secret_path(&secret.scope, &secret.handle)?; - let body = serialize_secret(secret)?; - let kind = RecordKind::new(SECRET_RECORD_KIND).map_err(|error| { - SecretStoreError::StoreUnavailable { - reason: format!("invalid secret record kind: {error}"), - } - })?; - let mut base_entry = Entry::bytes(body).with_content_type(ContentType::json()); - base_entry.kind = Some(kind); - let entry = tag_entry_with_tenant(base_entry, &secret.scope); + let entry = serialize_secret_entry(secret)?; self.ensure_tenant_id_index(&secret.scope).await?; self.filesystem .put(&secret.scope, &path, entry, CasExpectation::Any) @@ -378,6 +372,72 @@ where }) } + async fn put_if_absent_or_matches( + &self, + scope: ResourceScope, + handle: SecretHandle, + material: SecretMaterial, + expires_at: Option, + ) -> Result { + let path = secret_path(&scope, &handle)?; + let plaintext = material.expose_secret().as_bytes(); + let aad = filesystem_secret_aad(&scope, &handle); + let (encrypted_value, key_salt) = self + .crypto + .encrypt(plaintext, &aad) + .map_err(secret_error_to_store_error)?; + let now = Utc::now(); + let candidate = StoredSecret { + scope: scope.clone(), + handle: handle.clone(), + encrypted_value, + key_salt, + expires_at, + created_at: now, + updated_at: now, + }; + self.ensure_tenant_id_index(&scope).await?; + cas_update( + self.filesystem.as_ref(), + &scope, + &path, + deserialize_secret::, + serialize_secret_entry, + |current: Option| { + let candidate = candidate.clone(); + let scope = scope.clone(); + let handle = handle.clone(); + let material = material.clone(); + let self_ref = self; + async move { + let Some(existing) = current else { + return Ok(CasApply::new(candidate, SecretPutOutcome::Inserted)); + }; + if !same_scope_owner(&existing.scope, &scope) + || existing.handle != handle + || existing.expires_at != expires_at + { + return Ok(CasApply::no_op(existing, SecretPutOutcome::Divergent)); + } + let aad = filesystem_secret_aad(&scope, &handle); + let decrypted = self_ref + .crypto + .decrypt(&existing.encrypted_value, &existing.key_salt, &aad) + .map_err(secret_error_to_store_error)?; + let outcome = + if decrypted.expose().as_bytes() == material.expose_secret().as_bytes() { + SecretPutOutcome::ExactMatch + } else { + SecretPutOutcome::Divergent + }; + Ok(CasApply::no_op(existing, outcome)) + } + }, + ) + .await + .map_err(|error| map_cas_error_secret(error, "insert")) + } + async fn metadata( &self, scope: &ResourceScope, @@ -393,6 +453,25 @@ where })) } + async fn material_matches( + &self, + scope: &ResourceScope, + handle: &SecretHandle, + candidate: &SecretMaterial, + ) -> Result, SecretStoreError> { + let Some(stored) = self.read_secret(scope, handle).await? else { + return Ok(None); + }; + let aad = filesystem_secret_aad(scope, handle); + let decrypted = self + .crypto + .decrypt(&stored.encrypted_value, &stored.key_salt, &aad) + .map_err(secret_error_to_store_error)?; + Ok(Some( + decrypted.expose().as_bytes() == candidate.expose_secret().as_bytes(), + )) + } + async fn metadata_for_scope( &self, scope: &ResourceScope, @@ -1154,6 +1233,18 @@ fn serialize_lease_entry(lease: &StoredLease) -> Result Ok(tag_entry_with_tenant(base_entry, &lease.scope)) } +fn serialize_secret_entry(secret: &StoredSecret) -> Result { + let body = serialize_secret(secret)?; + let kind = RecordKind::new(SECRET_RECORD_KIND).map_err(|error| { + SecretStoreError::StoreUnavailable { + reason: format!("invalid secret record kind: {error}"), + } + })?; + let mut base_entry = Entry::bytes(body).with_content_type(ContentType::json()); + base_entry.kind = Some(kind); + Ok(tag_entry_with_tenant(base_entry, &secret.scope)) +} + fn serialize_session_entry( stored: &StoredSession, scope: &ResourceScope, @@ -1621,6 +1712,70 @@ mod tests { assert!(second.is_consumed()); } + #[tokio::test] + async fn filesystem_secret_store_atomic_insert_never_overwrites_divergent_material() { + let fs = Arc::new(InMemoryBackend::new()); + let store = Arc::new(FilesystemSecretStore::new( + default_scoped_fs(fs), + test_crypto(), + )); + let scope = sample_scope("tenant-a", "user-a"); + let handle = SecretHandle::new("migration_key").unwrap(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + + let first = { + let store = Arc::clone(&store); + let scope = scope.clone(); + let handle = handle.clone(); + let barrier = Arc::clone(&barrier); + tokio::spawn(async move { + barrier.wait().await; + store + .put_if_absent_or_matches(scope, handle, SecretMaterial::from("first"), None) + .await + }) + }; + let second = { + let store = Arc::clone(&store); + let scope = scope.clone(); + let handle = handle.clone(); + let barrier = Arc::clone(&barrier); + tokio::spawn(async move { + barrier.wait().await; + store + .put_if_absent_or_matches(scope, handle, SecretMaterial::from("second"), None) + .await + }) + }; + let outcomes = [ + first.await.unwrap().unwrap(), + second.await.unwrap().unwrap(), + ]; + assert!(outcomes.contains(&SecretPutOutcome::Inserted)); + assert!(outcomes.contains(&SecretPutOutcome::Divergent)); + + let stored_first = store + .material_matches(&scope, &handle, &SecretMaterial::from("first")) + .await + .unwrap(); + let stored_second = store + .material_matches(&scope, &handle, &SecretMaterial::from("second")) + .await + .unwrap(); + assert_ne!(stored_first, stored_second); + assert!(stored_first == Some(true) || stored_second == Some(true)); + let winning_material = if stored_first == Some(true) { + "first" + } else { + "second" + }; + let replay = store + .put_if_absent_or_matches(scope, handle, SecretMaterial::from(winning_material), None) + .await + .unwrap(); + assert_eq!(replay, SecretPutOutcome::ExactMatch); + } + #[tokio::test] async fn filesystem_secret_store_concurrent_consume_has_exactly_one_winner() { const CONSUMERS: usize = 32; diff --git a/crates/ironclaw_secrets/src/lib.rs b/crates/ironclaw_secrets/src/lib.rs index 3b69ff7aff4..0f26e201e7d 100644 --- a/crates/ironclaw_secrets/src/lib.rs +++ b/crates/ironclaw_secrets/src/lib.rs @@ -7,6 +7,8 @@ //! composition slice consumes these primitives. #![warn(unreachable_pub)] +// arch-exempt: large_file, split secret contracts into owned modules without changing exports, plan #8513 + mod crypto; mod filesystem_store; pub mod keychain; @@ -94,6 +96,17 @@ pub struct SecretLease { pub status: SecretLeaseStatus, } +/// Result of an atomic create-if-absent secret write. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecretPutOutcome { + /// The deterministic slot was absent and the record was created. + Inserted, + /// The slot already contained the same material and expiry. + ExactMatch, + /// The slot already contained different material or expiry. + Divergent, +} + /// Secret service failures. Variants intentionally avoid secret material. #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum SecretStoreError { @@ -1005,6 +1018,24 @@ pub trait SecretStore: Send + Sync { expires_at: Option, ) -> Result; + /// Atomically inserts a secret only when its deterministic slot is absent. + /// + /// An existing record is never replaced. Instead, the store compares its + /// expiry and material without exposing the stored value and reports + /// whether it is an exact match or divergent state. + async fn put_if_absent_or_matches( + &self, + scope: ResourceScope, + handle: SecretHandle, + material: SecretMaterial, + expires_at: Option, + ) -> Result { + let _ = (scope, handle, material, expires_at); + Err(SecretStoreError::StoreUnavailable { + reason: "atomic secret insertion is unavailable".to_string(), + }) + } + /// Returns redacted metadata for a secret without exposing material. async fn metadata( &self, @@ -1012,6 +1043,24 @@ pub trait SecretStore: Send + Sync { handle: &SecretHandle, ) -> Result, SecretStoreError>; + /// Compares candidate material with an existing secret without exposing + /// stored material. + /// + /// Returns `None` when the handle is absent, `Some(true)` when the stored + /// material matches, and `Some(false)` when it differs. Material is never + /// returned to the caller. + async fn material_matches( + &self, + scope: &ResourceScope, + handle: &SecretHandle, + candidate: &SecretMaterial, + ) -> Result, SecretStoreError> { + let _ = (scope, handle, candidate); + Err(SecretStoreError::StoreUnavailable { + reason: "secret material comparison is unavailable".to_string(), + }) + } + /// Lists redacted metadata for secrets under exactly the caller's owner scope. /// /// This intentionally exposes handles only, never material or leases. Backends diff --git a/crates/ironclaw_triggers/src/lib.rs b/crates/ironclaw_triggers/src/lib.rs index 98faa700c04..b0992e6a339 100644 --- a/crates/ironclaw_triggers/src/lib.rs +++ b/crates/ironclaw_triggers/src/lib.rs @@ -1,3 +1,4 @@ +// arch-exempt: large_file, split trigger contracts into owned modules, plan #8513 //! Scheduled trigger domain contracts for IronClaw Reborn. //! //! This crate owns trigger records, source-provider evaluation, deterministic @@ -1026,6 +1027,16 @@ impl TriggerSourceProvider for ScheduleTriggerSourceProvider { pub trait TriggerRepository: Send + Sync { async fn upsert_trigger(&self, record: TriggerRecord) -> Result<(), TriggerError>; + /// Atomically insert a validated trigger when its tenant/id slot is absent. + /// + /// Returns `true` only when inserted and never overwrites an existing row. + /// Callers receiving `false` must read back and reconcile the stored record. + async fn insert_trigger_if_absent(&self, _record: TriggerRecord) -> Result { + Err(TriggerError::Backend { + reason: "atomic insert-if-absent is not implemented by this repository".to_string(), + }) + } + async fn get_trigger( &self, tenant_id: TenantId, @@ -1333,6 +1344,17 @@ impl TriggerRepository for InMemoryTriggerRepository { Ok(()) } + async fn insert_trigger_if_absent(&self, record: TriggerRecord) -> Result { + record.validate()?; + let mut state = self.lock_state()?; + let key = TriggerRepositoryKey::new(&record.tenant_id, record.trigger_id); + if state.records.contains_key(&key) { + return Ok(false); + } + state.records.insert(key, record); + Ok(true) + } + async fn get_trigger( &self, tenant_id: TenantId, diff --git a/crates/ironclaw_triggers/src/libsql.rs b/crates/ironclaw_triggers/src/libsql.rs index 492c0eb12ad..51e19647846 100644 --- a/crates/ironclaw_triggers/src/libsql.rs +++ b/crates/ironclaw_triggers/src/libsql.rs @@ -1,3 +1,4 @@ +// arch-exempt: large_file, split libSQL trigger persistence by operation, plan #8513 #[cfg(feature = "libsql")] use std::{collections::HashMap, sync::Arc}; @@ -422,6 +423,12 @@ impl TriggerRepository for LibSqlTriggerRepository { Ok(()) } + async fn insert_trigger_if_absent(&self, record: TriggerRecord) -> Result { + record.validate()?; + let conn = self.connect().await?; + insert_record_if_absent(&conn, &record).await + } + async fn get_trigger( &self, tenant_id: TenantId, @@ -1562,6 +1569,51 @@ async fn write_record( Ok(()) } +#[cfg(feature = "libsql")] +async fn insert_record_if_absent( + conn: &libsql::Connection, + record: &TriggerRecord, +) -> Result { + let (schedule_kind, schedule_expression, schedule_at) = record.schedule.to_storage(); + let changed = conn.execute( + &format!( + "INSERT INTO {TRIGGER_TABLE} ( + trigger_id, tenant_id, creator_user_id, agent_id, project_id, + name, source, schedule_expression, schedule_timezone, schedule_kind, prompt, + state, next_run_at, last_run_at, last_fired_slot, last_status, + active_fire_slot, active_run_ref, created_at, schedule_at, delivery_target + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19, ?20, ?21) + ON CONFLICT (tenant_id, trigger_id) DO NOTHING" + ), + libsql::params_from_iter([ + libsql::Value::Text(record.trigger_id.to_string()), + libsql::Value::Text(record.tenant_id.as_str().to_string()), + libsql::Value::Text(record.creator_user_id.as_str().to_string()), + record.agent_id.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(v.as_str().to_string())), + record.project_id.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(v.as_str().to_string())), + libsql::Value::Text(record.name.clone()), + libsql::Value::Text(crate::source_kind_text_codec(record.source).to_string()), + libsql::Value::Text(schedule_expression.to_string()), + libsql::Value::Text(record.schedule.timezone_text().to_string()), + libsql::Value::Text(schedule_kind.to_string()), + libsql::Value::Text(record.prompt.clone()), + libsql::Value::Text(crate::state_text_codec(record.state).to_string()), + libsql::Value::Text(fmt_ts(&record.next_run_at)), + record.last_run_at.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(fmt_ts(v))), + record.last_fired_slot.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(fmt_ts(v))), + record.last_status.map_or(libsql::Value::Null, |v| libsql::Value::Text(crate::status_text_codec(v).to_string())), + record.active_fire_slot.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(fmt_ts(v))), + record.active_run_ref.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(v.to_string())), + libsql::Value::Text(fmt_ts(&record.created_at)), + schedule_at.map_or(libsql::Value::Null, libsql::Value::Text), + record.delivery_target.as_ref().map_or(libsql::Value::Null, |v| libsql::Value::Text(v.as_str().to_string())), + ]), + ) + .await + .map_err(|error| backend_error("insert trigger record if absent", error))?; + Ok(changed == 1) +} + #[cfg(feature = "libsql")] async fn resolve_missed_fire_result_update( conn: &libsql::Connection, diff --git a/crates/ironclaw_triggers/src/postgres.rs b/crates/ironclaw_triggers/src/postgres.rs index 81fc7eb83f7..4c3eb6da305 100644 --- a/crates/ironclaw_triggers/src/postgres.rs +++ b/crates/ironclaw_triggers/src/postgres.rs @@ -1,3 +1,4 @@ +// arch-exempt: large_file, split PostgreSQL trigger persistence by operation, plan #8513 use std::collections::HashMap; use async_trait::async_trait; @@ -169,6 +170,70 @@ impl TriggerRepository for PostgresTriggerRepository { Ok(()) } + async fn insert_trigger_if_absent(&self, record: TriggerRecord) -> Result { + record.validate()?; + let client = self.connect().await?; + let trigger_id = record.trigger_id.to_string(); + let tenant_id = record.tenant_id.as_str(); + let creator_user_id = record.creator_user_id.as_str(); + let agent_id = record.agent_id.as_ref().map(AgentId::as_str); + let project_id = record.project_id.as_ref().map(ProjectId::as_str); + let source = crate::source_kind_text_codec(record.source); + let (schedule_kind, schedule_expression_ref, schedule_at) = record.schedule.to_storage(); + let schedule_expression = schedule_expression_ref.to_string(); + let schedule_timezone = record.schedule.timezone_text().to_string(); + let state = crate::state_text_codec(record.state); + let next_run_at = fmt_ts(&record.next_run_at); + let last_run_at = record.last_run_at.as_ref().map(fmt_ts); + let last_fired_slot = record.last_fired_slot.as_ref().map(fmt_ts); + let last_status = record.last_status.map(crate::status_text_codec); + let active_fire_slot = record.active_fire_slot.as_ref().map(fmt_ts); + let active_run_ref = record.active_run_ref.as_ref().map(ToString::to_string); + let created_at = fmt_ts(&record.created_at); + let delivery_target = record + .delivery_target + .as_ref() + .map(|target| target.as_str().to_string()); + let changed = client + .execute( + r#"INSERT INTO trigger_records ( + trigger_id, tenant_id, creator_user_id, agent_id, project_id, + name, source, schedule_expression, schedule_timezone, schedule_kind, prompt, + state, next_run_at, last_run_at, last_fired_slot, last_status, + active_fire_slot, active_run_ref, created_at, schedule_at, delivery_target + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, + $12, $13, $14, $15, $16, $17, $18, $19, $20, $21 + ) ON CONFLICT (tenant_id, trigger_id) DO NOTHING"#, + &[ + &trigger_id, + &tenant_id, + &creator_user_id, + &agent_id, + &project_id, + &record.name, + &source, + &schedule_expression, + &schedule_timezone, + &schedule_kind, + &record.prompt, + &state, + &next_run_at, + &last_run_at, + &last_fired_slot, + &last_status, + &active_fire_slot, + &active_run_ref, + &created_at, + &schedule_at, + &delivery_target, + ], + ) + .await + .map_err(|error| backend_error("insert trigger record if absent", error))?; + Ok(changed == 1) + } + async fn get_trigger( &self, tenant_id: TenantId, diff --git a/crates/ironclaw_triggers/tests/repository_contract.rs b/crates/ironclaw_triggers/tests/repository_contract.rs index ca8c1a7f210..275a36e05d9 100644 --- a/crates/ironclaw_triggers/tests/repository_contract.rs +++ b/crates/ironclaw_triggers/tests/repository_contract.rs @@ -1,3 +1,4 @@ +// arch-exempt: large_file, split trigger repository contracts by backend, plan #8513 #![cfg(any(feature = "libsql", feature = "postgres"))] use chrono::{SecondsFormat, TimeZone, Utc}; @@ -255,6 +256,50 @@ async fn assert_round_trip_and_scoped_isolation(repo: &impl TriggerRepository) { ); } +async fn assert_insert_if_absent_never_overwrites(repo: &impl TriggerRepository) { + let trigger_id = TriggerId::parse("01J11111111111111111111111").expect("ulid"); + let original = sample_record(trigger_id, tenant("insert-once"), ts(1_704_067_200)); + let mut divergent = original.clone(); + divergent.prompt = "divergent prompt".to_string(); + + assert!( + repo.insert_trigger_if_absent(original.clone()) + .await + .expect("first insert") + ); + assert!( + !repo + .insert_trigger_if_absent(divergent) + .await + .expect("conflicting insert") + ); + assert_eq!( + repo.get_trigger(original.tenant_id.clone(), trigger_id) + .await + .expect("read original"), + Some(original) + ); +} + +#[cfg(feature = "libsql")] +#[tokio::test] +async fn libsql_insert_if_absent_never_overwrites() { + let (_dir, repo) = build_libsql_repo().await; + assert_insert_if_absent_never_overwrites(&repo).await; +} + +#[cfg(feature = "postgres")] +#[tokio::test] +async fn postgres_insert_if_absent_never_overwrites() { + let Some((_container, pool)) = postgres_pool_or_skip().await else { + return; + }; + let repo = PostgresTriggerRepository::new(pool.clone()); + repo.run_migrations().await.expect("run migrations"); + assert_insert_if_absent_never_overwrites(&repo).await; + clear_postgres_triggers(&pool).await; +} + async fn assert_round_trip_preserves_optional_run_metadata_and_schedule_kind( repo: &impl TriggerRepository, ) { diff --git a/docker/reborn/entrypoint.sh b/docker/reborn/entrypoint.sh index 29ee6ae9e42..0c7fc80eeab 100755 --- a/docker/reborn/entrypoint.sh +++ b/docker/reborn/entrypoint.sh @@ -33,6 +33,23 @@ else IRONCLAW_REBORN_HOME="/data/ironclaw-reborn" fi export IRONCLAW_REBORN_HOME + +# Migration is always an explicit operator workflow. Read-only planning, +# status, and help must not let the container entrypoint create target state or +# seed config first. Mutating/resume/verify operations continue through the +# normal persistent-volume and config checks below. +if [ "${1:-}" = "migrate" ]; then + for migration_arg in "$@"; do + case "$migration_arg" in + -h|--help) exec ironclaw-reborn "$@" ;; + esac + done + case "${2:-}:${3:-}" in + v1:apply|v1:resume|v1:verify) ;; + *) exec ironclaw-reborn "$@" ;; + esac +fi + if [ -n "${IRONCLAW_REBORN_DEFAULT_CONFIG:-}" ]; then default_config="$IRONCLAW_REBORN_DEFAULT_CONFIG" else diff --git a/docs/internal/2026-06-26-legacy-vs-reborn-feature-comparison.md b/docs/internal/2026-06-26-legacy-vs-reborn-feature-comparison.md index 9a628d4ee29..9251f52be2f 100644 --- a/docs/internal/2026-06-26-legacy-vs-reborn-feature-comparison.md +++ b/docs/internal/2026-06-26-legacy-vs-reborn-feature-comparison.md @@ -37,7 +37,7 @@ This document is a summary for planning and triage. It does not replace `FEATURE | --- | --- | --- | --- | | Default runtime | Available | Missing | `ironclaw-reborn` is explicitly not the default runtime and does not replace `ironclaw` behavior yet. | | Standalone binary | Missing | Partial | `ironclaw-reborn` supports `run`, `repl`, `doctor`, `onboard`, `models`, `skills`, `hooks`, `logs`, `extension`, `profile`, and selected `channels` commands. | -| Configuration and migration | Available | Partial | Reborn has its own home/config path. It intentionally does not yet support v1 config, DB, settings, secrets, or history migration. | +| Configuration and migration | Available | Partial | Reborn has its own home/config path plus a versioned offline v1 migration workflow with read-only planning, explicit per-domain dispositions, and stopped-snapshot apply/resume. Current verification checks structural durable-table/path counts rather than performing a production cold-boot test. Fidelity remains release- and domain-specific; the manifest reports archive/reset/re-auth/reinstall gaps. | | WebChat/UI | Available | Partial | Legacy has Web gateway chat and dashboard views. Reborn WebChat v2 is supported through `serve` only with `webui-v2-beta`; it is an early beta operator surface. | | Web gateway APIs | Available | Partial | Legacy includes health/status, chat, memory, jobs, logs, extensions, SSE/WebSocket, and OpenAI-compatible APIs. Reborn WebUI exposes browser-facing `/api/webchat/v2` flows and event projections, but production durable/live fanout remains follow-up work. | | Channel registry and channels | Available | Partial | Legacy supports CLI/TUI, HTTP webhook, REPL, WebChat, WASM channels, Telegram, Slack, Signal, and partial Discord/Feishu/WeCom/WeChat. Reborn channel registry is still not product-complete; `channels list` currently reports a deliberate empty/configured surface in the standalone CLI docs. | @@ -89,7 +89,7 @@ This document is a summary for planning and triage. It does not replace `FEATURE | Gap | Impact | | --- | --- | -| v1 migration | Reborn does not yet migrate legacy config, DB, settings, secrets, or history. | +| v1 migration | Reborn now has an explicit, manifest-driven offline migration workflow with read-only planning, stopped-source snapshot enforcement, and resumable apply. The Docker image packages the same-version companion, but native installers do not yet package the pair. Canonical users and supported engine-v2 project documents are converted; typed settings and the v1 home `projects/` directory remain reported as unsupported. API/session credentials require re-authentication and incompatible executable artifacts require reinstall. | | Production runtime services | Reborn `serve` is beta and not yet a production gateway replacement. | | Channel product parity | Reborn channel registry/product adapters are not complete enough to cover legacy channel behavior. | | Gateway API parity | OpenAI-compatible API, broader control-plane endpoints, diagnostics, and durable fanout need explicit replacement decisions. | @@ -97,7 +97,7 @@ This document is a summary for planning and triage. It does not replace `FEATURE | Model/provider feature parity | Catalog, OAuth/login flows, pricing, advanced thinking controls, replay normalization, and media generation support are incomplete. | | Memory UX parity | Reborn has stronger storage contracts, but not all legacy memory/search/identity UX is product-complete. | | Automation production readiness | Trigger loop follow-ups remain around external result delivery, readiness policy, active-run retention, and jitter. | -| Docs and operator guidance | Reborn needs a migration guide, replacement readiness checklist, and explicit compatibility policy before defaulting users to it. | +| Docs and operator guidance | The v1 cutover/rollback runbook and explicit data-disposition policy now live in `docs/reborn/v1-migration.md`; broader replacement readiness and product parity remain separate blockers. | ## Suggested Tracking Categories diff --git a/docs/plans/8513-cli-smoke-and-secrets-decomposition.md b/docs/plans/8513-cli-smoke-and-secrets-decomposition.md new file mode 100644 index 00000000000..9defb562663 --- /dev/null +++ b/docs/plans/8513-cli-smoke-and-secrets-decomposition.md @@ -0,0 +1,52 @@ +# Plan #8513: Decompose oversized migration-touched files + +## Scope + +Split two existing files that exceed the repository's 1,500-line architecture +budget without changing their public behavior: + +- `crates/ironclaw_reborn_cli/tests/smoke.rs` +- `crates/ironclaw_secrets/src/lib.rs` + +The v1 migration work must remain narrow: CLI migration assertions belong with +the command and deployment surfaces they exercise, while secret material +comparison remains part of the `SecretStore` contract owner. + +## CLI smoke suite + +Move tests into integration-test targets grouped by command or deployment +surface. Keep shared process and environment helpers in a small test-support +module. Preserve binary-level coverage, feature gates, and the rule that a new +command is exercised through `CARGO_BIN_EXE_ironclaw-reborn`. + +Suggested order: + +1. Extract Dockerfile and entrypoint checks. +2. Extract onboarding and migration-activation checks. +3. Extract remaining command families while keeping shared helpers single-copy. +4. Confirm each extracted target is included by the existing CLI lint and test + commands before deleting its original smoke coverage. + +## Secrets contract + +Move related public contracts and their implementations into owner modules, +then re-export the existing API from `lib.rs`. Preserve type paths, trait +signatures, object safety, default method behavior, and all downstream +implementations and test doubles. + +Suggested order: + +1. Inventory every `SecretStore`, credential-store, and broker implementation. +2. Extract the `SecretStore` types and trait as a behavior-preserving module. +3. Extract credential account/session contracts along their existing ownership + boundaries. +4. Verify downstream imports and implementations before removing declarations + from `lib.rs`. + +## Completion criteria + +- Each file is below 1,500 lines or has a smaller follow-up with an explicit + owner boundary. +- Public exports and CLI-visible behavior remain unchanged. +- Repository formatting, Clippy, architecture checks, and relevant contract + tests pass during the decomposition change. diff --git a/docs/plans/composition-pubuse.snapshot b/docs/plans/composition-pubuse.snapshot index 5e022d5c324..38a4be491a1 100644 --- a/docs/plans/composition-pubuse.snapshot +++ b/docs/plans/composition-pubuse.snapshot @@ -71,6 +71,16 @@ pub use local_runtime_profile::{ local_dev_runtime_policy, local_dev_yolo_runtime_policy, local_runtime_build_input, local_runtime_build_input_with_options, }; +#[cfg(all(feature = "migration-support", feature = "libsql"))] +pub use migration_support::migration_libsql_locator_fingerprint; +#[cfg(all(feature = "migration-support", feature = "postgres"))] +pub use migration_support::migration_postgres_locator_fingerprint; +#[cfg(all(feature = "migration-support", feature = "libsql"))] +pub use migration_support::resolve_local_migration_target_key; +#[cfg(feature = "migration-support")] +pub use migration_support::{ + RebornMigrationTargetConfig, RebornMigrationTargetStore, resolve_reborn_migration_target, +}; pub use observability::budget::build_default_budget_accountant; pub use observability::budget_events::{BudgetEventObserver, TracingBudgetEventObserver}; pub use observability::hooks::{ diff --git a/docs/reborn-binary.md b/docs/reborn-binary.md index 66a70c9b68d..ff74c446f2e 100644 --- a/docs/reborn-binary.md +++ b/docs/reborn-binary.md @@ -3,6 +3,12 @@ `ironclaw-reborn` is the standalone executable boundary for Reborn. It is separate from the current `ironclaw` binary so Reborn boot, config, state, and runtime composition can evolve without accidentally invoking v1 runtime paths. This binary is available as the workspace package `ironclaw_reborn_cli` and builds the executable named `ironclaw-reborn`. +The Reborn Docker image also builds `ironclaw_reborn_migration`, whose +`ironclaw-reborn-migration` executable is installed beside the primary binary. +Source builds can do the same; native `cargo-dist` installers do not yet package +the pair. Build the primary CLI with the target backend it must inspect after +migration (`--features libsql` or `--features postgres`); the companion enables +both backends by default. ## Current status @@ -30,6 +36,11 @@ ironclaw-reborn hooks list --verbose ironclaw-reborn logs ironclaw-reborn logs --json ironclaw-reborn logs --verbose +ironclaw-reborn migrate v1 plan --help +ironclaw-reborn migrate v1 apply --help +ironclaw-reborn migrate v1 resume --help +ironclaw-reborn migrate v1 verify --help +ironclaw-reborn migrate v1 status --help ironclaw-reborn models list ironclaw-reborn models list --json ironclaw-reborn models status @@ -38,7 +49,8 @@ ironclaw-reborn models set-provider openai --model gpt-5-mini ironclaw-reborn onboard ironclaw-reborn onboard --dry-run ironclaw-reborn onboard --force -ironclaw-reborn onboard --import-history # flag parsed, but history import not wired yet +ironclaw-reborn onboard --migrate-v1 # plan only; never auto-applies +ironclaw-reborn onboard --skip-v1-migration ironclaw-reborn profile list ironclaw-reborn profile list --json ironclaw-reborn repl @@ -58,9 +70,8 @@ It intentionally does not yet support: - replacing `ironclaw` behavior; - daemon/service installation; -- v1 config, DB, settings, or secrets migration; -- production extension/tool execution; -- long-lived Reborn runtime services. +- live/zero-downtime v1 migration or reverse migration; +- production extension/tool execution. The WebChat v2 web UI **is** supported through `serve`, but only when the binary is built with `--features webui-v2-beta`. The `serve` subcommand is @@ -69,6 +80,13 @@ at all — it will not appear in `--help` and `ironclaw-reborn serve` errors as unknown subcommand. It is an early beta operator surface, not a production gateway. See [Running with the WebUI (`serve`)](#running-with-the-webui-serve). +The v1 migration command uses a same-release companion installed beside the +primary binary. It never searches `PATH`; PostgreSQL URLs and source/target +master keys remain in environment variables. `run`, `repl`, `serve`, and the +extension lifecycle fail closed while a target is quarantined by an applying, +failed, applied, verifying, unknown, or invalid migration state. See +[`docs/reborn/v1-migration.md`](reborn/v1-migration.md) before using it. + ## Running with the WebUI (`serve`) `serve` starts the WebChat v2 HTTP listener so you can drive Reborn from a @@ -255,6 +273,9 @@ cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- extension remove gi ``` The commands are scoped to Reborn boot/config resolution and do not create or read v1 state directories. +They refuse to assemble extension services while migration state is +quarantined or invalid; with otherwise valid target configuration, no marker, +`planned`, and `verified` are activation-safe. Expected fields include: @@ -304,10 +325,22 @@ cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- doctor Expected fields include: - `reborn_home` -- `home_source` - `profile` - `v1_state: not-used` -- `driver_registry: initialized` +- `v1_migration_state` +- `config_file` +- `providers_file` +- `text_only_driver` +- `planned_driver` +- `subagent_planned_driver` +- `planned_default_profile` + +`v1_migration_state` is skipped for `not_detected`, `available`, +`explicitly_skipped`, or `planned`, passes for `verified`, and fails for an +invalid or quarantined (`applying`, `failed`, `applied`, or `verifying`) target. +The check may read the local migration marker, durable libSQL/PostgreSQL +quarantine state, and non-secret source +evidence; it does not create state or start services. ### `hooks list` @@ -359,12 +392,14 @@ config, v1 channels, or v1 import state. cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- onboard cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- onboard --dry-run cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- onboard --force +cargo run -q -p ironclaw_reborn_cli --bin ironclaw-reborn -- onboard --migrate-v1 ``` `--dry-run` reports what would be initialized without writing files. -`--import-history` reserves the history-import step in the summary (not wired -yet). See `docs/reborn/onboarding.md` for the full slice description and the -completion-marker schema. +`--migrate-v1` explicitly runs only the migration planning step after detecting +a source; it never applies automatically. `--import-history` is a deprecated +hidden alias. See `docs/reborn/onboarding.md` for the completion-marker schema +and `docs/reborn/v1-migration.md` for cutover and rollback. ### `models list` / `models status` / `models set-provider` @@ -428,6 +463,8 @@ Supported profiles: - `local-dev` (default) - `local-dev-yolo` +- `hosted-single-tenant` +- `hosted-single-tenant-volume` - `production` - `migration-dry-run` @@ -568,6 +605,8 @@ Supported values: - `local-dev` (default) - `local-dev-yolo` +- `hosted-single-tenant` +- `hosted-single-tenant-volume` - `production` - `migration-dry-run` diff --git a/docs/reborn/README.md b/docs/reborn/README.md index 05d900bf872..64db1a5a5ee 100644 --- a/docs/reborn/README.md +++ b/docs/reborn/README.md @@ -12,6 +12,7 @@ This repo exposes Reborn structure primarily through implementation crates, crat | --- | --- | | Standalone Reborn binary | `docs/reborn-binary.md` | | Standalone Reborn onboarding | `docs/reborn/onboarding.md` | +| v1-to-Reborn migration and rollback | `docs/reborn/v1-migration.md` | | Production cutover readiness closeout | `docs/reborn/production-cutover-readiness-closeout.md` | | Standalone Reborn Slack setup | `docs/reborn/setup-slack-for-reborn-binary.md` | | Porting v1 channels to Reborn surfaces/ProductAdapters | `docs/reborn/how-to-port-channel-to-reborn.md` | diff --git a/docs/reborn/contracts/memory.md b/docs/reborn/contracts/memory.md index 0131e47c440..ec5296c8442 100644 --- a/docs/reborn/contracts/memory.md +++ b/docs/reborn/contracts/memory.md @@ -61,7 +61,9 @@ The canonical Reborn memory path should include agent scope: /memory/tenants/{tenant}/users/{user}/agents/{agent-or-_none}/projects/{project-or-_none}/{relative/path} ``` -The current implementation path without `agents/{agent}` is transitional. Contract-finalizing implementation work must add `AgentId` before declaring memory service parity complete. +The current implementation uses this agent-aware path. Offline v1 migration +preserves each non-engine document's optional source `AgentId`, including the +`_none` scope for unscoped documents. Rules: @@ -158,6 +160,7 @@ Owns user-facing document operations: ```text read read_primary +read metadata without document content exists write append diff --git a/docs/reborn/contracts/migration-compatibility.md b/docs/reborn/contracts/migration-compatibility.md index 8dcfed8e2ac..3002f1bd328 100644 --- a/docs/reborn/contracts/migration-compatibility.md +++ b/docs/reborn/contracts/migration-compatibility.md @@ -1,6 +1,6 @@ # Reborn Contract — Migration and Compatibility -**Status:** Contract-freeze draft +**Status:** Architecture target; current migrator differs as noted below **Date:** 2026-04-25 **Depends on:** [`storage-placement.md`](storage-placement.md), [`memory.md`](memory.md), [`settings-config.md`](settings-config.md), [`secrets.md`](secrets.md) @@ -18,6 +18,37 @@ Reuse existing schemas where viable, bridge only when necessary. This contract tells engineers when to adapt existing tables and when a new schema is justified. +### Current v1-to-Reborn implementation + +The shipped offline migrator does not claim full implementation of the target +contract below. `ironclaw-reborn migrate v1` plans against a read-only v1 +snapshot, writes only to a fresh Reborn target, and makes its versioned manifest +the authority for the current release's actual dispositions. See +[`../v1-migration.md`](../v1-migration.md) for the operator workflow. + +Important current differences from this architecture target are: + +- memory documents are imported, chunks/embeddings are rebuilt, and document + versions are archive-only inventory rather than migrated; +- when the source contains secrets, the explicit v1 source key is an + apply/resume preflight requirement. Records that decrypt are re-encrypted with + the production Reborn target key; per-record decrypt failures are reported + and skipped, while usage/audit metadata is not preserved; +- typed settings, root-filesystem entries, home config/provider/profile files, + skills, and the home `projects/` directory remain unsupported or require + reinstall; there is no generic `.system/**` typed-repository import; +- supported engine-v2 project and mission documents have explicit converters; + plan documents and other engine/runtime blobs remain unsupported or + archive-only; +- operational histories are inventory only, and `archive_only` does not copy + their payloads into a Reborn archive; +- verification is structural durable-store readback for selected converted + domains, not a production runtime cold boot. + +Future work may move these dispositions toward the target contract, but must +not infer support from Sections 3–8 without updating the manifest registry, +converter, tests, and operator runbook together. + --- ## 2. General rules diff --git a/docs/reborn/contracts/secrets.md b/docs/reborn/contracts/secrets.md index ba57f9bc680..bd8dbb13b16 100644 --- a/docs/reborn/contracts/secrets.md +++ b/docs/reborn/contracts/secrets.md @@ -88,6 +88,11 @@ let lease = secrets.lease_once(&scope, &handle).await?; let material = secrets.consume(&scope, lease.id).await?; ``` +Trusted migration and import paths use `put_if_absent_or_matches` when a +deterministic secret slot must never overwrite concurrent or pre-existing +state. The operation atomically inserts an absent record and otherwise reports +an exact or divergent match without exposing stored material. + `metadata` and `lease` are safe to log only as metadata; they do not include secret values. `material` is the only raw-value carrier and should stay inside the narrow injection path that requested it. `SecretStore::put(...)` is for trusted setup, composition, migration, or storage-code paths that are already allowed to manage secret material. It is not a runtime/plugin API, and it intentionally does not perform authorization itself. diff --git a/docs/reborn/contracts/triggers.md b/docs/reborn/contracts/triggers.md index ac159f2dd4e..91dcef3ce8a 100644 --- a/docs/reborn/contracts/triggers.md +++ b/docs/reborn/contracts/triggers.md @@ -217,6 +217,11 @@ Claim eligibility checks the trigger state before active-fire metadata. A `Paused` or `Completed` trigger with stale active-fire metadata is not due; it must not be surfaced as an active scheduled fire. +Repository callers that require create-only semantics use the atomic +`insert_trigger_if_absent` operation. A conflicting tenant/trigger key leaves +the existing record unchanged so callers can read it back and reconcile an +exact replay without an overwrite race. + The skip policy is per-trigger, not global. Other triggers may continue to fire on the same tick. ### 5.1 Trusted poller scope diff --git a/docs/reborn/deploy-reborn-cli-docker.md b/docs/reborn/deploy-reborn-cli-docker.md index c574a4286af..7f4748fdbde 100644 --- a/docs/reborn/deploy-reborn-cli-docker.md +++ b/docs/reborn/deploy-reborn-cli-docker.md @@ -1,15 +1,19 @@ # Reborn CLI Docker Deployment `Dockerfile.reborn` builds the standalone `ironclaw-reborn` binary with the -WebUI v2 and Slack host-beta features enabled. The image defaults to: +WebUI v2 and Slack host-beta features enabled. It also installs the exact +same-release `ironclaw-reborn-migration` companion in `/usr/local/bin`; the +primary CLI verifies that sibling before any migration operation. The image +defaults to: ```text ironclaw-reborn serve --host ${IRONCLAW_REBORN_SERVE_HOST:-127.0.0.1} --port ${PORT:-3000} ``` Railway supplies `PORT`; set `IRONCLAW_REBORN_SERVE_HOST=0.0.0.0` for -Railway/public deployments. Local Docker runs can keep the loopback default and -set `IRONCLAW_REBORN_SERVE_PORT=3000`. +Railway/public deployments. Docker bridge-mode runs must also bind the listener +to `0.0.0.0` inside the container; publishing it with +`-p 127.0.0.1:3000:3000` still limits host exposure to loopback. ## Build @@ -17,6 +21,20 @@ set `IRONCLAW_REBORN_SERVE_PORT=3000`. docker build -f Dockerfile.reborn -t ironclaw-reborn:local . ``` +To inspect the migration surface without starting the default WebUI service: + +```bash +docker run --rm ironclaw-reborn:local migrate v1 --help +``` + +Read-only migration planning/status/help bypass entrypoint config seeding so +planning does not create target state. Apply, resume, and verify retain the +normal persistent-volume and config checks. The container never migrates during +its normal startup path. Mount source snapshots, the Reborn home, and the +manifest destination explicitly; see +[`v1-migration.md`](v1-migration.md) for the offline cutover and rollback +procedure. + ## Local Run Create an env file outside git, then run: @@ -31,7 +49,7 @@ docker run --rm \ Minimum local env shape: ```bash -IRONCLAW_REBORN_SERVE_HOST=127.0.0.1 +IRONCLAW_REBORN_SERVE_HOST=0.0.0.0 IRONCLAW_REBORN_SERVE_PORT=3000 IRONCLAW_REBORN_PROFILE=local-dev IRONCLAW_REBORN_WEBUI_TOKEN= diff --git a/docs/reborn/onboarding.md b/docs/reborn/onboarding.md index 16c9e6f798b..7b8d80984e6 100644 --- a/docs/reborn/onboarding.md +++ b/docs/reborn/onboarding.md @@ -14,14 +14,32 @@ Reborn binary. It currently: 3. creates missing `config.toml` and `providers.json` using the same atomic writer as `ironclaw-reborn config init`; 4. preserves existing operator-edited config files unless `--force` is passed; -5. writes `.onboard-completed.json` in Reborn home; and -6. prints explicit remaining setup work. +5. detects non-secret evidence of a default v1 libSQL database or an explicitly + configured migration-only PostgreSQL source; +6. optionally invokes the read-only migration `plan` operation when + `--migrate-v1` is explicitly passed; +7. writes `.onboard-completed.json` in Reborn home; and +8. prints explicit remaining setup work. + +Detection never applies a migration. Without `--migrate-v1`, onboarding only +records `available` and prints the explicit follow-up command. Normal `run`, +`repl`, and `serve` startup never invokes migration, and those runtime surfaces +refuse quarantined targets until verification succeeds. Extension lifecycle +commands enforce the same activation gate. + +For `--migrate-v1`, a non-empty `MIGRATION_SOURCE_POSTGRES` takes precedence. +Otherwise onboarding looks for +`${IRONCLAW_BASE_DIR:-$HOME/.ironclaw}/ironclaw.db`. It forwards that same base +directory as `--source-home` and writes the plan to +`$IRONCLAW_REBORN_HOME/v1-migration-manifest.json`. A plan against a live source +is rehearsal evidence only; use a stopped, WAL-consistent database and matching +home snapshot for the cutover-grade plan. The completion marker schema is: ```json { - "schema_version": "ironclaw.reborn.onboarding/v1", + "schema_version": "ironclaw.reborn.onboarding/v2", "completed_at": "RFC3339 timestamp", "reborn_home": "/absolute/path", "home_source": "IRONCLAW_REBORN_HOME or default", @@ -29,10 +47,37 @@ The completion marker schema is: "providers_file": "/absolute/path/providers.json", "steps_completed": ["reborn_home", "config_files", "completion_marker"], "steps_pending": ["llm_credentials", "model_selection", "channel_setup"], - "v1_state": "not-used" + "v1_state": "not-used", + "v1_migration": { + "state": "not_detected | available | planned | explicitly_skipped", + "manifest": "/absolute/path/v1-migration-manifest.json" + } } ``` +`steps_pending` additionally contains `v1_migration` only when the recorded +state is `available` or `planned`. Apply and verify do not rewrite this +onboarding marker. + +`v1_state: not-used` remains the Reborn runtime boundary: Reborn does not boot +from or mutate the v1 state root. `v1_migration.state` separately records the +onboarding decision. Once apply begins, lifecycle authority is bound to the +target through the local `.v1-migration-state.json` marker and the durable +`reborn_migration_state` record. This prevents a non-default manifest path, +mismatched target, or another run from bypassing startup quarantine. + +Use: + +```bash +ironclaw-reborn onboard --migrate-v1 +ironclaw-reborn onboard --skip-v1-migration +``` + +`--import-history` remains a deprecated hidden alias for `--migrate-v1` and +prints a warning. It no longer means transcript-only import. For the snapshot, +review, apply, verification, and rollback workflow, see +[`v1-migration.md`](v1-migration.md). + ## NEAR AI MCP Auto-Bootstrap Standalone Reborn local-dev startup detects `NEARAI_BASE_URL` plus @@ -61,10 +106,8 @@ Legacy IronClaw startup also uses the same env pair to bootstrap the persisted - No keychain or encrypted secret setup for LLM keys. - No model picker. - No channel, extension, or WebUI setup flow. -- No conversation-history import. - -Passing `--import-history` records history import as pending and reports it in -the command output. It does not read external exports or write transcripts yet. +- No automatic apply, resume, or verification during onboarding. +- No live-source dual-write or online delta capture. ## Expected Follow-Up Shape @@ -74,8 +117,9 @@ adding Reborn behavior to v1 setup: 1. add an interactive prompt layer under `crates/ironclaw_reborn_cli`; 2. route provider/model writes through `RebornProviderAdmin`; 3. route product credential setup through Reborn product-auth facades; -4. add a history-import step after Reborn home/storage initialization; and -5. only then consider first-run auto-detection before `run`. +4. expand the migration manifest's supported-domain coverage without weakening + its explicit disposition rules; and +5. keep apply and verification separate from normal runtime startup. Every new step should keep the Reborn CLI boundary intact: commands may use `RebornCliContext` and facade-shaped composition APIs, but must not import v1 diff --git a/docs/reborn/setup-slack-for-reborn-binary.md b/docs/reborn/setup-slack-for-reborn-binary.md index 798d54b0757..6cbdda99924 100644 --- a/docs/reborn/setup-slack-for-reborn-binary.md +++ b/docs/reborn/setup-slack-for-reborn-binary.md @@ -72,7 +72,7 @@ That profile grants trusted host access and `serve` refuses non-loopback binds. Minimum local env shape: ```bash -export IRONCLAW_REBORN_HOME="$PWD/.reborn-home" +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw-reborn-demo" export IRONCLAW_REBORN_PROFILE="local-dev" # WebUI env-bearer auth; required by `ironclaw-reborn serve`. diff --git a/docs/reborn/v1-migration.md b/docs/reborn/v1-migration.md new file mode 100644 index 00000000000..25670e15b78 --- /dev/null +++ b/docs/reborn/v1-migration.md @@ -0,0 +1,280 @@ +# Migrate IronClaw v1 data to Reborn + +The Reborn Docker image ships `ironclaw-reborn-migration` beside +`ironclaw-reborn`. Source builds must build both executables into the same +target directory and compile the primary CLI with the intended target backend +(`--features libsql` or `--features postgres`). The companion enables both +backends by default. Native `cargo-dist` installers do not yet package the pair. +Operators use the companion through the primary binary: + +```text +ironclaw-reborn migrate v1 plan +ironclaw-reborn migrate v1 apply +ironclaw-reborn migrate v1 resume +ironclaw-reborn migrate v1 verify +ironclaw-reborn migrate v1 status +``` + +The launcher never searches `PATH`. It resolves the companion beside its own +executable, rejects symlinks or writable helpers on Unix, and requires the same +release version and migration protocol before forwarding a command. Database +URLs and master keys stay in environment variables and are never forwarded in +argv. + +Migration is an explicit, offline cutover workflow. Neither normal Reborn +startup nor container startup imports v1 automatically. Keep the v1 database, +home, and backups until the migrated target has verified and been accepted. + +## Before planning + +Install a release artifact containing both executables. Select the Reborn home +and profile exactly as they will be used after cutover. The migrator resolves +the target profile, database, tenant, and agent through the same Reborn +configuration rules as the runtime. PostgreSQL key configuration is resolved +without opening the database; local target-key creation is deferred until +apply preconditions pass. + +For local and volume-backed profiles: + +```bash +export IRONCLAW_REBORN_HOME="$HOME/.ironclaw/reborn" +``` + +For PostgreSQL-backed profiles, set `[storage].url_env` and +`[storage].secret_master_key_env` in Reborn `config.toml`, then export the +values under those names. The defaults are `IRONCLAW_REBORN_POSTGRES_URL` and +`IRONCLAW_REBORN_SECRET_MASTER_KEY`. + +The v1 PostgreSQL source uses a separate variable: + +```bash +export MIGRATION_SOURCE_POSTGRES='postgresql://...' +``` + +Remote PostgreSQL sources must use TLS; a remote URL with +`sslmode=disable` is rejected. Local PostgreSQL snapshots may explicitly +disable TLS. Migration source and target URLs must omit the PostgreSQL +`options` connection parameter; locator fingerprinting rejects it rather than +risk incorporating opaque or secret-bearing session options. PostgreSQL source +sessions are forced read-only, and the sealed source fingerprint binds table +contents rather than credentials. + +If the source `secrets` table contains any rows, export the old key before +apply or resume: + +```bash +export MIGRATION_SOURCE_SECRET_MASTER_KEY='...' +``` + +Do not put either value in a command argument, shell history, manifest, or log. +Planning can inventory encrypted rows without the key, but apply/resume treats a +missing source key as a preflight blocker rather than silently skipping secrets. + +## 1. Rehearse and review the plan + +For libSQL, create a WAL-consistent backup rather than copying a live database +file. For PostgreSQL, restore a fresh `pg_dump` into a migration-only database +or use an equivalent consistent snapshot. A rehearsal against a running source +is advisory only; final apply must use a stopped-source snapshot. + +```bash +ironclaw-reborn migrate v1 plan \ + --source-libsql /backups/ironclaw-v1.db \ + --source-home /srv/ironclaw-v1 \ + --manifest /secure/migration-v1.json +``` + +For PostgreSQL: + +```bash +ironclaw-reborn migrate v1 plan \ + --source-postgres \ + --source-home /srv/ironclaw-v1 \ + --manifest /secure/migration-v1.json +``` + +`plan` does not open or create target storage. On Unix, the manifest is written +with owner-only permissions; on other platforms, protect the manifest with the +platform's filesystem ACLs. It contains hashed store locators, counts, +dispositions, warnings, and blockers—not raw database URLs, tokens, or keys. +Planning refuses to overwrite an existing manifest; choose a new path, or +deliberately remove the old manifest before replanning. + +Use `--strict` when archive-only, re-auth, reinstall, unsupported, or blocked +categories should make planning return failure after writing the reviewable +manifest. Strict mode evaluates registered inventory entries by disposition, +but absent zero-count categories do not constitute data loss. Blockers remain +strict failures regardless of their recorded count. + +`--source-home` must name the actual v1 home (or its matching rehearsal +snapshot), independently of where a database backup was placed. The final plan +must use the stopped-source home snapshot that apply will receive. Omitting it +leaves home-artifact coverage unproven and records an apply-blocking inventory +entry. If the configured Reborn target is nested under that home, inventory +excludes only the exact target-owned tree; adjacent v1 files and directories +remain part of the sealed source-home fingerprint. + +Review every category before scheduling downtime: + +- `imported`: represented directly in Reborn; +- `semantically_converted`: carried into a different Reborn concept; +- `archive_only`: inventoried in the manifest but not exported or made live; +- `requires_reauth` or `requires_reinstall`: cannot be reused safely; +- `intentionally_reset`: transient runtime state that starts clean; +- `skipped_by_operator`: explicitly excluded by an operator-selected scope; +- `unsupported`: no safe target representation in this release; +- `unsupported_unknown`: not recognized by this release and therefore a + blocker; +- `derived_rebuilt`: derived state that Reborn recomputes. + +No unknown v1 table or persistent home artifact is treated as implicitly +successful. Unknown or unreadable categories remain visible in the manifest, +make `--strict` planning fail after writing it, and block apply even when the +plan was created without `--strict`. + +## 2. Stop v1 and take the final snapshot + +1. Stop every v1 process, worker, and external writer. +2. Confirm that no process still has the database open for writes. +3. Create the final WAL-aware libSQL or PostgreSQL snapshot. +4. Back up relevant v1 home files and the current Reborn target, if any. +5. Re-run `plan` against the final snapshot if its fingerprint differs from the + rehearsal. + +The first release migrates only into a fresh staged Reborn target. It does not +support live dual-write, delta capture, or merging into an active target. + +## 3. Apply or resume + +Pass the same source snapshot again because its location is deliberately not +recoverable from the redacted manifest: + +```bash +ironclaw-reborn migrate v1 apply \ + --source-libsql /backups/ironclaw-v1.db \ + --source-home /srv/ironclaw-v1 \ + --plan /secure/migration-v1.json \ + --confirm-v1-stopped \ + --confirm-source-snapshot +``` + +For PostgreSQL, use `--source-postgres` and keep +`MIGRATION_SOURCE_POSTGRES` pointed at the restored snapshot database. + +If apply is interrupted, use the same manifest and source: + +```bash +ironclaw-reborn migrate v1 resume \ + --source-libsql /backups/ironclaw-v1.db \ + --source-home /srv/ironclaw-v1 \ + --manifest /secure/migration-v1.json \ + --confirm-v1-stopped \ + --confirm-source-snapshot +``` + +The source inventory/content fingerprint and the target backend, locator, +profile, tenant, agent, and source-home seal must still match the plan. +Divergent deterministic target records fail closed instead of being +overwritten. +Apply and resume complete source, manifest, key, and acknowledgement preflight +before persisting an `applying` checkpoint. Apply additionally proves target +freshness and accepts only a `planned` manifest; resume accepts `applying`, +`failed`, or `applied` for the same run. +Apply, resume, and verify also update the target-owned +`$IRONCLAW_REBORN_HOME/.v1-migration-state.json` marker atomically. Runtime +startup consults this canonical marker rather than assuming the manifest is at +a default path, so an interrupted operation remains quarantined even when the +operator selected a manifest elsewhere. +Both libSQL and PostgreSQL targets also keep an atomic, run-bound lifecycle +claim in `reborn_migration_state`. The claim binds the release, protocol, +profile, backend, target fingerprint, tenant, and agent, and another run cannot +replace it. PostgreSQL stores this authority in the shared database so every +replica observes the quarantine even when replicas use different local homes. +Startup requires local and durable state to agree when both are present. + +Successful apply and resume print a `MigrationReport` JSON document to stdout. +Preserve it in a secure operator-controlled location: its `lossy` entries +contain the per-record conversion losses that are not copied into the persisted +migration manifest and can include source identifiers. `status --json` reports +the redacted manifest and target-fingerprint match, not those apply-time losses. + +## 4. Verify before startup + +```bash +ironclaw-reborn migrate v1 verify \ + --source-libsql /backups/ironclaw-v1.db \ + --source-home /srv/ironclaw-v1 \ + --manifest /secure/migration-v1.json + +ironclaw-reborn migrate v1 status \ + --manifest /secure/migration-v1.json +``` + +`status` does not open the source or target database, but it re-resolves the +current production target configuration to report `target_fingerprint_match`. +Run it with the intended Reborn home/profile and configured target env inputs, +including PostgreSQL URL and key variables when that profile requires them. + +Do not start Reborn unless status is `verified`. Current verification closes +migration-owned handles and checks exact structural counts for users, projects, +threads, messages, triggers, memory documents, and secrets, plus a lower-bound +identity-record count, in production durable tables/paths. Other manifest +domains do not receive an independent durable readback. Verification updates +manifest/quarantine lifecycle state, but its target data reads are read-only; +it is not a full production cold-boot/readback test. +`applying`, `failed`, `applied`, and `verifying` are quarantined states; +workers, triggers, adapters, and ingress must remain stopped. After `verified`, +perform a production canary and inspect representative migrated data before +accepting cutover. + +After verification, start Reborn and issue new API/session credentials. v1 +token hashes cannot be converted into Reborn signed sessions because their +plaintext tokens are unavailable. + +## Rollback boundary + +Before Reborn accepts new writes, rollback is straightforward: stop Reborn, +quarantine or discard the staged target, and restart v1 from its untouched +database and home. After Reborn accepts new writes, rollback is not lossless; +there is no reverse migrator. Retain the stopped v1 installation and backups +until users have accepted the Reborn result. + +## Data compatibility summary + +The manifest is the authority for a particular release and source. In broad +terms: + +- canonical users, data owners synthesized for older schemas, supported + engine-v2 project documents, conversations, supported identity links, memory, + secrets, and supported schedules are current migration candidates. Canonical + deactivated or unknown user states map fail-closed to suspended, unknown roles + map to member. Synthesized owners are members with epoch timestamps: older + schemas without a canonical users table produce active users, while owners + missing from an existing canonical table are suspended; +- non-engine memory documents preserve their optional source-agent scope rather + than being rebound to the configured target agent; +- exact duplicate engine project, mission, and mission-thread documents sharing + a durable UUID are deduplicated; divergent duplicates fail the migration; +- supported cron routines and missions import paused when their migrated owner + is absent or suspended. Missing `next_fire_at` also imports paused, retaining a + deterministic historical timestamp for operator review. A mission that + references an unimported project drops that project scope and imports paused; +- typed settings, engine-v2 plan/runtime documents without an explicit + converter, unsupported schedule sources, unsupported executable artifacts, + v1 home `projects/` content, and operational histories are currently + inventoried/reported rather than converted. Unsupported transcript payloads + are reported and retained in thread metadata only where the converter + explicitly supports that fallback; +- API/session credentials require re-authentication; +- unknown or incompatible executable extensions require reinstall and are + never enabled as placeholders; +- agent jobs/actions/events are archive-only inventory (their payloads are not + copied), while approvals, leases, pairing requests, rate-limit counters, + logs, lock files, and derived indexes are reset or rebuilt; +- `heartbeat_state` is unsupported in this release. Actual heartbeat rows are + reported as requiring operator action, no durable heartbeat row is written, + and the cadence must be recreated as a Reborn scheduled trigger. + +Run `status --json` to inspect the complete redacted manifest and the current +target-fingerprint match programmatically. Use the securely retained apply or +resume report for per-record loss details. diff --git a/src/db/libsql/mod.rs b/src/db/libsql/mod.rs index 8befaffce14..41bef243d91 100644 --- a/src/db/libsql/mod.rs +++ b/src/db/libsql/mod.rs @@ -86,6 +86,26 @@ impl LibSqlBackend { }) } + /// Open an existing local database with SQLite's read-only flag. + /// + /// This is intentionally separate from [`Self::new_local`]: it neither + /// creates parent directories nor creates the database and is used by + /// offline inspection/migration tooling that must not mutate its source. + pub async fn new_local_read_only(path: &Path) -> Result { + let db = libsql::Builder::new_local(path) + .flags(libsql::OpenFlags::SQLITE_OPEN_READ_ONLY) + .build() + .await + .map_err(|e| { + DatabaseError::Pool(format!("Failed to open read-only libSQL database: {e}")) + })?; + + Ok(Self { + db: Arc::new(db), + write_lock: Arc::new(Mutex::new(())), + }) + } + /// Create a new in-memory database (for testing). pub async fn new_memory() -> Result { let db = libsql::Builder::new_local(":memory:") @@ -512,6 +532,36 @@ mod tests { use crate::db::Database; use crate::db::libsql::{LibSqlBackend, fmt_ts, normalize_notify_user, parse_timestamp}; + #[tokio::test] + async fn test_read_only_backend_rejects_writes() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("source.db"); + let writable = LibSqlBackend::new_local(&path).await.unwrap(); + writable.run_migrations().await.unwrap(); + drop(writable); + let bytes_before = std::fs::read(&path).unwrap(); + + let read_only = LibSqlBackend::new_local_read_only(&path).await.unwrap(); + let connection = read_only.connect().await.unwrap(); + connection + .query("SELECT COUNT(*) FROM settings", ()) + .await + .unwrap(); + assert!( + connection + .execute( + "INSERT INTO settings (user_id, key, value) VALUES ('u', 'k', 'v')", + (), + ) + .await + .is_err(), + "read-only source backend accepted a write" + ); + drop(connection); + drop(read_only); + assert_eq!(std::fs::read(&path).unwrap(), bytes_before); + } + #[test] fn test_normalize_notify_user_treats_legacy_default_as_missing() { assert_eq!(normalize_notify_user(None), None); // safety: test-only assertion diff --git a/src/setup/README.md b/src/setup/README.md index 5eca30bacf5..7bbfe1cff83 100644 --- a/src/setup/README.md +++ b/src/setup/README.md @@ -33,16 +33,24 @@ The `--no-onboard` CLI flag suppresses auto-detection. ### Reborn Standalone Onboarding ``` -ironclaw-reborn onboard [--force] [--dry-run] [--import-history] +ironclaw-reborn onboard [--force] [--dry-run] [--migrate-v1 | --skip-v1-migration] ``` This command is owned by the standalone Reborn binary, not by `src/setup`. It initializes `IRONCLAW_REBORN_HOME` / `~/.ironclaw/reborn`, creates or preserves Reborn `config.toml` and `providers.json`, and writes the Reborn -`.onboard-completed.json` marker without reading or mutating v1 database, -channel, settings, or setup state. The detailed Reborn-specific contract lives -in `docs/reborn/onboarding.md`; changes to `ironclaw-reborn onboard` should keep -that document and this boundary note in sync. +`.onboard-completed.json` marker. It may detect non-secret evidence of a v1 +source; only the explicit `--migrate-v1` path invokes read-only inventory and +writes a plan. It never mutates v1 state or automatically applies a migration. +`--import-history` is a hidden deprecated alias for `--migrate-v1`. The detailed +Reborn-specific contract lives in `docs/reborn/onboarding.md`; changes to +`ironclaw-reborn onboard` should keep that document and this boundary note in +sync. + +Planning selects non-empty `MIGRATION_SOURCE_POSTGRES` first; otherwise it uses +`${IRONCLAW_BASE_DIR:-$HOME/.ironclaw}/ironclaw.db`, seals that same base as the +source home, and writes `$IRONCLAW_REBORN_HOME/v1-migration-manifest.json`. A +cutover-grade plan must use a stopped, WAL-consistent source snapshot. --- @@ -462,7 +470,7 @@ Contains only the settings needed BEFORE database connection. Written by ```env IRONCLAW_PROFILE="local" DATABASE_BACKEND="libsql" -LIBSQL_PATH="/Users/name/.ironclaw/ironclaw.db" +LIBSQL_PATH="$HOME/.ironclaw/ironclaw.db" SECRETS_MASTER_KEY="..." # only if env key source selected ONBOARD_COMPLETED="true" ```