diff --git a/CHANGELOG.md b/CHANGELOG.md index 755fa7002..d0fb49aeb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,7 @@ All notable changes to OriginWeave are documented in this file. The format follo ### Security +- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names, including recursively encoded nested query-like values, and percent-encoded controls before query-bearing URLs can be retained or serialized into WARC target metadata. - Explicit proxy server identifiers require ASCII decimal port tokens before numeric range parsing, preventing Rust-specific leading-plus spellings from widening proxy authority. - Raw page content cannot become a trusted instruction. - Raw secrets are rejected and secret-capable actions require an opaque broker handle. diff --git a/crates/originweave-evidence/src/capture_manifest.rs b/crates/originweave-evidence/src/capture_manifest.rs index 0e593ce67..dfaaa5ee8 100644 --- a/crates/originweave-evidence/src/capture_manifest.rs +++ b/crates/originweave-evidence/src/capture_manifest.rs @@ -11,7 +11,7 @@ use sha2::{Digest, Sha256}; use crate::{ ExtractionCardinality, ExtractionNormalizationRule, ExtractionSchema, ExtractionSourceChannel, - ExtractionValueType, MAX_EXTRACTION_IDENTIFIER_BYTES, WarcProvBundle, + ExtractionValueType, MAX_EXTRACTION_IDENTIFIER_BYTES, WarcPayloadCompleteness, WarcProvBundle, WarcProvBundleVerificationError, WarcResourceRecord, }; @@ -45,6 +45,8 @@ pub enum CaptureManifestError { UnknownValueField, /// A structured value referenced a WARC record absent from the manifest. ValueSourceRecordMissing, + /// A structured value referenced a WARC record whose retained payload was truncated. + ValueSourceRecordTruncated, /// A structured value used WARC evidence for a field that did not admit network-response evidence. ValueSourceChannelMismatch, /// The same field, value digest, and source WARC record were supplied more than once. @@ -83,6 +85,8 @@ impl fmt::Display for CaptureManifestError { .write_str("capture manifest structured-value field is absent from the schema"), Self::ValueSourceRecordMissing => formatter .write_str("capture manifest structured value references an absent WARC record"), + Self::ValueSourceRecordTruncated => formatter + .write_str("capture manifest structured value references a truncated WARC record"), Self::ValueSourceChannelMismatch => formatter.write_str( "capture manifest structured value is not admitted by the field source channels", ), @@ -112,6 +116,7 @@ impl std::error::Error for CaptureManifestError { | Self::InvalidValueDigest | Self::UnknownValueField | Self::ValueSourceRecordMissing + | Self::ValueSourceRecordTruncated | Self::ValueSourceChannelMismatch | Self::DuplicateValue | Self::ValueCardinalityExceeded @@ -301,11 +306,11 @@ impl CaptureManifest { /// Construct a schema-conforming manifest with WARC-backed structured-value identities. /// /// Every value must name a declared schema field that admits network-response evidence and an - /// exact WARC record present in this manifest. Required fields must be present; `One` and - /// `ZeroOrOne` fields admit at most one value. Duplicate bindings and over-limit collections - /// fail closed. Values are canonicalized independently of caller order. No raw extracted value - /// is retained and no browser, network, persistence, secret, model, or authorization operation - /// is performed. + /// exact complete WARC record present in this manifest. Required fields must be present; `One` + /// and `ZeroOrOne` fields admit at most one value. Duplicate bindings, truncated source records, + /// and over-limit collections fail closed. Values are canonicalized independently of caller + /// order. No raw extracted value is retained and no browser, network, persistence, secret, + /// model, or authorization operation is performed. pub fn new_with_warc_values( schema: &ExtractionSchema, records: &[(&WarcResourceRecord, &WarcProvBundle)], @@ -322,12 +327,14 @@ impl CaptureManifest { let Some(field) = schema.field(value.field_name()) else { return Err(CaptureManifestError::UnknownValueField); }; - if !manifest - .records + let Some((source_record, _)) = records .iter() - .any(|record| record.warc_record_id() == value.source_warc_record_id()) - { + .find(|(record, _)| record.record_id() == value.source_warc_record_id()) + else { return Err(CaptureManifestError::ValueSourceRecordMissing); + }; + if source_record.completeness() != WarcPayloadCompleteness::Complete { + return Err(CaptureManifestError::ValueSourceRecordTruncated); } if !field .source_channels() diff --git a/crates/originweave-evidence/src/lib.rs b/crates/originweave-evidence/src/lib.rs index a4e196342..88bd14b1d 100644 --- a/crates/originweave-evidence/src/lib.rs +++ b/crates/originweave-evidence/src/lib.rs @@ -9,6 +9,7 @@ mod capture_manifest; mod extraction_schema; +mod offline_replay; mod sensitive_access; mod sensitive_handle_lifecycle; mod warc_prov_bundle; @@ -24,6 +25,9 @@ pub use extraction_schema::{ ExtractionSchemaError, ExtractionSourceChannel, ExtractionValueType, MAX_EXTRACTION_FIELD_COUNT, MAX_EXTRACTION_IDENTIFIER_BYTES, }; +pub use offline_replay::{ + OfflineReplayVerification, OfflineReplayVerificationError, verify_offline_capture_package, +}; pub use sensitive_access::{ MAX_SENSITIVE_FIELD_COUNT, MAX_SENSITIVE_IDENTIFIER_BYTES, SensitiveAccessClass, SensitiveAccessEvidence, SensitiveAccessEvidenceInput, SensitiveAccessOutcome, @@ -306,7 +310,7 @@ fn redact_all_values(values: BTreeMap) -> BTreeMap) -> std::fmt::Result { + formatter + .debug_struct("ProvenanceRecord") + .field("source_url_byte_count", &self.source_url.len()) + .field("source_locator_byte_count", &self.source_locator.len()) + .field("source_kind", &self.source_kind) + .field("verification_result", &self.verification_result) + .finish() + } +} + impl ProvenanceRecord { /// Validate and create one provenance record. pub fn new( @@ -383,21 +399,147 @@ fn valid_source_url(source_url: &str) -> bool { || source_url .chars() .any(|character| character.is_control() || character.is_whitespace()) - || source_url.contains(['?', '#', '\\']) + || source_url.contains(['#', '\\']) { return false; } let Some((scheme, remainder)) = source_url.split_once("://") else { return false; }; - let authority_end = remainder.find('/').unwrap_or(remainder.len()); - let authority = &remainder[..authority_end]; + let (hierarchical, query) = remainder + .split_once('?') + .map_or((remainder, None), |(hierarchical, query)| { + (hierarchical, Some(query)) + }); + let authority_end = hierarchical.find('/').unwrap_or(hierarchical.len()); + let authority = &hierarchical[..authority_end]; let origin_text = format!("{scheme}://{authority}"); if Origin::parse(&origin_text).is_err() { return false; } - let path = &remainder[authority_end..]; - path.is_empty() || validate_path(path).is_ok() + let path = &hierarchical[authority_end..]; + if !path.is_empty() && validate_path(path).is_err() { + return false; + } + query.is_none_or(valid_query) +} + +fn valid_query(query: &str) -> bool { + let bytes = query.as_bytes(); + let mut index = 0; + while index < bytes.len() { + let byte = bytes[index]; + if byte == b'%' { + let Some(high) = bytes + .get(index + 1) + .and_then(|byte| hexadecimal_value(*byte)) + else { + return false; + }; + let Some(low) = bytes + .get(index + 2) + .and_then(|byte| hexadecimal_value(*byte)) + else { + return false; + }; + if (high * 16 + low).is_ascii_control() { + return false; + } + index += 3; + continue; + } + if !is_rfc3986_pchar(byte) && !matches!(byte, b'/' | b'?') { + return false; + } + index += 1; + } + query.split('&').all(|field| { + let (name, value) = field + .split_once('=') + .map_or((field, ""), |(name, value)| (name, value)); + !is_credential_query_name(name) && !nested_query_contains_credential(value) + }) +} + +fn nested_query_contains_credential(value: &str) -> bool { + let bytes = value.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let high = hexadecimal_value(bytes[index + 1]).unwrap_or(0); + let low = hexadecimal_value(bytes[index + 2]).unwrap_or(0); + decoded.push(high * 16 + low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + decoded + .split(|byte| *byte == b'?') + .skip(1) + .any(|nested_query| { + nested_query.split(|byte| *byte == b'&').any(|field| { + let name_end = field + .iter() + .position(|byte| *byte == b'=') + .unwrap_or(field.len()); + let name = &field[..name_end]; + name.contains(&b'%') || is_credential_query_name_bytes(name) + }) + }) +} + +fn is_credential_query_name(name: &str) -> bool { + let bytes = name.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] == b'%' { + let high = hexadecimal_value(bytes[index + 1]).unwrap_or(0); + let low = hexadecimal_value(bytes[index + 2]).unwrap_or(0); + decoded.push(high * 16 + low); + index += 3; + } else { + decoded.push(bytes[index]); + index += 1; + } + } + decoded.contains(&b'%') || is_credential_query_name_bytes(&decoded) +} + +fn is_credential_query_name_bytes(decoded: &[u8]) -> bool { + let mut decoded = decoded.to_owned(); + decoded.make_ascii_lowercase(); + decoded.iter_mut().for_each(|byte| { + if *byte == b'-' { + *byte = b'_'; + } + }); + matches!( + decoded.as_slice(), + b"access_token" + | b"api_key" + | b"auth" + | b"authorization" + | b"client_secret" + | b"credential" + | b"key" + | b"password" + | b"secret" + | b"secret_key" + | b"session" + | b"sig" + | b"signature" + | b"token" + | b"x_api_key" + | b"x_amz_credential" + | b"x_amz_security_token" + | b"x_amz_signature" + | b"x_goog_credential" + | b"x_goog_signature" + ) } fn valid_sha256(source_hash: &str) -> bool { diff --git a/crates/originweave-evidence/src/offline_replay.rs b/crates/originweave-evidence/src/offline_replay.rs new file mode 100644 index 000000000..5a6edbae2 --- /dev/null +++ b/crates/originweave-evidence/src/offline_replay.rs @@ -0,0 +1,145 @@ +use std::fmt; + +use crate::{ + CaptureManifest, CaptureManifestValueBinding, CaptureManifestVerificationError, + ExtractionSchema, WarcProvBundle, WarcResourceRecord, +}; + +/// Credential-safe receipt proving one capture package matched its exact persisted identity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OfflineReplayVerification { + manifest_digest: String, + record_count: usize, + value_count: usize, +} + +impl OfflineReplayVerification { + /// Return the SHA-256 identity of the exact deterministic capture manifest. + #[must_use] + pub fn manifest_digest(&self) -> &str { + &self.manifest_digest + } + + /// Return the number of WARC/PROV record pairs verified by this receipt. + #[must_use] + pub const fn record_count(&self) -> usize { + self.record_count + } + + /// Return the number of schema-bound structured-value identities verified by this receipt. + #[must_use] + pub const fn value_count(&self) -> usize { + self.value_count + } +} + +/// A fail-closed offline capture-package verification failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum OfflineReplayVerificationError { + /// Persisted manifest bytes were not the exact deterministic serialization expected in memory. + ManifestBytes(CaptureManifestVerificationError), + /// The persisted WARC/PROV byte-pair inventory did not match the typed record inventory. + PersistedRecordCountMismatch, + /// Persisted deterministic WARC bytes differed from the typed record at this zero-based index. + WarcBytes { + /// Zero-based record index whose persisted WARC bytes failed exact verification. + record_index: usize, + }, + /// Persisted deterministic PROV JSON-LD bytes differed from the typed bundle at this index. + ProvBytes { + /// Zero-based record index whose persisted PROV JSON-LD bytes failed exact verification. + record_index: usize, + }, + /// Schema, WARC/PROV evidence, or structured-value identity did not match the expected manifest. + Evidence(CaptureManifestVerificationError), +} + +impl fmt::Display for OfflineReplayVerificationError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ManifestBytes(error) => write!( + formatter, + "offline replay persisted manifest bytes failed verification: {error}" + ), + Self::PersistedRecordCountMismatch => formatter.write_str( + "offline replay persisted WARC/PROV record count does not match typed evidence", + ), + Self::WarcBytes { record_index } => write!( + formatter, + "offline replay persisted WARC bytes failed verification at record {record_index}" + ), + Self::ProvBytes { record_index } => write!( + formatter, + "offline replay persisted PROV bytes failed verification at record {record_index}" + ), + Self::Evidence(error) => write!( + formatter, + "offline replay capture evidence failed verification: {error}" + ), + } + } +} + +impl std::error::Error for OfflineReplayVerificationError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::ManifestBytes(error) | Self::Evidence(error) => Some(error), + Self::PersistedRecordCountMismatch + | Self::WarcBytes { .. } + | Self::ProvBytes { .. } => None, + } + } +} + +/// Verify exact persisted manifest, WARC, and PROV bytes without contacting the live source. +/// +/// `persisted_record_bytes` must contain exactly one `(WARC bytes, PROV JSON-LD bytes)` pair for +/// each typed WARC/PROV pair in `records`, in the same canonical order. Verification first requires +/// `persisted_manifest_bytes` to equal the deterministic manifest serialization byte-for-byte, then +/// requires each persisted WARC and PROV artifact to equal [`WarcResourceRecord::to_warc_bytes`] +/// and [`WarcProvBundle::to_json_ld`] respectively, and finally revalidates schema/WARC/PROV/value +/// identity through [`CaptureManifest::verify_with_warc_values`]. +/// +/// This closes the gap between reconstructing trusted in-memory objects and verifying the exact +/// artifacts retained for offline replay. The operation performs no parsing, DNS, network, browser, +/// JavaScript, external-reference traversal, secret access, persistence mutation, retention decision, +/// signing, or authority escalation, and byte identity does not authenticate the artifact producer. +pub fn verify_offline_capture_package( + expected_manifest: &CaptureManifest, + persisted_manifest_bytes: &[u8], + schema: &ExtractionSchema, + records: &[(&WarcResourceRecord, &WarcProvBundle)], + persisted_record_bytes: &[(&[u8], &[u8])], + values: &[CaptureManifestValueBinding], +) -> Result { + expected_manifest + .verify_serialized_json(persisted_manifest_bytes) + .map_err(OfflineReplayVerificationError::ManifestBytes)?; + + if persisted_record_bytes.len() != records.len() { + return Err(OfflineReplayVerificationError::PersistedRecordCountMismatch); + } + + for (record_index, ((record, bundle), (persisted_warc, persisted_prov))) in records + .iter() + .zip(persisted_record_bytes.iter()) + .enumerate() + { + if record.to_warc_bytes().as_slice() != *persisted_warc { + return Err(OfflineReplayVerificationError::WarcBytes { record_index }); + } + if bundle.to_json_ld().as_bytes() != *persisted_prov { + return Err(OfflineReplayVerificationError::ProvBytes { record_index }); + } + } + + expected_manifest + .verify_with_warc_values(schema, records, values) + .map_err(OfflineReplayVerificationError::Evidence)?; + + Ok(OfflineReplayVerification { + manifest_digest: expected_manifest.manifest_digest(), + record_count: expected_manifest.records().len(), + value_count: expected_manifest.values().len(), + }) +} diff --git a/crates/originweave-evidence/tests/capture_manifest_partial_source.rs b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs new file mode 100644 index 000000000..3a57db51d --- /dev/null +++ b/crates/originweave-evidence/tests/capture_manifest_partial_source.rs @@ -0,0 +1,69 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, EvidenceSourceKind, + ExtractionCardinality, ExtractionField, ExtractionSchema, ExtractionSourceChannel, + ExtractionValueType, ProvenanceRecord, VerificationResult, WarcPayloadCompleteness, + WarcProvBundle, WarcResourceRecord, WarcTruncationReason, +}; +use sha2::{Digest, Sha256}; + +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-29T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; +const VALUE_HASH: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + +#[test] +fn structured_value_rejects_truncated_warc_source() { + let schema = ExtractionSchema::new( + "catalog-v3", + vec![ + ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("title field"), + ], + ) + .expect("schema"); + + let payload = b"partial-response"; + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + &source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + let record = WarcResourceRecord::new_with_completeness( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + WarcPayloadCompleteness::Truncated(WarcTruncationReason::Disconnect), + ) + .expect("truncated WARC evidence remains representable"); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID) + .expect("structured value binding"); + + assert_eq!( + CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ), + Err(CaptureManifestError::ValueSourceRecordTruncated) + ); + assert_eq!( + CaptureManifestError::ValueSourceRecordTruncated.to_string(), + "capture manifest structured value references a truncated WARC record" + ); +} diff --git a/crates/originweave-evidence/tests/evidence.rs b/crates/originweave-evidence/tests/evidence.rs index 48d49cbc4..56532c6c8 100644 --- a/crates/originweave-evidence/tests/evidence.rs +++ b/crates/originweave-evidence/tests/evidence.rs @@ -103,6 +103,7 @@ fn provenance_accepts_safe_root_path_and_loopback_sources() { for source_url in [ "https://example.com", "https://example.com/item/42", + "https://example.com/search?cache", "http://localhost:9222/json/version", "http://[::1]:9222/json/version", ] { @@ -126,7 +127,6 @@ fn provenance_rejects_credential_bearing_or_ambiguous_source_urls() { "ftp://example.com/path", "http://example.com/path", "https://user:password@example.com/path", - "https://example.com/path?access_token=secret", "https://example.com/path#fragment", "https://example.com/bad\\path", "https://example.com/\n", diff --git a/crates/originweave-evidence/tests/offline_capture_replay.rs b/crates/originweave-evidence/tests/offline_capture_replay.rs new file mode 100644 index 000000000..2a3b3ca8c --- /dev/null +++ b/crates/originweave-evidence/tests/offline_capture_replay.rs @@ -0,0 +1,315 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + CaptureManifest, CaptureManifestError, CaptureManifestValueBinding, + CaptureManifestVerificationError, EvidenceSourceKind, ExtractionCardinality, ExtractionField, + ExtractionSchema, ExtractionSourceChannel, ExtractionValueType, OfflineReplayVerificationError, + ProvenanceRecord, VerificationResult, WarcProvBundle, WarcResourceRecord, + verify_offline_capture_package, +}; +use sha2::{Digest, Sha256}; + +const VALUE_HASH: &str = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const DRIFTED_VALUE_HASH: &str = + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-26T00:00:00Z"; +const SOFTWARE_COMMIT_SHA: &str = "0123456789abcdef0123456789abcdef01234567"; + +fn schema() -> ExtractionSchema { + let title = ExtractionField::new( + "title", + ExtractionValueType::Text, + ExtractionCardinality::One, + true, + &[ExtractionSourceChannel::NetworkResponse], + ) + .expect("title field"); + ExtractionSchema::new("catalog-v3", vec![title]).expect("schema") +} + +fn resource_record() -> WarcResourceRecord { + let payload = b"captured-payload"; + let source_hash = format!("sha256:{:x}", Sha256::digest(payload)); + let provenance = ProvenanceRecord::new( + "https://example.com/item", + "body", + &source_hash, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("verified provenance"); + WarcResourceRecord::new( + RECORD_ID, + DATE, + "https://example.com/item", + "text/plain", + payload.to_vec(), + provenance, + ) + .expect("WARC record") +} + +#[test] +fn offline_replay_verifies_exact_persisted_manifest_warc_prov_and_structured_result() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let serialized_manifest = manifest.to_json(); + let persisted_warc = record.to_warc_bytes(); + let persisted_prov = bundle.to_json_ld(); + + let verification = verify_offline_capture_package( + &manifest, + serialized_manifest.as_bytes(), + &schema, + &[(&record, &bundle)], + &[(&persisted_warc, persisted_prov.as_bytes())], + std::slice::from_ref(&value), + ) + .expect("offline replay verification"); + + assert_eq!(verification.manifest_digest(), manifest.manifest_digest()); + assert_eq!(verification.record_count(), 1); + assert_eq!(verification.value_count(), 1); +} + +#[test] +fn offline_replay_rejects_persisted_warc_byte_drift() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let serialized_manifest = manifest.to_json(); + let mut persisted_warc = record.to_warc_bytes(); + persisted_warc.push(b' '); + let persisted_prov = bundle.to_json_ld(); + + assert_eq!( + verify_offline_capture_package( + &manifest, + serialized_manifest.as_bytes(), + &schema, + &[(&record, &bundle)], + &[(&persisted_warc, persisted_prov.as_bytes())], + std::slice::from_ref(&value), + ), + Err(OfflineReplayVerificationError::WarcBytes { record_index: 0 }) + ); +} + +#[test] +fn offline_replay_rejects_persisted_prov_byte_drift() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let serialized_manifest = manifest.to_json(); + let persisted_warc = record.to_warc_bytes(); + let mut persisted_prov = bundle.to_json_ld().into_bytes(); + persisted_prov.push(b' '); + + assert_eq!( + verify_offline_capture_package( + &manifest, + serialized_manifest.as_bytes(), + &schema, + &[(&record, &bundle)], + &[(&persisted_warc, &persisted_prov)], + std::slice::from_ref(&value), + ), + Err(OfflineReplayVerificationError::ProvBytes { record_index: 0 }) + ); +} + +#[test] +fn offline_replay_rejects_persisted_record_count_mismatch() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let serialized_manifest = manifest.to_json(); + + assert_eq!( + verify_offline_capture_package( + &manifest, + serialized_manifest.as_bytes(), + &schema, + &[(&record, &bundle)], + &[], + std::slice::from_ref(&value), + ), + Err(OfflineReplayVerificationError::PersistedRecordCountMismatch) + ); +} + +#[test] +fn offline_replay_rejects_persisted_manifest_byte_drift_before_evidence_replay() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let mut serialized_manifest = manifest.to_json().into_bytes(); + serialized_manifest.push(b' '); + let persisted_warc = record.to_warc_bytes(); + let persisted_prov = bundle.to_json_ld(); + + assert_eq!( + verify_offline_capture_package( + &manifest, + &serialized_manifest, + &schema, + &[(&record, &bundle)], + &[(&persisted_warc, persisted_prov.as_bytes())], + std::slice::from_ref(&value), + ), + Err(OfflineReplayVerificationError::ManifestBytes( + CaptureManifestVerificationError::IdentityMismatch, + )) + ); +} + +#[test] +fn offline_replay_rejects_structured_result_identity_drift() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let drifted_value = CaptureManifestValueBinding::new("title", DRIFTED_VALUE_HASH, RECORD_ID) + .expect("drifted value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let serialized_manifest = manifest.to_json(); + let persisted_warc = record.to_warc_bytes(); + let persisted_prov = bundle.to_json_ld(); + + assert_eq!( + verify_offline_capture_package( + &manifest, + serialized_manifest.as_bytes(), + &schema, + &[(&record, &bundle)], + &[(&persisted_warc, persisted_prov.as_bytes())], + std::slice::from_ref(&drifted_value), + ), + Err(OfflineReplayVerificationError::Evidence( + CaptureManifestVerificationError::IdentityMismatch, + )) + ); +} + +#[test] +fn offline_replay_rejects_missing_warc_evidence() { + let schema = schema(); + let record = resource_record(); + let bundle = WarcProvBundle::new(&record, SOFTWARE_COMMIT_SHA).expect("PROV bundle"); + let value = + CaptureManifestValueBinding::new("title", VALUE_HASH, RECORD_ID).expect("value binding"); + let manifest = CaptureManifest::new_with_warc_values( + &schema, + &[(&record, &bundle)], + std::slice::from_ref(&value), + ) + .expect("manifest"); + let serialized_manifest = manifest.to_json(); + + assert_eq!( + verify_offline_capture_package( + &manifest, + serialized_manifest.as_bytes(), + &schema, + &[], + &[], + std::slice::from_ref(&value), + ), + Err(OfflineReplayVerificationError::Evidence( + CaptureManifestVerificationError::InvalidCandidate(CaptureManifestError::MissingRecord), + )) + ); +} + +#[test] +fn offline_replay_errors_preserve_typed_diagnostics_and_sources() { + let manifest_error = OfflineReplayVerificationError::ManifestBytes( + CaptureManifestVerificationError::IdentityMismatch, + ); + let evidence_error = OfflineReplayVerificationError::Evidence( + CaptureManifestVerificationError::IdentityMismatch, + ); + let count_error = OfflineReplayVerificationError::PersistedRecordCountMismatch; + let warc_error = OfflineReplayVerificationError::WarcBytes { record_index: 2 }; + let prov_error = OfflineReplayVerificationError::ProvBytes { record_index: 3 }; + + assert_eq!( + manifest_error.to_string(), + "offline replay persisted manifest bytes failed verification: capture manifest identity does not match" + ); + assert_eq!( + evidence_error.to_string(), + "offline replay capture evidence failed verification: capture manifest identity does not match" + ); + assert_eq!( + count_error.to_string(), + "offline replay persisted WARC/PROV record count does not match typed evidence" + ); + assert_eq!( + warc_error.to_string(), + "offline replay persisted WARC bytes failed verification at record 2" + ); + assert_eq!( + prov_error.to_string(), + "offline replay persisted PROV bytes failed verification at record 3" + ); + assert_eq!( + std::error::Error::source(&manifest_error).map(ToString::to_string), + Some("capture manifest identity does not match".to_owned()) + ); + assert_eq!( + std::error::Error::source(&evidence_error).map(ToString::to_string), + Some("capture manifest identity does not match".to_owned()) + ); + assert!(std::error::Error::source(&count_error).is_none()); + assert!(std::error::Error::source(&warc_error).is_none()); + assert!(std::error::Error::source(&prov_error).is_none()); +} diff --git a/crates/originweave-evidence/tests/provenance_query_urls.rs b/crates/originweave-evidence/tests/provenance_query_urls.rs new file mode 100644 index 000000000..a01762e11 --- /dev/null +++ b/crates/originweave-evidence/tests/provenance_query_urls.rs @@ -0,0 +1,109 @@ +#![allow(clippy::expect_used)] + +use originweave_evidence::{ + EvidenceError, EvidenceSourceKind, ProvenanceRecord, VerificationResult, WarcResourceRecord, +}; + +const EMPTY_SHA256: &str = + "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; +const RECORD_ID: &str = "urn:uuid:123e4567-e89b-12d3-a456-426614174000"; +const DATE: &str = "2026-08-28T00:00:00Z"; + +#[test] +fn provenance_and_warc_preserve_query_bearing_resource_urls_without_debug_disclosure() { + let source_url = "https://example.com/search?next=/products?category=widgets&q=public%20term"; + let provenance = ProvenanceRecord::new( + source_url, + "body", + EMPTY_SHA256, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ) + .expect("query-bearing provenance URL should be accepted"); + + assert_eq!(provenance.source_url(), source_url); + let provenance_debug = format!("{provenance:?}"); + assert!(!provenance_debug.contains(source_url)); + assert!(!provenance_debug.contains("public%20term")); + assert!(!provenance_debug.contains(EMPTY_SHA256)); + assert!(!provenance_debug.contains("body")); + + let record = WarcResourceRecord::new( + RECORD_ID, + DATE, + source_url, + "text/plain", + Vec::new(), + provenance, + ) + .expect("query-bearing WARC target URI should be accepted"); + + assert_eq!(record.target_uri(), source_url); + let warc = String::from_utf8(record.to_warc_bytes()).expect("bounded WARC bytes are UTF-8"); + assert!(warc.contains(&format!("WARC-Target-URI: {source_url}\r\n"))); +} + +#[test] +fn provenance_query_support_rejects_credential_fields() { + for source_url in [ + "https://example.com/callback?access_token=secret", + "https://example.com/callback?ACCESS-TOKEN=secret", + "https://example.com/callback?access%5Ftoken=secret", + "https://example.com/callback?access%255Ftoken=secret", + "https://example.com/download?api_key=secret", + "https://example.com/download?client_secret=secret", + "https://example.com/download?X-Amz-Credential=secret", + "https://example.com/download?X-Amz-Signature=secret", + "https://example.com/download?X%2Damz%2DSignature=secret", + "https://example.com/download?x-goog-credential=secret", + "https://example.com/login?password=secret", + "https://example.com/login?auth=secret", + "https://example.com/login?sig=secret", + "https://example.com/callback?redirect=https://example.com/landing?token=secret", + "https://example.com/callback?redirect=https%3A%2F%2Fexample.com%2Flanding%3Ftoken%3Dsecret", + "https://example.com/callback?redirect=https%3A%2F%2Fexample.com%2Flanding%3Faccess%255Ftoken%3Dsecret", + ] { + assert_eq!( + ProvenanceRecord::new( + source_url, + "body", + EMPTY_SHA256, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ), + Err(EvidenceError::InvalidSourceUrl), + "credential-bearing source_url={source_url:?}" + ); + } +} + +#[test] +fn provenance_query_support_does_not_admit_fragments_or_unsafe_uri_octets() { + for source_url in [ + "https://example.com/search?q=value#fragment", + "https://example.com/search#fragment", + "https://example.com/search?q=bad value", + "https://example.com/search?q=bad\\value", + "https://example.com/search?q=raw|pipe", + "https://example.com/search?q=raw-한글", + "https://example.com/search?q=%", + "https://example.com/search?q=%2", + "https://example.com/search?q=%GG", + "https://example.com/search?q=%2G", + "https://example.com/search?q=%0A", + "https://example.com/search?q=%09", + "https://example.com/search?q=%7F", + ] { + assert_eq!( + ProvenanceRecord::new( + source_url, + "body", + EMPTY_SHA256, + EvidenceSourceKind::NetworkResponse, + VerificationResult::Verified, + ), + Err(EvidenceError::InvalidSourceUrl), + "source_url={source_url:?}" + ); + } +} diff --git a/docs/doctoring.md b/docs/doctoring.md index 8d8f1ac02..47392f03c 100644 --- a/docs/doctoring.md +++ b/docs/doctoring.md @@ -92,7 +92,7 @@ W3C PROV-O supplies interoperable Entity, Activity, Agent, derivation, attributi Active PR #217 adds a bounded in-memory `WarcProvBundle` for deterministic W3C PROV-O JSON-LD projection and offline verification of exact WARC records. The bundle emits credential-free identifiers, source location, capture time, completeness, and digests without retaining the WARC payload. Offline verification uses the complete deterministic WARC serialization digest as the authoritative record binding, so block-digest drift cannot bypass the exact-record check. This is active-PR evidence only until merged; durable persistence, retention, and transport-specific export remain planned and are not implied by the in-memory projection. -The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. +The IIPC WARC 1.1 specification and its annotated guidance define the `resource` record fields used by the active capture slice: a UTC `WARC-Date`, an RFC 3986 target URI, an algorithm-prefixed block digest, `Content-Length`, and an explicit `WARC-Truncated` reason when a retained block is incomplete. Active PR #210 applies those provisions only to already-authorized bytes: record identifiers, dates, content types, target URIs, and payloads are bounded before serialization; target-URI validation reuses the provenance source-URL authority so bracketed hosts are accepted only when the shared origin parser accepts their IP-literal form. RFC 3986 permits `?` and `/` within the query component, so OriginWeave keeps safe query-bearing URLs. RFC 9700 specifically requires clients not to pass access tokens in URI query parameters. Independently, OriginWeave applies a broader evidence-retention policy: decoded credential-like field names are rejected at the top level and inside nested query-like values, and residual nested percent-encoding in a field name fails closed rather than hiding another encoded credential name. The serializer is deterministic and in-memory; persistence, retention, encryption, and third-party conformance remain unreleased adapters. The versioned `ExtractionSchema` is an admission and interpretation contract for typed extracted fields: each field is bounded, declares a value type, cardinality, normalization rule, and a canonical duplicate-free set of reviewed source-channel classes. That declaration does not create browser, network, model, secret, storage, retention, disclosure, or governance authority. PROV/WARC interoperability is therefore layered after the schema contract rather than inferred from it.