Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
101 commits
Select commit Hold shift + click to select a range
f1b65d8
feat(evidence): add bounded WARC resource records
seonghobae Aug 21, 2026
7607c25
test(evidence): cover WARC validation branches
seonghobae Aug 21, 2026
3c6b3ba
test(evidence): reject impossible WARC dates
seonghobae Aug 21, 2026
999979a
fix(evidence): validate WARC calendar dates
seonghobae Aug 21, 2026
a55dd93
test(warc): reject impossible leap-second placement
seonghobae Aug 21, 2026
9fd8c05
fix(warc): reject unverifiable leap-second timestamps
seonghobae Aug 21, 2026
68fa94b
docs(adr): carry extraction binding into WARC stack
seonghobae Aug 21, 2026
a5d0ecd
chore(stack): sync WARC slice with extraction ADR
seonghobae Aug 21, 2026
58ba97a
test(evidence): reject malformed WARC media types
seonghobae Aug 21, 2026
fc0a059
fix(evidence): validate WARC MIME media types
seonghobae Aug 21, 2026
7c0d6e3
style(evidence): apply rustfmt to MIME validator
seonghobae Aug 21, 2026
a93de9e
fix(evidence): satisfy strict MIME validator lint
seonghobae Aug 21, 2026
ba11c69
test(evidence): preserve quoted MIME parameter delimiters
seonghobae Aug 21, 2026
9a73dd7
fix(evidence): preserve quoted MIME parameter syntax
seonghobae Aug 21, 2026
020e241
test(evidence): cover quoted MIME parser branches
seonghobae Aug 21, 2026
a6d765b
test(evidence): cover rejected quoted-pair controls
seonghobae Aug 21, 2026
61c4877
test(evidence): reject ambiguous WARC target URI formatting
seonghobae Aug 21, 2026
adb5ac8
fix(evidence): reject ambiguous WARC target URI formatting
seonghobae Aug 21, 2026
ad1192b
test(evidence): cover WARC target presentation branches
seonghobae Aug 21, 2026
4d34db9
docs(evidence): record WARC target URI presentation hardening
seonghobae Aug 21, 2026
b6fb3a5
style(evidence): apply canonical rustfmt
seonghobae Aug 21, 2026
577346e
test(evidence): require credential-safe WARC debug output
seonghobae Aug 21, 2026
faa2e2d
fix(evidence): redact sensitive WARC debug fields
seonghobae Aug 21, 2026
d17f9c0
style(evidence): apply canonical Rust formatting
seonghobae Aug 21, 2026
f84e6ba
test(evidence): align debug regression with strict clippy
seonghobae Aug 21, 2026
d16e648
test(evidence): classify oversized WARC fields as limits
seonghobae Aug 21, 2026
894dbec
test(evidence): apply canonical formatting
seonghobae Aug 21, 2026
c9af0ca
fix(evidence): preserve WARC field limit errors
seonghobae Aug 21, 2026
a38f9e5
test(evidence): align WARC limit error contract
seonghobae Aug 21, 2026
84642d9
test(evidence): require RFC 3986 WARC target URIs
seonghobae Aug 21, 2026
f4cd9df
test(evidence): cover RFC 3986 target URI syntax
seonghobae Aug 21, 2026
05df0b7
fix(evidence): enforce RFC 3986 WARC target syntax
seonghobae Aug 21, 2026
c00f168
chore(evidence): apply canonical RFC3986 formatting
seonghobae Aug 21, 2026
1609b91
test(evidence): require standard WARC error contract
seonghobae Aug 21, 2026
6bfcadc
fix(evidence): expose standard WARC error contract
seonghobae Aug 21, 2026
920dd19
docs: record WARC error integration
seonghobae Aug 21, 2026
ff1d27a
test(evidence): retain extraction error contract in WARC stack
seonghobae Aug 21, 2026
f5084a6
fix(evidence): converge WARC stack on schema error contract
seonghobae Aug 21, 2026
67dad69
docs(changelog): converge WARC stack schema error contract
seonghobae Aug 21, 2026
e4aeacb
test(evidence): preserve parent source-channel set identity
seonghobae Aug 22, 2026
57db990
fix(evidence): preserve parent extraction source-set identity
seonghobae Aug 22, 2026
20a516e
test(evidence): align extraction error contract with parent
seonghobae Aug 22, 2026
08733c5
chore(evidence): converge WARC stack onto live extraction head
seonghobae Aug 22, 2026
e0a0917
test(evidence): require explicit WARC truncation state
seonghobae Aug 22, 2026
634c4b7
style(evidence): apply canonical rustfmt to WARC truncation regression
seonghobae Aug 22, 2026
9d193df
feat(evidence): preserve explicit WARC truncation state
seonghobae Aug 22, 2026
ce3d054
feat(evidence): export typed WARC truncation contract
seonghobae Aug 22, 2026
b22a262
style(evidence): apply canonical rustfmt to truncation exports
seonghobae Aug 22, 2026
e2fdb93
docs(evidence): record WARC truncation semantics truthfully
seonghobae Aug 22, 2026
9d19c78
docs(evidence): record WARC truncation standard decision
seonghobae Aug 22, 2026
5717508
docs(evidence): integrate ExtractionSchema authority basis
seonghobae Aug 22, 2026
e86cc40
merge(stack): integrate current extraction-schema parent
seonghobae Aug 22, 2026
bb2332b
test(evidence): reject malformed WARC percent encoding
seonghobae Aug 22, 2026
a865207
test(evidence): reach WARC percent validation boundary
seonghobae Aug 22, 2026
af1c29c
fix(evidence): validate WARC percent encoding
seonghobae Aug 22, 2026
0ac2052
test(evidence): reject RFC 3986 gen-delims in WARC paths
seonghobae Aug 22, 2026
c3fd33d
test(evidence): preserve bracketed IPv6 WARC authority
seonghobae Aug 22, 2026
e8a73fd
fix(evidence): enforce WARC path delimiter syntax
seonghobae Aug 22, 2026
f4dba5e
test(evidence): reject nil WARC record identity
seonghobae Aug 23, 2026
65dbad3
fix(evidence): reject nil WARC record identifiers
seonghobae Aug 23, 2026
597f3a6
style(evidence): preserve canonical Rust newline
seonghobae Aug 23, 2026
529d11a
chore(evidence): realign WARC stack to extraction schema
seonghobae Aug 23, 2026
06cdec9
chore(evidence): realign WARC stack to current extraction schema
seonghobae Aug 25, 2026
ddb840b
fix(evidence): restore current prerequisite tree before WARC integration
seonghobae Aug 25, 2026
7851b8b
feat(evidence): restore WARC dependency and ADR bindings
seonghobae Aug 25, 2026
760d5c3
feat(evidence): expose current-stack WARC resource contract
seonghobae Aug 25, 2026
efb5dfa
test(evidence): adapt WARC URI regressions to stricter provenance
seonghobae Aug 25, 2026
3e5174e
Merge efb5dfae50e2a32e74ac92cf76bdcf252dece98c into f658f329c83a106b6…
seonghobae Aug 27, 2026
a18429b
chore(evidence): reconcile WARC slice with current main
seonghobae Aug 27, 2026
a639a30
fix(evidence): redact WARC debug media parameters
seonghobae Aug 28, 2026
9a73cc8
fix(evidence): bound WARC target URI validation
seonghobae Aug 28, 2026
43ad019
test(evidence): require WARC payload provenance binding
seonghobae Aug 28, 2026
bc52f53
style(evidence): canonicalize WARC provenance RED test
seonghobae Aug 28, 2026
baab6e2
fix(evidence): bind WARC payload to verified provenance
seonghobae Aug 28, 2026
2305d8d
test(evidence): bind WARC debug fixtures to payload digests
seonghobae Aug 28, 2026
1c77db8
test(evidence): bind WARC limit fixtures to empty payload
seonghobae Aug 28, 2026
cef0e03
test(evidence): bind WARC URI fixtures to empty payload
seonghobae Aug 28, 2026
cc5f0d7
test(evidence): bind WARC truncation fixtures to retained bytes
seonghobae Aug 28, 2026
717fa4a
test(evidence): bind WARC fixtures to exact payload identity
seonghobae Aug 28, 2026
0572dfd
test(evidence): cover query-bearing provenance URLs
seonghobae Aug 28, 2026
5474b19
test(evidence): retire blanket query rejection
seonghobae Aug 28, 2026
34936a7
fix(evidence): preserve safe query-bearing provenance URLs
seonghobae Aug 28, 2026
5e66715
fix(evidence): satisfy strict query URL lint
seonghobae Aug 28, 2026
2edf09a
test(evidence): close query parser branch coverage
seonghobae Aug 28, 2026
926529c
test(evidence): reject credential-bearing provenance queries
seonghobae Aug 28, 2026
356e063
fix(evidence): reject credential query names
seonghobae Aug 28, 2026
6a0e6ab
test(evidence): cover bare query fields
seonghobae Aug 28, 2026
d83748a
fix(evidence): reject nested credential query values
seonghobae Aug 28, 2026
83459c4
test(evidence): reject double-encoded nested credential names
seonghobae Aug 28, 2026
c8a9376
fix(evidence): reject recursively encoded nested credential names
seonghobae Aug 28, 2026
61243c9
docs(evidence): separate RFC 9700 from credential policy
seonghobae Aug 28, 2026
0341079
fix(evidence): reject encoded query controls
seonghobae Aug 28, 2026
6ecb3b0
test(evidence): reject recursively encoded credential names
seonghobae Aug 28, 2026
9f0117c
fix(evidence): reject recursively encoded credential names
seonghobae Aug 28, 2026
bea6564
style(evidence): preserve rustfmt newline
seonghobae Aug 28, 2026
66f360c
feat(evidence): bind WARC records to PROV JSON-LD
seonghobae Aug 28, 2026
5f59947
docs: mark merged WARC child attribution
seonghobae Aug 28, 2026
92951df
test(evidence): reject recursively encoded query controls
seonghobae Aug 28, 2026
2f7eb11
fix(evidence): reject recursively encoded query controls
seonghobae Aug 28, 2026
6c4794d
fix(evidence): reject recursively encoded query controls
seonghobae Aug 28, 2026
7946dce
test(evidence): cover residual query escapes
seonghobae Aug 28, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ Owns validated task budgets and deterministic cumulative mitigation plans. Platf

### `originweave-evidence`

Owns universally value-redacted network evidence and source-bound provenance records. Generic network records retain only bounded method, canonical origin, unambiguous bounded path, and bounded field names. Body capture, typed metadata values, WARC serialization, object storage, retention, encryption, and legal policy remain future bounded modules.
Owns universally value-redacted network evidence and source-bound provenance records. Generic network records retain only bounded method, canonical origin, unambiguous bounded path, and bounded field names. Active PR #210 adds a bounded in-memory WARC 1.1 `resource` serializer over already-authorized bytes, with target URIs capped by the shared path ceiling and validated by the shared source-URL authority. This is active-PR evidence only; body persistence, retention, encryption, and legal policy remain future bounded modules.

## 6. Planned modules

Expand Down
8 changes: 7 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ All notable changes to OriginWeave are documented in this file. The format follo
- Authoritative product documentation graph spanning PRD, TRD, ADR lifecycle/index, product-wide UML, conceptual ERD, requirement/decision traceability, threat modeling, product-wide test strategy, operability, API/protocol, release/rollback, and current primary-source standards doctoring, with machine-checkable repository contracts that keep conversation-derived future work distinct from protected-main implementation claims.
- Purpose-bound data-governance and privacy baseline that rejects both blanket masking and ambient raw-value propagation, defines field-scoped just-in-time disclosure, opaque-handle/trusted-broker boundaries, model/provider/region policy, retention/deletion/residency/break-glass controls, truthful CSAP/SOC 2 readiness language, and machine-checkable documentation contracts without inventing an OriginWeave-owned production database.
- Proposed product-wide target-architecture ADRs for the Rust control plane, isolated execution modes, typed actions, semantic observation/stale-node authority, prompt-injection and secret separation, resource-governor priority, provenance evidence, browser/protocol adapters, crawler policy, and hourly automation operational closure; these remain Proposed rather than shipped claims until protected review and merge.
- The WARC/PROV child slice from PR #217, now integrated into the unprotected #210 feature branch, adds a bounded in-memory `WarcProvBundle` for deterministic W3C PROV-O JSON-LD projection and offline verification of exact WARC records; durable persistence, retention, and transport-specific export remain planned rather than shipped.
- Connected the active WARC/PROV bundle evidence to the primary standards doctoring record, preserving the in-memory-only and non-shipped durable-adapter boundary.
- Made WARC/PROV offline verification use the complete deterministic WARC digest as the authoritative block-digest binding, preserving fail-closed exact-record checks while keeping the public verifier surface minimal.

### Changed

Expand All @@ -70,6 +73,7 @@ All notable changes to OriginWeave are documented in this file. The format follo

### Security

- Provenance source URLs now reject case-insensitive and percent-encoded credential query-field names, including recursively encoded nested query-like values, and singly or recursively percent-encoded controls before query-bearing URLs can be retained or serialized into WARC target metadata.
- Explicit proxy server identifiers require ASCII decimal port tokens before numeric range parsing, preventing Rust-specific leading-plus spellings from widening proxy authority.
- Raw page content cannot become a trusted instruction.
- Raw secrets are rejected and secret-capable actions require an opaque broker handle.
Expand Down Expand Up @@ -101,5 +105,7 @@ All notable changes to OriginWeave are documented in this file. The format follo
- Hard RAM and VRAM pressure pauses the active agent and rejects new admission; hard VRAM pressure also offloads a resident local model.
- The hourly product agent has no Git metadata or repository authority. A separate post-verification publisher opens one PR and cannot approve or merge it.
- The unprivileged OpenCode user is restricted to loopback egress during model execution, preventing runner-wide allow-listed endpoints from becoming direct source-exfiltration channels.
- WARC resource-record debug output now omits untrusted Content-Type parameters while retaining the full validated media type for WARC serialization.
- Active PR #210 now bounds WARC target URIs at the shared path ceiling and reuses the shared source-URL parser, rejecting non-IP-literal bracketed authorities before provenance comparison; this remains active-PR evidence only.

[Unreleased]: https://github.com/ContextualWisdomLab/OriginWeave/compare/main...HEAD
[Unreleased]: https://github.com/ContextualWisdomLab/OriginWeave/compare/main...HEAD
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions crates/originweave-evidence/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ publish = false

[dependencies]
originweave-core = { path = "../originweave-core" }
sha2 = "=0.10.9"

[lints]
workspace = true
211 changes: 205 additions & 6 deletions crates/originweave-evidence/src/lib.rs
Comment thread
seonghobae marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
mod extraction_schema;
Comment thread
seonghobae marked this conversation as resolved.
mod sensitive_access;
mod sensitive_handle_lifecycle;
mod warc_prov_bundle;
mod warc_resource_record;

pub use extraction_schema::{
ExtractionCardinality, ExtractionField, ExtractionNormalizationRule, ExtractionSchema,
Expand All @@ -24,6 +26,15 @@ pub use sensitive_access::{
pub use sensitive_handle_lifecycle::{
SensitiveHandleLifecycleEvidence, SensitiveHandleLifecycleEvidenceInput,
};
pub use warc_prov_bundle::{
MAX_PROV_SOFTWARE_COMMIT_SHA_BYTES, WarcProvBundle, WarcProvBundleError,
WarcProvBundleVerificationError,
};
pub use warc_resource_record::{
MAX_WARC_CONTENT_TYPE_BYTES, MAX_WARC_DATE_BYTES, MAX_WARC_PAYLOAD_BYTES,
MAX_WARC_RECORD_ID_BYTES, MAX_WARC_TARGET_URI_BYTES, WarcPayloadCompleteness,
WarcResourceRecord, WarcResourceRecordError, WarcTruncationReason,
};

use std::collections::BTreeMap;

Expand Down Expand Up @@ -289,7 +300,7 @@ fn redact_all_values(values: BTreeMap<String, String>) -> BTreeMap<String, Strin
}

/// A provenance pointer from an extracted assertion to its exact evidence.
#[derive(Debug, Clone, PartialEq, Eq)]
#[derive(Clone, PartialEq, Eq)]
pub struct ProvenanceRecord {
source_url: String,
source_locator: String,
Expand All @@ -298,6 +309,18 @@ pub struct ProvenanceRecord {
verification_result: VerificationResult,
}

impl std::fmt::Debug for ProvenanceRecord {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("ProvenanceRecord")
.field("source_url_byte_count", &self.source_url.len())
.field("source_locator_byte_count", &self.source_locator.len())
.field("source_kind", &self.source_kind)
.field("verification_result", &self.verification_result)
.finish()
}
}

impl ProvenanceRecord {
/// Validate and create one provenance record.
pub fn new(
Expand Down Expand Up @@ -366,21 +389,197 @@ fn valid_source_url(source_url: &str) -> bool {
|| source_url
.chars()
.any(|character| character.is_control() || character.is_whitespace())
|| source_url.contains(['?', '#', '\\'])
|| source_url.contains(['#', '\\'])
{
return false;
}
let Some((scheme, remainder)) = source_url.split_once("://") else {
return false;
};
let authority_end = remainder.find('/').unwrap_or(remainder.len());
let authority = &remainder[..authority_end];
let (hierarchical, query) = remainder
.split_once('?')
.map_or((remainder, None), |(hierarchical, query)| {
(hierarchical, Some(query))
});
let authority_end = hierarchical.find('/').unwrap_or(hierarchical.len());
let authority = &hierarchical[..authority_end];
Comment thread
seonghobae marked this conversation as resolved.
let origin_text = format!("{scheme}://{authority}");
if Origin::parse(&origin_text).is_err() {
return false;
}
let path = &remainder[authority_end..];
path.is_empty() || validate_path(path).is_ok()
let path = &hierarchical[authority_end..];
if !path.is_empty() && validate_path(path).is_err() {
return false;
}
query.is_none_or(valid_query)
Comment thread
seonghobae marked this conversation as resolved.
}

fn valid_query(query: &str) -> bool {
let bytes = query.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
let byte = bytes[index];
if byte == b'%' {
let Some(high) = bytes
.get(index + 1)
.and_then(|byte| hexadecimal_value(*byte))
else {
return false;
};
let Some(low) = bytes
.get(index + 2)
.and_then(|byte| hexadecimal_value(*byte))
else {
return false;
};
let decoded_byte = high * 16 + low;
if decoded_byte.is_ascii_control() {
return false;
}
decoded.push(decoded_byte);
index += 3;
continue;
}
if !is_rfc3986_pchar(byte) && !matches!(byte, b'/' | b'?') {
return false;
}
decoded.push(byte);
index += 1;
}
Comment thread
seonghobae marked this conversation as resolved.
if recursively_encoded_control(&decoded) {
return false;
}
query.split('&').all(|field| {
let (name, value) = field
.split_once('=')
.map_or((field, ""), |(name, value)| (name, value));
!is_credential_query_name(name) && !nested_query_contains_credential(value)
})
}

fn recursively_encoded_control(bytes: &[u8]) -> bool {
let mut candidate = bytes.to_owned();
loop {
let mut decoded = Vec::with_capacity(candidate.len());
let mut index = 0;
let mut found_escape = false;
while index < candidate.len() {
if candidate[index] == b'%' {
let Some(high) = candidate
.get(index + 1)
.and_then(|byte| hexadecimal_value(*byte))
else {
decoded.push(candidate[index]);
index += 1;
continue;
};
let Some(low) = candidate
.get(index + 2)
.and_then(|byte| hexadecimal_value(*byte))
else {
decoded.push(candidate[index]);
index += 1;
continue;
};
let decoded_byte = high * 16 + low;
if decoded_byte.is_ascii_control() {
return true;
}
decoded.push(decoded_byte);
index += 3;
found_escape = true;
continue;
}
decoded.push(candidate[index]);
index += 1;
}
if !found_escape {
return false;
}
candidate = decoded;
}
}
Comment thread
seonghobae marked this conversation as resolved.

fn nested_query_contains_credential(value: &str) -> bool {
let bytes = value.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
if bytes[index] == b'%' {
let high = hexadecimal_value(bytes[index + 1]).unwrap_or(0);
let low = hexadecimal_value(bytes[index + 2]).unwrap_or(0);
decoded.push(high * 16 + low);
index += 3;
} else {
decoded.push(bytes[index]);
index += 1;
}
}
decoded
.split(|byte| *byte == b'?')
.skip(1)
.any(|nested_query| {
nested_query.split(|byte| *byte == b'&').any(|field| {
let name_end = field
.iter()
.position(|byte| *byte == b'=')
.unwrap_or(field.len());
let name = &field[..name_end];
name.contains(&b'%') || is_credential_query_name_bytes(name)
})
Comment thread
coderabbitai[bot] marked this conversation as resolved.
})
Comment thread
seonghobae marked this conversation as resolved.
}

fn is_credential_query_name(name: &str) -> bool {
let bytes = name.as_bytes();
let mut decoded = Vec::with_capacity(bytes.len());
let mut index = 0;
while index < bytes.len() {
if bytes[index] == b'%' {
let high = hexadecimal_value(bytes[index + 1]).unwrap_or(0);
let low = hexadecimal_value(bytes[index + 2]).unwrap_or(0);
decoded.push(high * 16 + low);
index += 3;
} else {
decoded.push(bytes[index]);
index += 1;
}
}
decoded.contains(&b'%') || is_credential_query_name_bytes(&decoded)
Comment thread
seonghobae marked this conversation as resolved.
}

fn is_credential_query_name_bytes(decoded: &[u8]) -> bool {
let mut decoded = decoded.to_owned();
decoded.make_ascii_lowercase();
decoded.iter_mut().for_each(|byte| {
if *byte == b'-' {
*byte = b'_';
}
});
Comment thread
seonghobae marked this conversation as resolved.
matches!(
decoded.as_slice(),
b"access_token"
| b"api_key"
| b"auth"
| b"authorization"
| b"client_secret"
| b"credential"
| b"key"
| b"password"
| b"secret"
| b"secret_key"
| b"session"
| b"sig"
| b"signature"
| b"token"
| b"x_api_key"
| b"x_amz_credential"
| b"x_amz_security_token"
| b"x_amz_signature"
| b"x_goog_credential"
| b"x_goog_signature"
)
}

fn valid_sha256(source_hash: &str) -> bool {
Expand Down
Loading
Loading