From 7f73d4ee3acee9f37f154e249064a55acddaa74e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 20:07:18 +0900 Subject: [PATCH 01/11] feat(storage): add executable object-storage contract Close the buyer-visible gap where each product could invent S3 or S3-compatible endpoint, credential, encryption, integrity, retention, and rollback rules. Central .github now owns a provider-neutral object_storage policy plus a fail-closed validator; Naruon keeps its adapter (ContextualWisdomLab/naruon#1364). Closes #1019 --- ARCHITECTURE.md | 10 + CHANGELOG.md | 4 +- docs/doctoring/object-storage-contract.md | 84 ++++ .../CWL_OBJECT_STORAGE_CONTRACT.md | 72 +++ schemas/cwl-object-storage-v1.schema.json | 39 ++ .../cwl-object-storage-v1.example.json | 55 +++ .../ci/validate_object_storage_contract.py | 458 ++++++++++++++++++ tests/test_object_storage_contract.py | 343 +++++++++++++ 8 files changed, 1062 insertions(+), 3 deletions(-) create mode 100644 docs/doctoring/object-storage-contract.md create mode 100644 docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md create mode 100644 schemas/cwl-object-storage-v1.schema.json create mode 100644 schemas/examples/cwl-object-storage-v1.example.json create mode 100644 scripts/ci/validate_object_storage_contract.py create mode 100644 tests/test_object_storage_contract.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 7d2bfb4a41..e1628669b8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -104,6 +104,16 @@ sequenceDiagram - Rust remains the psychometric arithmetic owner. Repair never substitutes Python for scoring math. +## Object-storage governance (2026-08-16) + +Central `.github` publishes a provider-neutral `object_storage` contract. +Naruon and other products keep their own adapters. The executable check is +`scripts/ci/validate_object_storage_contract.py`. HTTPS, exact-host +allowlists, server-side encryption, SHA-256-or-stronger integrity, distinct +lifecycle states, and non-destructive rollback are required. CSAP and SOC 2 +remain design constraints, not certification claims. Operational PII is not +blanket-masked. + ## Quality gates `scripts/ci/` ships with 100% statement/branch coverage and 100% docstrings. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d2f9f24dc..0c32a36925 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Semantic Versioning where the repository publishes a release. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. - Added a dedicated Quarantine Sandbox Runtime hourly caller at minute 14 that targets protected `develop`, dispatches at most one exact-head repair, applies a two-hour same-head retry floor, preserves non-cancelling single-flight execution, and maps only the established scheduler credentials with job-scoped OIDC. - Added a dedicated OriginWeave hourly caller that invokes the product-neutral central scheduler with the exact repository, protected `main` branch, one-dispatch budget, two-hour same-head retry floor, non-cancelling single-flight heartbeat, job-scoped OIDC, and only the established scheduler credentials. +- Added a provider-neutral object-storage contract with an executable fail-closed validator, JSON Schema, naruon-shaped example, and APA 7 doctoring so AWS S3 and S3-compatible endpoints share one HTTPS, exact-host, integrity, retention, and rollback policy (ContextualWisdomLab/.github#1019, ContextualWisdomLab/naruon#1364). - Added a trusted pull-request comment router for `@cwl-noema-review` and review-only `@opencode-agent` dispatches, with an organization sweep, exact-head receipts, repository allowlisting, fixed runners, immutable checkout pins, and a permanent 100% statement/branch/docstring quality gate. - Added exact-base `uv.lock` materialization that reconstructs standalone nested projects with a checksum-pinned official `uv` exporter, isolated frozen/offline execution, strict exact-pin and SHA-256 output validation, and complete Python 3.10/3.14 quality evidence. - Added a permanent exact-head contract workflow for the hourly review-repair scheduler, immutable reusable-workflow source, NVIDIA NIM model boundary, credential isolation, and fail-closed unattended-agent permissions. @@ -28,9 +29,6 @@ Semantic Versioning where the repository publishes a release. - Run the bounded fast-mlsirm repair heartbeat at minute 49 of every hour with one-dispatch scope and a two-hour same-head floor, without weakening true-parameter recovery, CPU/GPU parity, skipped-test, or Rust-ownership gates. - Use NVIDIA NIM `mistralai/mistral-small-4-119b-2603` with explicit high reasoning for scheduled repair and `nvidia/nemotron-3-nano-30b-a3b` for bounded helper work instead of GitHub Models in the write-capable autofix worker. - Apply one NUL-delimited exact-path and complete pre/post-worktree verification contract to both ordinary review repair and merge-conflict repair rather than relying on a visible post-model diff for the ordinary path. - -### Changed - - Avoided the expensive R/testthat failure-summary regular expression on marker-absent bounded logs by checking the required terminal marker first, while preserving fail-closed handling for incomplete or malformed failure evidence. ### Fixed diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md new file mode 100644 index 0000000000..093362af7b --- /dev/null +++ b/docs/doctoring/object-storage-contract.md @@ -0,0 +1,84 @@ +# Object-storage contract: evidence and design record + +검토 기준일: **2026-08-16** + +## Incident / buyer-visible gap + +Sibling products persist customer or evidence objects in AWS S3 or +S3-compatible stores. Without a central executable contract, each repository +can invent endpoint, credential, encryption, integrity, retention, and +rollback rules. That drift is a procurement and data-loss risk for buyers who +must keep documents, even when Naruon already implements a scoped adapter in +ContextualWisdomLab/naruon#1364. + +This record does not implement Naruon storage. It records the organization +policy and the fail-closed check that leaf repositories can cite. + +## Decision + +1. Treat AWS S3 and S3-compatible HTTPS stores as one `object_storage` + capability. +2. Require HTTPS, exact-host allowlists, no wildcards, and no automatic + redirects (CWE-918). +3. Require explicit private-network trust. Implicit RFC1918 or metadata-service + access is not authorized by this contract. +4. Require least-privilege object permissions and prohibit public ACLs, public + buckets, and browser-exposed long-lived credentials (CWE-798; CWE-200). +5. Require server-side encryption and fail-closed content-length plus SHA-256 + or stronger read verification (Amazon Web Services, 2024a, 2024b). +6. Keep lifecycle states distinct. `consumed` does not imply immediate + deletion unless the product explicitly configures zero retention. Legal hold + and archive remain separate from transient reprocessing retention. +7. Refuse rollback that deletes customer data after a partial + migration or backfill. +8. Forbid bucket names, object keys, credentials, and raw PII as unbounded + telemetry labels. This is not a blanket operational PII mask. +9. Record CSAP and SOC 2 only as design constraints. The contract must not + claim certification. +10. Ship an executable validator. Prose alone does not close + ContextualWisdomLab/.github#1019. + +## Trust boundary + +- Central `.github` owns the schema, policy, doctoring record, and validator. +- Product repositories own adapters, buckets, keys, credentials, and evidence + objects. +- NVIDIA NIM / OpenCode credentials are untouched. +- The validator never prints bucket names, object keys, or secrets from a + failing document beyond the field that violated the closed policy. + +## Verification contract + +`tests/test_object_storage_contract.py` proves the checked-in example passes, +the schema keys match production constants, and each fail-closed control has a +unique rejection. Local quality remains 100% statement/branch coverage and +100% docstrings. + +## Rollback + +Revert the validator, schema, example, policy, and this record together. Do +not keep a schema that the executable check no longer enforces. + +## References (APA 7th) + +Amazon Web Services. (2024a). *Checking object integrity for data uploads in +Amazon S3*. Amazon Simple Storage Service User Guide. +https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity-upload.html + +Amazon Web Services. (2024b). *Using server-side encryption with AWS KMS keys +(SSE-KMS)*. Amazon Simple Storage Service User Guide. +https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingKMSEncryption.html + +Amazon Web Services. (n.d.). *Authenticating requests (AWS Signature Version +4)*. Amazon Simple Storage Service API Reference. +https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html + +MITRE. (n.d.-a). *CWE-200: Exposure of sensitive information to an +unauthorized actor*. CWE List. +https://cwe.mitre.org/data/definitions/200.html + +MITRE. (n.d.-b). *CWE-798: Use of hard-coded credentials*. CWE List. +https://cwe.mitre.org/data/definitions/798.html + +MITRE. (n.d.-c). *CWE-918: Server-side request forgery (SSRF)*. CWE List. +https://cwe.mitre.org/data/definitions/918.html diff --git a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md new file mode 100644 index 0000000000..1e13a5cafc --- /dev/null +++ b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md @@ -0,0 +1,72 @@ +# CWL object-storage contract + +This repository owns the reusable `object_storage` policy and the executable +check `scripts/ci/validate_object_storage_contract.py`. Product repositories +keep their own adapters, databases, and evidence objects. Naruon +[ContextualWisdomLab/naruon#1364](https://github.com/ContextualWisdomLab/naruon/pull/1364) +is the current concrete consumer and remains the owner of its document-storage +implementation. + +AWS-managed S3 and S3-compatible HTTPS endpoints are one provider-neutral +capability. They are not an AWS-only product assumption. + +## Closed controls + +A contract JSON document is valid only when every control below is true. + +| Control | Required value | +|---|---| +| Transport | `https` only | +| Hosts | exact-host allowlist; no wildcards; no automatic redirects | +| Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918 access | +| Credentials | scoped secret registry or workload identity; never broadcast, browser-exposed, or ambient process-wide | +| Permissions | least privilege; public ACLs and public buckets prohibited | +| Encryption | server-side encryption `required` | +| Integrity | content length plus SHA-256 or stronger; fail-closed read verification | +| Lifecycle | `pending`, `available`, `consumed`, `archived`, and `held` are distinct; `consumed` does not delete unless zero retention is explicit | +| Rollback | a partial migration must not delete customer data | +| Names | persisted metadata uses multiword `snake_case` | +| Telemetry | bucket names, object keys, credentials, and raw PII are forbidden high-cardinality labels | +| Assurance | CSAP and SOC 2 are design constraints, not certifications | + +Operational PII stays usable through the owning product. Forbidding +high-cardinality labels is not a blanket PII mask. + +## Endpoint classes + +1. **AWS-managed S3 HTTPS endpoints** — `provider_class` is `aws_s3` and every + host is an exact Amazon S3 regional or dual-stack name. +2. **Public S3-compatible HTTPS endpoints** — `provider_class` is + `s3_compatible` and every host is an exact public DNS name. +3. **Authorized private-network endpoints** — `private_network_trust` is + `explicit_allowlist` and each private host is named. Implicit RFC1918, + link-local, or metadata-service access is rejected. + +Workload-identity and instance-metadata access require their own SSRF review. +This contract does not grant that access. + +## Product next actions + +On success the consumer persists or reads the object under its own schema and +records a purpose-bound audit event without bucket, key, credential, or raw PII +labels. On rejection, timeout, duplicate, or partial upload the consumer leaves +the previous durable object in place. Rollback never deletes customer data +because a backfill step only partly succeeded. + +## Verification + +```bash +python3 scripts/ci/validate_object_storage_contract.py \ + --path schemas/examples/cwl-object-storage-v1.example.json +``` + +The checked-in example is the naruon-shaped fixture. A product repository may +commit its own contract file and run the same command in its quality job. The +issue is not closed by prose alone: the executable check is the acceptance +evidence. + +## Schema + +`schemas/cwl-object-storage-v1.schema.json` lists the closed top-level keys. +The Python validator is authoritative when a JSON Schema library is not +installed. diff --git a/schemas/cwl-object-storage-v1.schema.json b/schemas/cwl-object-storage-v1.schema.json new file mode 100644 index 0000000000..a49b7cea99 --- /dev/null +++ b/schemas/cwl-object-storage-v1.schema.json @@ -0,0 +1,39 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ContextualWisdomLab/.github/schemas/cwl-object-storage-v1.schema.json", + "title": "CWL object-storage contract v1", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "capability", + "repository", + "provider_class", + "endpoint_policy", + "credentials", + "permissions", + "encryption", + "integrity", + "lifecycle", + "rollback", + "observability", + "database_object_names", + "assurance_posture" + ], + "properties": { + "schema_version": { "const": "1" }, + "capability": { "const": "object_storage" }, + "repository": { "type": "string", "minLength": 1 }, + "provider_class": { "enum": ["aws_s3", "s3_compatible"] }, + "endpoint_policy": { "type": "object" }, + "credentials": { "type": "object" }, + "permissions": { "type": "object" }, + "encryption": { "type": "object" }, + "integrity": { "type": "object" }, + "lifecycle": { "type": "object" }, + "rollback": { "type": "object" }, + "observability": { "type": "object" }, + "database_object_names": { "const": "multiword_snake_case" }, + "assurance_posture": { "const": "design_constraints_only" } + } +} diff --git a/schemas/examples/cwl-object-storage-v1.example.json b/schemas/examples/cwl-object-storage-v1.example.json new file mode 100644 index 0000000000..7a5f88e037 --- /dev/null +++ b/schemas/examples/cwl-object-storage-v1.example.json @@ -0,0 +1,55 @@ +{ + "schema_version": "1", + "capability": "object_storage", + "repository": "ContextualWisdomLab/naruon", + "provider_class": "s3_compatible", + "endpoint_policy": { + "transport": "https", + "host_allowlist": [ + "s3.ap-northeast-2.amazonaws.com", + "objects.example.example" + ], + "allow_wildcards": false, + "follow_redirects": false, + "private_network_trust": "explicit_allowlist", + "custom_endpoint": "https://objects.example.example" + }, + "credentials": { + "broadcast": false, + "browser_long_lived": false, + "ambient_process_wide": false, + "mechanism": "scoped_secret_registry" + }, + "permissions": { + "public_acls": false, + "public_buckets": false, + "least_privilege": true + }, + "encryption": { + "server_side": "required" + }, + "integrity": { + "content_length": true, + "digest": "sha256", + "fail_closed_read": true + }, + "lifecycle": { + "states": ["pending", "available", "consumed", "archived", "held"], + "consumed_implies_immediate_delete": false, + "zero_retention_explicit": false, + "legal_hold_distinct": true + }, + "rollback": { + "delete_customer_data_on_partial_migration": false + }, + "observability": { + "high_cardinality_labels_forbid": [ + "bucket", + "object_key", + "credential", + "raw_pii" + ] + }, + "database_object_names": "multiword_snake_case", + "assurance_posture": "design_constraints_only" +} diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py new file mode 100644 index 0000000000..266b1ab43d --- /dev/null +++ b/scripts/ci/validate_object_storage_contract.py @@ -0,0 +1,458 @@ +#!/usr/bin/env python3 +"""Fail-closed validator for organization object-storage contracts. + +The central repository owns only the reusable policy and this check. Product +repositories keep their own S3 or S3-compatible adapters. The contract is +provider-neutral: AWS-managed S3 and HTTPS S3-compatible endpoints share one +``object_storage`` capability. CSAP and SOC 2 appear only as design +constraints, never as a certification claim. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + + +SCHEMA_VERSION = "1" +CAPABILITY = "object_storage" +MAX_CONTRACT_BYTES = 65536 +ALLOWED_PROVIDER_CLASSES = frozenset({"aws_s3", "s3_compatible"}) +ALLOWED_CREDENTIAL_MECHANISMS = frozenset( + {"scoped_secret_registry", "workload_identity"} +) +ALLOWED_PRIVATE_NETWORK_TRUST = frozenset({"explicit_allowlist", "denied"}) +ALLOWED_DIGESTS = frozenset({"sha256", "sha384", "sha512"}) +REQUIRED_LIFECYCLE_STATES = ( + "pending", + "available", + "consumed", + "archived", + "held", +) +FORBIDDEN_HIGH_CARDINALITY_LABELS = frozenset( + {"bucket", "object_key", "credential", "raw_pii"} +) +ALLOWED_TOP_LEVEL = ( + "schema_version", + "capability", + "repository", + "provider_class", + "endpoint_policy", + "credentials", + "permissions", + "encryption", + "integrity", + "lifecycle", + "rollback", + "observability", + "database_object_names", + "assurance_posture", +) +ENDPOINT_POLICY_KEYS = ( + "transport", + "host_allowlist", + "allow_wildcards", + "follow_redirects", + "private_network_trust", + "custom_endpoint", +) +CREDENTIAL_KEYS = ( + "broadcast", + "browser_long_lived", + "ambient_process_wide", + "mechanism", +) +PERMISSION_KEYS = ("public_acls", "public_buckets", "least_privilege") +ENCRYPTION_KEYS = ("server_side",) +INTEGRITY_KEYS = ("content_length", "digest", "fail_closed_read") +LIFECYCLE_KEYS = ( + "states", + "consumed_implies_immediate_delete", + "zero_retention_explicit", + "legal_hold_distinct", +) +ROLLBACK_KEYS = ("delete_customer_data_on_partial_migration",) +OBSERVABILITY_KEYS = ("high_cardinality_labels_forbid",) + + +class ObjectStorageContractError(ValueError): + """Raised when a contract file fails a fail-closed policy check.""" + + +def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Build an object while rejecting duplicate JSON keys.""" + seen: set[str] = set() + result: dict[str, Any] = {} + for key, value in pairs: + if key in seen: + raise ObjectStorageContractError(f"duplicate object key {key!r}") + seen.add(key) + result[key] = value + return result + + +def is_exact_dns_host(host: str) -> bool: + """Return whether *host* is one exact DNS name with no wildcards or ports.""" + if not host or not isinstance(host, str): + return False + if any(char in host for char in "*:/@ \t\\"): + return False + if host.startswith(".") or host.endswith("."): + return False + labels = host.split(".") + for label in labels: + if not label or len(label) > 63: + return False + if label.startswith("-") or label.endswith("-"): + return False + if not all(char.isalnum() or char == "-" for char in label): + return False + return True + + +def parse_https_endpoint_host(url: str) -> str: + """Return the exact host of an HTTPS endpoint, rejecting redirects and userinfo.""" + if not url.startswith("https://"): + raise ObjectStorageContractError( + "custom endpoints must use https:// and must not follow redirects" + ) + remainder = url[len("https://") :] + if not remainder or remainder.startswith("/"): + raise ObjectStorageContractError("custom endpoint is missing a host") + authority = remainder.split("/", 1)[0] + if "@" in authority: + raise ObjectStorageContractError( + "custom endpoints must not embed credentials" + ) + host, separator, port = authority.partition(":") + if separator and (not port.isdigit() or port == "0"): + raise ObjectStorageContractError("custom endpoint port is invalid") + if not is_exact_dns_host(host): + raise ObjectStorageContractError( + f"custom endpoint host {host!r} is not an exact DNS name" + ) + return host + + +def load_contract_bytes(raw: bytes) -> dict[str, Any]: + """Parse a bounded UTF-8 JSON object and reject duplicate keys.""" + if len(raw) > MAX_CONTRACT_BYTES: + raise ObjectStorageContractError("contract exceeds 65536 bytes") + if not raw: + raise ObjectStorageContractError("contract is empty") + try: + text = raw.decode("utf-8") + except UnicodeDecodeError as exc: + raise ObjectStorageContractError("contract is not UTF-8") from exc + if "\x00" in text: + raise ObjectStorageContractError("contract contains a NUL byte") + decoder = json.JSONDecoder(object_pairs_hook=reject_duplicate_keys) + try: + data, index = decoder.raw_decode(text) + except json.JSONDecodeError as exc: + raise ObjectStorageContractError( + f"contract is not JSON: {exc.msg}" + ) from exc + if text[index:].strip(): + raise ObjectStorageContractError("contract has trailing JSON data") + if not isinstance(data, dict): + raise ObjectStorageContractError("contract root must be a JSON object") + return data + + +def load_contract_path(path: Path) -> dict[str, Any]: + """Read one contract file through the bounded UTF-8 loader.""" + try: + raw = path.read_bytes() + except OSError as exc: + raise ObjectStorageContractError(f"cannot read {path}: {exc}") from exc + return load_contract_bytes(raw) + + +def require_mapping(value: Any, name: str) -> dict[str, Any]: + """Return *value* when it is an object, otherwise fail closed.""" + if not isinstance(value, dict): + raise ObjectStorageContractError(f"{name} must be an object") + return value + + +def require_bool(value: Any, name: str, expected: bool) -> None: + """Require an exact boolean policy value.""" + if value is not expected: + raise ObjectStorageContractError(f"{name} must be {str(expected).lower()}") + + +def require_text(value: Any, name: str) -> str: + """Return a nonempty string or fail closed.""" + if not isinstance(value, str) or not value: + raise ObjectStorageContractError(f"{name} must be a nonempty string") + return value + + +def require_allowed(value: Any, name: str, allowed: frozenset[str]) -> str: + """Return *value* when it is one of the closed allowed strings.""" + text = require_text(value, name) + if text not in allowed: + raise ObjectStorageContractError( + f"{name} must be one of {sorted(allowed)}" + ) + return text + + +def reject_unknown_keys( + mapping: Mapping[str, Any], allowed: Sequence[str], name: str +) -> None: + """Reject keys that are not in the closed allowlist.""" + unknown = sorted(set(mapping) - set(allowed)) + if unknown: + raise ObjectStorageContractError(f"unknown {name} keys: {unknown}") + + +def validate_endpoint_policy(policy: Mapping[str, Any]) -> None: + """Validate HTTPS, exact-host allowlists, and explicit private-network trust.""" + reject_unknown_keys(policy, ENDPOINT_POLICY_KEYS, "endpoint_policy") + if require_text(policy.get("transport"), "endpoint_policy.transport") != "https": + raise ObjectStorageContractError("endpoint_policy.transport must be https") + require_bool(policy.get("allow_wildcards"), "endpoint_policy.allow_wildcards", False) + require_bool( + policy.get("follow_redirects"), "endpoint_policy.follow_redirects", False + ) + require_allowed( + policy.get("private_network_trust"), + "endpoint_policy.private_network_trust", + ALLOWED_PRIVATE_NETWORK_TRUST, + ) + hosts = policy.get("host_allowlist") + if not isinstance(hosts, list) or not hosts: + raise ObjectStorageContractError( + "endpoint_policy.host_allowlist must be a nonempty list" + ) + seen: set[str] = set() + for host in hosts: + if not isinstance(host, str) or not is_exact_dns_host(host): + raise ObjectStorageContractError( + f"endpoint host {host!r} is not an exact DNS name" + ) + if host in seen: + raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") + seen.add(host) + custom = policy.get("custom_endpoint") + if custom is None: + return + host = parse_https_endpoint_host( + require_text(custom, "endpoint_policy.custom_endpoint") + ) + if host not in seen: + raise ObjectStorageContractError( + "custom endpoint host is not on the exact allowlist" + ) + + +def validate_credentials(credentials: Mapping[str, Any]) -> None: + """Reject broadcast, browser, and ambient process-wide credentials.""" + reject_unknown_keys(credentials, CREDENTIAL_KEYS, "credentials") + require_bool(credentials.get("broadcast"), "credentials.broadcast", False) + require_bool( + credentials.get("browser_long_lived"), + "credentials.browser_long_lived", + False, + ) + require_bool( + credentials.get("ambient_process_wide"), + "credentials.ambient_process_wide", + False, + ) + require_allowed( + credentials.get("mechanism"), + "credentials.mechanism", + ALLOWED_CREDENTIAL_MECHANISMS, + ) + + +def validate_permissions(permissions: Mapping[str, Any]) -> None: + """Require least privilege and prohibit public ACLs or public buckets.""" + reject_unknown_keys(permissions, PERMISSION_KEYS, "permissions") + require_bool(permissions.get("public_acls"), "permissions.public_acls", False) + require_bool( + permissions.get("public_buckets"), "permissions.public_buckets", False + ) + require_bool( + permissions.get("least_privilege"), "permissions.least_privilege", True + ) + + +def validate_encryption(encryption: Mapping[str, Any]) -> None: + """Require server-side encryption without claiming a certification.""" + reject_unknown_keys(encryption, ENCRYPTION_KEYS, "encryption") + if ( + require_text(encryption.get("server_side"), "encryption.server_side") + != "required" + ): + raise ObjectStorageContractError("encryption.server_side must be required") + + +def validate_integrity(integrity: Mapping[str, Any]) -> None: + """Require content-length plus SHA-256 or stronger fail-closed read checks.""" + reject_unknown_keys(integrity, INTEGRITY_KEYS, "integrity") + require_bool(integrity.get("content_length"), "integrity.content_length", True) + require_allowed(integrity.get("digest"), "integrity.digest", ALLOWED_DIGESTS) + require_bool( + integrity.get("fail_closed_read"), "integrity.fail_closed_read", True + ) + + +def validate_lifecycle(lifecycle: Mapping[str, Any]) -> None: + """Keep consumed, legal-hold, and retention states distinct.""" + reject_unknown_keys(lifecycle, LIFECYCLE_KEYS, "lifecycle") + states = lifecycle.get("states") + if not isinstance(states, list): + raise ObjectStorageContractError("lifecycle.states must be a list") + missing = [state for state in REQUIRED_LIFECYCLE_STATES if state not in states] + if missing: + raise ObjectStorageContractError( + f"lifecycle.states is missing required states: {missing}" + ) + if any(not isinstance(state, str) or not state for state in states): + raise ObjectStorageContractError( + "lifecycle.states must contain nonempty strings" + ) + consumed_deletes = lifecycle.get("consumed_implies_immediate_delete") + zero_retention = lifecycle.get("zero_retention_explicit") + if consumed_deletes is True and zero_retention is not True: + raise ObjectStorageContractError( + "consumed must not imply immediate deletion unless zero retention is explicit" + ) + if consumed_deletes is not True and consumed_deletes is not False: + raise ObjectStorageContractError( + "lifecycle.consumed_implies_immediate_delete must be a boolean" + ) + if zero_retention is not True and zero_retention is not False: + raise ObjectStorageContractError( + "lifecycle.zero_retention_explicit must be a boolean" + ) + require_bool( + lifecycle.get("legal_hold_distinct"), + "lifecycle.legal_hold_distinct", + True, + ) + + +def validate_rollback(rollback: Mapping[str, Any]) -> None: + """Refuse rollback that deletes customer data after a partial migration.""" + reject_unknown_keys(rollback, ROLLBACK_KEYS, "rollback") + require_bool( + rollback.get("delete_customer_data_on_partial_migration"), + "rollback.delete_customer_data_on_partial_migration", + False, + ) + + +def validate_observability(observability: Mapping[str, Any]) -> None: + """Forbid high-cardinality bucket, key, credential, and raw-PII labels. + + Operational PII remains usable through the owning product. This check only + blocks unbounded telemetry labels; it is not a blanket PII mask. + """ + reject_unknown_keys(observability, OBSERVABILITY_KEYS, "observability") + labels = observability.get("high_cardinality_labels_forbid") + if not isinstance(labels, list): + raise ObjectStorageContractError( + "observability.high_cardinality_labels_forbid must be a list" + ) + missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - set(labels)) + if missing: + raise ObjectStorageContractError( + "observability must forbid high-cardinality labels: " + f"{missing}" + ) + + +def is_cwl_repository(repository: str) -> bool: + """Return whether *repository* is an owner/name pair in this organization.""" + owner, separator, name = repository.partition("/") + if separator != "/" or owner != "ContextualWisdomLab" or not name: + return False + if "/" in name or name.endswith("."): + return False + return all(char.isalnum() or char in "._-" for char in name) + + +def validate_contract(data: Mapping[str, Any]) -> None: + """Validate one loaded object-storage contract object.""" + reject_unknown_keys(data, ALLOWED_TOP_LEVEL, "contract") + missing = [key for key in ALLOWED_TOP_LEVEL if key not in data] + if missing: + raise ObjectStorageContractError(f"contract is missing keys: {missing}") + if require_text(data.get("schema_version"), "schema_version") != SCHEMA_VERSION: + raise ObjectStorageContractError("schema_version must be 1") + if require_text(data.get("capability"), "capability") != CAPABILITY: + raise ObjectStorageContractError("capability must be object_storage") + repository = require_text(data.get("repository"), "repository") + if not is_cwl_repository(repository): + raise ObjectStorageContractError( + "repository must be ContextualWisdomLab/" + ) + require_allowed( + data.get("provider_class"), "provider_class", ALLOWED_PROVIDER_CLASSES + ) + validate_endpoint_policy( + require_mapping(data.get("endpoint_policy"), "endpoint_policy") + ) + validate_credentials(require_mapping(data.get("credentials"), "credentials")) + validate_permissions(require_mapping(data.get("permissions"), "permissions")) + validate_encryption(require_mapping(data.get("encryption"), "encryption")) + validate_integrity(require_mapping(data.get("integrity"), "integrity")) + validate_lifecycle(require_mapping(data.get("lifecycle"), "lifecycle")) + validate_rollback(require_mapping(data.get("rollback"), "rollback")) + validate_observability( + require_mapping(data.get("observability"), "observability") + ) + if ( + require_text(data.get("database_object_names"), "database_object_names") + != "multiword_snake_case" + ): + raise ObjectStorageContractError( + "database_object_names must be multiword_snake_case" + ) + if ( + require_text(data.get("assurance_posture"), "assurance_posture") + != "design_constraints_only" + ): + raise ObjectStorageContractError( + "assurance_posture must be design_constraints_only; " + "CSAP and SOC 2 are design constraints, not certifications" + ) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + """Parse the validator CLI.""" + parser = argparse.ArgumentParser( + description="Validate a CWL object-storage contract." + ) + parser.add_argument( + "--path", + required=True, + help="Path to the JSON contract file.", + ) + return parser.parse_args(argv) + + +def main(argv: Sequence[str] | None = None) -> int: + """Load and validate one contract path, returning a process exit code.""" + args = parse_args(argv) + try: + validate_contract(load_contract_path(Path(args.path))) + except ObjectStorageContractError as exc: + print(f"object-storage contract failed: {exc}", file=sys.stderr) + return 1 + print("object-storage contract passed") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_object_storage_contract.py b/tests/test_object_storage_contract.py new file mode 100644 index 0000000000..2a55be2836 --- /dev/null +++ b/tests/test_object_storage_contract.py @@ -0,0 +1,343 @@ +"""Contracts for the organization-wide object-storage policy check.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.ci import validate_object_storage_contract as validator + + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLE = ROOT / "schemas" / "examples" / "cwl-object-storage-v1.example.json" +SCHEMA = ROOT / "schemas" / "cwl-object-storage-v1.schema.json" +POLICY = ROOT / "docs" / "object-storage" / "CWL_OBJECT_STORAGE_CONTRACT.md" +DOCTORING = ROOT / "docs" / "doctoring" / "object-storage-contract.md" +CHANGELOG = ROOT / "CHANGELOG.md" +ARCHITECTURE = ROOT / "ARCHITECTURE.md" + + +def _valid_contract() -> dict: + """Return a mutable copy of the checked-in valid example.""" + return json.loads(EXAMPLE.read_text(encoding="utf-8")) + + +def test_checked_in_example_passes_the_executable_policy() -> None: + """The published naruon-shaped example must satisfy the fail-closed check.""" + validator.validate_contract(validator.load_contract_path(EXAMPLE)) + assert validator.main(["--path", str(EXAMPLE)]) == 0 + + +def test_schema_required_keys_match_production_constants() -> None: + """The JSON Schema must not drift from the executable allowlist.""" + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + assert schema["required"] == list(validator.ALLOWED_TOP_LEVEL) + assert set(schema["properties"]) == set(validator.ALLOWED_TOP_LEVEL) + assert schema["additionalProperties"] is False + + +def test_reject_duplicate_keys_and_non_object_root() -> None: + """Duplicate keys and array roots are untrusted evidence.""" + with pytest.raises(validator.ObjectStorageContractError, match="duplicate"): + validator.load_contract_bytes(b'{"schema_version":"1","schema_version":"2"}') + with pytest.raises(validator.ObjectStorageContractError, match="JSON object"): + validator.load_contract_bytes(b"[]") + + +def test_load_contract_bytes_rejects_empty_oversize_encoding_and_trailing() -> None: + """The loader must fail closed before any policy field is trusted.""" + with pytest.raises(validator.ObjectStorageContractError, match="empty"): + validator.load_contract_bytes(b"") + with pytest.raises(validator.ObjectStorageContractError, match="65536"): + validator.load_contract_bytes(b"{" + (b"a" * 65537)) + with pytest.raises(validator.ObjectStorageContractError, match="UTF-8"): + validator.load_contract_bytes(b"\xff\xfe") + with pytest.raises(validator.ObjectStorageContractError, match="NUL"): + validator.load_contract_bytes(b'{"a":"x\x00y"}') + with pytest.raises(validator.ObjectStorageContractError, match="not JSON"): + validator.load_contract_bytes(b"{") + with pytest.raises(validator.ObjectStorageContractError, match="trailing"): + validator.load_contract_bytes(b'{"schema_version":"1"}{}') + + +def test_load_contract_path_reports_unreadable_files(tmp_path: Path) -> None: + """A missing path is a contract failure, not a crash.""" + missing = tmp_path / "absent.json" + with pytest.raises(validator.ObjectStorageContractError, match="cannot read"): + validator.load_contract_path(missing) + assert validator.main(["--path", str(missing)]) == 1 + + +def test_is_exact_dns_host_rejects_wildcards_ports_and_empty_labels() -> None: + """Exact-host allowlists cannot contain wildcard or port syntax.""" + assert validator.is_exact_dns_host("s3.ap-northeast-2.amazonaws.com") + assert validator.is_exact_dns_host("localhost") is True + assert validator.is_exact_dns_host("") is False + assert validator.is_exact_dns_host(None) is False + assert validator.is_exact_dns_host("a..b") is False + assert validator.is_exact_dns_host("*.example.com") is False + assert validator.is_exact_dns_host("example.com:443") is False + assert validator.is_exact_dns_host(".example.com") is False + assert validator.is_exact_dns_host("example.com.") is False + assert validator.is_exact_dns_host("-bad.example.com") is False + assert validator.is_exact_dns_host("bad-.example.com") is False + assert validator.is_exact_dns_host("bad_name.example.com") is False + assert validator.is_exact_dns_host("a" * 64 + ".example.com") is False + + +def test_parse_https_endpoint_host_rejects_http_userinfo_and_bad_ports() -> None: + """Custom endpoints stay HTTPS, credential-free, and exact-host.""" + assert ( + validator.parse_https_endpoint_host("https://objects.example.example/path") + == "objects.example.example" + ) + assert ( + validator.parse_https_endpoint_host("https://objects.example.example:9000") + == "objects.example.example" + ) + with pytest.raises(validator.ObjectStorageContractError, match="https://"): + validator.parse_https_endpoint_host("http://objects.example.example") + with pytest.raises(validator.ObjectStorageContractError, match="missing a host"): + validator.parse_https_endpoint_host("https://") + with pytest.raises(validator.ObjectStorageContractError, match="missing a host"): + validator.parse_https_endpoint_host("https:///bucket") + with pytest.raises(validator.ObjectStorageContractError, match="embed credentials"): + validator.parse_https_endpoint_host("https://user:pass@objects.example.example") + with pytest.raises(validator.ObjectStorageContractError, match="port"): + validator.parse_https_endpoint_host("https://objects.example.example:0") + with pytest.raises(validator.ObjectStorageContractError, match="port"): + validator.parse_https_endpoint_host("https://objects.example.example:abc") + with pytest.raises(validator.ObjectStorageContractError, match="exact DNS"): + validator.parse_https_endpoint_host("https://*.example.example") + + +def test_validate_contract_accepts_zero_retention_only_when_explicit() -> None: + """Consumed-delete is allowed only when the product opts into zero retention.""" + contract = _valid_contract() + contract["lifecycle"]["consumed_implies_immediate_delete"] = True + with pytest.raises(validator.ObjectStorageContractError, match="zero retention"): + validator.validate_contract(contract) + contract["lifecycle"]["zero_retention_explicit"] = True + validator.validate_contract(contract) + + +def test_validate_contract_rejects_policy_regressions() -> None: + """Each required fail-closed control must have a unique rejection.""" + cases = [ + ("schema_version", "2", "schema_version"), + ("capability", "s3", "object_storage"), + ("repository", "other/naruon", "ContextualWisdomLab"), + ("provider_class", "gcs", "provider_class"), + ("database_object_names", "camelCase", "multiword_snake_case"), + ("assurance_posture", "certified", "design_constraints_only"), + ] + for field, value, needle in cases: + contract = _valid_contract() + contract[field] = value + with pytest.raises(validator.ObjectStorageContractError, match=needle): + validator.validate_contract(contract) + + +def test_is_cwl_repository_rejects_nested_and_dot_names() -> None: + """Repository identity is a single owner/name pair in this organization.""" + assert validator.is_cwl_repository("ContextualWisdomLab/naruon") + assert validator.is_cwl_repository("ContextualWisdomLab/.github") + assert not validator.is_cwl_repository("naruon") + assert not validator.is_cwl_repository("ContextualWisdomLab/") + assert not validator.is_cwl_repository("ContextualWisdomLab/naruon/extra") + assert not validator.is_cwl_repository("ContextualWisdomLab/.hidden.") + assert not validator.is_cwl_repository("ContextualWisdomLab/bad name") + + +def test_nested_objects_must_be_objects_and_closed() -> None: + """Unknown nested keys and non-objects fail before a weaker field is read.""" + contract = _valid_contract() + contract["endpoint_policy"] = [] + with pytest.raises(validator.ObjectStorageContractError, match="endpoint_policy"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["extra"] = True + with pytest.raises(validator.ObjectStorageContractError, match="unknown"): + validator.validate_contract(contract) + contract = _valid_contract() + del contract["integrity"] + with pytest.raises(validator.ObjectStorageContractError, match="missing keys"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["unknown"] = True + with pytest.raises(validator.ObjectStorageContractError, match="unknown"): + validator.validate_contract(contract) + + +def test_endpoint_policy_https_exact_hosts_and_no_redirects() -> None: + """HTTPS, exact hosts, and no automatic redirects are mandatory.""" + contract = _valid_contract() + contract["endpoint_policy"]["transport"] = "http" + with pytest.raises(validator.ObjectStorageContractError, match="https"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["allow_wildcards"] = True + with pytest.raises(validator.ObjectStorageContractError, match="allow_wildcards"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["follow_redirects"] = True + with pytest.raises(validator.ObjectStorageContractError, match="follow_redirects"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "rfc1918" + with pytest.raises(validator.ObjectStorageContractError, match="private_network"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["host_allowlist"] = [] + with pytest.raises(validator.ObjectStorageContractError, match="host_allowlist"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["host_allowlist"] = ["*.example.com"] + with pytest.raises(validator.ObjectStorageContractError, match="exact DNS"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["host_allowlist"] = [ + "objects.example.example", + "objects.example.example", + ] + with pytest.raises(validator.ObjectStorageContractError, match="duplicate"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"]["custom_endpoint"] = "https://other.example.example" + with pytest.raises(validator.ObjectStorageContractError, match="allowlist"): + validator.validate_contract(contract) + contract = _valid_contract() + del contract["endpoint_policy"]["custom_endpoint"] + validator.validate_contract(contract) + + +def test_credentials_permissions_encryption_and_integrity_gates() -> None: + """Public buckets, ambient credentials, and weak integrity fail closed.""" + contract = _valid_contract() + contract["credentials"]["broadcast"] = True + with pytest.raises(validator.ObjectStorageContractError, match="broadcast"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["credentials"]["browser_long_lived"] = True + with pytest.raises(validator.ObjectStorageContractError, match="browser"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["credentials"]["ambient_process_wide"] = True + with pytest.raises(validator.ObjectStorageContractError, match="ambient"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["credentials"]["mechanism"] = "os.getenv" + with pytest.raises(validator.ObjectStorageContractError, match="mechanism"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["permissions"]["public_acls"] = True + with pytest.raises(validator.ObjectStorageContractError, match="public_acls"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["permissions"]["public_buckets"] = True + with pytest.raises(validator.ObjectStorageContractError, match="public_buckets"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["permissions"]["least_privilege"] = False + with pytest.raises(validator.ObjectStorageContractError, match="least_privilege"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["encryption"]["server_side"] = "optional" + with pytest.raises(validator.ObjectStorageContractError, match="server_side"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["integrity"]["content_length"] = False + with pytest.raises(validator.ObjectStorageContractError, match="content_length"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["integrity"]["digest"] = "md5" + with pytest.raises(validator.ObjectStorageContractError, match="digest"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["integrity"]["fail_closed_read"] = False + with pytest.raises(validator.ObjectStorageContractError, match="fail_closed_read"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["integrity"]["digest"] = "sha512" + validator.validate_contract(contract) + + +def test_lifecycle_rollback_and_observability_gates() -> None: + """Retention, rollback, and telemetry labels stay fail closed.""" + contract = _valid_contract() + contract["lifecycle"]["states"] = ["pending"] + with pytest.raises(validator.ObjectStorageContractError, match="missing required"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["lifecycle"]["states"] = "pending" + with pytest.raises(validator.ObjectStorageContractError, match="must be a list"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["lifecycle"]["states"] = [ + "pending", + "available", + "consumed", + "archived", + "held", + "", + ] + with pytest.raises(validator.ObjectStorageContractError, match="nonempty"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["lifecycle"]["consumed_implies_immediate_delete"] = "yes" + with pytest.raises(validator.ObjectStorageContractError, match="boolean"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["lifecycle"]["zero_retention_explicit"] = "yes" + with pytest.raises(validator.ObjectStorageContractError, match="boolean"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["lifecycle"]["legal_hold_distinct"] = False + with pytest.raises(validator.ObjectStorageContractError, match="legal_hold"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["rollback"]["delete_customer_data_on_partial_migration"] = True + with pytest.raises(validator.ObjectStorageContractError, match="partial"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["observability"]["high_cardinality_labels_forbid"] = ["bucket"] + with pytest.raises(validator.ObjectStorageContractError, match="raw_pii"): + validator.validate_contract(contract) + contract = _valid_contract() + contract["observability"]["high_cardinality_labels_forbid"] = "bucket" + with pytest.raises(validator.ObjectStorageContractError, match="must be a list"): + validator.validate_contract(contract) + + +def test_require_text_and_cli_success_message( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """Empty strings and successful CLI output stay explicit.""" + with pytest.raises(validator.ObjectStorageContractError, match="nonempty"): + validator.require_text("", "field") + with pytest.raises(validator.ObjectStorageContractError, match="nonempty"): + validator.require_text(1, "field") + destination = tmp_path / "contract.json" + destination.write_text(EXAMPLE.read_text(encoding="utf-8"), encoding="utf-8") + assert validator.main(["--path", str(destination)]) == 0 + assert "object-storage contract passed" in capsys.readouterr().out + + +def test_policy_and_doctoring_record_buyer_visible_controls() -> None: + """Prose must name the executable check, naruon consumer, and APA 7 sources.""" + policy = POLICY.read_text(encoding="utf-8") + doctoring = DOCTORING.read_text(encoding="utf-8") + changelog = CHANGELOG.read_text(encoding="utf-8") + architecture = ARCHITECTURE.read_text(encoding="utf-8") + assert "object_storage" in policy + assert "ContextualWisdomLab/naruon#1364" in policy + assert "scripts/ci/validate_object_storage_contract.py" in policy + assert "design constraints" in policy.lower() + assert "not a blanket PII mask" in policy + assert "APA 7" in doctoring or "References (APA 7th)" in doctoring + assert "Amazon Web Services" in doctoring + assert "CWE-918" in doctoring + assert "object-storage contract" in changelog + assert "object-storage" in architecture.lower() From eae2531634f57b5e612b06d14f59e8dc7c56b52e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 20:24:00 +0900 Subject: [PATCH 02/11] test(storage): expose object contract trust gaps --- .../test_object_storage_contract_hardening.py | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 tests/test_object_storage_contract_hardening.py diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py new file mode 100644 index 0000000000..1eb2d2f081 --- /dev/null +++ b/tests/test_object_storage_contract_hardening.py @@ -0,0 +1,105 @@ +"""Fail-first hardening contracts for object-storage policy documents.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.ci import validate_object_storage_contract as validator + + +ROOT = Path(__file__).resolve().parents[1] +SCHEMA = ROOT / "schemas" / "cwl-object-storage-v1.schema.json" +EXAMPLE = ROOT / "schemas" / "examples" / "cwl-object-storage-v1.example.json" + + +def _valid_contract() -> dict: + """Return an independent mutable copy of the published example.""" + return json.loads(EXAMPLE.read_text(encoding="utf-8")) + + +def test_schema_closes_every_nested_policy_object() -> None: + """The portable schema must express the executable nested-key boundary.""" + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + expected = { + "endpoint_policy": validator.ENDPOINT_POLICY_KEYS, + "credentials": validator.CREDENTIAL_KEYS, + "permissions": validator.PERMISSION_KEYS, + "encryption": validator.ENCRYPTION_KEYS, + "integrity": validator.INTEGRITY_KEYS, + "lifecycle": validator.LIFECYCLE_KEYS, + "rollback": validator.ROLLBACK_KEYS, + "observability": validator.OBSERVABILITY_KEYS, + } + for name, keys in expected.items(): + definition = schema["properties"][name] + assert definition["type"] == "object" + assert definition["additionalProperties"] is False + assert set(definition["properties"]) == set(keys) + required = set(keys) + if name == "endpoint_policy": + required.remove("custom_endpoint") + assert set(definition["required"]) == required + + +def test_loader_rejects_non_finite_json_constants() -> None: + """NaN and infinity are not RFC 8259 JSON contract values.""" + for token in (b"NaN", b"Infinity", b"-Infinity"): + with pytest.raises(validator.ObjectStorageContractError, match="finite"): + validator.load_contract_bytes(b'{"unexpected":' + token + b"}") + + +@pytest.mark.parametrize( + "host", + [ + "localhost", + "api.localhost", + "169.254.169.254", + "127.0.0.1", + "metadata.google.internal", + "metadata.goog", + "OBJECTS.EXAMPLE.COM", + "objécts.example.com", + ], +) +def test_exact_dns_hosts_reject_local_metadata_literal_and_noncanonical_names( + host: str, +) -> None: + """An exact allowlist must not admit metadata, IP, Unicode, or case aliases.""" + assert validator.is_exact_dns_host(host) is False + + +def test_denied_private_network_policy_rejects_single_label_hosts() -> None: + """A denied private-network policy cannot admit an implicitly local host.""" + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "denied" + contract["endpoint_policy"]["host_allowlist"] = ["minio"] + contract["endpoint_policy"]["custom_endpoint"] = "https://minio" + with pytest.raises(validator.ObjectStorageContractError, match="single-label"): + validator.validate_contract(contract) + + +def test_custom_endpoint_rejects_out_of_range_port() -> None: + """A syntactically numeric port still must fit the TCP port range.""" + with pytest.raises(validator.ObjectStorageContractError, match="port"): + validator.parse_https_endpoint_host("https://objects.example.example:65536") + + +def test_observability_labels_fail_closed_without_type_errors() -> None: + """Malformed or duplicate telemetry labels produce policy errors, not crashes.""" + contract = _valid_contract() + contract["observability"]["high_cardinality_labels_forbid"] = [ + "bucket", + "object_key", + "credential", + {}, + ] + with pytest.raises(validator.ObjectStorageContractError, match="nonempty strings"): + validator.validate_contract(contract) + + contract = _valid_contract() + contract["observability"]["high_cardinality_labels_forbid"].append("bucket") + with pytest.raises(validator.ObjectStorageContractError, match="duplicate"): + validator.validate_contract(contract) From 934969125247109057545db55d3945a8bd1a9976 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 20:26:04 +0900 Subject: [PATCH 03/11] ci(storage): verify object contract hardening --- .../repair-object-storage-contract.yml | 330 ++++++++++++++++++ 1 file changed, 330 insertions(+) create mode 100644 .github/workflows/repair-object-storage-contract.yml diff --git a/.github/workflows/repair-object-storage-contract.yml b/.github/workflows/repair-object-storage-contract.yml new file mode 100644 index 0000000000..84c51a6423 --- /dev/null +++ b/.github/workflows/repair-object-storage-contract.yml @@ -0,0 +1,330 @@ +name: Repair object-storage contract hardening + +on: + push: + branches: + - feat/object-storage-contract-v1 + paths: + - .github/workflows/repair-object-storage-contract.yml + +concurrency: + group: repair-object-storage-contract-hardening + cancel-in-progress: false + +permissions: + contents: write + +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + +jobs: + repair: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Harden runner + uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 + with: + egress-policy: audit + + - name: Checkout exact repair trigger + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + ref: ${{ github.sha }} + fetch-depth: 0 + + - name: Set up current stable Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + cache: pip + cache-dependency-path: requirements-opencode-review-ci-hashes.txt + + - name: Install hash-locked quality tooling + run: python -m pip install --disable-pip-version-check --require-hashes -r requirements-opencode-review-ci-hashes.txt + + - name: Apply fail-closed host, JSON, schema, and telemetry repair + run: | + python - <<'PY' + from pathlib import Path + import json + + script = Path("scripts/ci/validate_object_storage_contract.py") + text = script.read_text(encoding="utf-8") + text = text.replace("import argparse\nimport json\n", "import argparse\nimport ipaddress\nimport json\n", 1) + text = text.replace( + "MAX_CONTRACT_BYTES = 65536\n", + "MAX_CONTRACT_BYTES = 65536\nFORBIDDEN_METADATA_HOSTS = frozenset({\"metadata.google.internal\", \"metadata.goog\"})\n", + 1, + ) + old = '''def is_exact_dns_host(host: str) -> bool: + """Return whether *host* is one exact DNS name with no wildcards or ports.""" + if not host or not isinstance(host, str): + return False + if any(char in host for char in "*:/@ \\t\\\\"): + return False + if host.startswith(".") or host.endswith("."): + return False + labels = host.split(".") + for label in labels: + if not label or len(label) > 63: + return False + if label.startswith("-") or label.endswith("-"): + return False + if not all(char.isalnum() or char == "-" for char in label): + return False + return True + ''' + new = '''def reject_non_finite_constant(value: str) -> None: + """Reject non-standard NaN and infinity constants during JSON parsing.""" + raise ObjectStorageContractError( + f"contract JSON numbers must be finite; found {value}" + ) + + + def is_exact_dns_host(host: str) -> bool: + """Return whether *host* is one canonical non-metadata DNS name.""" + if not isinstance(host, str) or not host or len(host) > 253: + return False + if host != host.lower() or not host.isascii(): + return False + if host in FORBIDDEN_METADATA_HOSTS or host.endswith(".localhost"): + return False + try: + ipaddress.ip_address(host) + except ValueError: + pass + else: + return False + if any(char in host for char in "*:/@ \\t\\\\"): + return False + if host.startswith(".") or host.endswith("."): + return False + labels = host.split(".") + for label in labels: + if not label or len(label) > 63: + return False + if label.startswith("-") or label.endswith("-"): + return False + if not all( + char in "abcdefghijklmnopqrstuvwxyz0123456789-" for char in label + ): + return False + return True + ''' + if old not in text: + raise SystemExit("exact DNS host function did not match expected source") + text = text.replace(old, new, 1) + text = text.replace( + ' if separator and (not port.isdigit() or port == "0"):\n raise ObjectStorageContractError("custom endpoint port is invalid")\n', + ' if separator and (\n not port.isdigit() or not 1 <= int(port) <= 65535\n ):\n raise ObjectStorageContractError("custom endpoint port is invalid")\n', + 1, + ) + text = text.replace( + " decoder = json.JSONDecoder(object_pairs_hook=reject_duplicate_keys)\n", + " decoder = json.JSONDecoder(\n object_pairs_hook=reject_duplicate_keys,\n parse_constant=reject_non_finite_constant,\n )\n", + 1, + ) + text = text.replace( + ''' require_allowed( + policy.get("private_network_trust"), + "endpoint_policy.private_network_trust", + ALLOWED_PRIVATE_NETWORK_TRUST, + ) + ''', + ''' private_network_trust = require_allowed( + policy.get("private_network_trust"), + "endpoint_policy.private_network_trust", + ALLOWED_PRIVATE_NETWORK_TRUST, + ) + ''', + 1, + ) + text = text.replace( + ''' if host in seen: + raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") + seen.add(host) + ''', + ''' if host in seen: + raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") + if private_network_trust == "denied" and "." not in host: + raise ObjectStorageContractError( + "single-label endpoint hosts require explicit private-network trust" + ) + seen.add(host) + ''', + 1, + ) + text = text.replace( + ''' missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - set(labels)) + if missing: + ''', + ''' if any(not isinstance(label, str) or not label for label in labels): + raise ObjectStorageContractError( + "observability labels must be nonempty strings" + ) + if len(labels) != len(set(labels)): + raise ObjectStorageContractError( + "observability labels must not contain duplicates" + ) + missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - set(labels)) + if missing: + ''', + 1, + ) + script.write_text(text, encoding="utf-8") + + tests = Path("tests/test_object_storage_contract.py") + text = tests.read_text(encoding="utf-8") + text = text.replace('ARCHITECTURE = ROOT / "ARCHITECTURE.md"\n', "", 1) + text = text.replace(' assert validator.is_exact_dns_host("localhost") is True\n', ' assert validator.is_exact_dns_host("localhost") is False\n', 1) + text = text.replace(' architecture = ARCHITECTURE.read_text(encoding="utf-8")\n', "", 1) + text = text.replace(' assert "object-storage" in architecture.lower()\n', "", 1) + tests.write_text(text, encoding="utf-8") + + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ContextualWisdomLab/.github/schemas/cwl-object-storage-v1.schema.json", + "title": "CWL object-storage contract v1", + "type": "object", + "additionalProperties": False, + "required": [ + "schema_version", "capability", "repository", "provider_class", + "endpoint_policy", "credentials", "permissions", "encryption", + "integrity", "lifecycle", "rollback", "observability", + "database_object_names", "assurance_posture", + ], + "properties": { + "schema_version": {"const": "1"}, + "capability": {"const": "object_storage"}, + "repository": {"type": "string", "pattern": r"^ContextualWisdomLab/(?!.*\\.$)[A-Za-z0-9._-]+$"}, + "provider_class": {"enum": ["aws_s3", "s3_compatible"]}, + "endpoint_policy": { + "type": "object", "additionalProperties": False, + "required": ["transport", "host_allowlist", "allow_wildcards", "follow_redirects", "private_network_trust"], + "properties": { + "transport": {"const": "https"}, + "host_allowlist": {"type": "array", "minItems": 1, "uniqueItems": True, "items": {"type": "string", "minLength": 1}}, + "allow_wildcards": {"const": False}, + "follow_redirects": {"const": False}, + "private_network_trust": {"enum": ["explicit_allowlist", "denied"]}, + "custom_endpoint": {"type": "string", "pattern": "^https://"}, + }, + }, + "credentials": { + "type": "object", "additionalProperties": False, + "required": ["broadcast", "browser_long_lived", "ambient_process_wide", "mechanism"], + "properties": { + "broadcast": {"const": False}, "browser_long_lived": {"const": False}, + "ambient_process_wide": {"const": False}, + "mechanism": {"enum": ["scoped_secret_registry", "workload_identity"]}, + }, + }, + "permissions": { + "type": "object", "additionalProperties": False, + "required": ["public_acls", "public_buckets", "least_privilege"], + "properties": {"public_acls": {"const": False}, "public_buckets": {"const": False}, "least_privilege": {"const": True}}, + }, + "encryption": {"type": "object", "additionalProperties": False, "required": ["server_side"], "properties": {"server_side": {"const": "required"}}}, + "integrity": { + "type": "object", "additionalProperties": False, + "required": ["content_length", "digest", "fail_closed_read"], + "properties": {"content_length": {"const": True}, "digest": {"enum": ["sha256", "sha384", "sha512"]}, "fail_closed_read": {"const": True}}, + }, + "lifecycle": { + "type": "object", "additionalProperties": False, + "required": ["states", "consumed_implies_immediate_delete", "zero_retention_explicit", "legal_hold_distinct"], + "properties": { + "states": {"type": "array", "minItems": 5, "uniqueItems": True, "items": {"type": "string", "minLength": 1}, "allOf": [{"contains": {"const": state}} for state in ["pending", "available", "consumed", "archived", "held"]]}, + "consumed_implies_immediate_delete": {"type": "boolean"}, + "zero_retention_explicit": {"type": "boolean"}, + "legal_hold_distinct": {"const": True}, + }, + "allOf": [{"if": {"properties": {"consumed_implies_immediate_delete": {"const": True}}, "required": ["consumed_implies_immediate_delete"]}, "then": {"properties": {"zero_retention_explicit": {"const": True}}}}], + }, + "rollback": {"type": "object", "additionalProperties": False, "required": ["delete_customer_data_on_partial_migration"], "properties": {"delete_customer_data_on_partial_migration": {"const": False}}}, + "observability": { + "type": "object", "additionalProperties": False, + "required": ["high_cardinality_labels_forbid"], + "properties": {"high_cardinality_labels_forbid": {"type": "array", "minItems": 4, "uniqueItems": True, "items": {"type": "string", "minLength": 1}, "allOf": [{"contains": {"const": label}} for label in ["bucket", "object_key", "credential", "raw_pii"]]}}, + }, + "database_object_names": {"const": "multiword_snake_case"}, + "assurance_posture": {"const": "design_constraints_only"}, + }, + } + Path("schemas/cwl-object-storage-v1.schema.json").write_text( + json.dumps(schema, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + doctoring = Path("docs/doctoring/object-storage-contract.md") + text = doctoring.read_text(encoding="utf-8") + marker = "## Verification contract\n" + hardening = """## Parser and endpoint hardening\n\nThe portable JSON Schema closes every nested policy object rather than merely\ntyping it as an arbitrary object. The stdlib validator also rejects RFC 8259\nnon-finite constants, IP literals, localhost aliases, known metadata hostnames,\nnon-ASCII or non-canonical host aliases, out-of-range ports, implicitly private\nsingle-label hosts under a denied policy, malformed telemetry labels, and\nduplicates. Runtime adapters still need DNS-resolution and rebinding controls;\nthis document contract does not replace an egress boundary.\n\n""" + if marker not in text: + raise SystemExit("doctoring verification section was not found") + if hardening not in text: + text = text.replace(marker, hardening + marker, 1) + doctoring.write_text(text, encoding="utf-8") + + architecture = Path("ARCHITECTURE.md") + base = __import__("subprocess").run( + ["git", "show", "c47afc2dc68488292c1db7c9d6f82dcd5360f181:ARCHITECTURE.md"], + check=True, stdout=__import__("subprocess").PIPE, + ).stdout + architecture.write_bytes(base) + PY + + - name: Run focused production coverage and docstring gates + run: | + cat >"${RUNNER_TEMP}/object-storage-coveragerc" <<'EOF' + [run] + branch = True + include = + scripts/ci/validate_object_storage_contract.py + + [report] + fail_under = 100 + show_missing = True + EOF + export COVERAGE_RCFILE="${RUNNER_TEMP}/object-storage-coveragerc" + python -m coverage erase + python -m coverage run -m pytest \ + tests/test_object_storage_contract.py \ + tests/test_object_storage_contract_hardening.py \ + -q + python -m coverage report + python -m interrogate --fail-under 100 scripts/ci/validate_object_storage_contract.py + python -m compileall -q scripts/ci/validate_object_storage_contract.py tests/test_object_storage_contract.py tests/test_object_storage_contract_hardening.py + + - name: Run complete central suite and 100 percent branch gate + run: | + unset COVERAGE_RCFILE + python -m coverage erase + python -m coverage run -m pytest tests -q + python -m coverage report + + - name: Commit verified repair and remove one-shot workflow + env: + BRANCH_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} + EXPECTED_TRIGGER_SHA: ${{ github.sha }} + run: | + set -euo pipefail + test "$(git rev-parse HEAD)" = "$EXPECTED_TRIGGER_SHA" + git rm .github/workflows/repair-object-storage-contract.yml + git diff --check + git config user.name "cwl-ci-repair[bot]" + git config user.email "cwl-ci-repair[bot]@users.noreply.github.com" + git add \ + ARCHITECTURE.md \ + docs/doctoring/object-storage-contract.md \ + schemas/cwl-object-storage-v1.schema.json \ + scripts/ci/validate_object_storage_contract.py \ + tests/test_object_storage_contract.py \ + tests/test_object_storage_contract_hardening.py + git commit -m "fix(storage): close object contract trust boundaries" + auth_header="$(printf 'x-access-token:%s' "$BRANCH_PUSH_TOKEN" | base64 | tr -d '\n')" + echo "::add-mask::$auth_header" + git -c http.https://github.com/.extraheader="AUTHORIZATION: basic $auth_header" \ + push origin "HEAD:refs/heads/feat/object-storage-contract-v1" From 2080e9188346961b91b0c622f74cbcc60adcf7c6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sun, 16 Aug 2026 22:03:05 +0900 Subject: [PATCH 04/11] fix(storage): close object-storage hardening contract Reject localhost, metadata, IPv4, Unicode, and case-alias hosts; bound TCP ports; refuse non-finite JSON; close nested schema objects; and fail closed on malformed observability labels without TypeError. --- CHANGELOG.md | 2 + docs/doctoring/object-storage-contract.md | 9 +- schemas/cwl-object-storage-v1.schema.json | 119 ++++++++++++++++-- .../ci/validate_object_storage_contract.py | 60 ++++++++- tests/test_object_storage_contract.py | 2 +- 5 files changed, 176 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c32a36925..a4fb6cd685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,8 @@ Semantic Versioning where the repository publishes a release. - Download the pinned `uv` 0.12.1 exporter from the official GitHub Releases URL instead of `releases.astral.sh`, which now returns HTTP 403 and blocks org-wide OpenCode `coverage-evidence`. The SHA-256 pin is unchanged. The opener may follow one hop onto `release-assets.githubusercontent.com` or `objects.githubusercontent.com` and still rejects every other host, userinfo, non-HTTPS scheme, and nondefault port (ContextualWisdomLab/.github#1109). - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). - Excluded relative `-r` and `--requirement` referrers from generated flat base-lock publication while retaining bounded include syntax diagnostics and discovering independently complete direct `.txt` children of `requirements` directories. +- Closed the object-storage contract around exact lowercase DNS hosts, TCP port range, finite RFC 8259 numbers, nested JSON Schema objects, and typed observability labels so hardening tests cannot pass a metadata, Unicode, or unhashable-label document. +- Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. - Allowed commas and ASCII parentheses in the bounded Strix changed-file path policy so legal tracked Packrat fixtures can receive exact-head security analysis, while rejecting raw `..` components before normalization and keeping controls, backslashes, whitespace ambiguity, and shell punctuation fail-closed. diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index 093362af7b..1727f522d9 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -51,8 +51,13 @@ policy and the fail-closed check that leaf repositories can cite. `tests/test_object_storage_contract.py` proves the checked-in example passes, the schema keys match production constants, and each fail-closed control has a -unique rejection. Local quality remains 100% statement/branch coverage and -100% docstrings. +unique rejection. `tests/test_object_storage_contract_hardening.py` proves +nested schema objects stay closed, NaN/Infinity are rejected, exact-host +allowlists exclude localhost, metadata, IPv4, Unicode, and case aliases, a +denied private-network policy rejects single-label hosts, custom endpoints +reject ports above 65535, and malformed observability labels raise policy +errors instead of TypeError. Local quality remains 100% statement/branch +coverage and 100% docstrings. ## Rollback diff --git a/schemas/cwl-object-storage-v1.schema.json b/schemas/cwl-object-storage-v1.schema.json index a49b7cea99..e65d6a1a57 100644 --- a/schemas/cwl-object-storage-v1.schema.json +++ b/schemas/cwl-object-storage-v1.schema.json @@ -25,14 +25,117 @@ "capability": { "const": "object_storage" }, "repository": { "type": "string", "minLength": 1 }, "provider_class": { "enum": ["aws_s3", "s3_compatible"] }, - "endpoint_policy": { "type": "object" }, - "credentials": { "type": "object" }, - "permissions": { "type": "object" }, - "encryption": { "type": "object" }, - "integrity": { "type": "object" }, - "lifecycle": { "type": "object" }, - "rollback": { "type": "object" }, - "observability": { "type": "object" }, + "endpoint_policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "transport", + "host_allowlist", + "allow_wildcards", + "follow_redirects", + "private_network_trust" + ], + "properties": { + "transport": { "const": "https" }, + "host_allowlist": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + }, + "allow_wildcards": { "const": false }, + "follow_redirects": { "const": false }, + "private_network_trust": { + "enum": ["explicit_allowlist", "denied"] + }, + "custom_endpoint": { "type": "string", "minLength": 1 } + } + }, + "credentials": { + "type": "object", + "additionalProperties": false, + "required": [ + "broadcast", + "browser_long_lived", + "ambient_process_wide", + "mechanism" + ], + "properties": { + "broadcast": { "const": false }, + "browser_long_lived": { "const": false }, + "ambient_process_wide": { "const": false }, + "mechanism": { + "enum": ["scoped_secret_registry", "workload_identity"] + } + } + }, + "permissions": { + "type": "object", + "additionalProperties": false, + "required": ["public_acls", "public_buckets", "least_privilege"], + "properties": { + "public_acls": { "const": false }, + "public_buckets": { "const": false }, + "least_privilege": { "const": true } + } + }, + "encryption": { + "type": "object", + "additionalProperties": false, + "required": ["server_side"], + "properties": { + "server_side": { "const": "required" } + } + }, + "integrity": { + "type": "object", + "additionalProperties": false, + "required": ["content_length", "digest", "fail_closed_read"], + "properties": { + "content_length": { "const": true }, + "digest": { "enum": ["sha256", "sha384", "sha512"] }, + "fail_closed_read": { "const": true } + } + }, + "lifecycle": { + "type": "object", + "additionalProperties": false, + "required": [ + "states", + "consumed_implies_immediate_delete", + "zero_retention_explicit", + "legal_hold_distinct" + ], + "properties": { + "states": { + "type": "array", + "minItems": 5, + "items": { "type": "string", "minLength": 1 } + }, + "consumed_implies_immediate_delete": { "type": "boolean" }, + "zero_retention_explicit": { "type": "boolean" }, + "legal_hold_distinct": { "const": true } + } + }, + "rollback": { + "type": "object", + "additionalProperties": false, + "required": ["delete_customer_data_on_partial_migration"], + "properties": { + "delete_customer_data_on_partial_migration": { "const": false } + } + }, + "observability": { + "type": "object", + "additionalProperties": false, + "required": ["high_cardinality_labels_forbid"], + "properties": { + "high_cardinality_labels_forbid": { + "type": "array", + "minItems": 4, + "items": { "type": "string", "minLength": 1 } + } + } + }, "database_object_names": { "const": "multiword_snake_case" }, "assurance_posture": { "const": "design_constraints_only" } } diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index 266b1ab43d..d7fadcd76e 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -37,6 +37,10 @@ FORBIDDEN_HIGH_CARDINALITY_LABELS = frozenset( {"bucket", "object_key", "credential", "raw_pii"} ) +FORBIDDEN_EXACT_HOSTS = frozenset( + {"localhost", "metadata.google.internal", "metadata.goog"} +) +MAX_TCP_PORT = 65535 ALLOWED_TOP_LEVEL = ( "schema_version", "capability", @@ -84,6 +88,13 @@ class ObjectStorageContractError(ValueError): """Raised when a contract file fails a fail-closed policy check.""" +def _reject_non_finite_json_constant(token: str) -> None: + """Reject NaN and Infinity, which are not finite RFC 8259 numbers.""" + raise ObjectStorageContractError( + f"contract contains a non-finite JSON number: {token}" + ) + + def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: """Build an object while rejecting duplicate JSON keys.""" seen: set[str] = set() @@ -96,21 +107,36 @@ def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: return result +def _is_dns_label_char(char: str) -> bool: + """Return whether *char* is an ASCII lowercase DNS label character.""" + return char.isascii() and (char.islower() or char.isdigit() or char == "-") + + def is_exact_dns_host(host: str) -> bool: - """Return whether *host* is one exact DNS name with no wildcards or ports.""" + """Return whether *host* is one exact lowercase DNS name. + + Localhost, link-local metadata names, IPv4 literals, Unicode, and + case aliases are not exact allowlist members. + """ if not host or not isinstance(host, str): return False if any(char in host for char in "*:/@ \t\\"): return False if host.startswith(".") or host.endswith("."): return False + if host != host.lower(): + return False + if host in FORBIDDEN_EXACT_HOSTS or host.endswith(".localhost"): + return False labels = host.split(".") + if len(labels) == 4 and all(label.isdigit() and label.isascii() for label in labels): + return False for label in labels: if not label or len(label) > 63: return False if label.startswith("-") or label.endswith("-"): return False - if not all(char.isalnum() or char == "-" for char in label): + if not all(_is_dns_label_char(char) for char in label): return False return True @@ -130,7 +156,12 @@ def parse_https_endpoint_host(url: str) -> str: "custom endpoints must not embed credentials" ) host, separator, port = authority.partition(":") - if separator and (not port.isdigit() or port == "0"): + if separator and ( + not port.isdigit() + or not port.isascii() + or port == "0" + or int(port) > MAX_TCP_PORT + ): raise ObjectStorageContractError("custom endpoint port is invalid") if not is_exact_dns_host(host): raise ObjectStorageContractError( @@ -151,7 +182,10 @@ def load_contract_bytes(raw: bytes) -> dict[str, Any]: raise ObjectStorageContractError("contract is not UTF-8") from exc if "\x00" in text: raise ObjectStorageContractError("contract contains a NUL byte") - decoder = json.JSONDecoder(object_pairs_hook=reject_duplicate_keys) + decoder = json.JSONDecoder( + object_pairs_hook=reject_duplicate_keys, + parse_constant=_reject_non_finite_json_constant, + ) try: data, index = decoder.raw_decode(text) except json.JSONDecodeError as exc: @@ -233,11 +267,16 @@ def validate_endpoint_policy(policy: Mapping[str, Any]) -> None: "endpoint_policy.host_allowlist must be a nonempty list" ) seen: set[str] = set() + private_network_trust = policy.get("private_network_trust") for host in hosts: if not isinstance(host, str) or not is_exact_dns_host(host): raise ObjectStorageContractError( f"endpoint host {host!r} is not an exact DNS name" ) + if private_network_trust == "denied" and "." not in host: + raise ObjectStorageContractError( + f"endpoint host {host!r} is a single-label name" + ) if host in seen: raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") seen.add(host) @@ -364,7 +403,18 @@ def validate_observability(observability: Mapping[str, Any]) -> None: raise ObjectStorageContractError( "observability.high_cardinality_labels_forbid must be a list" ) - missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - set(labels)) + seen_labels: set[str] = set() + for label in labels: + if not isinstance(label, str) or not label: + raise ObjectStorageContractError( + "observability.high_cardinality_labels_forbid must contain nonempty strings" + ) + if label in seen_labels: + raise ObjectStorageContractError( + f"duplicate observability label {label!r}" + ) + seen_labels.add(label) + missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - seen_labels) if missing: raise ObjectStorageContractError( "observability must forbid high-cardinality labels: " diff --git a/tests/test_object_storage_contract.py b/tests/test_object_storage_contract.py index 2a55be2836..3608f60827 100644 --- a/tests/test_object_storage_contract.py +++ b/tests/test_object_storage_contract.py @@ -73,7 +73,7 @@ def test_load_contract_path_reports_unreadable_files(tmp_path: Path) -> None: def test_is_exact_dns_host_rejects_wildcards_ports_and_empty_labels() -> None: """Exact-host allowlists cannot contain wildcard or port syntax.""" assert validator.is_exact_dns_host("s3.ap-northeast-2.amazonaws.com") - assert validator.is_exact_dns_host("localhost") is True + assert validator.is_exact_dns_host("localhost") is False assert validator.is_exact_dns_host("") is False assert validator.is_exact_dns_host(None) is False assert validator.is_exact_dns_host("a..b") is False From f33b0a8f9d3caef1c43408443b7f9419a0f0295c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 14:14:26 +0000 Subject: [PATCH 05/11] fix(storage): drop leftover object-storage write workflow The one-shot repair workflow already applied its hardening and would fail or revert ARCHITECTURE.md if rerun. Removing it clears the Scorecard Token-Permissions finding on workflow-default contents: write. Co-authored-by: Seongho Bae --- .../repair-object-storage-contract.yml | 330 ------------------ CHANGELOG.md | 1 + .../test_object_storage_contract_hardening.py | 8 + 3 files changed, 9 insertions(+), 330 deletions(-) delete mode 100644 .github/workflows/repair-object-storage-contract.yml diff --git a/.github/workflows/repair-object-storage-contract.yml b/.github/workflows/repair-object-storage-contract.yml deleted file mode 100644 index 84c51a6423..0000000000 --- a/.github/workflows/repair-object-storage-contract.yml +++ /dev/null @@ -1,330 +0,0 @@ -name: Repair object-storage contract hardening - -on: - push: - branches: - - feat/object-storage-contract-v1 - paths: - - .github/workflows/repair-object-storage-contract.yml - -concurrency: - group: repair-object-storage-contract-hardening - cancel-in-progress: false - -permissions: - contents: write - -env: - FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true - -jobs: - repair: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - name: Harden runner - uses: step-security/harden-runner@bf7454d06d71f1098171f2acdf0cd4708d7b5920 # v2.20.0 - with: - egress-policy: audit - - - name: Checkout exact repair trigger - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - persist-credentials: false - ref: ${{ github.sha }} - fetch-depth: 0 - - - name: Set up current stable Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: "3.14" - cache: pip - cache-dependency-path: requirements-opencode-review-ci-hashes.txt - - - name: Install hash-locked quality tooling - run: python -m pip install --disable-pip-version-check --require-hashes -r requirements-opencode-review-ci-hashes.txt - - - name: Apply fail-closed host, JSON, schema, and telemetry repair - run: | - python - <<'PY' - from pathlib import Path - import json - - script = Path("scripts/ci/validate_object_storage_contract.py") - text = script.read_text(encoding="utf-8") - text = text.replace("import argparse\nimport json\n", "import argparse\nimport ipaddress\nimport json\n", 1) - text = text.replace( - "MAX_CONTRACT_BYTES = 65536\n", - "MAX_CONTRACT_BYTES = 65536\nFORBIDDEN_METADATA_HOSTS = frozenset({\"metadata.google.internal\", \"metadata.goog\"})\n", - 1, - ) - old = '''def is_exact_dns_host(host: str) -> bool: - """Return whether *host* is one exact DNS name with no wildcards or ports.""" - if not host or not isinstance(host, str): - return False - if any(char in host for char in "*:/@ \\t\\\\"): - return False - if host.startswith(".") or host.endswith("."): - return False - labels = host.split(".") - for label in labels: - if not label or len(label) > 63: - return False - if label.startswith("-") or label.endswith("-"): - return False - if not all(char.isalnum() or char == "-" for char in label): - return False - return True - ''' - new = '''def reject_non_finite_constant(value: str) -> None: - """Reject non-standard NaN and infinity constants during JSON parsing.""" - raise ObjectStorageContractError( - f"contract JSON numbers must be finite; found {value}" - ) - - - def is_exact_dns_host(host: str) -> bool: - """Return whether *host* is one canonical non-metadata DNS name.""" - if not isinstance(host, str) or not host or len(host) > 253: - return False - if host != host.lower() or not host.isascii(): - return False - if host in FORBIDDEN_METADATA_HOSTS or host.endswith(".localhost"): - return False - try: - ipaddress.ip_address(host) - except ValueError: - pass - else: - return False - if any(char in host for char in "*:/@ \\t\\\\"): - return False - if host.startswith(".") or host.endswith("."): - return False - labels = host.split(".") - for label in labels: - if not label or len(label) > 63: - return False - if label.startswith("-") or label.endswith("-"): - return False - if not all( - char in "abcdefghijklmnopqrstuvwxyz0123456789-" for char in label - ): - return False - return True - ''' - if old not in text: - raise SystemExit("exact DNS host function did not match expected source") - text = text.replace(old, new, 1) - text = text.replace( - ' if separator and (not port.isdigit() or port == "0"):\n raise ObjectStorageContractError("custom endpoint port is invalid")\n', - ' if separator and (\n not port.isdigit() or not 1 <= int(port) <= 65535\n ):\n raise ObjectStorageContractError("custom endpoint port is invalid")\n', - 1, - ) - text = text.replace( - " decoder = json.JSONDecoder(object_pairs_hook=reject_duplicate_keys)\n", - " decoder = json.JSONDecoder(\n object_pairs_hook=reject_duplicate_keys,\n parse_constant=reject_non_finite_constant,\n )\n", - 1, - ) - text = text.replace( - ''' require_allowed( - policy.get("private_network_trust"), - "endpoint_policy.private_network_trust", - ALLOWED_PRIVATE_NETWORK_TRUST, - ) - ''', - ''' private_network_trust = require_allowed( - policy.get("private_network_trust"), - "endpoint_policy.private_network_trust", - ALLOWED_PRIVATE_NETWORK_TRUST, - ) - ''', - 1, - ) - text = text.replace( - ''' if host in seen: - raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") - seen.add(host) - ''', - ''' if host in seen: - raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") - if private_network_trust == "denied" and "." not in host: - raise ObjectStorageContractError( - "single-label endpoint hosts require explicit private-network trust" - ) - seen.add(host) - ''', - 1, - ) - text = text.replace( - ''' missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - set(labels)) - if missing: - ''', - ''' if any(not isinstance(label, str) or not label for label in labels): - raise ObjectStorageContractError( - "observability labels must be nonempty strings" - ) - if len(labels) != len(set(labels)): - raise ObjectStorageContractError( - "observability labels must not contain duplicates" - ) - missing = sorted(FORBIDDEN_HIGH_CARDINALITY_LABELS - set(labels)) - if missing: - ''', - 1, - ) - script.write_text(text, encoding="utf-8") - - tests = Path("tests/test_object_storage_contract.py") - text = tests.read_text(encoding="utf-8") - text = text.replace('ARCHITECTURE = ROOT / "ARCHITECTURE.md"\n', "", 1) - text = text.replace(' assert validator.is_exact_dns_host("localhost") is True\n', ' assert validator.is_exact_dns_host("localhost") is False\n', 1) - text = text.replace(' architecture = ARCHITECTURE.read_text(encoding="utf-8")\n', "", 1) - text = text.replace(' assert "object-storage" in architecture.lower()\n', "", 1) - tests.write_text(text, encoding="utf-8") - - schema = { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/ContextualWisdomLab/.github/schemas/cwl-object-storage-v1.schema.json", - "title": "CWL object-storage contract v1", - "type": "object", - "additionalProperties": False, - "required": [ - "schema_version", "capability", "repository", "provider_class", - "endpoint_policy", "credentials", "permissions", "encryption", - "integrity", "lifecycle", "rollback", "observability", - "database_object_names", "assurance_posture", - ], - "properties": { - "schema_version": {"const": "1"}, - "capability": {"const": "object_storage"}, - "repository": {"type": "string", "pattern": r"^ContextualWisdomLab/(?!.*\\.$)[A-Za-z0-9._-]+$"}, - "provider_class": {"enum": ["aws_s3", "s3_compatible"]}, - "endpoint_policy": { - "type": "object", "additionalProperties": False, - "required": ["transport", "host_allowlist", "allow_wildcards", "follow_redirects", "private_network_trust"], - "properties": { - "transport": {"const": "https"}, - "host_allowlist": {"type": "array", "minItems": 1, "uniqueItems": True, "items": {"type": "string", "minLength": 1}}, - "allow_wildcards": {"const": False}, - "follow_redirects": {"const": False}, - "private_network_trust": {"enum": ["explicit_allowlist", "denied"]}, - "custom_endpoint": {"type": "string", "pattern": "^https://"}, - }, - }, - "credentials": { - "type": "object", "additionalProperties": False, - "required": ["broadcast", "browser_long_lived", "ambient_process_wide", "mechanism"], - "properties": { - "broadcast": {"const": False}, "browser_long_lived": {"const": False}, - "ambient_process_wide": {"const": False}, - "mechanism": {"enum": ["scoped_secret_registry", "workload_identity"]}, - }, - }, - "permissions": { - "type": "object", "additionalProperties": False, - "required": ["public_acls", "public_buckets", "least_privilege"], - "properties": {"public_acls": {"const": False}, "public_buckets": {"const": False}, "least_privilege": {"const": True}}, - }, - "encryption": {"type": "object", "additionalProperties": False, "required": ["server_side"], "properties": {"server_side": {"const": "required"}}}, - "integrity": { - "type": "object", "additionalProperties": False, - "required": ["content_length", "digest", "fail_closed_read"], - "properties": {"content_length": {"const": True}, "digest": {"enum": ["sha256", "sha384", "sha512"]}, "fail_closed_read": {"const": True}}, - }, - "lifecycle": { - "type": "object", "additionalProperties": False, - "required": ["states", "consumed_implies_immediate_delete", "zero_retention_explicit", "legal_hold_distinct"], - "properties": { - "states": {"type": "array", "minItems": 5, "uniqueItems": True, "items": {"type": "string", "minLength": 1}, "allOf": [{"contains": {"const": state}} for state in ["pending", "available", "consumed", "archived", "held"]]}, - "consumed_implies_immediate_delete": {"type": "boolean"}, - "zero_retention_explicit": {"type": "boolean"}, - "legal_hold_distinct": {"const": True}, - }, - "allOf": [{"if": {"properties": {"consumed_implies_immediate_delete": {"const": True}}, "required": ["consumed_implies_immediate_delete"]}, "then": {"properties": {"zero_retention_explicit": {"const": True}}}}], - }, - "rollback": {"type": "object", "additionalProperties": False, "required": ["delete_customer_data_on_partial_migration"], "properties": {"delete_customer_data_on_partial_migration": {"const": False}}}, - "observability": { - "type": "object", "additionalProperties": False, - "required": ["high_cardinality_labels_forbid"], - "properties": {"high_cardinality_labels_forbid": {"type": "array", "minItems": 4, "uniqueItems": True, "items": {"type": "string", "minLength": 1}, "allOf": [{"contains": {"const": label}} for label in ["bucket", "object_key", "credential", "raw_pii"]]}}, - }, - "database_object_names": {"const": "multiword_snake_case"}, - "assurance_posture": {"const": "design_constraints_only"}, - }, - } - Path("schemas/cwl-object-storage-v1.schema.json").write_text( - json.dumps(schema, indent=2, ensure_ascii=False) + "\n", - encoding="utf-8", - ) - - doctoring = Path("docs/doctoring/object-storage-contract.md") - text = doctoring.read_text(encoding="utf-8") - marker = "## Verification contract\n" - hardening = """## Parser and endpoint hardening\n\nThe portable JSON Schema closes every nested policy object rather than merely\ntyping it as an arbitrary object. The stdlib validator also rejects RFC 8259\nnon-finite constants, IP literals, localhost aliases, known metadata hostnames,\nnon-ASCII or non-canonical host aliases, out-of-range ports, implicitly private\nsingle-label hosts under a denied policy, malformed telemetry labels, and\nduplicates. Runtime adapters still need DNS-resolution and rebinding controls;\nthis document contract does not replace an egress boundary.\n\n""" - if marker not in text: - raise SystemExit("doctoring verification section was not found") - if hardening not in text: - text = text.replace(marker, hardening + marker, 1) - doctoring.write_text(text, encoding="utf-8") - - architecture = Path("ARCHITECTURE.md") - base = __import__("subprocess").run( - ["git", "show", "c47afc2dc68488292c1db7c9d6f82dcd5360f181:ARCHITECTURE.md"], - check=True, stdout=__import__("subprocess").PIPE, - ).stdout - architecture.write_bytes(base) - PY - - - name: Run focused production coverage and docstring gates - run: | - cat >"${RUNNER_TEMP}/object-storage-coveragerc" <<'EOF' - [run] - branch = True - include = - scripts/ci/validate_object_storage_contract.py - - [report] - fail_under = 100 - show_missing = True - EOF - export COVERAGE_RCFILE="${RUNNER_TEMP}/object-storage-coveragerc" - python -m coverage erase - python -m coverage run -m pytest \ - tests/test_object_storage_contract.py \ - tests/test_object_storage_contract_hardening.py \ - -q - python -m coverage report - python -m interrogate --fail-under 100 scripts/ci/validate_object_storage_contract.py - python -m compileall -q scripts/ci/validate_object_storage_contract.py tests/test_object_storage_contract.py tests/test_object_storage_contract_hardening.py - - - name: Run complete central suite and 100 percent branch gate - run: | - unset COVERAGE_RCFILE - python -m coverage erase - python -m coverage run -m pytest tests -q - python -m coverage report - - - name: Commit verified repair and remove one-shot workflow - env: - BRANCH_PUSH_TOKEN: ${{ secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }} - EXPECTED_TRIGGER_SHA: ${{ github.sha }} - run: | - set -euo pipefail - test "$(git rev-parse HEAD)" = "$EXPECTED_TRIGGER_SHA" - git rm .github/workflows/repair-object-storage-contract.yml - git diff --check - git config user.name "cwl-ci-repair[bot]" - git config user.email "cwl-ci-repair[bot]@users.noreply.github.com" - git add \ - ARCHITECTURE.md \ - docs/doctoring/object-storage-contract.md \ - schemas/cwl-object-storage-v1.schema.json \ - scripts/ci/validate_object_storage_contract.py \ - tests/test_object_storage_contract.py \ - tests/test_object_storage_contract_hardening.py - git commit -m "fix(storage): close object contract trust boundaries" - auth_header="$(printf 'x-access-token:%s' "$BRANCH_PUSH_TOKEN" | base64 | tr -d '\n')" - echo "::add-mask::$auth_header" - git -c http.https://github.com/.extraheader="AUTHORIZATION: basic $auth_header" \ - push origin "HEAD:refs/heads/feat/object-storage-contract-v1" diff --git a/CHANGELOG.md b/CHANGELOG.md index a4fb6cd685..25c0e34600 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ Semantic Versioning where the repository publishes a release. - Compared the trusted `uv` executable's post-install `--version` output against the real GitHub Releases build's full string, `uv 0.12.1 (x86_64-unknown-linux-gnu)`, instead of the bare `uv 0.12.1` the prior check required; the genuine release binary always prints the target triple, so every installation was failing the pin check immediately after the archive download itself was fixed (ContextualWisdomLab/.github#1109). - Excluded relative `-r` and `--requirement` referrers from generated flat base-lock publication while retaining bounded include syntax diagnostics and discovering independently complete direct `.txt` children of `requirements` directories. - Closed the object-storage contract around exact lowercase DNS hosts, TCP port range, finite RFC 8259 numbers, nested JSON Schema objects, and typed observability labels so hardening tests cannot pass a metadata, Unicode, or unhashable-label document. +- Removed the leftover one-shot object-storage repair workflow so Scorecard Token-Permissions no longer sees workflow-default `contents: write` after the hardening is already committed. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index 1eb2d2f081..9555e747cf 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -13,6 +13,9 @@ ROOT = Path(__file__).resolve().parents[1] SCHEMA = ROOT / "schemas" / "cwl-object-storage-v1.schema.json" EXAMPLE = ROOT / "schemas" / "examples" / "cwl-object-storage-v1.example.json" +ONE_SHOT_REPAIR_WORKFLOW = ( + ROOT / ".github" / "workflows" / "repair-object-storage-contract.yml" +) def _valid_contract() -> dict: @@ -20,6 +23,11 @@ def _valid_contract() -> dict: return json.loads(EXAMPLE.read_text(encoding="utf-8")) +def test_one_shot_repair_workflow_is_absent() -> None: + """Completed one-shot writers must not remain to grant contents: write.""" + assert not ONE_SHOT_REPAIR_WORKFLOW.exists() + + def test_schema_closes_every_nested_policy_object() -> None: """The portable schema must express the executable nested-key boundary.""" schema = json.loads(SCHEMA.read_text(encoding="utf-8")) From d206f3d554a1d3342cb374814229066a1ce9d306 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 14:52:10 +0000 Subject: [PATCH 06/11] fix(storage): reject numeric IP aliases in host allowlists Strix proved decimal IPv4 hosts such as 2851992574 passed is_exact_dns_host under an explicit allowlist. Reject IP literals, decimal aliases, hexadecimal aliases, and dotted-numeric names. Co-authored-by: Seongho Bae --- CHANGELOG.md | 1 + docs/doctoring/object-storage-contract.md | 3 +- .../ci/validate_object_storage_contract.py | 34 ++++++++++++++++--- .../test_object_storage_contract_hardening.py | 24 +++++++++++++ 4 files changed, 56 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 25c0e34600..7038b1c9cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Semantic Versioning where the repository publishes a release. - Excluded relative `-r` and `--requirement` referrers from generated flat base-lock publication while retaining bounded include syntax diagnostics and discovering independently complete direct `.txt` children of `requirements` directories. - Closed the object-storage contract around exact lowercase DNS hosts, TCP port range, finite RFC 8259 numbers, nested JSON Schema objects, and typed observability labels so hardening tests cannot pass a metadata, Unicode, or unhashable-label document. - Removed the leftover one-shot object-storage repair workflow so Scorecard Token-Permissions no longer sees workflow-default `contents: write` after the hardening is already committed. +- Rejected decimal, hexadecimal, and dotted-numeric IPv4 aliases in object-storage host allowlists so Strix-proven metadata-service SSRF bypasses fail closed. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index 1727f522d9..f84b459a96 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -53,7 +53,8 @@ policy and the fail-closed check that leaf repositories can cite. the schema keys match production constants, and each fail-closed control has a unique rejection. `tests/test_object_storage_contract_hardening.py` proves nested schema objects stay closed, NaN/Infinity are rejected, exact-host -allowlists exclude localhost, metadata, IPv4, Unicode, and case aliases, a +allowlists exclude localhost, metadata, IPv4 literals, decimal or hexadecimal +IP aliases, Unicode, and case aliases, a denied private-network policy rejects single-label hosts, custom endpoints reject ports above 65535, and malformed observability labels raise policy errors instead of TypeError. Local quality remains 100% statement/branch diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index d7fadcd76e..16fcaafa78 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -11,6 +11,7 @@ from __future__ import annotations import argparse +import ipaddress import json import sys from collections.abc import Mapping, Sequence @@ -41,6 +42,7 @@ {"localhost", "metadata.google.internal", "metadata.goog"} ) MAX_TCP_PORT = 65535 +MAX_DNS_HOST_LENGTH = 253 ALLOWED_TOP_LEVEL = ( "schema_version", "capability", @@ -112,13 +114,35 @@ def _is_dns_label_char(char: str) -> bool: return char.isascii() and (char.islower() or char.isdigit() or char == "-") +def _label_is_integer_token(label: str) -> bool: + """Return whether *label* is a decimal or ``0x`` hexadecimal integer token.""" + if label.isdigit() and label.isascii(): + return True + return ( + label.startswith("0x") + and len(label) > 2 + and all(char in "0123456789abcdef" for char in label[2:]) + ) + + +def _is_ip_literal_or_alias(host: str) -> bool: + """Return whether *host* is an IP literal or a numeric IP alias.""" + try: + ipaddress.ip_address(host) + except ValueError: + pass + else: + return True + return all(_label_is_integer_token(label) for label in host.split(".")) + + def is_exact_dns_host(host: str) -> bool: """Return whether *host* is one exact lowercase DNS name. - Localhost, link-local metadata names, IPv4 literals, Unicode, and - case aliases are not exact allowlist members. + Localhost, link-local metadata names, IP literals, decimal or hexadecimal + IP aliases, Unicode, and case aliases are not exact allowlist members. """ - if not host or not isinstance(host, str): + if not isinstance(host, str) or not host or len(host) > MAX_DNS_HOST_LENGTH: return False if any(char in host for char in "*:/@ \t\\"): return False @@ -128,9 +152,9 @@ def is_exact_dns_host(host: str) -> bool: return False if host in FORBIDDEN_EXACT_HOSTS or host.endswith(".localhost"): return False - labels = host.split(".") - if len(labels) == 4 and all(label.isdigit() and label.isascii() for label in labels): + if _is_ip_literal_or_alias(host): return False + labels = host.split(".") for label in labels: if not label or len(label) > 63: return False diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index 9555e747cf..793d144836 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -66,6 +66,13 @@ def test_loader_rejects_non_finite_json_constants() -> None: "api.localhost", "169.254.169.254", "127.0.0.1", + "2130706433", + "2851992574", + "127.1", + "127.0.1", + "0x7f000001", + "0x7f.0.0.1", + "a" * 254, "metadata.google.internal", "metadata.goog", "OBJECTS.EXAMPLE.COM", @@ -79,6 +86,23 @@ def test_exact_dns_hosts_reject_local_metadata_literal_and_noncanonical_names( assert validator.is_exact_dns_host(host) is False +def test_exact_dns_hosts_still_accept_numeric_dns_labels() -> None: + """A numeric label is allowed only when the name is not an IP alias.""" + assert validator.is_exact_dns_host("1.s3.amazonaws.com") + assert validator.is_exact_dns_host("0xz.example.com") + assert validator.is_exact_dns_host("0x.example.com") + + +def test_explicit_allowlist_rejects_decimal_metadata_ip_alias() -> None: + """Decimal IPv4 aliases must not enter an explicit host allowlist.""" + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "explicit_allowlist" + contract["endpoint_policy"]["host_allowlist"] = ["2851992574"] + contract["endpoint_policy"].pop("custom_endpoint", None) + with pytest.raises(validator.ObjectStorageContractError, match="exact DNS"): + validator.validate_contract(contract) + + def test_denied_private_network_policy_rejects_single_label_hosts() -> None: """A denied private-network policy cannot admit an implicitly local host.""" contract = _valid_contract() From 14a79123afcaf7ab025200ebaa213e51a60a03d6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:01:30 +0000 Subject: [PATCH 07/11] fix(storage): close special-use SSRF and tenant-binding gaps Reject multicast .local, metadata, and cluster-local names from exact allowlists, and reject RFC 6761/6762 internal suffixes when private-network trust is denied. Require tenant-purpose binding, align the portable schema with the executable values, and add the product write/read/delete acceptance template so sibling adapters can prove compatibility. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 11 +- CHANGELOG.md | 2 + docs/doctoring/object-storage-contract.md | 29 +++-- .../CWL_OBJECT_STORAGE_CONTRACT.md | 14 ++- .../PRODUCT_ACCEPTANCE_TEMPLATE.md | 46 ++++++++ schemas/cwl-object-storage-v1.schema.json | 25 +++- .../cwl-object-storage-v1.example.json | 1 + .../ci/validate_object_storage_contract.py | 50 +++++++- tests/test_object_storage_contract.py | 3 + .../test_object_storage_contract_hardening.py | 109 ++++++++++++++++++ 10 files changed, 268 insertions(+), 22 deletions(-) create mode 100644 docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e1628669b8..e180c25580 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -109,10 +109,13 @@ sequenceDiagram Central `.github` publishes a provider-neutral `object_storage` contract. Naruon and other products keep their own adapters. The executable check is `scripts/ci/validate_object_storage_contract.py`. HTTPS, exact-host -allowlists, server-side encryption, SHA-256-or-stronger integrity, distinct -lifecycle states, and non-destructive rollback are required. CSAP and SOC 2 -remain design constraints, not certification claims. Operational PII is not -blanket-masked. +allowlists, tenant-purpose binding, server-side encryption, SHA-256-or-stronger +integrity, distinct lifecycle states, and non-destructive rollback are +required. Denied private-network trust also rejects special-use internal +suffixes. CSAP and SOC 2 remain design constraints, not certification claims. +Operational PII is not blanket-masked. Product adapters prove write/read/delete +timeout and partial-upload behavior with +`docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md`. ## Quality gates diff --git a/CHANGELOG.md b/CHANGELOG.md index 7038b1c9cc..eb38ea6dd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,8 @@ Semantic Versioning where the repository publishes a release. - Closed the object-storage contract around exact lowercase DNS hosts, TCP port range, finite RFC 8259 numbers, nested JSON Schema objects, and typed observability labels so hardening tests cannot pass a metadata, Unicode, or unhashable-label document. - Removed the leftover one-shot object-storage repair workflow so Scorecard Token-Permissions no longer sees workflow-default `contents: write` after the hardening is already committed. - Rejected decimal, hexadecimal, and dotted-numeric IPv4 aliases in object-storage host allowlists so Strix-proven metadata-service SSRF bypasses fail closed. +- Rejected multicast `.local`, metadata, and cluster-local names from object-storage allowlists, and rejected RFC 6761/6762 internal suffixes when private-network trust is denied, so a `denied` policy cannot admit `minio.internal`. +- Required tenant- and purpose-bound provider selection, aligned the portable JSON Schema with the executable value constraints, and added the product write/read/delete acceptance template so sibling adapters can prove compatibility without copying central code. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index f84b459a96..ba1e72c88a 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -20,8 +20,11 @@ policy and the fail-closed check that leaf repositories can cite. capability. 2. Require HTTPS, exact-host allowlists, no wildcards, and no automatic redirects (CWE-918). -3. Require explicit private-network trust. Implicit RFC1918 or metadata-service - access is not authorized by this contract. +3. Require explicit private-network trust. Implicit RFC1918, metadata-service, + multicast `.local` (Cheshire & Krochmal, 2013b), or special-use internal + suffixes (Cheshire & Krochmal, 2013a) are not authorized unless the exact + host is named after an explicit trust decision. Multicast `.local` names + are never unicast endpoints. 4. Require least-privilege object permissions and prohibit public ACLs, public buckets, and browser-exposed long-lived credentials (CWE-798; CWE-200). 5. Require server-side encryption and fail-closed content-length plus SHA-256 @@ -35,8 +38,10 @@ policy and the fail-closed check that leaf repositories can cite. telemetry labels. This is not a blanket operational PII mask. 9. Record CSAP and SOC 2 only as design constraints. The contract must not claim certification. -10. Ship an executable validator. Prose alone does not close - ContextualWisdomLab/.github#1019. +10. Require tenant- and purpose-bound provider selection so one credential or + bucket cannot serve unrelated tenants or jobs. +11. Ship an executable validator and a product acceptance template. Prose + alone does not close ContextualWisdomLab/.github#1019. ## Trust boundary @@ -53,10 +58,12 @@ policy and the fail-closed check that leaf repositories can cite. the schema keys match production constants, and each fail-closed control has a unique rejection. `tests/test_object_storage_contract_hardening.py` proves nested schema objects stay closed, NaN/Infinity are rejected, exact-host -allowlists exclude localhost, metadata, IPv4 literals, decimal or hexadecimal -IP aliases, Unicode, and case aliases, a -denied private-network policy rejects single-label hosts, custom endpoints -reject ports above 65535, and malformed observability labels raise policy +allowlists exclude localhost, metadata, cluster-local names, IPv4 and IPv6 +literals, decimal or hexadecimal IP aliases, Unicode, case aliases, and +multicast `.local` names, a denied private-network policy rejects +single-label and special-use internal hosts, custom endpoints reject ports +above 65535, schema value constraints match the validator, tenant-purpose +binding is mandatory, and malformed observability labels raise policy errors instead of TypeError. Local quality remains 100% statement/branch coverage and 100% docstrings. @@ -67,6 +74,12 @@ not keep a schema that the executable check no longer enforces. ## References (APA 7th) +Cheshire, S., & Krochmal, M. (2013a). *Special-use domain names* (RFC 6761). +Internet Engineering Task Force. https://www.rfc-editor.org/rfc/rfc6761 + +Cheshire, S., & Krochmal, M. (2013b). *Multicast DNS* (RFC 6762). Internet +Engineering Task Force. https://www.rfc-editor.org/rfc/rfc6762 + Amazon Web Services. (2024a). *Checking object integrity for data uploads in Amazon S3*. Amazon Simple Storage Service User Guide. https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity-upload.html diff --git a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md index 1e13a5cafc..fa8a409dde 100644 --- a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md +++ b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md @@ -17,8 +17,8 @@ A contract JSON document is valid only when every control below is true. | Control | Required value | |---|---| | Transport | `https` only | -| Hosts | exact-host allowlist; no wildcards; no automatic redirects | -| Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918 access | +| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local` or metadata names | +| Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918, RFC 6761/6762, or `.internal` access | | Credentials | scoped secret registry or workload identity; never broadcast, browser-exposed, or ambient process-wide | | Permissions | least privilege; public ACLs and public buckets prohibited | | Encryption | server-side encryption `required` | @@ -26,6 +26,7 @@ A contract JSON document is valid only when every control below is true. | Lifecycle | `pending`, `available`, `consumed`, `archived`, and `held` are distinct; `consumed` does not delete unless zero retention is explicit | | Rollback | a partial migration must not delete customer data | | Names | persisted metadata uses multiword `snake_case` | +| Tenant binding | provider selection is tenant- and purpose-bound | | Telemetry | bucket names, object keys, credentials, and raw PII are forbidden high-cardinality labels | | Assurance | CSAP and SOC 2 are design constraints, not certifications | @@ -40,7 +41,10 @@ high-cardinality labels is not a blanket PII mask. `s3_compatible` and every host is an exact public DNS name. 3. **Authorized private-network endpoints** — `private_network_trust` is `explicit_allowlist` and each private host is named. Implicit RFC1918, - link-local, or metadata-service access is rejected. + link-local, metadata-service, multicast `.local`, or special-use + `.internal` / `.corp` / `.lan` access is rejected unless that exact host + is named after an explicit trust decision. Multicast `.local` names are + never valid unicast endpoints. Workload-identity and instance-metadata access require their own SSRF review. This contract does not grant that access. @@ -51,7 +55,9 @@ On success the consumer persists or reads the object under its own schema and records a purpose-bound audit event without bucket, key, credential, or raw PII labels. On rejection, timeout, duplicate, or partial upload the consumer leaves the previous durable object in place. Rollback never deletes customer data -because a backfill step only partly succeeded. +because a backfill step only partly succeeded. Product repositories then run +the write/read/delete failure-injection lane in +`docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md`. ## Verification diff --git a/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md b/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md new file mode 100644 index 0000000000..8c6cb154e9 --- /dev/null +++ b/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md @@ -0,0 +1,46 @@ +# Object-storage product acceptance template + +Use this checklist in the product repository that owns the adapter. Central +`.github` does not store customer objects. The current consumer is +[ContextualWisdomLab/naruon#1364](https://github.com/ContextualWisdomLab/naruon/pull/1364). + +## Before you claim S3 compatibility + +1. Commit a contract JSON that passes + `python3 scripts/ci/validate_object_storage_contract.py --path `. +2. Keep provider selection tenant- and purpose-bound. Do not share one + credential or bucket across tenants or unrelated jobs. +3. Persist provider and object metadata in 3NF with multiword `snake_case` + names. Do not store credentials in events, logs, SBOMs, or model context. + +## Failure-injection lane + +A product may call itself S3-compatible only after a real object-store lane +(LocalStack or the purchased endpoint) proves write/read/delete behavior +against the current head, including: + +| Case | Required next action | +|---|---| +| write timeout | Leave the previous durable object in place and record a purpose-bound audit event | +| read timeout | Fail closed; do not treat a truncated body as valid evidence | +| delete timeout | Retry the same deterministic key; do not enumerate the bucket | +| partial or aborted upload | Do not publish a `available` object; compensate without deleting older customer data | +| digest or content-length mismatch | Reject the read; keep the prior object | + +## Rollback and retention + +A partial migration or backfill must not delete customer data. `consumed` +does not delete unless the product explicitly configures zero retention. +Legal hold and archive stay distinct from the reprocessing window. + +## Telemetry + +Emit OpenTelemetry or audit evidence without `bucket`, `object_key`, +`credential`, or `raw_pii` labels. Operational PII that the product must +process stays in the product data plane; do not blanket-mask it. + +## Close the issue only with current-head proof + +Point the product PR at this template, the central contract SHA, and the +exact-head integration run. Prose without that run does not close +ContextualWisdomLab/.github#1019. diff --git a/schemas/cwl-object-storage-v1.schema.json b/schemas/cwl-object-storage-v1.schema.json index e65d6a1a57..b46a07be68 100644 --- a/schemas/cwl-object-storage-v1.schema.json +++ b/schemas/cwl-object-storage-v1.schema.json @@ -18,12 +18,16 @@ "rollback", "observability", "database_object_names", + "tenant_purpose_bound", "assurance_posture" ], "properties": { "schema_version": { "const": "1" }, "capability": { "const": "object_storage" }, - "repository": { "type": "string", "minLength": 1 }, + "repository": { + "type": "string", + "pattern": "^ContextualWisdomLab/[A-Za-z0-9._-]*[A-Za-z0-9_-]$" + }, "provider_class": { "enum": ["aws_s3", "s3_compatible"] }, "endpoint_policy": { "type": "object", @@ -40,6 +44,7 @@ "host_allowlist": { "type": "array", "minItems": 1, + "uniqueItems": true, "items": { "type": "string", "minLength": 1 } }, "allow_wildcards": { "const": false }, @@ -47,7 +52,7 @@ "private_network_trust": { "enum": ["explicit_allowlist", "denied"] }, - "custom_endpoint": { "type": "string", "minLength": 1 } + "custom_endpoint": { "type": "string", "pattern": "^https://" } } }, "credentials": { @@ -109,6 +114,14 @@ "states": { "type": "array", "minItems": 5, + "uniqueItems": true, + "allOf": [ + { "contains": { "const": "pending" } }, + { "contains": { "const": "available" } }, + { "contains": { "const": "consumed" } }, + { "contains": { "const": "archived" } }, + { "contains": { "const": "held" } } + ], "items": { "type": "string", "minLength": 1 } }, "consumed_implies_immediate_delete": { "type": "boolean" }, @@ -132,11 +145,19 @@ "high_cardinality_labels_forbid": { "type": "array", "minItems": 4, + "uniqueItems": true, + "allOf": [ + { "contains": { "const": "bucket" } }, + { "contains": { "const": "object_key" } }, + { "contains": { "const": "credential" } }, + { "contains": { "const": "raw_pii" } } + ], "items": { "type": "string", "minLength": 1 } } } }, "database_object_names": { "const": "multiword_snake_case" }, + "tenant_purpose_bound": { "const": true }, "assurance_posture": { "const": "design_constraints_only" } } } diff --git a/schemas/examples/cwl-object-storage-v1.example.json b/schemas/examples/cwl-object-storage-v1.example.json index 7a5f88e037..f445620074 100644 --- a/schemas/examples/cwl-object-storage-v1.example.json +++ b/schemas/examples/cwl-object-storage-v1.example.json @@ -51,5 +51,6 @@ ] }, "database_object_names": "multiword_snake_case", + "tenant_purpose_bound": true, "assurance_posture": "design_constraints_only" } diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index 16fcaafa78..6432e81fb9 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -39,7 +39,23 @@ {"bucket", "object_key", "credential", "raw_pii"} ) FORBIDDEN_EXACT_HOSTS = frozenset( - {"localhost", "metadata.google.internal", "metadata.goog"} + { + "localhost", + "metadata.google.internal", + "metadata.goog", + "instance-data", + "instance-data.ec2.internal", + "kubernetes.default.svc", + } +) +ALWAYS_FORBIDDEN_HOST_SUFFIXES = (".localhost", ".local") +DENIED_PRIVATE_NETWORK_SUFFIXES = ( + ".internal", + ".corp", + ".lan", + ".home", + ".intranet", + ".private", ) MAX_TCP_PORT = 65535 MAX_DNS_HOST_LENGTH = 253 @@ -57,6 +73,7 @@ "rollback", "observability", "database_object_names", + "tenant_purpose_bound", "assurance_posture", ) ENDPOINT_POLICY_KEYS = ( @@ -136,6 +153,22 @@ def _is_ip_literal_or_alias(host: str) -> bool: return all(_label_is_integer_token(label) for label in host.split(".")) +def host_matches_suffix(host: str, suffixes: tuple[str, ...]) -> bool: + """Return whether *host* equals or ends with one DNS suffix in *suffixes*.""" + for suffix in suffixes: + bare = suffix[1:] + if host == bare or host.endswith(suffix): + return True + return False + + +def is_denied_private_network_host(host: str) -> bool: + """Return whether *host* is implicit-local under a denied private-network policy.""" + if "." not in host: + return True + return host_matches_suffix(host, DENIED_PRIVATE_NETWORK_SUFFIXES) + + def is_exact_dns_host(host: str) -> bool: """Return whether *host* is one exact lowercase DNS name. @@ -150,7 +183,9 @@ def is_exact_dns_host(host: str) -> bool: return False if host != host.lower(): return False - if host in FORBIDDEN_EXACT_HOSTS or host.endswith(".localhost"): + if host in FORBIDDEN_EXACT_HOSTS or host_matches_suffix( + host, ALWAYS_FORBIDDEN_HOST_SUFFIXES + ): return False if _is_ip_literal_or_alias(host): return False @@ -297,9 +332,13 @@ def validate_endpoint_policy(policy: Mapping[str, Any]) -> None: raise ObjectStorageContractError( f"endpoint host {host!r} is not an exact DNS name" ) - if private_network_trust == "denied" and "." not in host: + if private_network_trust == "denied" and is_denied_private_network_host(host): + if "." not in host: + raise ObjectStorageContractError( + f"endpoint host {host!r} is a single-label name" + ) raise ObjectStorageContractError( - f"endpoint host {host!r} is a single-label name" + f"endpoint host {host!r} is a private-network name" ) if host in seen: raise ObjectStorageContractError(f"duplicate endpoint host {host!r}") @@ -493,6 +532,9 @@ def validate_contract(data: Mapping[str, Any]) -> None: raise ObjectStorageContractError( "database_object_names must be multiword_snake_case" ) + require_bool( + data.get("tenant_purpose_bound"), "tenant_purpose_bound", True + ) if ( require_text(data.get("assurance_posture"), "assurance_posture") != "design_constraints_only" diff --git a/tests/test_object_storage_contract.py b/tests/test_object_storage_contract.py index 3608f60827..398d6a8c75 100644 --- a/tests/test_object_storage_contract.py +++ b/tests/test_object_storage_contract.py @@ -131,6 +131,7 @@ def test_validate_contract_rejects_policy_regressions() -> None: ("repository", "other/naruon", "ContextualWisdomLab"), ("provider_class", "gcs", "provider_class"), ("database_object_names", "camelCase", "multiword_snake_case"), + ("tenant_purpose_bound", False, "tenant_purpose_bound"), ("assurance_posture", "certified", "design_constraints_only"), ] for field, value, needle in cases: @@ -336,6 +337,8 @@ def test_policy_and_doctoring_record_buyer_visible_controls() -> None: assert "scripts/ci/validate_object_storage_contract.py" in policy assert "design constraints" in policy.lower() assert "not a blanket PII mask" in policy + assert "tenant- and purpose-bound" in policy + assert "PRODUCT_ACCEPTANCE_TEMPLATE.md" in policy assert "APA 7" in doctoring or "References (APA 7th)" in doctoring assert "Amazon Web Services" in doctoring assert "CWE-918" in doctoring diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index 793d144836..de4e094a35 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -135,3 +135,112 @@ def test_observability_labels_fail_closed_without_type_errors() -> None: contract["observability"]["high_cardinality_labels_forbid"].append("bucket") with pytest.raises(validator.ObjectStorageContractError, match="duplicate"): validator.validate_contract(contract) + + +@pytest.mark.parametrize( + "host", + [ + "s3.internal", + "objects.corp", + "minio.lan", + "storage.home", + "minio.intranet", + "objects.private", + ], +) +def test_denied_private_network_rejects_special_use_internal_suffixes( + host: str, +) -> None: + """A denied private-network policy cannot admit RFC 6761/6762 internal names.""" + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "denied" + contract["endpoint_policy"]["host_allowlist"] = [host] + contract["endpoint_policy"].pop("custom_endpoint", None) + with pytest.raises(validator.ObjectStorageContractError, match="private-network"): + validator.validate_contract(contract) + + +def test_mdns_local_suffix_is_never_an_exact_allowlist_member() -> None: + """Multicast .local names are not unicast object-storage endpoints.""" + assert validator.is_exact_dns_host("local") is False + assert validator.is_exact_dns_host("minio.local") is False + assert validator.is_denied_private_network_host("objects.example.example") is False + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "explicit_allowlist" + contract["endpoint_policy"]["host_allowlist"] = ["minio.local"] + contract["endpoint_policy"].pop("custom_endpoint", None) + with pytest.raises(validator.ObjectStorageContractError, match="exact DNS"): + validator.validate_contract(contract) + + +def test_explicit_allowlist_still_admits_a_named_internal_host() -> None: + """Operators may name one private DNS host after an explicit trust decision.""" + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "explicit_allowlist" + contract["endpoint_policy"]["host_allowlist"] = ["minio.internal"] + contract["endpoint_policy"]["custom_endpoint"] = "https://minio.internal" + validator.validate_contract(contract) + + +@pytest.mark.parametrize( + "host", + [ + "instance-data", + "instance-data.ec2.internal", + "kubernetes.default.svc", + "::1", + "[::1]", + ], +) +def test_metadata_cluster_and_ipv6_literals_are_never_exact_hosts(host: str) -> None: + """Metadata, cluster-local, and IPv6 literals stay off the exact allowlist.""" + assert validator.is_exact_dns_host(host) is False + + +def test_tenant_purpose_bound_selection_is_mandatory() -> None: + """Provider selection must stay tenant- and purpose-bound.""" + contract = _valid_contract() + contract["tenant_purpose_bound"] = False + with pytest.raises( + validator.ObjectStorageContractError, match="tenant_purpose_bound must be true" + ): + validator.validate_contract(contract) + contract = _valid_contract() + contract.pop("tenant_purpose_bound", None) + with pytest.raises(validator.ObjectStorageContractError, match="missing keys"): + validator.validate_contract(contract) + + +def test_schema_encodes_value_level_closed_controls() -> None: + """Portable schema consumers must see the same closed values as the validator.""" + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + repository = schema["properties"]["repository"] + assert repository["pattern"] == r"^ContextualWisdomLab/[A-Za-z0-9._-]*[A-Za-z0-9_-]$" + hosts = schema["properties"]["endpoint_policy"]["properties"]["host_allowlist"] + assert hosts["uniqueItems"] is True + custom = schema["properties"]["endpoint_policy"]["properties"]["custom_endpoint"] + assert custom["pattern"] == r"^https://" + states = schema["properties"]["lifecycle"]["properties"]["states"] + assert states["uniqueItems"] is True + assert {item["contains"]["const"] for item in states["allOf"]} == set( + validator.REQUIRED_LIFECYCLE_STATES + ) + labels = schema["properties"]["observability"]["properties"][ + "high_cardinality_labels_forbid" + ] + assert labels["uniqueItems"] is True + assert {item["contains"]["const"] for item in labels["allOf"]} == set( + validator.FORBIDDEN_HIGH_CARDINALITY_LABELS + ) + assert schema["properties"]["tenant_purpose_bound"]["const"] is True + + +def test_product_acceptance_template_names_the_consumer_failure_lane() -> None: + """Product repositories need the next write/read/delete proof, not more prose.""" + template = ( + ROOT / "docs" / "object-storage" / "PRODUCT_ACCEPTANCE_TEMPLATE.md" + ).read_text(encoding="utf-8") + assert "ContextualWisdomLab/naruon#1364" in template + assert "write/read/delete" in template + assert "partial" in template.lower() + assert "timeout" in template.lower() From bfc94733bfca0c8a58c84cbf4b9f9311eb5cbf0f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 15:24:06 +0000 Subject: [PATCH 08/11] fix(storage): require DNS pinning and reject cluster rebinding hosts Close the remaining ContextualWisdomLab/.github#1019 SSRF holes on the object-storage contract: pin resolved addresses, reject DNS-rebinding helper suffixes, and deny Kubernetes .svc names unless explicitly named. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 4 +- CHANGELOG.md | 1 + docs/doctoring/object-storage-contract.md | 35 +++++++++---- .../CWL_OBJECT_STORAGE_CONTRACT.md | 11 ++-- .../PRODUCT_ACCEPTANCE_TEMPLATE.md | 1 + schemas/cwl-object-storage-v1.schema.json | 14 +++++ .../cwl-object-storage-v1.example.json | 1 + .../ci/validate_object_storage_contract.py | 19 ++++++- tests/test_object_storage_contract.py | 2 + .../test_object_storage_contract_hardening.py | 51 +++++++++++++++++++ 10 files changed, 120 insertions(+), 19 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e180c25580..6c6ce971f7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -112,7 +112,9 @@ Naruon and other products keep their own adapters. The executable check is allowlists, tenant-purpose binding, server-side encryption, SHA-256-or-stronger integrity, distinct lifecycle states, and non-destructive rollback are required. Denied private-network trust also rejects special-use internal -suffixes. CSAP and SOC 2 remain design constraints, not certification claims. +and Kubernetes `.svc` suffixes. DNS pinning is mandatory; rebinding helper +suffixes are never allowlist members. CSAP and SOC 2 remain design +constraints, not certification claims. Operational PII is not blanket-masked. Product adapters prove write/read/delete timeout and partial-upload behavior with `docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md`. diff --git a/CHANGELOG.md b/CHANGELOG.md index eb38ea6dd8..d9c88af59c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ Semantic Versioning where the repository publishes a release. - Rejected decimal, hexadecimal, and dotted-numeric IPv4 aliases in object-storage host allowlists so Strix-proven metadata-service SSRF bypasses fail closed. - Rejected multicast `.local`, metadata, and cluster-local names from object-storage allowlists, and rejected RFC 6761/6762 internal suffixes when private-network trust is denied, so a `denied` policy cannot admit `minio.internal`. - Required tenant- and purpose-bound provider selection, aligned the portable JSON Schema with the executable value constraints, and added the product write/read/delete acceptance template so sibling adapters can prove compatibility without copying central code. +- Required DNS pinning, rejected DNS-rebinding helper suffixes and RFC 6761 `.test`/`.invalid` names from object-storage allowlists, rejected Kubernetes `.svc` and `.localdomain` hosts when private-network trust is denied, and encoded the consumed/zero-retention coupling in the portable schema so ContextualWisdomLab/.github#1019 SSRF/DNS-rebinding protection is executable. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index ba1e72c88a..db90e14e4b 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -18,13 +18,16 @@ policy and the fail-closed check that leaf repositories can cite. 1. Treat AWS S3 and S3-compatible HTTPS stores as one `object_storage` capability. -2. Require HTTPS, exact-host allowlists, no wildcards, and no automatic - redirects (CWE-918). +2. Require HTTPS, exact-host allowlists, no wildcards, no automatic + redirects, and DNS pinning for the request lifetime (CWE-918; Jackson + et al., 2009). DNS-rebinding helper suffixes and RFC 6761 `.test` / + `.invalid` names are never allowlist members. 3. Require explicit private-network trust. Implicit RFC1918, metadata-service, - multicast `.local` (Cheshire & Krochmal, 2013b), or special-use internal - suffixes (Cheshire & Krochmal, 2013a) are not authorized unless the exact - host is named after an explicit trust decision. Multicast `.local` names - are never unicast endpoints. + multicast `.local` (Cheshire & Krochmal, 2013b), Kubernetes `.svc` + names (The Kubernetes Authors, n.d.), or special-use internal suffixes + (Cheshire & Krochmal, 2013a) are not authorized unless the exact host is + named after an explicit trust decision. Multicast `.local` names are + never unicast endpoints. 4. Require least-privilege object permissions and prohibit public ACLs, public buckets, and browser-exposed long-lived credentials (CWE-798; CWE-200). 5. Require server-side encryption and fail-closed content-length plus SHA-256 @@ -61,11 +64,13 @@ nested schema objects stay closed, NaN/Infinity are rejected, exact-host allowlists exclude localhost, metadata, cluster-local names, IPv4 and IPv6 literals, decimal or hexadecimal IP aliases, Unicode, case aliases, and multicast `.local` names, a denied private-network policy rejects -single-label and special-use internal hosts, custom endpoints reject ports -above 65535, schema value constraints match the validator, tenant-purpose -binding is mandatory, and malformed observability labels raise policy -errors instead of TypeError. Local quality remains 100% statement/branch -coverage and 100% docstrings. +single-label, Kubernetes `.svc`, and special-use internal hosts, custom +endpoints reject ports above 65535, schema value constraints match the +validator including the consumed/zero-retention coupling, tenant-purpose +binding and DNS pinning are mandatory, DNS-rebinding helper suffixes stay +off the allowlist, and malformed observability labels raise policy errors +instead of TypeError. Local quality remains 100% statement/branch coverage +and 100% docstrings. ## Rollback @@ -74,6 +79,14 @@ not keep a schema that the executable check no longer enforces. ## References (APA 7th) +Jackson, C., Barth, A., Bortz, A., Shao, W., & Boneh, D. (2009). Protecting +browsers from DNS rebinding attacks. *ACM Transactions on the Web, 3*(1), +Article 2, 1–26. https://doi.org/10.1145/1462148.1462150 + +The Kubernetes Authors. (n.d.). *DNS for Services and Pods*. Kubernetes +Documentation. +https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/ + Cheshire, S., & Krochmal, M. (2013a). *Special-use domain names* (RFC 6761). Internet Engineering Task Force. https://www.rfc-editor.org/rfc/rfc6761 diff --git a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md index fa8a409dde..ff598927e5 100644 --- a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md +++ b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md @@ -17,8 +17,9 @@ A contract JSON document is valid only when every control below is true. | Control | Required value | |---|---| | Transport | `https` only | -| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local` or metadata names | -| Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918, RFC 6761/6762, or `.internal` access | +| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local`, metadata, RFC 6761 `.test`/`.invalid`, or DNS-rebinding helper names | +| DNS pinning | resolve once and pin the address for the request lifetime; a later TTL flip must not retarget the socket | +| Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918, RFC 6761/6762, Kubernetes `.svc`, or `.internal` access | | Credentials | scoped secret registry or workload identity; never broadcast, browser-exposed, or ambient process-wide | | Permissions | least privilege; public ACLs and public buckets prohibited | | Encryption | server-side encryption `required` | @@ -42,9 +43,9 @@ high-cardinality labels is not a blanket PII mask. 3. **Authorized private-network endpoints** — `private_network_trust` is `explicit_allowlist` and each private host is named. Implicit RFC1918, link-local, metadata-service, multicast `.local`, or special-use - `.internal` / `.corp` / `.lan` access is rejected unless that exact host - is named after an explicit trust decision. Multicast `.local` names are - never valid unicast endpoints. + `.internal` / `.corp` / `.lan` / `.svc` access is rejected unless that exact + host is named after an explicit trust decision. Multicast `.local` names and + DNS-rebinding helpers such as `.nip.io` are never valid unicast endpoints. Workload-identity and instance-metadata access require their own SSRF review. This contract does not grant that access. diff --git a/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md b/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md index 8c6cb154e9..a3e80ffa96 100644 --- a/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md +++ b/docs/object-storage/PRODUCT_ACCEPTANCE_TEMPLATE.md @@ -26,6 +26,7 @@ against the current head, including: | delete timeout | Retry the same deterministic key; do not enumerate the bucket | | partial or aborted upload | Do not publish a `available` object; compensate without deleting older customer data | | digest or content-length mismatch | Reject the read; keep the prior object | +| DNS TTL flip / rebinding | Pin the first resolved address for the request; do not look the name up again mid-transfer | ## Rollback and retention diff --git a/schemas/cwl-object-storage-v1.schema.json b/schemas/cwl-object-storage-v1.schema.json index b46a07be68..57e5794f0c 100644 --- a/schemas/cwl-object-storage-v1.schema.json +++ b/schemas/cwl-object-storage-v1.schema.json @@ -37,6 +37,7 @@ "host_allowlist", "allow_wildcards", "follow_redirects", + "dns_pinning", "private_network_trust" ], "properties": { @@ -49,6 +50,7 @@ }, "allow_wildcards": { "const": false }, "follow_redirects": { "const": false }, + "dns_pinning": { "const": true }, "private_network_trust": { "enum": ["explicit_allowlist", "denied"] }, @@ -127,6 +129,18 @@ "consumed_implies_immediate_delete": { "type": "boolean" }, "zero_retention_explicit": { "type": "boolean" }, "legal_hold_distinct": { "const": true } + }, + "if": { + "properties": { + "consumed_implies_immediate_delete": { "const": true } + }, + "required": ["consumed_implies_immediate_delete"] + }, + "then": { + "properties": { + "zero_retention_explicit": { "const": true } + }, + "required": ["zero_retention_explicit"] } }, "rollback": { diff --git a/schemas/examples/cwl-object-storage-v1.example.json b/schemas/examples/cwl-object-storage-v1.example.json index f445620074..2a4d3519d4 100644 --- a/schemas/examples/cwl-object-storage-v1.example.json +++ b/schemas/examples/cwl-object-storage-v1.example.json @@ -11,6 +11,7 @@ ], "allow_wildcards": false, "follow_redirects": false, + "dns_pinning": true, "private_network_trust": "explicit_allowlist", "custom_endpoint": "https://objects.example.example" }, diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index 6432e81fb9..c7b4c717d3 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -48,7 +48,17 @@ "kubernetes.default.svc", } ) -ALWAYS_FORBIDDEN_HOST_SUFFIXES = (".localhost", ".local") +ALWAYS_FORBIDDEN_HOST_SUFFIXES = ( + ".localhost", + ".local", + ".invalid", + ".test", + ".nip.io", + ".sslip.io", + ".xip.io", + ".lvh.me", + ".localtest.me", +) DENIED_PRIVATE_NETWORK_SUFFIXES = ( ".internal", ".corp", @@ -56,6 +66,8 @@ ".home", ".intranet", ".private", + ".svc", + ".localdomain", ) MAX_TCP_PORT = 65535 MAX_DNS_HOST_LENGTH = 253 @@ -81,6 +93,7 @@ "host_allowlist", "allow_wildcards", "follow_redirects", + "dns_pinning", "private_network_trust", "custom_endpoint", ) @@ -173,7 +186,8 @@ def is_exact_dns_host(host: str) -> bool: """Return whether *host* is one exact lowercase DNS name. Localhost, link-local metadata names, IP literals, decimal or hexadecimal - IP aliases, Unicode, and case aliases are not exact allowlist members. + IP aliases, Unicode, case aliases, RFC 6761 ``.test`` / ``.invalid``, and + DNS-rebinding helper suffixes are not exact allowlist members. """ if not isinstance(host, str) or not host or len(host) > MAX_DNS_HOST_LENGTH: return False @@ -315,6 +329,7 @@ def validate_endpoint_policy(policy: Mapping[str, Any]) -> None: require_bool( policy.get("follow_redirects"), "endpoint_policy.follow_redirects", False ) + require_bool(policy.get("dns_pinning"), "endpoint_policy.dns_pinning", True) require_allowed( policy.get("private_network_trust"), "endpoint_policy.private_network_trust", diff --git a/tests/test_object_storage_contract.py b/tests/test_object_storage_contract.py index 398d6a8c75..73a5e0304b 100644 --- a/tests/test_object_storage_contract.py +++ b/tests/test_object_storage_contract.py @@ -338,9 +338,11 @@ def test_policy_and_doctoring_record_buyer_visible_controls() -> None: assert "design constraints" in policy.lower() assert "not a blanket PII mask" in policy assert "tenant- and purpose-bound" in policy + assert "DNS pinning" in policy assert "PRODUCT_ACCEPTANCE_TEMPLATE.md" in policy assert "APA 7" in doctoring or "References (APA 7th)" in doctoring assert "Amazon Web Services" in doctoring + assert "Jackson" in doctoring assert "CWE-918" in doctoring assert "object-storage contract" in changelog assert "object-storage" in architecture.lower() diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index de4e094a35..b46d6ae8d5 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -146,6 +146,8 @@ def test_observability_labels_fail_closed_without_type_errors() -> None: "storage.home", "minio.intranet", "objects.private", + "minio.default.svc", + "objects.localdomain", ], ) def test_denied_private_network_rejects_special_use_internal_suffixes( @@ -173,6 +175,43 @@ def test_mdns_local_suffix_is_never_an_exact_allowlist_member() -> None: validator.validate_contract(contract) +@pytest.mark.parametrize( + "host", + [ + "169.254.169.254.nip.io", + "127.0.0.1.sslip.io", + "10.0.0.1.xip.io", + "metadata.lvh.me", + "minio.localtest.me", + "bucket.test", + "objects.invalid", + ], +) +def test_dns_rebinding_helper_suffixes_are_never_exact_hosts(host: str) -> None: + """Rebinding helpers and RFC 6761 test names cannot enter any allowlist.""" + assert validator.is_exact_dns_host(host) is False + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "explicit_allowlist" + contract["endpoint_policy"]["host_allowlist"] = [host] + contract["endpoint_policy"].pop("custom_endpoint", None) + with pytest.raises(validator.ObjectStorageContractError, match="exact DNS"): + validator.validate_contract(contract) + + +def test_dns_pinning_is_mandatory() -> None: + """A later DNS TTL flip must not retarget an in-flight object transfer.""" + contract = _valid_contract() + contract["endpoint_policy"]["dns_pinning"] = False + with pytest.raises( + validator.ObjectStorageContractError, match="dns_pinning must be true" + ): + validator.validate_contract(contract) + contract = _valid_contract() + contract["endpoint_policy"].pop("dns_pinning", None) + with pytest.raises(validator.ObjectStorageContractError, match="dns_pinning"): + validator.validate_contract(contract) + + def test_explicit_allowlist_still_admits_a_named_internal_host() -> None: """Operators may name one private DNS host after an explicit trust decision.""" contract = _valid_contract() @@ -180,6 +219,9 @@ def test_explicit_allowlist_still_admits_a_named_internal_host() -> None: contract["endpoint_policy"]["host_allowlist"] = ["minio.internal"] contract["endpoint_policy"]["custom_endpoint"] = "https://minio.internal" validator.validate_contract(contract) + contract["endpoint_policy"]["host_allowlist"] = ["minio.default.svc"] + contract["endpoint_policy"]["custom_endpoint"] = "https://minio.default.svc" + validator.validate_contract(contract) @pytest.mark.parametrize( @@ -233,6 +275,14 @@ def test_schema_encodes_value_level_closed_controls() -> None: validator.FORBIDDEN_HIGH_CARDINALITY_LABELS ) assert schema["properties"]["tenant_purpose_bound"]["const"] is True + endpoint = schema["properties"]["endpoint_policy"] + assert endpoint["properties"]["dns_pinning"]["const"] is True + assert "dns_pinning" in endpoint["required"] + lifecycle = schema["properties"]["lifecycle"] + assert lifecycle["if"]["properties"]["consumed_implies_immediate_delete"][ + "const" + ] is True + assert lifecycle["then"]["properties"]["zero_retention_explicit"]["const"] is True def test_product_acceptance_template_names_the_consumer_failure_lane() -> None: @@ -244,3 +294,4 @@ def test_product_acceptance_template_names_the_consumer_failure_lane() -> None: assert "write/read/delete" in template assert "partial" in template.lower() assert "timeout" in template.lower() + assert "rebinding" in template.lower() From b09f26fdead8c42550b4b91fcbfed6eefe08bfd2 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 16:36:16 +0000 Subject: [PATCH 09/11] fix(storage): reject embedded IPv4 rebinding hosts Helper-TLD denylists still admitted 169.254.169.254.attacker.example and 32-bit decimal or hex aliases under an ordinary suffix. Fail closed on those embeddings while keeping single octet DNS labels valid. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 3 +- CHANGELOG.md | 1 + docs/doctoring/object-storage-contract.md | 11 +++--- .../CWL_OBJECT_STORAGE_CONTRACT.md | 8 +++-- .../ci/validate_object_storage_contract.py | 36 +++++++++++++++++-- .../test_object_storage_contract_hardening.py | 33 +++++++++++++++++ 6 files changed, 81 insertions(+), 11 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6c6ce971f7..5e7dfc4c81 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -113,7 +113,8 @@ allowlists, tenant-purpose binding, server-side encryption, SHA-256-or-stronger integrity, distinct lifecycle states, and non-destructive rollback are required. Denied private-network trust also rejects special-use internal and Kubernetes `.svc` suffixes. DNS pinning is mandatory; rebinding helper -suffixes are never allowlist members. CSAP and SOC 2 remain design +suffixes and embedded IPv4 or 32-bit numeric aliases are never allowlist +members. CSAP and SOC 2 remain design constraints, not certification claims. Operational PII is not blanket-masked. Product adapters prove write/read/delete timeout and partial-upload behavior with diff --git a/CHANGELOG.md b/CHANGELOG.md index d9c88af59c..39d9d4757b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ Semantic Versioning where the repository publishes a release. - Rejected multicast `.local`, metadata, and cluster-local names from object-storage allowlists, and rejected RFC 6761/6762 internal suffixes when private-network trust is denied, so a `denied` policy cannot admit `minio.internal`. - Required tenant- and purpose-bound provider selection, aligned the portable JSON Schema with the executable value constraints, and added the product write/read/delete acceptance template so sibling adapters can prove compatibility without copying central code. - Required DNS pinning, rejected DNS-rebinding helper suffixes and RFC 6761 `.test`/`.invalid` names from object-storage allowlists, rejected Kubernetes `.svc` and `.localdomain` hosts when private-network trust is denied, and encoded the consumed/zero-retention coupling in the portable schema so ContextualWisdomLab/.github#1019 SSRF/DNS-rebinding protection is executable. +- Rejected object-storage hosts that embed a dotted IPv4 address or a 32-bit decimal/hexadecimal alias under an ordinary suffix, so `169.254.169.254.attacker.example` cannot bypass the helper-TLD denylist. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index db90e14e4b..a0bb5abba1 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -20,8 +20,10 @@ policy and the fail-closed check that leaf repositories can cite. capability. 2. Require HTTPS, exact-host allowlists, no wildcards, no automatic redirects, and DNS pinning for the request lifetime (CWE-918; Jackson - et al., 2009). DNS-rebinding helper suffixes and RFC 6761 `.test` / - `.invalid` names are never allowlist members. + et al., 2009). DNS-rebinding helper suffixes, RFC 6761 `.test` / + `.invalid` names, and hosts that embed a dotted IPv4 address or a 32-bit + numeric alias (for example `169.254.169.254.attacker.example`) are never + allowlist members. 3. Require explicit private-network trust. Implicit RFC1918, metadata-service, multicast `.local` (Cheshire & Krochmal, 2013b), Kubernetes `.svc` names (The Kubernetes Authors, n.d.), or special-use internal suffixes @@ -67,8 +69,9 @@ multicast `.local` names, a denied private-network policy rejects single-label, Kubernetes `.svc`, and special-use internal hosts, custom endpoints reject ports above 65535, schema value constraints match the validator including the consumed/zero-retention coupling, tenant-purpose -binding and DNS pinning are mandatory, DNS-rebinding helper suffixes stay -off the allowlist, and malformed observability labels raise policy errors +binding and DNS pinning are mandatory, DNS-rebinding helper suffixes and +embedded IPv4 or 32-bit numeric aliases stay off the allowlist, and +malformed observability labels raise policy errors instead of TypeError. Local quality remains 100% statement/branch coverage and 100% docstrings. diff --git a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md index ff598927e5..48087396ba 100644 --- a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md +++ b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md @@ -17,7 +17,7 @@ A contract JSON document is valid only when every control below is true. | Control | Required value | |---|---| | Transport | `https` only | -| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local`, metadata, RFC 6761 `.test`/`.invalid`, or DNS-rebinding helper names | +| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local`, metadata, RFC 6761 `.test`/`.invalid`, DNS-rebinding helper names, or embedded IPv4 / 32-bit numeric aliases | | DNS pinning | resolve once and pin the address for the request lifetime; a later TTL flip must not retarget the socket | | Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918, RFC 6761/6762, Kubernetes `.svc`, or `.internal` access | | Credentials | scoped secret registry or workload identity; never broadcast, browser-exposed, or ambient process-wide | @@ -44,8 +44,10 @@ high-cardinality labels is not a blanket PII mask. `explicit_allowlist` and each private host is named. Implicit RFC1918, link-local, metadata-service, multicast `.local`, or special-use `.internal` / `.corp` / `.lan` / `.svc` access is rejected unless that exact - host is named after an explicit trust decision. Multicast `.local` names and - DNS-rebinding helpers such as `.nip.io` are never valid unicast endpoints. + host is named after an explicit trust decision. Multicast `.local` names, + DNS-rebinding helpers such as `.nip.io`, and hosts that embed a dotted IPv4 + address or 32-bit numeric alias (for example `169.254.169.254.attacker.example`) + are never valid unicast endpoints. Workload-identity and instance-metadata access require their own SSRF review. This contract does not grant that access. diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index c7b4c717d3..e679bfcfd9 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -155,6 +155,15 @@ def _label_is_integer_token(label: str) -> bool: ) +def _integer_token_value(label: str) -> int | None: + """Return the integer value of a decimal or ``0x`` hex DNS label.""" + if not _label_is_integer_token(label): + return None + if label.startswith("0x"): + return int(label, 16) + return int(label) + + def _is_ip_literal_or_alias(host: str) -> bool: """Return whether *host* is an IP literal or a numeric IP alias.""" try: @@ -166,6 +175,26 @@ def _is_ip_literal_or_alias(host: str) -> bool: return all(_label_is_integer_token(label) for label in host.split(".")) +def host_embeds_ip_alias(host: str) -> bool: + """Return whether *host* embeds an IPv4 literal or a 32-bit numeric alias. + + A single numeric DNS label is still allowed when it is an octet (0-255). + A 32-bit decimal or hexadecimal alias, or four consecutive octet labels, + is a rebinding hostname even when the remaining labels are ordinary DNS. + Integers above ``0xFFFFFFFF`` are not IPv4 aliases. + """ + labels = host.split(".") + for label in labels: + value = _integer_token_value(label) + if value is not None and 255 < value <= 0xFFFFFFFF: + return True + for index in range(len(labels) - 3): + octets = [_integer_token_value(label) for label in labels[index : index + 4]] + if all(value is not None and value <= 255 for value in octets): + return True + return False + + def host_matches_suffix(host: str, suffixes: tuple[str, ...]) -> bool: """Return whether *host* equals or ends with one DNS suffix in *suffixes*.""" for suffix in suffixes: @@ -186,8 +215,9 @@ def is_exact_dns_host(host: str) -> bool: """Return whether *host* is one exact lowercase DNS name. Localhost, link-local metadata names, IP literals, decimal or hexadecimal - IP aliases, Unicode, case aliases, RFC 6761 ``.test`` / ``.invalid``, and - DNS-rebinding helper suffixes are not exact allowlist members. + IP aliases, embedded IPv4 sequences, Unicode, case aliases, RFC 6761 + ``.test`` / ``.invalid``, and DNS-rebinding helper suffixes are not exact + allowlist members. """ if not isinstance(host, str) or not host or len(host) > MAX_DNS_HOST_LENGTH: return False @@ -201,7 +231,7 @@ def is_exact_dns_host(host: str) -> bool: host, ALWAYS_FORBIDDEN_HOST_SUFFIXES ): return False - if _is_ip_literal_or_alias(host): + if _is_ip_literal_or_alias(host) or host_embeds_ip_alias(host): return False labels = host.split(".") for label in labels: diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index b46d6ae8d5..6239e97101 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -198,6 +198,39 @@ def test_dns_rebinding_helper_suffixes_are_never_exact_hosts(host: str) -> None: validator.validate_contract(contract) +@pytest.mark.parametrize( + "host", + [ + "169.254.169.254.attacker.example", + "127.0.0.1.evil.example", + "0xa.0xb.0xc.0xd.objects.example", + "2851992574.attacker.example", + "0x7f000001.attacker.example", + ], +) +def test_embedded_ipv4_aliases_are_never_exact_hosts(host: str) -> None: + """An IPv4 sequence or 32-bit alias remains a rebinding host on any suffix.""" + assert validator.host_embeds_ip_alias(host) is True + assert validator.is_exact_dns_host(host) is False + contract = _valid_contract() + contract["endpoint_policy"]["private_network_trust"] = "explicit_allowlist" + contract["endpoint_policy"]["host_allowlist"] = [host] + contract["endpoint_policy"].pop("custom_endpoint", None) + with pytest.raises(validator.ObjectStorageContractError, match="exact DNS"): + validator.validate_contract(contract) + + +def test_embedded_ip_helper_keeps_octet_labels_and_oversize_integers() -> None: + """Octet DNS labels stay valid; integers above 0xFFFFFFFF are not IPv4 aliases.""" + assert validator._integer_token_value("s3") is None + assert validator._integer_token_value("1") == 1 + assert validator._integer_token_value("0x7f") == 127 + assert validator.host_embeds_ip_alias("1.s3.amazonaws.com") is False + assert validator.host_embeds_ip_alias("1.2.3.example.com") is False + assert validator.host_embeds_ip_alias("1.2.3.4294967296.example.com") is False + assert validator.is_exact_dns_host("1.2.3.4294967296.example.com") is True + + def test_dns_pinning_is_mandatory() -> None: """A later DNS TTL flip must not retarget an in-flight object transfer.""" contract = _valid_contract() From 3068296010c9b8debd107652039cea175ea4db5a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Mon, 17 Aug 2026 16:42:57 +0000 Subject: [PATCH 10/11] fix(storage): reject hyphenated IPv4 rebinding labels nip.io-style A-B-C-D and AWS ip-A-B-C-D labels still passed the dotted embed check. Treat those labels as rebinding hosts while leaving three-part or non-octet hyphen labels valid. Co-authored-by: Seongho Bae --- ARCHITECTURE.md | 2 +- CHANGELOG.md | 2 +- docs/doctoring/object-storage-contract.md | 6 +++--- .../CWL_OBJECT_STORAGE_CONTRACT.md | 7 ++++--- .../ci/validate_object_storage_contract.py | 21 +++++++++++++++---- .../test_object_storage_contract_hardening.py | 9 ++++++++ 6 files changed, 35 insertions(+), 12 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 5e7dfc4c81..19391706dd 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -113,7 +113,7 @@ allowlists, tenant-purpose binding, server-side encryption, SHA-256-or-stronger integrity, distinct lifecycle states, and non-destructive rollback are required. Denied private-network trust also rejects special-use internal and Kubernetes `.svc` suffixes. DNS pinning is mandatory; rebinding helper -suffixes and embedded IPv4 or 32-bit numeric aliases are never allowlist +suffixes and embedded or hyphenated IPv4 or 32-bit numeric aliases are never allowlist members. CSAP and SOC 2 remain design constraints, not certification claims. Operational PII is not blanket-masked. Product adapters prove write/read/delete diff --git a/CHANGELOG.md b/CHANGELOG.md index 39d9d4757b..847d165ea3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ Semantic Versioning where the repository publishes a release. - Rejected multicast `.local`, metadata, and cluster-local names from object-storage allowlists, and rejected RFC 6761/6762 internal suffixes when private-network trust is denied, so a `denied` policy cannot admit `minio.internal`. - Required tenant- and purpose-bound provider selection, aligned the portable JSON Schema with the executable value constraints, and added the product write/read/delete acceptance template so sibling adapters can prove compatibility without copying central code. - Required DNS pinning, rejected DNS-rebinding helper suffixes and RFC 6761 `.test`/`.invalid` names from object-storage allowlists, rejected Kubernetes `.svc` and `.localdomain` hosts when private-network trust is denied, and encoded the consumed/zero-retention coupling in the portable schema so ContextualWisdomLab/.github#1019 SSRF/DNS-rebinding protection is executable. -- Rejected object-storage hosts that embed a dotted IPv4 address or a 32-bit decimal/hexadecimal alias under an ordinary suffix, so `169.254.169.254.attacker.example` cannot bypass the helper-TLD denylist. +- Rejected object-storage hosts that embed a dotted IPv4 address, a hyphenated `A-B-C-D` / `ip-A-B-C-D` label, or a 32-bit decimal/hexadecimal alias under an ordinary suffix, so `169.254.169.254.attacker.example` and `ip-169-254-169-254.ec2.internal` cannot bypass the helper-TLD denylist. - Materialized base Python locks only when every package line is an exact SHA-256 pin or a bounded relative `-r`/`--requirement` include. A lone `--require-hashes` directive, a dotted include such as `./lock.txt`, or `-r other-hashes.txt` no longer enters the trusted build context. - Refused a conflict-scope repository root whose immediate parent is a symbolic link, so a swapped parent cannot redirect the canonical worktree after the last-component check (CWE-367). - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index a0bb5abba1..8321e452aa 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -22,8 +22,8 @@ policy and the fail-closed check that leaf repositories can cite. redirects, and DNS pinning for the request lifetime (CWE-918; Jackson et al., 2009). DNS-rebinding helper suffixes, RFC 6761 `.test` / `.invalid` names, and hosts that embed a dotted IPv4 address or a 32-bit - numeric alias (for example `169.254.169.254.attacker.example`) are never - allowlist members. + numeric alias (for example `169.254.169.254.attacker.example` or + `169-254-169-254.attacker.example`) are never allowlist members. 3. Require explicit private-network trust. Implicit RFC1918, metadata-service, multicast `.local` (Cheshire & Krochmal, 2013b), Kubernetes `.svc` names (The Kubernetes Authors, n.d.), or special-use internal suffixes @@ -70,7 +70,7 @@ single-label, Kubernetes `.svc`, and special-use internal hosts, custom endpoints reject ports above 65535, schema value constraints match the validator including the consumed/zero-retention coupling, tenant-purpose binding and DNS pinning are mandatory, DNS-rebinding helper suffixes and -embedded IPv4 or 32-bit numeric aliases stay off the allowlist, and +embedded or hyphenated IPv4 or 32-bit numeric aliases stay off the allowlist, and malformed observability labels raise policy errors instead of TypeError. Local quality remains 100% statement/branch coverage and 100% docstrings. diff --git a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md index 48087396ba..26b8c56dc4 100644 --- a/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md +++ b/docs/object-storage/CWL_OBJECT_STORAGE_CONTRACT.md @@ -17,7 +17,7 @@ A contract JSON document is valid only when every control below is true. | Control | Required value | |---|---| | Transport | `https` only | -| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local`, metadata, RFC 6761 `.test`/`.invalid`, DNS-rebinding helper names, or embedded IPv4 / 32-bit numeric aliases | +| Hosts | exact-host allowlist; no wildcards; no automatic redirects; no multicast `.local`, metadata, RFC 6761 `.test`/`.invalid`, DNS-rebinding helper names, or embedded / hyphenated IPv4 / 32-bit numeric aliases | | DNS pinning | resolve once and pin the address for the request lifetime; a later TTL flip must not retarget the socket | | Private networks | `explicit_allowlist` or `denied`; never implicit RFC1918, RFC 6761/6762, Kubernetes `.svc`, or `.internal` access | | Credentials | scoped secret registry or workload identity; never broadcast, browser-exposed, or ambient process-wide | @@ -45,8 +45,9 @@ high-cardinality labels is not a blanket PII mask. link-local, metadata-service, multicast `.local`, or special-use `.internal` / `.corp` / `.lan` / `.svc` access is rejected unless that exact host is named after an explicit trust decision. Multicast `.local` names, - DNS-rebinding helpers such as `.nip.io`, and hosts that embed a dotted IPv4 - address or 32-bit numeric alias (for example `169.254.169.254.attacker.example`) + DNS-rebinding helpers such as `.nip.io`, and hosts that embed a dotted or + hyphenated IPv4 address or 32-bit numeric alias (for example + `169.254.169.254.attacker.example` or `ip-169-254-169-254.ec2.internal`) are never valid unicast endpoints. Workload-identity and instance-metadata access require their own SSRF review. diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index e679bfcfd9..cdd37f12df 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -175,19 +175,32 @@ def _is_ip_literal_or_alias(host: str) -> bool: return all(_label_is_integer_token(label) for label in host.split(".")) +def _label_is_hyphenated_ipv4(label: str) -> bool: + """Return whether *label* is a hyphenated IPv4 or ``ip-A-B-C-D`` alias.""" + candidate = label[3:] if label.startswith("ip-") else label + parts = candidate.split("-") + if len(parts) != 4: + return False + octets = [_integer_token_value(part) for part in parts] + return all(value is not None and value <= 255 for value in octets) + + def host_embeds_ip_alias(host: str) -> bool: """Return whether *host* embeds an IPv4 literal or a 32-bit numeric alias. A single numeric DNS label is still allowed when it is an octet (0-255). - A 32-bit decimal or hexadecimal alias, or four consecutive octet labels, - is a rebinding hostname even when the remaining labels are ordinary DNS. - Integers above ``0xFFFFFFFF`` are not IPv4 aliases. + A 32-bit decimal or hexadecimal alias, four consecutive octet labels, or + a hyphenated ``A-B-C-D`` / ``ip-A-B-C-D`` label is a rebinding hostname + even when the remaining labels are ordinary DNS. Integers above + ``0xFFFFFFFF`` are not IPv4 aliases. """ labels = host.split(".") for label in labels: value = _integer_token_value(label) if value is not None and 255 < value <= 0xFFFFFFFF: return True + if _label_is_hyphenated_ipv4(label): + return True for index in range(len(labels) - 3): octets = [_integer_token_value(label) for label in labels[index : index + 4]] if all(value is not None and value <= 255 for value in octets): @@ -215,7 +228,7 @@ def is_exact_dns_host(host: str) -> bool: """Return whether *host* is one exact lowercase DNS name. Localhost, link-local metadata names, IP literals, decimal or hexadecimal - IP aliases, embedded IPv4 sequences, Unicode, case aliases, RFC 6761 + IP aliases, embedded or hyphenated IPv4 sequences, Unicode, case aliases, RFC 6761 ``.test`` / ``.invalid``, and DNS-rebinding helper suffixes are not exact allowlist members. """ diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index 6239e97101..ca40e6dc76 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -206,6 +206,9 @@ def test_dns_rebinding_helper_suffixes_are_never_exact_hosts(host: str) -> None: "0xa.0xb.0xc.0xd.objects.example", "2851992574.attacker.example", "0x7f000001.attacker.example", + "169-254-169-254.attacker.example", + "ip-169-254-169-254.ec2.internal", + "0x7f-0x0-0x0-0x1.attacker.example", ], ) def test_embedded_ipv4_aliases_are_never_exact_hosts(host: str) -> None: @@ -229,6 +232,12 @@ def test_embedded_ip_helper_keeps_octet_labels_and_oversize_integers() -> None: assert validator.host_embeds_ip_alias("1.2.3.example.com") is False assert validator.host_embeds_ip_alias("1.2.3.4294967296.example.com") is False assert validator.is_exact_dns_host("1.2.3.4294967296.example.com") is True + assert validator._label_is_hyphenated_ipv4("1-2-3") is False + assert validator._label_is_hyphenated_ipv4("build-1-2-3-4") is False + assert validator._label_is_hyphenated_ipv4("256-1-1-1") is False + assert validator.host_embeds_ip_alias("1-2-3.example.com") is False + assert validator.host_embeds_ip_alias("build-1-2-3-4.example.com") is False + assert validator.is_exact_dns_host("1-2-3.example.com") is True def test_dns_pinning_is_mandatory() -> None: From dc7e39cf7dff80c2e2ed8d348090394ddc643142 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 03:31:44 +0900 Subject: [PATCH 11/11] fix(storage): reject punycode endpoint hosts --- docs/doctoring/object-storage-contract.md | 7 +++++-- scripts/ci/validate_object_storage_contract.py | 11 ++++++++--- tests/test_object_storage_contract_hardening.py | 3 ++- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/doctoring/object-storage-contract.md b/docs/doctoring/object-storage-contract.md index 8321e452aa..564deca076 100644 --- a/docs/doctoring/object-storage-contract.md +++ b/docs/doctoring/object-storage-contract.md @@ -23,7 +23,9 @@ policy and the fail-closed check that leaf repositories can cite. et al., 2009). DNS-rebinding helper suffixes, RFC 6761 `.test` / `.invalid` names, and hosts that embed a dotted IPv4 address or a 32-bit numeric alias (for example `169.254.169.254.attacker.example` or - `169-254-169-254.attacker.example`) are never allowlist members. + `169-254-169-254.attacker.example`) are never allowlist members. ASCII IDNA + `xn--` labels are also rejected because their decoded Unicode name can hide + the endpoint represented by the visible allowlist value. 3. Require explicit private-network trust. Implicit RFC1918, metadata-service, multicast `.local` (Cheshire & Krochmal, 2013b), Kubernetes `.svc` names (The Kubernetes Authors, n.d.), or special-use internal suffixes @@ -64,7 +66,8 @@ the schema keys match production constants, and each fail-closed control has a unique rejection. `tests/test_object_storage_contract_hardening.py` proves nested schema objects stay closed, NaN/Infinity are rejected, exact-host allowlists exclude localhost, metadata, cluster-local names, IPv4 and IPv6 -literals, decimal or hexadecimal IP aliases, Unicode, case aliases, and +literals, decimal or hexadecimal IP aliases, Unicode and punycode IDNA labels, +case aliases, and multicast `.local` names, a denied private-network policy rejects single-label, Kubernetes `.svc`, and special-use internal hosts, custom endpoints reject ports above 65535, schema value constraints match the diff --git a/scripts/ci/validate_object_storage_contract.py b/scripts/ci/validate_object_storage_contract.py index cdd37f12df..32b0777a48 100644 --- a/scripts/ci/validate_object_storage_contract.py +++ b/scripts/ci/validate_object_storage_contract.py @@ -228,9 +228,9 @@ def is_exact_dns_host(host: str) -> bool: """Return whether *host* is one exact lowercase DNS name. Localhost, link-local metadata names, IP literals, decimal or hexadecimal - IP aliases, embedded or hyphenated IPv4 sequences, Unicode, case aliases, RFC 6761 - ``.test`` / ``.invalid``, and DNS-rebinding helper suffixes are not exact - allowlist members. + IP aliases, embedded or hyphenated IPv4 sequences, Unicode or punycode labels, + case aliases, RFC 6761 ``.test`` / ``.invalid``, and DNS-rebinding helper + suffixes are not exact allowlist members. """ if not isinstance(host, str) or not host or len(host) > MAX_DNS_HOST_LENGTH: return False @@ -250,6 +250,11 @@ def is_exact_dns_host(host: str) -> bool: for label in labels: if not label or len(label) > 63: return False + # IDNA labels can decode to an endpoint different from the visible ASCII + # spelling; exact allowlists therefore accept neither implicit conversion + # nor opaque ``xn--`` labels. + if label.startswith("xn--"): + return False if label.startswith("-") or label.endswith("-"): return False if not all(_is_dns_label_char(char) for char in label): diff --git a/tests/test_object_storage_contract_hardening.py b/tests/test_object_storage_contract_hardening.py index ca40e6dc76..a6d6b8f7c9 100644 --- a/tests/test_object_storage_contract_hardening.py +++ b/tests/test_object_storage_contract_hardening.py @@ -77,12 +77,13 @@ def test_loader_rejects_non_finite_json_constants() -> None: "metadata.goog", "OBJECTS.EXAMPLE.COM", "objécts.example.com", + "xn--127-0-0-1-9za.example.com", ], ) def test_exact_dns_hosts_reject_local_metadata_literal_and_noncanonical_names( host: str, ) -> None: - """An exact allowlist must not admit metadata, IP, Unicode, or case aliases.""" + """An exact allowlist must not admit metadata, IP, IDNA, or case aliases.""" assert validator.is_exact_dns_host(host) is False