diff --git a/bin/fm-cognee-lookup.sh b/bin/fm-cognee-lookup.sh index c0b806c2d7f..80f5a518d66 100755 --- a/bin/fm-cognee-lookup.sh +++ b/bin/fm-cognee-lookup.sh @@ -1,17 +1,27 @@ #!/usr/bin/env bash -# Local dry-run wrapper for future Cognee lookup integration. +# Local and live read-only wrapper for Cognee lookup integration. # -# This script deliberately does not call Cognee. It accepts a local answer -# fixture, treats it as an untrusted hint, and asks the local manifest checker to -# prove whether any cited source can be reopened and checksum-verified. +# Dry-run mode accepts a local answer fixture. Live mode calls only the read-only +# Cognee search endpoint, treats the response as an untrusted hint, and asks the +# local manifest checker to prove whether any cited source can be reopened and +# checksum-verified. set -eu usage() { cat >&2 <<'USAGE' -usage: fm-cognee-lookup.sh --dry-run --query [--manifest --answer-file ] +usage: fm-cognee-lookup.sh [--dry-run] --query [--manifest --answer-file ] + fm-cognee-lookup.sh -No live mode exists yet. Without --dry-run this command fails closed before any -network, environment, MCP, or config access can happen. +Live mode uses only already-exported environment variables: + COGNEE_BASE_URL + COGNEE_API_KEY + FM_COGNEE_MANIFEST or --manifest + +It can be used through: + FM_COGNEE_LOOKUP_CMD=/absolute/path/to/bin/fm-cognee-lookup.sh + +Live mode calls only POST /api/v1/search and never creates datasets, imports, +cognifies, deletes, syncs, mutates config, mutates MCP, or writes env files. USAGE } @@ -27,6 +37,215 @@ DRY_RUN=false QUERY= MANIFEST= ANSWER_FILE= +POSITIONAL=() + +safe_label() { + printf '%s' "$1" | sed -E 's/[^A-Za-z0-9_.:-]+/_/g; s/^_+//; s/_+$//' | cut -c 1-120 +} + +dataset_alias() { + printf '%s' "${FM_COGNEE_DATASET_ALIAS:-${COGNEE_DATASET_ALIAS:-firstmate-curated-memory-0629}}" +} + +dataset_id_hash() { + if [ -n "${COGNEE_DATASET_ID:-}" ]; then + printf 'sha256:%s' "$(printf '%s' "$COGNEE_DATASET_ID" | sha256sum | awk '{print $1}')" + fi +} + +live_telemetry_log() { + local status=$1 error_class=$2 http_status=$3 retryable=$4 retry_count=$5 latency_ms=$6 parsed_source_count=$7 verification_outcome=$8 + local telemetry_file dataset_alias_value dataset_id_hash_value + telemetry_file=${FM_COGNEE_TELEMETRY_FILE:-$(fm_cognee_telemetry_default_path)} + dataset_alias_value=$(dataset_alias) + dataset_id_hash_value=$(dataset_id_hash) + ( + set +e + mkdir -p "$(dirname "$telemetry_file")" >/dev/null 2>&1 || exit 0 + FM_COGNEE_LIVE_TELEMETRY_FILE=$telemetry_file \ + FM_COGNEE_LIVE_STATUS=$(safe_label "$status") \ + FM_COGNEE_LIVE_ERROR=$(safe_label "$error_class") \ + FM_COGNEE_LIVE_HTTP_STATUS=$http_status \ + FM_COGNEE_LIVE_RETRYABLE=$retryable \ + FM_COGNEE_LIVE_RETRY_COUNT=$retry_count \ + FM_COGNEE_LIVE_LATENCY_MS=$latency_ms \ + FM_COGNEE_LIVE_PARSED_SOURCE_COUNT=$parsed_source_count \ + FM_COGNEE_LIVE_VERIFICATION=$(safe_label "$verification_outcome") \ + FM_COGNEE_LIVE_DATASET_ALIAS=$(safe_label "$dataset_alias_value") \ + FM_COGNEE_LIVE_DATASET_ID_HASH=$dataset_id_hash_value \ + FM_COGNEE_LIVE_SEARCH_TYPE=$(safe_label "${FM_COGNEE_SEARCH_TYPE:-RAG_COMPLETION}") \ + FM_COGNEE_LIVE_TOP_K=${FM_COGNEE_TOP_K:-8} \ + python3 - <<'PY' >/dev/null 2>&1 +import datetime as dt +import json +import os +from pathlib import Path + + +def integer(name, default=0): + try: + return max(int(os.environ.get(name, "") or default), 0) + except ValueError: + return default + + +def maybe_int(name): + value = os.environ.get(name, "") + if value == "": + return None + try: + return int(value) + except ValueError: + return None + + +def boolean(name): + return (os.environ.get(name, "") or "").lower() == "true" + + +now = dt.datetime.now(dt.timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") +event = { + "schema_version": "cognee_live_lookup.v1", + "ts_utc": now, + "event_type": "api_attempt", + "operation_name": "cognee_lookup", + "mode": "live", + "dataset": { + "dataset_alias": os.environ.get("FM_COGNEE_LIVE_DATASET_ALIAS") or "unknown", + "dataset_id_hash": os.environ.get("FM_COGNEE_LIVE_DATASET_ID_HASH") or None, + }, + "operation": { + "operation_name": "search", + "endpoint_template": "/api/v1/search", + "http_method": "POST", + "mutates_remote": False, + "search_type": os.environ.get("FM_COGNEE_LIVE_SEARCH_TYPE") or "RAG_COMPLETION", + "topK": integer("FM_COGNEE_LIVE_TOP_K", 8), + }, + "status": { + "status": os.environ.get("FM_COGNEE_LIVE_STATUS") or "unknown", + "error_class": os.environ.get("FM_COGNEE_LIVE_ERROR") or "none", + "http_status": maybe_int("FM_COGNEE_LIVE_HTTP_STATUS"), + "retryable": boolean("FM_COGNEE_LIVE_RETRYABLE"), + }, + "attempt": { + "retry_count": integer("FM_COGNEE_LIVE_RETRY_COUNT", 0), + }, + "latency": { + "duration_ms": integer("FM_COGNEE_LIVE_LATENCY_MS", 0), + }, + "results": { + "parsed_source_count": integer("FM_COGNEE_LIVE_PARSED_SOURCE_COUNT", 0), + "answer_body_logged": False, + }, + "source_verification_outcome": os.environ.get("FM_COGNEE_LIVE_VERIFICATION") or "not_attempted", + "external_action_authorized": False, +} +try: + path = Path(os.environ["FM_COGNEE_LIVE_TELEMETRY_FILE"]) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, sort_keys=True) + "\n") +except Exception: + pass +PY + ) || true +} + +json_payload() { + local output=$1 + FM_COGNEE_QUERY=$QUERY \ + FM_COGNEE_SEARCH_TYPE=${FM_COGNEE_SEARCH_TYPE:-RAG_COMPLETION} \ + FM_COGNEE_TOP_K=${FM_COGNEE_TOP_K:-8} \ + python3 - "$output" <<'PY' +import json +import os +import sys + +try: + top_k = int(os.environ.get("FM_COGNEE_TOP_K") or 8) +except ValueError: + top_k = 8 +payload = { + "query": os.environ.get("FM_COGNEE_QUERY", ""), + "searchType": os.environ.get("FM_COGNEE_SEARCH_TYPE") or "RAG_COMPLETION", + "topK": top_k, + "includeReferences": True, +} +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PY +} + +extract_answer_text() { + local response_file=$1 answer_file=$2 count_file=$3 + python3 - "$response_file" "$answer_file" "$count_file" <<'PY' +import json +import re +import sys +from pathlib import Path + + +response_path, answer_path, count_path = map(Path, sys.argv[1:]) +SOURCE_RE = re.compile(r"\b(?:SOURCE_ID|SOURCE_PATH|SEED_FILE)\s*[:=]") + +try: + data = json.loads(response_path.read_text(encoding="utf-8")) +except Exception: + data = response_path.read_text(encoding="utf-8", errors="replace") + +strings = [] + + +def walk(value, key=""): + if isinstance(value, dict): + for child_key, child in value.items(): + walk(child, str(child_key)) + elif isinstance(value, list): + for child in value: + walk(child, key) + elif isinstance(value, str): + if key in {"search_result", "answer", "text", "content", "result"} or SOURCE_RE.search(value): + strings.append(value) + + +walk(data) +if not strings and isinstance(data, str): + strings.append(data) +answer = "\n".join(strings) +answer_path.write_text(answer, encoding="utf-8") +count_path.write_text(str(len(SOURCE_RE.findall(answer))) + "\n", encoding="utf-8") +PY +} + +verification_outcome() { + local file=$1 + python3 - "$file" <<'PY' +import json +import sys +from pathlib import Path + +text = Path(sys.argv[1]).read_text(encoding="utf-8", errors="replace") +lines = [line for line in text.splitlines() if line.strip()] +for line in lines: + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + result = obj.get("verification_result", {}) + if result.get("outcome"): + print(result["outcome"]) + raise SystemExit +for line in lines: + for token in line.split(): + if token.startswith("reason="): + print(token.split("=", 1)[1]) + raise SystemExit + if token.startswith("label="): + print(token.split("=", 1)[1]) + raise SystemExit +print("not_attempted") +PY +} while [ $# -gt 0 ]; do case "$1" in @@ -54,19 +273,126 @@ while [ $# -gt 0 ]; do exit 0 ;; *) - usage - exit 1 + POSITIONAL+=("$1") + shift ;; esac done +if [ -z "$QUERY" ] && [ "${#POSITIONAL[@]}" -gt 0 ]; then + QUERY=${POSITIONAL[*]} +fi + if ! "$DRY_RUN"; then - fm_cognee_telemetry_log \ - cognee_lookup live blocked live_cognee_lookup_not_implemented 0 \ - "$(fm_cognee_telemetry_latency_ms "$TELEMETRY_START_MS")" \ - "" "" not_attempted "" unknown_vendor_cost "" unknown_vendor_cost - echo "label=blocked_missing_proof reason=live_cognee_lookup_not_implemented external_action_authorized=false" >&2 - exit 2 + [ -n "$QUERY" ] || die "--query is required in live mode" + MANIFEST=${MANIFEST:-${FM_COGNEE_MANIFEST:-}} + + missing_env= + [ -n "${COGNEE_BASE_URL:-}" ] || missing_env="${missing_env:+$missing_env,}COGNEE_BASE_URL" + [ -n "${COGNEE_API_KEY:-}" ] || missing_env="${missing_env:+$missing_env,}COGNEE_API_KEY" + if [ -n "$missing_env" ]; then + live_telemetry_log blocked missing_required_env "" false 0 \ + "$(fm_cognee_telemetry_latency_ms "$TELEMETRY_START_MS")" 0 missing_required_env + echo "label=blocked_missing_proof reason=missing_required_env missing_env=$missing_env external_action_authorized=false" >&2 + exit 2 + fi + + if [ -z "$MANIFEST" ]; then + live_telemetry_log blocked missing_manifest "" false 0 \ + "$(fm_cognee_telemetry_latency_ms "$TELEMETRY_START_MS")" 0 missing_manifest + echo "label=blocked_missing_proof reason=missing_manifest external_action_authorized=false" >&2 + exit 2 + fi + [ -f "$MANIFEST" ] || die "manifest not found: $MANIFEST" + + TMP_DIR=$(mktemp -d "${TMPDIR:-/tmp}/fm-cognee-live.XXXXXX") + cleanup_live() { rm -rf "$TMP_DIR"; } + trap cleanup_live EXIT + PAYLOAD="$TMP_DIR/search.json" + BODY="$TMP_DIR/body.json" + ANSWER="$TMP_DIR/answer.txt" + VERIFY_OUT="$TMP_DIR/verify.out" + COUNT_FILE="$TMP_DIR/source-count.txt" + CURL_ERR="$TMP_DIR/curl.err" + json_payload "$PAYLOAD" + + base=${COGNEE_BASE_URL%/} + endpoint="$base/api/v1/search" + max_attempts=${FM_COGNEE_MAX_ATTEMPTS:-3} + case "$max_attempts" in ''|*[!0-9]*) max_attempts=3 ;; esac + [ "$max_attempts" -ge 1 ] || max_attempts=1 + attempt=1 + http_status=0 + retryable=false + curl_rc=0 + while [ "$attempt" -le "$max_attempts" ]; do + : > "$BODY" + : > "$CURL_ERR" + set +e + http_status=$(curl -sS -o "$BODY" -w '%{http_code}' \ + -X POST "$endpoint" \ + -H "X-Api-Key: $COGNEE_API_KEY" \ + -H "Content-Type: application/json" \ + --data-binary "@$PAYLOAD" 2> "$CURL_ERR") + curl_rc=$? + set -e + retryable=false + if [ "$curl_rc" -ne 0 ]; then + http_status=0 + retryable=true + else + case "$http_status" in + 429|500|502|503|504) retryable=true ;; + esac + fi + if ! "$retryable" || [ "$attempt" -ge "$max_attempts" ]; then + break + fi + attempt=$((attempt + 1)) + done + + retry_count=$((attempt - 1)) + latency=$(fm_cognee_telemetry_latency_ms "$TELEMETRY_START_MS") + if [ "$curl_rc" -ne 0 ] || [ "$http_status" -lt 200 ] 2>/dev/null || [ "$http_status" -ge 300 ] 2>/dev/null; then + live_telemetry_log blocked http_or_transport_failure "$http_status" "$retryable" "$retry_count" "$latency" 0 http_or_transport_failure + echo "label=blocked_missing_proof reason=http_or_transport_failure http_status=$http_status retry_count=$retry_count retryable=$retryable external_action_authorized=false" >&2 + exit 2 + fi + + extract_answer_text "$BODY" "$ANSWER" "$COUNT_FILE" + parsed_source_count=$(cat "$COUNT_FILE") + set +e + case "$MANIFEST" in + *.jsonl) + "$SCRIPT_DIR/fm-cognee-verify-source.sh" --manifest "$MANIFEST" --answer "$ANSWER" > "$VERIFY_OUT" + ;; + *) + "$SCRIPT_DIR/fm-cognee-manifest-check.sh" --manifest "$MANIFEST" --answer-file "$ANSWER" > "$VERIFY_OUT" + ;; + esac + verify_rc=$? + set -e + cat "$VERIFY_OUT" + source_outcome=$(verification_outcome "$VERIFY_OUT") + if [ "$verify_rc" -eq 0 ]; then + tel_status=verified + tel_error=none + else + tel_status=blocked + tel_error=$source_outcome + fi + live_telemetry_log "$tel_status" "$tel_error" "$http_status" false "$retry_count" "$latency" "$parsed_source_count" "$source_outcome" + echo "mode=live" + echo "dataset_alias=$(safe_label "$(dataset_alias)")" + echo "endpoint=/api/v1/search" + echo "http_status=$http_status" + echo "retry_count=$retry_count" + echo "retryable=false" + echo "parsed_source_count=$parsed_source_count" + echo "cognee_answer_status=hint_only" + echo "source_verification_outcome=$source_outcome" + echo "external_action_authorized=false" + exit "$verify_rc" fi [ -n "$QUERY" ] || die "--query is required in dry-run mode" diff --git a/bin/fm-cognee-manifest-check.sh b/bin/fm-cognee-manifest-check.sh index 875d905c2e7..f1ba15dbabc 100755 --- a/bin/fm-cognee-manifest-check.sh +++ b/bin/fm-cognee-manifest-check.sh @@ -149,6 +149,9 @@ row_matches_ref() { [ -n "$summary_path" ] && [ "$value" = "$summary_path" ] && return 0 ;; SEED_FILE) + # A bare "report.md" appears across many imported report rows. It is + # useful as hint text, but too generic to prove exact attribution. + [ "$value" = "report.md" ] && return 1 [ "$value" = "$seed_name" ] && return 0 case "$import_text" in *"SEED_FILE=$value"*|*"SEED_FILE: $value"*) return 0 ;; esac ;; @@ -291,14 +294,32 @@ sanitize_ref() { REFS=$(mktemp "${TMPDIR:-/tmp}/fm-cognee-refs.XXXXXX") if [ -n "$ANSWER_FILE" ]; then - { - grep -Eoh 'SOURCE_ID[[:space:]]*[:=][[:space:]]*[^[:space:],;)]+' "$ANSWER_FILE" 2>/dev/null \ - | sed -E 's/^SOURCE_ID[[:space:]]*[:=][[:space:]]*//' | sanitize_ref | awk 'NF {print "SOURCE_ID\t"$0}' - grep -Eoh 'SOURCE_PATH[[:space:]]*[:=][[:space:]]*[^[:space:],;)]+' "$ANSWER_FILE" 2>/dev/null \ - | sed -E 's/^SOURCE_PATH[[:space:]]*[:=][[:space:]]*//' | sanitize_ref | awk 'NF {print "SOURCE_PATH\t"$0}' - grep -Eoh 'SEED_FILE[[:space:]]*[:=][[:space:]]*[^[:space:],;)]+' "$ANSWER_FILE" 2>/dev/null \ - | sed -E 's/^SEED_FILE[[:space:]]*[:=][[:space:]]*//' | sanitize_ref | awk 'NF {print "SEED_FILE\t"$0}' - } | sort -u > "$REFS" + python3 - "$ANSWER_FILE" > "$REFS" <<'PY' +import re +import sys +from pathlib import Path + + +label_re = re.compile( + r"\b(SOURCE_ID|SOURCE_PATH|SEED_FILE)\s*[:=]\s*[*_`\u202f\s]*" + r"(?:\"([^\"]*)\"|'([^']*)'|([^\s,;\]\)]+))" +) + +try: + text = Path(sys.argv[1]).read_text(encoding="utf-8") +except Exception: + raise SystemExit + +refs = set() +for match in label_re.finditer(text): + value = next(group for group in match.groups()[1:] if group is not None).strip() + value = value.strip("`\"'[],;.)") + if value: + refs.add((match.group(1), value)) + +for kind, value in sorted(refs): + print(f"{kind}\t{value}") +PY fi if "$VALIDATE"; then diff --git a/bin/fm-cognee-telemetry-lib.sh b/bin/fm-cognee-telemetry-lib.sh index 99461e4a99f..379db207f0a 100755 --- a/bin/fm-cognee-telemetry-lib.sh +++ b/bin/fm-cognee-telemetry-lib.sh @@ -111,6 +111,7 @@ event = { "vendor_estimated_cost_usd": number_or_none("FM_COGNEE_T_VENDOR_ESTIMATED_COST_USD"), "vendor_cost_status": label("FM_COGNEE_T_VENDOR_COST_STATUS"), "currency": "USD", + "external_action_authorized": False, } try: path = Path(os.environ["FM_COGNEE_T_FILE"]) diff --git a/bin/fm-cognee-verify-source.sh b/bin/fm-cognee-verify-source.sh index fa5c9f22a3f..da7dba9a8db 100755 --- a/bin/fm-cognee-verify-source.sh +++ b/bin/fm-cognee-verify-source.sh @@ -63,6 +63,7 @@ UUID_RE = re.compile( ) LABEL_RE = re.compile( r"\b(SOURCE_ID|SOURCE_PATH|SEED_FILE|DATA_ID|DATA_UUID|CHUNK_ID|CHUNK_UUID)\s*[:=]\s*" + r"[*_`\u202f\s]*" r"(?:\"([^\"]*)\"|'([^']*)'|([^\s,;\]\)]+))" ) @@ -201,17 +202,25 @@ def _parse_answer(text): "data_ids": set(), "chunk_ids": set(), "uuid_mentions": set(), + "_source_ids_ordered": [], + "_source_paths_ordered": [], + "_seed_files_ordered": [], } malformed = 0 for match in LABEL_RE.finditer(text): label = match.group(1) cleaned = next(group for group in match.groups()[1:] if group is not None).strip() + cleaned = cleaned.split("\\n", 1)[0].splitlines()[0].strip() + cleaned = cleaned.strip("`\"'[],;.)") if label == "SOURCE_ID": labels["source_ids"].add(cleaned) + labels["_source_ids_ordered"].append(cleaned) elif label == "SOURCE_PATH": labels["source_paths"].add(cleaned) + labels["_source_paths_ordered"].append(cleaned) elif label == "SEED_FILE": labels["seed_files"].add(cleaned) + labels["_seed_files_ordered"].append(cleaned) elif label in ("DATA_ID", "DATA_UUID"): if UUID_RE.fullmatch(cleaned): labels["data_ids"].add(cleaned.lower()) @@ -259,6 +268,11 @@ def _manifest_checksum(row): return row.get("checksum_sha256") or row.get("sha256") or row.get("checksum") +def _row_source_ids(row): + values = _field_set(row, "source_id") | _field_set(row, "row_id") | _field_set(row, "cognee_source_id") + return {value for value in values if value and value != "None"} + + def _resolve_path(row): raw = _source_path(row) if not raw: @@ -282,12 +296,15 @@ def _sha256(path): def _find_row(rows, parsed): - source_ids = parsed["source_ids"] - if source_ids: + for source_id in parsed.get("_source_ids_ordered", []): for row in rows: - if str(row.get("source_id")) in source_ids: + if source_id in _row_source_ids(row): + return row + + for source_path in parsed.get("_source_paths_ordered", []): + for row in rows: + if _source_path(row) == source_path or str(_resolve_path(row)) == source_path: return row - return None source_paths = parsed["source_paths"] seed_files = parsed["seed_files"] @@ -341,15 +358,22 @@ def _verify(): source_path = _resolve_path(row) local = {"opened": False, "readable": False, "checksum_match": None} - row_source_id = {str(row.get("source_id"))} - if parsed["source_ids"] - row_source_id: + row_source_id = _row_source_ids(row) + if parsed["source_ids"] and not (parsed["source_ids"] & row_source_id): errors.append("SOURCE_ID does not match manifest row") + elif parsed["source_ids"] - row_source_id: + warnings.append("extra_source_ids_ignored") row_raw_path = _source_path(row) row_resolved_path = str(source_path) if source_path else "" - if parsed["source_paths"] - {row_raw_path, row_resolved_path}: + row_paths = {row_raw_path, row_resolved_path} + if parsed["source_paths"] and not (parsed["source_paths"] & row_paths): errors.append("SOURCE_PATH does not match manifest row") - if parsed["seed_files"] - {_seed_file(row)}: + elif parsed["source_paths"] - row_paths: + warnings.append("extra_source_paths_ignored") + if parsed["seed_files"] and not (parsed["seed_files"] & {_seed_file(row)}): errors.append("SEED_FILE does not match manifest row") + elif parsed["seed_files"] - {_seed_file(row)}: + warnings.append("extra_seed_files_ignored") row_data_ids = _lower_field_set(row, "data_ids") row_chunk_ids = _lower_field_set(row, "chunk_ids") @@ -359,7 +383,7 @@ def _verify(): errors.append("CHUNK_ID does not match manifest row") unknown_uuid_mentions = parsed["uuid_mentions"] - row_data_ids - row_chunk_ids if unknown_uuid_mentions: - errors.append("UUID mention does not match manifest row") + warnings.append("unlabelled_uuid_mentions_ignored") if not source_path or not source_path.is_file(): errors.append("local source file is missing") @@ -386,7 +410,7 @@ def _verify(): warnings.append(f"stale_risk={stale_risk}") raw_status = str(row.get("raw_readback_status") or row.get("raw_status") or "ok").lower() - raw_blocked = raw_status not in {"", "ok", "passed", "available", "readable", "200"} + raw_blocked = raw_status not in {"", "ok", "passed", "available", "readable", "200", "not_attempted", "not_trusted", "not_applicable"} if raw_blocked: errors.append(f"raw_readback_status={raw_status}") diff --git a/bin/fm-memory-lookup.sh b/bin/fm-memory-lookup.sh index a55a2337207..4a86758132a 100755 --- a/bin/fm-memory-lookup.sh +++ b/bin/fm-memory-lookup.sh @@ -5,7 +5,9 @@ # firstmate invokes it by hand. Cognee output is treated as an untrusted hint; # only local source files that can be opened are eligible for brief attachment. # Configure a read-only lookup backend with: -# FM_COGNEE_LOOKUP_CMD=/absolute/path/to/read-only-lookup +# FM_COGNEE_LOOKUP_CMD=/absolute/path/to/bin/fm-cognee-lookup.sh +# FM_COGNEE_MANIFEST=/absolute/path/to/manifest.tsv +# COGNEE_BASE_URL and COGNEE_API_KEY already exported in the process # The backend is executed as: "$FM_COGNEE_LOOKUP_CMD" "$query" set -eu diff --git a/docs/scripts.md b/docs/scripts.md index 107ce1dbe3b..7734481cdaa 100644 --- a/docs/scripts.md +++ b/docs/scripts.md @@ -20,7 +20,7 @@ Each file also starts with a short header comment. | `fm-route.sh` | Classify a task into a deterministic route profile, harness, model, effort, reason, override, and risk flags without changing spawn behavior | | `fm-merge-local.sh` | Fast-forward a `local-only` project's local default branch after approval | | `fm-review-diff.sh` | Review a crewmate branch against the authoritative base, with optional `--stat` output | -| `fm-cognee-lookup.sh` | Local-only dry-run wrapper for future Cognee lookup; treats answer fixtures as hints and delegates source proof to the manifest checker | +| `fm-cognee-lookup.sh` | Read-only Cognee lookup wrapper with dry-run fixtures and guarded live `POST /api/v1/search`; treats answers as hints and delegates source proof to local manifest/source verification | | `fm-cognee-manifest-check.sh` | Validate TSV Cognee manifest rows and verify `SOURCE_ID`, `SOURCE_PATH`, or `SEED_FILE` answer references against reopened local files | | `fm-marker-lib.sh` | Shared from-firstmate request marker and detector sourced by `fm-send.sh`, `fm-brief.sh`, and tests | | `fm-watch-arm.sh` | Verified per-home watcher re-arm; reports `started`, `healthy`, or `FAILED`; `--restart` relaunches only this home's watcher | diff --git a/tests/fm-cognee-lookup.test.sh b/tests/fm-cognee-lookup.test.sh index ee993e3a26b..de693736d5d 100755 --- a/tests/fm-cognee-lookup.test.sh +++ b/tests/fm-cognee-lookup.test.sh @@ -20,13 +20,86 @@ write_manifest() { } > "$manifest" } +test_live_missing_env_fails_closed_without_secret_values() { + local out code telemetry secret + telemetry="$TMP_ROOT/missing-env-telemetry.jsonl" + secret="SECRET_API_KEY_SHOULD_NOT_PRINT" + + set +e + out=$(env -u COGNEE_BASE_URL COGNEE_API_KEY="$secret" FM_COGNEE_TELEMETRY_FILE="$telemetry" \ + "$ROOT/bin/fm-cognee-lookup.sh" --query "find cognee proof" 2>&1) + code=$? + set -e + expect_code 2 "$code" "live lookup missing env should fail closed" + assert_contains "$out" "reason=missing_required_env" "missing env reason is explicit" + assert_contains "$out" "missing_env=COGNEE_BASE_URL" "missing env names are reported" + assert_contains "$out" "external_action_authorized=false" "missing env cannot authorize action" + assert_not_contains "$out" "$secret" "missing env output must not print secret values" + assert_present "$telemetry" "missing env should write telemetry" + assert_not_contains "$(cat "$telemetry")" "$secret" "telemetry must not contain API key" + assert_contains "$(cat "$telemetry")" '"external_action_authorized": false' "telemetry keeps action authorization false" + pass "live lookup missing env fails closed without printing secret values" +} + +test_live_fake_search_parses_verifies_and_writes_redacted_telemetry() { + local dir source manifest fakebin out code sha telemetry secret + dir="$TMP_ROOT/live-fake" + mkdir -p "$dir" + source="$dir/source.md" + manifest="$dir/manifest.tsv" + fakebin=$(fm_fakebin "$dir") + telemetry="$dir/telemetry.jsonl" + secret="SECRET_LIVE_API_KEY_DO_NOT_LOG" + printf 'local source truth from fake live search\n' > "$source" + sha=$(sha256sum "$source" | awk '{print $1}') + write_manifest "$manifest" batch-live-01 "$source" "$sha" + cat > "$fakebin/curl" < "\$out" <<'JSON' +[{"dataset_name":"firstmate-curated-memory-0629","search_result":["**SOURCE_ID:** batch-live-01\\n**SOURCE_PATH:** $source\\nEvidence mentions generated ids 123e4567-e89b-12d3-a456-426614174000 and Markdown, but local proof comes from the manifest row."]}] +JSON +printf '200' +SH + chmod +x "$fakebin/curl" + + set +e + out=$(PATH="$fakebin:$PATH" COGNEE_BASE_URL="https://cognee.invalid" COGNEE_API_KEY="$secret" \ + FM_COGNEE_TELEMETRY_FILE="$telemetry" "$ROOT/bin/fm-cognee-lookup.sh" \ + --query "which source matters" --manifest "$manifest") + code=$? + set -e + expect_code 0 "$code" "fake live lookup should verify" + assert_contains "$out" "mode=live" "live mode should be visible" + assert_contains "$out" "http_status=200" "HTTP status should be reported" + assert_contains "$out" "parsed_source_count=2" "source labels should be counted" + assert_contains "$out" "label=verified_local_source" "manifest plus local reopen verifies source" + assert_contains "$out" "external_action_authorized=false" "verified lookup still cannot authorize action" + assert_not_contains "$out" "$secret" "live output must not print API key" + assert_present "$telemetry" "live lookup should write telemetry" + assert_not_contains "$(cat "$telemetry")" "$secret" "live telemetry must not contain API key" + assert_contains "$(cat "$telemetry")" '"endpoint_template": "/api/v1/search"' "telemetry records read-only search endpoint" + assert_contains "$(cat "$telemetry")" '"http_status": 200' "telemetry records HTTP status" + assert_contains "$(cat "$telemetry")" '"parsed_source_count": 2' "telemetry records parsed source count" + assert_contains "$(cat "$telemetry")" '"source_verification_outcome": "verified_local_source"' "telemetry records verification outcome" + assert_contains "$(cat "$telemetry")" '"external_action_authorized": false' "telemetry never authorizes action" + pass "fake live search is parsed, verified locally, and redacted in telemetry" +} + test_dry_run_blocks_live_mode() { local out code out=$("$ROOT/bin/fm-cognee-lookup.sh" --query "find cognee proof" 2>&1) code=$? - expect_code 2 "$code" "live lookup fails closed" - assert_contains "$out" "reason=live_cognee_lookup_not_implemented" "live mode states why it is blocked" - pass "cognee lookup has no live/API mode" + expect_code 2 "$code" "live lookup without env fails closed" + assert_contains "$out" "reason=missing_required_env" "live mode states missing env names" + assert_contains "$out" "external_action_authorized=false" "live failure cannot authorize action" + pass "cognee lookup live mode fails closed without env" } test_dry_run_verifies_local_source_without_echoing_answer() { @@ -99,6 +172,34 @@ test_source_path_and_seed_file_references_verify() { pass "SOURCE_PATH and SEED_FILE references can verify local sources" } +test_generic_report_seed_file_does_not_verify_by_itself() { + local dir source_a source_b manifest answer out code sha_a sha_b + dir="$TMP_ROOT/generic-seed" + mkdir -p "$dir/a" "$dir/b" + source_a="$dir/a/report.md" + source_b="$dir/b/report.md" + manifest="$dir/manifest.tsv" + answer="$dir/answer.txt" + printf 'local source truth a\n' > "$source_a" + printf 'local source truth b\n' > "$source_b" + sha_a=$(sha256sum "$source_a" | awk '{print $1}') + sha_b=$(sha256sum "$source_b" | awk '{print $1}') + write_manifest "$manifest" batch-report-a "$source_a" "$sha_a" + tail -n 1 "$manifest" > "$dir/row-a.tsv" + write_manifest "$dir/manifest-b.tsv" batch-report-b "$source_b" "$sha_b" + tail -n 1 "$dir/manifest-b.tsv" >> "$manifest" + printf 'Generated answer cites only SEED_FILE=report.md and a Markdown UUID 123e4567-e89b-12d3-a456-426614174000.\n' > "$answer" + + set +e + out=$("$ROOT/bin/fm-cognee-lookup.sh" --dry-run --query "lookup" --manifest "$manifest" --answer-file "$answer") + code=$? + set -e + expect_code 3 "$code" "generic report.md seed file should not prove a source" + assert_contains "$out" "reason=manifest_reference_not_found" "generic seed file is not precise attribution" + assert_not_contains "$out" "label=verified_local_source" "generic seed file cannot verify" + pass "generic report.md seed references do not create false proof" +} + test_checksum_mismatch_blocks_proof() { local dir source manifest answer out code sha dir="$TMP_ROOT/checksum" @@ -186,3 +287,6 @@ test_checksum_mismatch_blocks_proof test_redaction_not_checked_blocks_proof test_secret_risk_path_blocks_without_echoing_path test_high_stale_risk_warns_after_local_verification +test_live_missing_env_fails_closed_without_secret_values +test_live_fake_search_parses_verifies_and_writes_redacted_telemetry +test_generic_report_seed_file_does_not_verify_by_itself diff --git a/tests/fm-cognee-source-verify.test.sh b/tests/fm-cognee-source-verify.test.sh index 7c58112f15d..9d764b156cc 100644 --- a/tests/fm-cognee-source-verify.test.sh +++ b/tests/fm-cognee-source-verify.test.sh @@ -129,8 +129,8 @@ test_checksum_mismatch_fails_closed() { [ "$(printf '%s' "$out" | jq -r '.manifest.manifest_checksum_match')" = "false" ] || fail "checksum match must be false" } -test_extra_source_id_fails_closed() { - local case_dir source manifest answer out code +test_extra_source_id_is_warning_when_exact_path_verifies() { + local case_dir source manifest answer out case_dir="$TMP_ROOT/extra-source-id" mkdir -p "$case_dir" source="$case_dir/redacted-source.md" @@ -145,13 +145,9 @@ SOURCE_PATH=$source SEED_FILE=seed/redacted-07.md EOF - set +e - out=$("$VERIFY" --manifest "$manifest" --answer "$answer") - code=$? - set -e - expect_code 2 "$code" "extra source id" - [ "$(printf '%s' "$out" | jq -r '.verification_result.status')" = "failed_closed" ] || fail "extra source id must fail closed" - [ "$(printf '%s' "$out" | jq -r '.verification_result.outcome')" = "failed_closed_missing_proof" ] || fail "extra source id outcome" + out=$("$VERIFY" --manifest "$manifest" --answer "$answer") || fail "exact path should verify despite extra generated source id" + [ "$(printf '%s' "$out" | jq -r '.verification_result.status')" = "verified" ] || fail "extra source id with exact path should verify" + [ "$(printf '%s' "$out" | jq -r '.verification_result.warnings[]' | grep -c '^extra_source_ids_ignored$')" = "1" ] || fail "extra source id should be warning-only" } test_raw_404_stays_durability_blocker() { @@ -200,12 +196,37 @@ EOF [ "$(printf '%s' "$out" | jq -r '.verification_result.outcome')" = "failed_closed_identifier_mismatch" ] || fail "unknown well formed UUID must fail closed" } +test_markdown_source_labels_verify_despite_unlabelled_uuid_noise() { + local case_dir source manifest answer out + case_dir="$TMP_ROOT/markdown-noise" + mkdir -p "$case_dir" + source="$case_dir/redacted-source.md" + manifest="$case_dir/manifest.jsonl" + answer="$case_dir/answer.txt" + printf 'Redacted report fixture.\nLocal proof line.\n' > "$source" + write_manifest "$manifest" "$source" "$(sha256_file "$source")" + cat > "$answer" < "$fake" <<'SH' +#!/usr/bin/env bash +printf '%s\n' 'label=blocked_missing_proof reason=missing_required_env external_action_authorized=false' >&2 +exit 2 +SH + chmod +x "$fake" + + set +e + out=$(FM_COGNEE_LOOKUP_CMD="$fake" "$LOOKUP" -- "lookup can fail") + rc=$? + set -e + expect_code 0 "$rc" "failed configured backend should not block dispatch" + assert_contains "$out" "lookup command failed; dispatch continues without memory context" "backend failure should be visible and non-blocking" + assert_contains "$out" "verified local source path:" "source section should still render" + assert_contains "$out" "none" "no source should be verified after backend failure" + pass "fm-memory-lookup: configured backend failure exits 0 for dispatch" +} + test_absent_cognee_is_non_blocking test_lookup_separates_hint_verified_path_and_warning test_brief_append_excludes_raw_hint test_absent_cognee_brief_note +test_configured_backend_failure_is_non_blocking