diff --git a/.env.example b/.env.example index 6125c0a43..cf533b479 100644 --- a/.env.example +++ b/.env.example @@ -743,8 +743,13 @@ OLLAMA_BASE_URL= # The credential pair supabase-storage accepts on its S3 protocol # endpoint. Do NOT set both halves to the same value: SigV4 transmits the # access key id in the clear inside the Authorization header, so an -# identical secret is disclosed on every request. Configure distinct -# credentials on the storage service and use those here. +# identical secret is disclosed on every request. Generate two unrelated +# values, for example `openssl rand -hex 16` and `openssl rand -base64 32`. +# Nothing else has to be configured on the storage service: the compose +# file hands these same two variables to supabase-storage as +# S3_PROTOCOL_ACCESS_KEY_ID and S3_PROTOCOL_ACCESS_KEY_SECRET, so the +# signing pair cannot drift from the verifying pair. Leaving them unset +# is what made every upload return 500 on the demo box (issue #1282). # # ── Four ENTERPRISE_* variables that are an auth boundary, not a detail ───── # @@ -1023,8 +1028,8 @@ ENTERPRISE_SSO_MICROSOFT_TENANT_URL=https://login.microsoftonline.com/ -# S3_SECRET_KEY= +# S3_ACCESS_KEY= +# S3_SECRET_KEY= # S3_REGION=local # S3_USE_SSL=false # S3_BUCKET_FILES=hive-files diff --git a/apps/edge-api/internal/rag/chat_handler_test.go b/apps/edge-api/internal/rag/chat_handler_test.go index f4c2252ad..d93bd7833 100644 --- a/apps/edge-api/internal/rag/chat_handler_test.go +++ b/apps/edge-api/internal/rag/chat_handler_test.go @@ -493,7 +493,15 @@ func TestHandleChat_SuppressedChunkUsageNotAccounted(t *testing.T) { t.Fatalf("expected 200 for streaming, got %d: %s", w.Code, w.Body.String()) } body := w.Body.String() - if strings.Contains(body, "SPURIOUS-POST-FINISH-MARKER") || strings.Contains(body, "999") { + // The usage check names the field, not the bare number. Every frame this + // handler emits carries a generated `ragchat-` id, and a hex uuid + // contains the substring "999" often enough to fail this test at random: + // it did on CI run 33238715016, on an id of ragchat-39990550-..., with + // nothing wrong in the code under test. A flake in a regression guard is + // worse than no guard, because the next red is assumed to be this one. + if strings.Contains(body, "SPURIOUS-POST-FINISH-MARKER") || + strings.Contains(body, `"total_tokens":999`) || + strings.Contains(body, `"prompt_tokens":999`) { t.Errorf("suppressed chunk (and its bogus usage) must never reach the client:\n%s", body) } diff --git a/deploy/docker/docker-compose.enterprise.yml b/deploy/docker/docker-compose.enterprise.yml index dc92c4c31..e3594c908 100644 --- a/deploy/docker/docker-compose.enterprise.yml +++ b/deploy/docker/docker-compose.enterprise.yml @@ -345,6 +345,36 @@ services: TENANT_ID: stub REGION: local GLOBAL_S3_BUCKET: stub + # The S3-compatible endpoint, which is the ONLY way edge-api and + # control-plane speak to Storage. Without these three variables Storage + # answers every signed request with 403 AccessDenied and the body + # "Missing S3 Protocol Access Key ID or Secret Key Environment + # variables", so POST /v1/files, every Batches input and output object, + # and RAG document upload all return 500 while the container reports + # healthy and the bucket rows exist. That was live on the demo box until + # 2026-08-29 (issue #1282). + # + # Deliberately the SAME two variables the consumers read + # (docker-compose.yml, edge-api and control-plane). The client's access + # key and secret must equal the server's pair exactly, since Storage + # recomputes the AWS SigV4 signature from them; two independent + # variables would let the halves drift apart with no boot error on + # either side. .env.example's older recipe of pointing S3_ACCESS_KEY and + # S3_SECRET_KEY at the service_role key never worked: a service_role JWT + # is not an S3 credential in single-tenant mode, and Storage compares + # against S3_PROTOCOL_ACCESS_KEY_ID/SECRET only. + S3_PROTOCOL_ACCESS_KEY_ID: ${S3_ACCESS_KEY:?set S3_ACCESS_KEY in .env (the S3 protocol access key edge-api and control-plane sign with)} + S3_PROTOCOL_ACCESS_KEY_SECRET: ${S3_SECRET_KEY:?set S3_SECRET_KEY in .env (the S3 protocol secret edge-api and control-plane sign with)} + # The prefix Caddyfile.supabase's `handle_path /storage/v1/*` STRIPS + # before this service sees the request. The signature covers the path the + # client signed, so Storage has to put the prefix back before it + # recomputes: canonicalUri is `s3ProtocolPrefix + request.url`. Leave it + # empty and every request fails SignatureDoesNotMatch instead, which + # looks like a wrong key rather than a wrong path. Must stay equal to the + # prefix in S3_ENDPOINT (http://caddy-supabase/storage/v1/s3) and to the + # gateway's handle_path; scripts/test_selfhost_supabase_seam.py fails if + # the three drift. + S3_PROTOCOL_PREFIX: /storage/v1 ENABLE_IMAGE_TRANSFORMATION: "true" IMGPROXY_URL: "" volumes: diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index 8e0fb3aa4..6f465d074 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -488,8 +488,14 @@ services: # Storage itself serves /s3 at its own root. Two earlier versions # of this comment named the service directly, one of them with an # /object/s3 path that no listener answers. - # S3_ACCESS_KEY= - # S3_SECRET_KEY= + # The two S3 credentials are a dedicated pair, NOT the service_role + # key: Storage compares them against its own + # S3_PROTOCOL_ACCESS_KEY_ID/SECRET, which the enterprise compose file + # feeds from these same variables. A service_role JWT here is refused, + # and the two halves must differ from each other because SigV4 sends + # the access key id in the clear. + # S3_ACCESS_KEY= + # S3_SECRET_KEY= # S3_REGION=local # S3_USE_SSL=false (internal compose network, no TLS) S3_ENDPOINT: ${S3_ENDPOINT} diff --git a/scripts/install.sh b/scripts/install.sh index 605d6825f..bfd969e20 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -336,11 +336,27 @@ setup_env() { prompt_value SUPABASE_SERVICE_ROLE_KEY "Supabase Service Role Key" required "" secret prompt_value SUPABASE_DB_URL "Supabase DB URL (postgres://...)" required "" secret - # ── Storage (required) ── + # ── Storage ── + # This installer starts the enterprise profile, which brings up + # supabase-storage in the stack, so all three values describe that + # in-stack service and none of them come from a hosted project. The + # endpoint is the gateway form because Caddy is what restores the + # /storage/v1 prefix Storage's own routes do not carry. + # + # The credential pair is generated rather than prompted for blank. It is + # a shared secret between the client and the server halves of one stack, + # which the compose file wires from these same two variables, so there is + # nothing for an operator to look up. Asking produced the failure this + # block now prevents: the prompt named a hosted project, so the value + # people reached for was the service_role key, which Storage does not + # accept as an S3 credential and which SigV4 would then have sent in the + # clear inside every Authorization header (issue #1282). printf '%s-- Supabase Storage (S3) --%s\n' "${BOLD}" "${RESET}" - prompt_value S3_ENDPOINT "S3 Endpoint (e.g. https://.supabase.co/storage/v1/s3)" required - prompt_value S3_ACCESS_KEY "S3 Access Key" required "" secret - prompt_value S3_SECRET_KEY "S3 Secret Key" required "" secret + _default_s3_access="$(command -v openssl >/dev/null 2>&1 && openssl rand -hex 16 || printf 'change-me-generate-with-openssl-rand-hex-16')" + _default_s3_secret="$(command -v openssl >/dev/null 2>&1 && openssl rand -base64 32 || printf 'change-me-generate-with-openssl-rand-base64-32')" + prompt_value S3_ENDPOINT "S3 Endpoint" optional "http://caddy-supabase/storage/v1/s3" + prompt_value S3_ACCESS_KEY "S3 Access Key" optional "$_default_s3_access" secret + prompt_value S3_SECRET_KEY "S3 Secret Key" optional "$_default_s3_secret" secret prompt_value S3_REGION "S3 Region" optional "us-east-1" # ── LLM Provider (at least one required) ── diff --git a/scripts/test_selfhost_supabase_seam.py b/scripts/test_selfhost_supabase_seam.py index 7d53cb9a2..6c0924cd2 100644 --- a/scripts/test_selfhost_supabase_seam.py +++ b/scripts/test_selfhost_supabase_seam.py @@ -284,6 +284,52 @@ def test_open_webui_pgvector_dsn_comes_from_the_libpq_flavour() -> None: assert not re.search(r"SUPABASE_DB_POOL_URL(?!_LIBPQ)", dsn), dsn +def test_storage_accepts_the_s3_credentials_the_consumers_sign_with() -> None: + """Storage verifies an S3 request by recomputing the SigV4 signature from + S3_PROTOCOL_ACCESS_KEY_ID and S3_PROTOCOL_ACCESS_KEY_SECRET. With neither + set it refuses every request with 403 AccessDenied and "Missing S3 Protocol + Access Key ID or Secret Key Environment variables", which is a 500 on POST + /v1/files, on every Batches object and on RAG document upload while the + container stays healthy and the bucket rows exist (issue #1282). + + They must come from the SAME two variables edge-api and control-plane sign + with. A separate pair would let the signing half and the verifying half + drift with no boot error on either side, and the only symptom is a 403 from + a service that looks configured.""" + # A plain substring would also match ${S3_ACCESS_KEY_ID}, a different + # variable that would leave the two halves signing with different values, + # which is exactly the drift this asserts against. The terminator is what + # makes the match the whole name. + def references(value: str, variable: str) -> bool: + return re.search(r"\$\{" + variable + r"[:}]", value) is not None + + storage = ENT_SERVICES["supabase-storage"] + key_id = env_value(storage, "S3_PROTOCOL_ACCESS_KEY_ID") + secret = env_value(storage, "S3_PROTOCOL_ACCESS_KEY_SECRET") + assert references(key_id, "S3_ACCESS_KEY"), key_id + assert references(secret, "S3_SECRET_KEY"), secret + for consumer in ("edge-api", "control-plane"): + block = BASE_SERVICES[consumer] + assert references(env_value(block, "S3_ACCESS_KEY"), "S3_ACCESS_KEY"), consumer + assert references(env_value(block, "S3_SECRET_KEY"), "S3_SECRET_KEY"), consumer + + +def test_the_storage_s3_prefix_matches_the_prefix_the_gateway_strips() -> None: + """The consumers sign the path they send to the gateway, and the gateway's + `handle_path` strips its prefix before Storage sees the request. Storage + puts S3_PROTOCOL_PREFIX back before recomputing the signature, so an empty + or mismatched value fails every request with SignatureDoesNotMatch, which + reads as a wrong key rather than a wrong path. Three places have to agree: + this variable, S3_ENDPOINT, and the Caddyfile route.""" + prefix = env_value(ENT_SERVICES["supabase-storage"], "S3_PROTOCOL_PREFIX") + assert prefix == "/storage/v1", prefix + caddy = (ROOT / "deploy" / "docker" / "Caddyfile.supabase").read_text() + assert f"handle_path {prefix}/*" in caddy, prefix + # The endpoint the consumers are told to use, in the file that documents it. + example = (ROOT / ".env.example").read_text() + assert f"S3_ENDPOINT=http://caddy-supabase{prefix}/s3" in example + + def test_no_hosted_supabase_host_is_hardcoded_in_the_data_plane() -> None: """The data plane IS the replacement for the hosted project. A hosted hostname appearing here is either a stale copy-paste or a repointing that