fix: give supabase-storage the S3 protocol credentials its clients sign with (#1282)

.env.example

@@ -743,8 +743,13 @@ OLLAMA_BASE_URL=
 #       The credential pair supabase-storage accepts on its S3 protocol
 #       endpoint. Do NOT set both halves to the same value: SigV4 transmits the
 #       access key id in the clear inside the Authorization header, so an
-#       identical secret is disclosed on every request. Configure distinct
-#       credentials on the storage service and use those here.
+#       identical secret is disclosed on every request. Generate two unrelated
+#       values, for example `openssl rand -hex 16` and `openssl rand -base64 32`.
+#       Nothing else has to be configured on the storage service: the compose
+#       file hands these same two variables to supabase-storage as
+#       S3_PROTOCOL_ACCESS_KEY_ID and S3_PROTOCOL_ACCESS_KEY_SECRET, so the
+#       signing pair cannot drift from the verifying pair. Leaving them unset
+#       is what made every upload return 500 on the demo box (issue #1282).
 #
 # ── Four ENTERPRISE_* variables that are an auth boundary, not a detail ─────
 #
@@ -1023,8 +1028,8 @@ ENTERPRISE_SSO_MICROSOFT_TENANT_URL=https://login.microsoftonline.com/<tenant-id
 # Storage: Supabase Storage API with local filesystem backend (no external S3).
 # The S3-compatible endpoint is at /storage/v1/s3 (matches hosted Supabase path).
 # S3_ENDPOINT=http://caddy-supabase/storage/v1/s3
-# S3_ACCESS_KEY=<ENTERPRISE_SERVICE_ROLE_KEY>
-# S3_SECRET_KEY=<ENTERPRISE_SERVICE_ROLE_KEY>
+# S3_ACCESS_KEY=<generate: openssl rand -hex 16>
+# S3_SECRET_KEY=<generate: openssl rand -base64 32>
 # S3_REGION=local
 # S3_USE_SSL=false
 # S3_BUCKET_FILES=hive-files

apps/edge-api/internal/rag/chat_handler_test.go

@@ -493,7 +493,15 @@ func TestHandleChat_SuppressedChunkUsageNotAccounted(t *testing.T) {
 		t.Fatalf("expected 200 for streaming, got %d: %s", w.Code, w.Body.String())
 	}
 	body := w.Body.String()
-	if strings.Contains(body, "SPURIOUS-POST-FINISH-MARKER") || strings.Contains(body, "999") {
+	// The usage check names the field, not the bare number. Every frame this
+	// handler emits carries a generated `ragchat-<uuid>` id, and a hex uuid
+	// contains the substring "999" often enough to fail this test at random:
+	// it did on CI run 33238715016, on an id of ragchat-39990550-..., with
+	// nothing wrong in the code under test. A flake in a regression guard is
+	// worse than no guard, because the next red is assumed to be this one.
+	if strings.Contains(body, "SPURIOUS-POST-FINISH-MARKER") ||
+		strings.Contains(body, `"total_tokens":999`) ||
+		strings.Contains(body, `"prompt_tokens":999`) {
 		t.Errorf("suppressed chunk (and its bogus usage) must never reach the client:\n%s", body)
 	}
 

deploy/docker/docker-compose.enterprise.yml

@@ -345,6 +345,36 @@ services:
       TENANT_ID: stub
       REGION: local
       GLOBAL_S3_BUCKET: stub
+      # The S3-compatible endpoint, which is the ONLY way edge-api and
+      # control-plane speak to Storage. Without these three variables Storage
+      # answers every signed request with 403 AccessDenied and the body
+      # "Missing S3 Protocol Access Key ID or Secret Key Environment
+      # variables", so POST /v1/files, every Batches input and output object,
+      # and RAG document upload all return 500 while the container reports
+      # healthy and the bucket rows exist. That was live on the demo box until
+      # 2026-08-29 (issue #1282).
+      #
+      # Deliberately the SAME two variables the consumers read
+      # (docker-compose.yml, edge-api and control-plane). The client's access
+      # key and secret must equal the server's pair exactly, since Storage
+      # recomputes the AWS SigV4 signature from them; two independent
+      # variables would let the halves drift apart with no boot error on
+      # either side. .env.example's older recipe of pointing S3_ACCESS_KEY and
+      # S3_SECRET_KEY at the service_role key never worked: a service_role JWT
+      # is not an S3 credential in single-tenant mode, and Storage compares
+      # against S3_PROTOCOL_ACCESS_KEY_ID/SECRET only.
+      S3_PROTOCOL_ACCESS_KEY_ID: ${S3_ACCESS_KEY:?set S3_ACCESS_KEY in .env (the S3 protocol access key edge-api and control-plane sign with)}
+      S3_PROTOCOL_ACCESS_KEY_SECRET: ${S3_SECRET_KEY:?set S3_SECRET_KEY in .env (the S3 protocol secret edge-api and control-plane sign with)}
+      # The prefix Caddyfile.supabase's `handle_path /storage/v1/*` STRIPS
+      # before this service sees the request. The signature covers the path the
+      # client signed, so Storage has to put the prefix back before it
+      # recomputes: canonicalUri is `s3ProtocolPrefix + request.url`. Leave it
+      # empty and every request fails SignatureDoesNotMatch instead, which
+      # looks like a wrong key rather than a wrong path. Must stay equal to the
+      # prefix in S3_ENDPOINT (http://caddy-supabase/storage/v1/s3) and to the
+      # gateway's handle_path; scripts/test_selfhost_supabase_seam.py fails if
+      # the three drift.
+      S3_PROTOCOL_PREFIX: /storage/v1
       ENABLE_IMAGE_TRANSFORMATION: "true"
       IMGPROXY_URL: ""
     volumes:

deploy/docker/docker-compose.yml

@@ -488,8 +488,14 @@ services:
       # Storage itself serves /s3 at its own root. Two earlier versions
       # of this comment named the service directly, one of them with an
       # /object/s3 path that no listener answers.
-      #   S3_ACCESS_KEY=<ENTERPRISE_SERVICE_ROLE_KEY>
-      #   S3_SECRET_KEY=<ENTERPRISE_SERVICE_ROLE_KEY>
+      # The two S3 credentials are a dedicated pair, NOT the service_role
+      # key: Storage compares them against its own
+      # S3_PROTOCOL_ACCESS_KEY_ID/SECRET, which the enterprise compose file
+      # feeds from these same variables. A service_role JWT here is refused,
+      # and the two halves must differ from each other because SigV4 sends
+      # the access key id in the clear.
+      #   S3_ACCESS_KEY=<openssl rand -hex 16>
+      #   S3_SECRET_KEY=<openssl rand -base64 32>
       #   S3_REGION=local
       #   S3_USE_SSL=false  (internal compose network, no TLS)
       S3_ENDPOINT: ${S3_ENDPOINT}

scripts/install.sh

@@ -336,11 +336,27 @@ setup_env() {
     prompt_value SUPABASE_SERVICE_ROLE_KEY "Supabase Service Role Key" required "" secret
     prompt_value SUPABASE_DB_URL       "Supabase DB URL (postgres://...)" required "" secret
 
-    # ── Storage (required) ──
+    # ── Storage ──
+    # This installer starts the enterprise profile, which brings up
+    # supabase-storage in the stack, so all three values describe that
+    # in-stack service and none of them come from a hosted project. The
+    # endpoint is the gateway form because Caddy is what restores the
+    # /storage/v1 prefix Storage's own routes do not carry.
+    #
+    # The credential pair is generated rather than prompted for blank. It is
+    # a shared secret between the client and the server halves of one stack,
+    # which the compose file wires from these same two variables, so there is
+    # nothing for an operator to look up. Asking produced the failure this
+    # block now prevents: the prompt named a hosted project, so the value
+    # people reached for was the service_role key, which Storage does not
+    # accept as an S3 credential and which SigV4 would then have sent in the
+    # clear inside every Authorization header (issue #1282).
     printf '%s-- Supabase Storage (S3) --%s\n' "${BOLD}" "${RESET}"
-    prompt_value S3_ENDPOINT  "S3 Endpoint (e.g. https://<ref>.supabase.co/storage/v1/s3)" required
-    prompt_value S3_ACCESS_KEY "S3 Access Key" required "" secret
-    prompt_value S3_SECRET_KEY "S3 Secret Key" required "" secret
+    _default_s3_access="$(command -v openssl >/dev/null 2>&1 && openssl rand -hex 16 || printf 'change-me-generate-with-openssl-rand-hex-16')"
+    _default_s3_secret="$(command -v openssl >/dev/null 2>&1 && openssl rand -base64 32 || printf 'change-me-generate-with-openssl-rand-base64-32')"
+    prompt_value S3_ENDPOINT  "S3 Endpoint" optional "http://caddy-supabase/storage/v1/s3"
+    prompt_value S3_ACCESS_KEY "S3 Access Key" optional "$_default_s3_access" secret
+    prompt_value S3_SECRET_KEY "S3 Secret Key" optional "$_default_s3_secret" secret
     prompt_value S3_REGION    "S3 Region" optional "us-east-1"
 
     # ── LLM Provider (at least one required) ──

scripts/test_selfhost_supabase_seam.py

@@ -284,6 +284,52 @@ def test_open_webui_pgvector_dsn_comes_from_the_libpq_flavour() -> None:
     assert not re.search(r"SUPABASE_DB_POOL_URL(?!_LIBPQ)", dsn), dsn
 
 
+def test_storage_accepts_the_s3_credentials_the_consumers_sign_with() -> None:
+    """Storage verifies an S3 request by recomputing the SigV4 signature from
+    S3_PROTOCOL_ACCESS_KEY_ID and S3_PROTOCOL_ACCESS_KEY_SECRET. With neither
+    set it refuses every request with 403 AccessDenied and "Missing S3 Protocol
+    Access Key ID or Secret Key Environment variables", which is a 500 on POST
+    /v1/files, on every Batches object and on RAG document upload while the
+    container stays healthy and the bucket rows exist (issue #1282).
+
+    They must come from the SAME two variables edge-api and control-plane sign
+    with. A separate pair would let the signing half and the verifying half
+    drift with no boot error on either side, and the only symptom is a 403 from
+    a service that looks configured."""
+    # A plain substring would also match ${S3_ACCESS_KEY_ID}, a different
+    # variable that would leave the two halves signing with different values,
+    # which is exactly the drift this asserts against. The terminator is what
+    # makes the match the whole name.
+    def references(value: str, variable: str) -> bool:
+        return re.search(r"\$\{" + variable + r"[:}]", value) is not None
+
+    storage = ENT_SERVICES["supabase-storage"]
+    key_id = env_value(storage, "S3_PROTOCOL_ACCESS_KEY_ID")
+    secret = env_value(storage, "S3_PROTOCOL_ACCESS_KEY_SECRET")
+    assert references(key_id, "S3_ACCESS_KEY"), key_id
+    assert references(secret, "S3_SECRET_KEY"), secret
+    for consumer in ("edge-api", "control-plane"):
+        block = BASE_SERVICES[consumer]
+        assert references(env_value(block, "S3_ACCESS_KEY"), "S3_ACCESS_KEY"), consumer
+        assert references(env_value(block, "S3_SECRET_KEY"), "S3_SECRET_KEY"), consumer
+
+
+def test_the_storage_s3_prefix_matches_the_prefix_the_gateway_strips() -> None:
+    """The consumers sign the path they send to the gateway, and the gateway's
+    `handle_path` strips its prefix before Storage sees the request. Storage
+    puts S3_PROTOCOL_PREFIX back before recomputing the signature, so an empty
+    or mismatched value fails every request with SignatureDoesNotMatch, which
+    reads as a wrong key rather than a wrong path. Three places have to agree:
+    this variable, S3_ENDPOINT, and the Caddyfile route."""
+    prefix = env_value(ENT_SERVICES["supabase-storage"], "S3_PROTOCOL_PREFIX")
+    assert prefix == "/storage/v1", prefix
+    caddy = (ROOT / "deploy" / "docker" / "Caddyfile.supabase").read_text()
+    assert f"handle_path {prefix}/*" in caddy, prefix
+    # The endpoint the consumers are told to use, in the file that documents it.
+    example = (ROOT / ".env.example").read_text()
+    assert f"S3_ENDPOINT=http://caddy-supabase{prefix}/s3" in example
+
+
 def test_no_hosted_supabase_host_is_hardcoded_in_the_data_plane() -> None:
     """The data plane IS the replacement for the hosted project. A hosted
     hostname appearing here is either a stale copy-paste or a repointing that