diff --git a/.env.example b/.env.example index 6a2c0578c..7068e44f6 100644 --- a/.env.example +++ b/.env.example @@ -412,6 +412,14 @@ OWUI_ADMIN_TOKEN= # 3144 prompt tokens against 52 without them. Set it to native only when the # alias in use routes to a tool capable model. OWUI_DEFAULT_FUNCTION_CALLING=legacy +# Catalog aliases the Open WebUI chat model picker must not list (#792). +# Comma-separated, exact alias ids. These stay on the gateway's own +# /v1/models, which is an OpenAI-contract surface direct API clients depend +# on; they are hidden only from the chat dropdown, where none of them can +# serve a completion. The image patch also hides whatever RAG_EMBEDDING_MODEL +# names, so the admin-selected embedding alias never needs a mention here. +# Leave unset to accept the compose default. +OWUI_PICKER_HIDDEN_ALIASES=hive-embedding-default,hive-stt,hive-tts OWUI_E2E_MODE=false # Open WebUI web search. Off by default -- this env var is shared with the # enterprise profile, which must not silently make live outbound search diff --git a/Makefile b/Makefile index 5aae66347..0aeb74f1b 100644 --- a/Makefile +++ b/Makefile @@ -27,3 +27,4 @@ test-scripts: python3 scripts/test_owui_rag_env_config.py python3 scripts/test_owui_ui_surfaces.py python3 scripts/test_caddy_owui_blocklist.py + python3 scripts/test_owui_model_picker_filter.py diff --git a/deploy/docker/Dockerfile.open-webui b/deploy/docker/Dockerfile.open-webui index cd84a302f..fe71e5a5e 100644 --- a/deploy/docker/Dockerfile.open-webui +++ b/deploy/docker/Dockerfile.open-webui @@ -160,6 +160,35 @@ RUN sed -i "s/or 'native'/or os.environ.get('HIVE_DEFAULT_FUNCTION_CALLING', 'na && ! grep -q "or 'native'" /app/backend/open_webui/main.py \ && python3 -c "import ast, pathlib; ast.parse(pathlib.Path('/app/backend/open_webui/main.py').read_text())" +# #792: keep the three non-chat catalog aliases (hive-embedding-default, +# hive-stt, hive-tts) out of the chat model picker, while the gateway keeps +# serving all six on /v1/models for direct API clients. +# +# Open WebUI's own per-model access control cannot do this job on this +# deployment, which is why #776 shipped inert. docker-compose.yml sets +# BYPASS_MODEL_ACCESS_CONTROL: "true" by design, and in this pinned image that +# skips get_filtered_models on /api/models for every role; even with the flag +# off, get_filtered_models exempts administrators whenever +# BYPASS_ADMIN_ACCESS_CONTROL is set, which defaults to true, and the +# tenant-role patch above makes every tenant owner an administrator. Measured +# on a booted v0.10.2 container: with the flag off an administrator still saw +# all six aliases while a member saw an empty picker and got HTTP 400 "Model +# not found" on every chat request, so turning the flag off buys nothing and +# breaks chat for members. +# +# The patch filters the /api/models response and nothing else. It does not +# touch request.app.state.MODELS, so chat, document RAG embeddings and +# text-to-speech all still resolve every alias. The hidden set comes from the +# environment (HIVE_PICKER_HIDDEN_MODEL_IDS plus Open WebUI's own +# RAG_EMBEDDING_MODEL / AUDIO_TTS_MODEL / AUDIO_STT_MODEL), never from a +# hardcoded list, so the admin-selectable embedding alias stays single-sourced +# (D-001). +COPY deploy/docker/owui-patches/hive_model_picker.py /app/backend/open_webui/utils/hive_model_picker.py +COPY deploy/docker/owui-patches/apply_model_picker_patch.py /tmp/apply_model_picker_patch.py +RUN python3 /tmp/apply_model_picker_patch.py \ + && grep -q 'hive_model_picker' /app/backend/open_webui/main.py \ + && python3 -c "import ast, pathlib; ast.parse(pathlib.Path('/app/backend/open_webui/main.py').read_text())" + # #771: drop the Settings > Integrations tab, the chat UI's only entry point to # "Manage Tool Servers" and "Open Terminal". Both accept an arbitrary URL and an # auth credential. The tool servers added there are direct ones, called by the diff --git a/deploy/docker/docker-compose.yml b/deploy/docker/docker-compose.yml index c4aa4a072..00ded7510 100644 --- a/deploy/docker/docker-compose.yml +++ b/deploy/docker/docker-compose.yml @@ -749,6 +749,26 @@ services: # never filtered anything; the filtering was Open WebUI's default access # control the whole time. BYPASS_MODEL_ACCESS_CONTROL: "true" + # #792: the one thing the gateway list cannot express. All six aliases + # are legitimately on /v1/models -- that endpoint is an OpenAI-contract + # surface and direct API clients need the embedding and audio ids -- but + # three of them serve no chat completion, so listing them in the chat + # dropdown only produces dead conversations. Dockerfile.open-webui's + # owui-patches/apply_model_picker_patch.py drops these ids from the + # /api/models response the picker reads, and from nothing else. + # + # Not Open WebUI's per-model access control, which is what #776 tried: + # the flag above disables it for every role, and it is admin-exempt + # regardless (BYPASS_ADMIN_ACCESS_CONTROL defaults true) while every + # tenant owner here is an Open WebUI admin. That fix could never fire. + # + # The patch also hides whatever RAG_EMBEDDING_MODEL, AUDIO_TTS_MODEL and + # AUDIO_STT_MODEL name, so changing the admin-selected embedding alias + # (D-001) does not need a second edit here. This list is for aliases + # Open WebUI itself never calls; hive-stt and hive-tts are named because + # this deployment routes voice through the gateway rather than through + # Open WebUI's own audio settings. + HIVE_PICKER_HIDDEN_MODEL_IDS: ${OWUI_PICKER_HIDDEN_ALIASES:-hive-embedding-default,hive-stt,hive-tts} ENABLE_EVALUATION_ARENA_MODELS: "false" # Web search. Opt-in via .env, default off (same pattern as # OWUI_E2E_MODE below) -- this service block is shared with the diff --git a/deploy/docker/owui-patches/apply_model_picker_patch.py b/deploy/docker/owui-patches/apply_model_picker_patch.py new file mode 100644 index 000000000..755d8e919 --- /dev/null +++ b/deploy/docker/owui-patches/apply_model_picker_patch.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Build-time splice: filter Hive's non-chat aliases out of the listing Open +WebUI's chat model picker reads (issue #792). + +The anchor is inside `main.py`'s `/api/models` handler, so the filter runs on +that response and on nothing else. It is inserted after `get_filtered_models` +rather than replacing it, and it is inserted UNCONDITIONALLY: the Hive filter +must not inherit upstream's access-control gating, which +`BYPASS_MODEL_ACCESS_CONTROL: "true"` disables for every role on this +deployment and which exempts administrators regardless. That gating is exactly +why #776's `access_control` mechanism shipped inert, and `assert_unconditional` +below is what stops this one from repeating it. + +Asserts its own effect and fails the build otherwise, the same posture as this +Dockerfile's other patches: a future open-webui digest bump whose `/api/models` +handler shifted breaks the build loudly rather than silently restoring the +three aliases to the dropdown. + +The transform lives in `patch()` so `scripts/test_owui_model_picker_filter.py` +can run the real thing against a checked-in excerpt of the pinned image's own +`main.py`. PR CI never builds this image, so without that the patch would be +verified only at deploy time. +""" + +import ast +import pathlib +import re +import sys + +TARGET = pathlib.Path("/app/backend/open_webui/main.py") + +SIGNATURE = "async def get_models(request: Request, refresh: bool = False, user=Depends(get_verified_user)):\n" +ANCHOR = " models = await get_filtered_models(models, user)\n" +RETURN = " return {'data': models}\n" +CALL = " models = _hive_filter_models(models, os.environ)\n" + +INSERT = ( + """ # hive #792: the three non-chat catalog aliases (embeddings, STT, TTS) + # must not appear in the chat model picker, while the gateway keeps + # serving all six on /v1/models for direct API clients. This runs on the + # response only -- request.app.state.MODELS is untouched, so every + # invocation path (chat, document RAG embeddings, text-to-speech) is + # unaffected. Unconditional by design: the access-control filter above is + # disabled deployment-wide by BYPASS_MODEL_ACCESS_CONTROL and is + # admin-exempt regardless, and every tenant owner here is an admin. + from open_webui.utils.hive_model_picker import filter_models as _hive_filter_models + +""" + + CALL +) + + +def handler_body(text: str) -> str: + """Return the source of main.py's /api/models handler.""" + assert text.count(SIGNATURE) == 1, ( + "the /api/models handler is not defined exactly once with the expected " + "signature -- upstream open-webui source shifted, patch needs updating" + ) + start = text.index(SIGNATURE) + len(SIGNATURE) + next_top_level = re.search(r"\n@|\n\S", text[start:]) + end = start + next_top_level.start() if next_top_level else len(text) + return text[start:end] + + +def assert_unconditional(body: str) -> None: + """Fail unless the Hive filter runs on every request to this handler. + + This is the guard #776 did not have. Its mechanism was real code that a + deployment flag switched off, and every test it shipped with still passed. + Any future edit that puts the call behind a flag, a role check, or an + access-control branch trips this, because the statement would no longer sit + at the handler's own indentation level. + """ + for line in body.splitlines(keepends=True): + if line.lstrip().startswith("models = _hive_filter_models("): + assert line == CALL.rstrip("\n") + "\n" or line == CALL, ( + "the hive picker filter is indented deeper than the handler " + "body, so something now gates it. It must run for every " + "request: BYPASS_MODEL_ACCESS_CONTROL and " + "BYPASS_ADMIN_ACCESS_CONTROL between them disable every " + "conditional Open WebUI offers here (issue #792)." + ) + return + raise AssertionError("the hive picker filter call is not in the /api/models handler") + + +def patch(text: str) -> str: + """Return main.py with the picker filter spliced into /api/models.""" + body = handler_body(text) + + assert text.count(ANCHOR) == 1, ( + "the 'models = await get_filtered_models(models, user)' anchor is not " + "present exactly once -- upstream open-webui source shifted, patch " + "needs updating" + ) + assert ANCHOR in body, ( + "the get_filtered_models anchor is not inside the /api/models " + "handler's own body -- upstream open-webui source shifted, patch needs " + "updating" + ) + assert RETURN in body, ( + "the /api/models handler no longer returns {'data': models} -- the " + "filter would run on a value the response does not use, patch needs " + "updating" + ) + assert "@app.get('/api/models')\n" in text, ( + "main.py no longer mounts GET /api/models -- patch needs updating" + ) + assert re.search(r"^import os$", text, re.MULTILINE), ( + "main.py no longer imports os -- patch needs updating" + ) + + patched = text.replace(ANCHOR, ANCHOR + INSERT, 1) + patched_body = handler_body(patched) + assert_unconditional(patched_body) + assert patched_body.index(CALL) > patched_body.index(ANCHOR), ( + "the filter must run after upstream's own filtering, not before" + ) + assert patched_body.index(CALL) < patched_body.index(RETURN), ( + "the filter must run before the handler returns, or the response is " + "unchanged -- which is precisely how #776 shipped inert" + ) + ast.parse(patched) # never write a main.py that cannot be imported + return patched + + +if __name__ == "__main__": + TARGET.write_text(patch(TARGET.read_text())) + sys.stdout.write("hive #792: /api/models picker filter spliced into main.py\n") diff --git a/deploy/docker/owui-patches/hive_model_picker.py b/deploy/docker/owui-patches/hive_model_picker.py new file mode 100644 index 000000000..a6dc015ba --- /dev/null +++ b/deploy/docker/owui-patches/hive_model_picker.py @@ -0,0 +1,92 @@ +"""Keep Hive's non-chat catalog aliases out of Open WebUI's chat model picker +(issues #772, #792). + +The gateway serves one model list for every client, and it must keep doing +that: `GET /v1/models` on the API origin is an OpenAI-contract surface that +`packages/openai-contract/matrix/support-matrix.json` marks supported, and +upstream OpenAI lists its embedding and audio model ids there too. Direct API +clients must keep seeing `hive-embedding-default`, `hive-stt` and `hive-tts`. +The chat picker is the only surface where listing them is wrong, because none +of the three serves chat completions and picking one produces a dead +conversation. + +So the filter belongs on the listing the picker reads and nowhere else. +`/api/models` in Open WebUI's `main.py` is exactly that listing: it is built +from `request.app.state.MODELS`, but it does not write it back, so removing an +entry here removes it from the dropdown while leaving every invocation path +untouched. Nothing in Open WebUI resolves a model for a chat completion, a RAG +embedding, or a text-to-speech call through this response. + +Why not Open WebUI's own per-model access control, which is what #776 tried: +`deploy/docker/docker-compose.yml` sets `BYPASS_MODEL_ACCESS_CONTROL: "true"` +by deliberate design (the gateway is the single source of truth for model +visibility, D-014), and in the pinned v0.10.2 image that flag makes +`main.py` skip `get_filtered_models` entirely, for every role. Even with the +flag off, `get_filtered_models` exempts administrators whenever +`BYPASS_ADMIN_ACCESS_CONTROL` is set, and that variable defaults to true +(`config.py:2029`) while this deployment promotes every tenant owner to an +Open WebUI administrator. Measured on a booted v0.10.2 container: with the +flag off, an administrator still saw all six aliases and a member saw an +empty picker and got HTTP 400 "Model not found" on every chat request. So an +access-control-shaped fix cannot hide anything from the people who use the +product, and turning the flag off to reach it breaks chat for everyone else. + +This filter is therefore unconditional. It reads no access-control flag, no +role, and no group membership, because every one of those is either bypassed +or admin-exempt on this deployment. + +The hidden set is never hardcoded here. It is the union of + + * `HIVE_PICKER_HIDDEN_MODEL_IDS`, a comma-separated list compose owns, and + * whichever aliases Open WebUI is itself configured to call for a non-chat + purpose (`RAG_EMBEDDING_MODEL`, `AUDIO_TTS_MODEL`, `AUDIO_STT_MODEL`). + +The second half matters because the RAG embedding alias is admin-selectable +and drives vector-store provisioning (D-001), so a deployment that changes it +must not have to remember to edit a second list to keep the picker clean. +An unset or blank variable contributes nothing, so a deployment that sets none +of them keeps upstream behaviour exactly. +""" + +# Comma-separated list of model ids compose wants out of the chat picker. +HIDDEN_IDS_ENV = "HIVE_PICKER_HIDDEN_MODEL_IDS" + +# Open WebUI's own settings for the three non-chat modalities. Anything named +# here is by definition not a chat model, so it is hidden without needing a +# second mention in HIDDEN_IDS_ENV. +MODALITY_MODEL_ENV = ( + "RAG_EMBEDDING_MODEL", + "AUDIO_TTS_MODEL", + "AUDIO_STT_MODEL", +) + + +def hidden_model_ids(environ) -> frozenset: + """Model ids the chat picker must not list, from the environment alone.""" + hidden = set() + + for entry in (environ.get(HIDDEN_IDS_ENV) or "").split(","): + entry = entry.strip() + if entry: + hidden.add(entry) + + for variable in MODALITY_MODEL_ENV: + value = (environ.get(variable) or "").strip() + if value: + hidden.add(value) + + return frozenset(hidden) + + +def filter_models(models, environ): + """Drop the hidden ids from a `/api/models` list. + + Returns the input unchanged when nothing is configured, so this is a no-op + on a deployment that sets none of the variables above. Entries without an + `id` are passed through rather than dropped: this filter exists to remove + three known aliases, not to police the shape of upstream's payload. + """ + hidden = hidden_model_ids(environ) + if not hidden: + return models + return [model for model in models if model.get("id") not in hidden] diff --git a/deploy/docker/owui-patches/pinned-main-excerpts.json b/deploy/docker/owui-patches/pinned-main-excerpts.json new file mode 100644 index 000000000..0c549192a --- /dev/null +++ b/deploy/docker/owui-patches/pinned-main-excerpts.json @@ -0,0 +1,7 @@ +{ + "image": "ghcr.io/open-webui/open-webui:v0.10.2@sha256:9fcea9c6e32ab60b0498f3986c6cdf651ddbe61db48d2213a3d28048ddd673d4", + "source": "/app/backend/open_webui/main.py", + "note": "Verbatim excerpt of the pinned image's /api/models handler, the listing Open WebUI's chat model picker reads. PR CI never builds Dockerfile.open-webui, so scripts/test_owui_model_picker_filter.py runs apply_model_picker_patch.patch against this instead of against an image nothing in CI has. Regenerate on any digest bump. The leading 'import os' is not from upstream's excerpt window; it stands in for main.py's own import so the excerpt parses on its own.", + "sha256": "3d26fd33a5ede734d1a2b9c828a63e470eb1ca4df1f8414b8141d7f3b08d8810", + "api_models_handler": "import os\n\n\n@app.get('/api/models')\n@app.get('/api/v1/models') # Experimental: Compatibility with OpenAI API\nasync def get_models(request: Request, refresh: bool = False, user=Depends(get_verified_user)):\n all_models = await get_all_models(request, refresh=refresh, user=user)\n\n models = []\n for model in all_models:\n # Filter out filter pipelines\n if 'pipeline' in model and model['pipeline'].get('type', None) == 'filter':\n continue\n\n # Remove profile image URL to reduce payload size\n if model.get('info', {}).get('meta', {}).get('profile_image_url'):\n model['info']['meta'].pop('profile_image_url', None)\n\n try:\n model_tags = [tag.get('name') for tag in model.get('info', {}).get('meta', {}).get('tags', [])]\n tags = [tag.get('name') for tag in model.get('tags', [])]\n\n tags = list(set(model_tags + tags))\n model['tags'] = [{'name': tag} for tag in tags]\n except Exception as e:\n log.debug(f'Error processing model tags: {e}')\n model['tags'] = []\n pass\n\n models.append(model)\n\n # Chat requests resolve models by ID from request.app.state.MODELS, where\n # duplicate IDs collapse to the last model. Return the same effective list.\n models = list({model['id']: model for model in models}.values())\n\n model_order_list = await Config.get('ui.model_order_list')\n if model_order_list:\n model_order_dict = {model_id: i for i, model_id in enumerate(model_order_list)}\n # Sort models by order list priority, with fallback for those not in the list\n models.sort(\n key=lambda model: (\n model_order_dict.get(model.get('id', ''), float('inf')),\n (model.get('name', '') or ''),\n )\n )\n\n models = await get_filtered_models(models, user)\n\n log.debug(\n f'/api/models returned filtered models accessible to the user: {json.dumps([model.get(\"id\") for model in models])}'\n )\n return {'data': models}\n\n\n" +} diff --git a/docs/proof/issue-792/01-before-picker-owner.png b/docs/proof/issue-792/01-before-picker-owner.png new file mode 100644 index 000000000..62fb6a8d0 Binary files /dev/null and b/docs/proof/issue-792/01-before-picker-owner.png differ diff --git a/docs/proof/issue-792/02-after-picker-owner.png b/docs/proof/issue-792/02-after-picker-owner.png new file mode 100644 index 000000000..8cbc5f5a3 Binary files /dev/null and b/docs/proof/issue-792/02-after-picker-owner.png differ diff --git a/docs/proof/issue-792/03-after-picker-listing-endpoint.png b/docs/proof/issue-792/03-after-picker-listing-endpoint.png new file mode 100644 index 000000000..fdc6d7b86 Binary files /dev/null and b/docs/proof/issue-792/03-after-picker-listing-endpoint.png differ diff --git a/docs/proof/issue-792/04-after-gateway-list-unfiltered.png b/docs/proof/issue-792/04-after-gateway-list-unfiltered.png new file mode 100644 index 000000000..565a46f12 Binary files /dev/null and b/docs/proof/issue-792/04-after-gateway-list-unfiltered.png differ diff --git a/docs/proof/issue-792/README.md b/docs/proof/issue-792/README.md new file mode 100644 index 000000000..83f73f820 --- /dev/null +++ b/docs/proof/issue-792/README.md @@ -0,0 +1,89 @@ +# Issue #792 evidence + +Everything here was captured against a booted Open WebUI, not against a unit +test. #776 passed its tests, merged, deployed, and changed nothing, so the bar +for this fix is a running stack. + +## Harness + +An isolated stack, deliberately not the shared one: the pinned Open WebUI image +plus a pgvector container and a stub gateway that serves exactly the six Hive +catalog aliases on `/v1/models` and answers the three upstream calls Open WebUI +makes (chat completions, RAG embeddings, `audio/speech` and +`audio/transcriptions`). Open WebUI's environment mirrors the `open-webui` +service in `deploy/docker/docker-compose.yml`, including +`BYPASS_MODEL_ACCESS_CONTROL: "true"`. + +Two personas, because the demo box promotes every tenant owner to an Open WebUI +administrator (#748) and Open WebUI's access control treats the two completely +differently: + +* `owner@harness.local` — Open WebUI role `admin`, the persona an actual demo + user has. +* `member@harness.local` — Open WebUI role `user`. + +Audio is wired to the gateway in the harness even though `docker-compose.yml` +does not set `AUDIO_*` today, because #792 names text-to-speech as a risk and a +risk that is not configured cannot be measured. + +## Screenshots + +| File | What it shows | +| --- | --- | +| `01-before-picker-owner.png` | Pre-fix picker, signed in as the owner persona: all six aliases, including `hive-embedding-default`, `hive-stt` and `hive-tts`. This is #792. | +| `02-after-picker-owner.png` | Same persona, same harness, patched image: three chat aliases. | +| `03-after-picker-listing-endpoint.png` | `/api/models`, the listing the picker reads, post-fix. | +| `04-after-gateway-list-unfiltered.png` | `/openai/models` in the same signed-in session post-fix: the gateway list still carries all six. Both halves of the requirement, from one running system, one moment. | + +## Probes + +`probe-*.json` are full runs across both personas: picker listing, gateway +listing, chat completion, document RAG upload plus ingest plus query, TTS and +STT. + +| File | Scenario | Result | +| --- | --- | --- | +| `probe-a-before-fix.json` | #776 merged, bypass on, i.e. the deployed box | Both personas see all six. Everything else works. The fix is inert. | +| `probe-b-bypass-removed.json` | Bypass removed, no other change | Admin still sees all six and chats fine. Member sees an **empty picker** and gets **HTTP 400 "Model not found"** on chat. RAG ingest, RAG query, TTS and STT are **unaffected for both personas**. | +| `probe-c-after-fix.json` | The fix, bypass left on as deployed | Both personas see three chat aliases; gateway list still six; chat, RAG, TTS and STT all still work. | + +Probe B is the measurement that decided the design. The risks #792 named, RAG +and text-to-speech, do not depend on the bypass at all: neither consults Open +WebUI's model registry. What the bypass actually protects is chat itself for +non-admin members. And removing it would not even have hidden the three aliases +from the people who use the demo, because they are all administrators. + +## Guard, red then green + +`scripts/test_owui_model_picker_filter.py` run against the same harness, the +only difference being which image the container runs: + +Pre-fix image, bypass on (the deployed world): + +``` +OWUI chat model picker filter regression (1 failure(s)): + run_live: the chat picker still lists ['hive-embedding-default', 'hive-stt', 'hive-tts'] (issue #792). The picker filter is not on the response path, or something is gating it. +exit=1 +``` + +Patched image, bypass still on: + +``` +live: gateway serves 6 models including all 3 non-chat aliases; picker lists 3 and none of them +OWUI chat model picker filter: 10 checks passed +exit=0 +``` + +The static half, which is what CI runs, goes red the same way with the +Dockerfile's `RUN` line for the patch neutered and compose left as it is: + +``` +OWUI chat model picker filter regression (2 failure(s)): + test_bypass_and_picker_filter_stay_coupled: docker-compose.yml sets BYPASS_MODEL_ACCESS_CONTROL: "true", which disables Open WebUI's per-model access control for every role, so an access_control-shaped fix cannot hide the non-chat aliases from the picker (this is #792). A picker filter that does not depend on access control must be present and must run in the image. + test_dockerfile_stages_and_runs_the_patch: Dockerfile.open-webui no longer RUNs apply_model_picker_patch.py, so the three non-chat aliases are back in the picker +exit=1 +``` + +That second failure is the one #776 could not have produced: it asserts the +mechanism reaches the image and is not the access-control mechanism the +deployment disables, rather than asserting that a value was written. diff --git a/docs/proof/issue-792/probe-a-before-fix.json b/docs/proof/issue-792/probe-a-before-fix.json new file mode 100644 index 000000000..a3d9ded1b --- /dev/null +++ b/docs/proof/issue-792/probe-a-before-fix.json @@ -0,0 +1,87 @@ +{ + "scenario": "A. before fix (#776 merged), BYPASS_MODEL_ACCESS_CONTROL=true (as deployed)", + "owner (owui role=admin)": { + "picker": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "gateway_list": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "chat": [ + 200, + "ok" + ], + "rag_upload": [ + 200, + "ok" + ], + "rag_ingest_status": [ + 200, + "{'status': 'completed'}" + ], + "rag_query": [ + 200, + "1 hit(s)" + ], + "tts": [ + 200, + "621 bytes" + ], + "stt": [ + 200, + "stub gateway transcript" + ] + }, + "member (owui role=user)": { + "picker": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "gateway_list": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "chat": [ + 200, + "ok" + ], + "rag_upload": [ + 200, + "ok" + ], + "rag_ingest_status": [ + 200, + "{'status': 'completed'}" + ], + "rag_query": [ + 200, + "1 hit(s)" + ], + "tts": [ + 200, + "621 bytes" + ], + "stt": [ + 200, + "stub gateway transcript" + ] + } +} diff --git a/docs/proof/issue-792/probe-b-bypass-removed.json b/docs/proof/issue-792/probe-b-bypass-removed.json new file mode 100644 index 000000000..6dabb6118 --- /dev/null +++ b/docs/proof/issue-792/probe-b-bypass-removed.json @@ -0,0 +1,73 @@ +{ + "scenario": "B. before fix, BYPASS_MODEL_ACCESS_CONTROL removed (the tempting fix)", + "owner (owui role=admin)": { + "picker": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "gateway_list": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "chat": [ + 200, + "ok" + ], + "rag_upload": [ + 200, + "ok" + ], + "rag_ingest_status": [ + 200, + "{'status': 'completed'}" + ], + "rag_query": [ + 200, + "1 hit(s)" + ], + "tts": [ + 200, + "621 bytes" + ], + "stt": [ + 200, + "stub gateway transcript" + ] + }, + "member (owui role=user)": { + "picker": [], + "gateway_list": [], + "chat": [ + 400, + "{\"detail\":\"Model not found\"}" + ], + "rag_upload": [ + 200, + "ok" + ], + "rag_ingest_status": [ + 200, + "{'status': 'completed'}" + ], + "rag_query": [ + 200, + "1 hit(s)" + ], + "tts": [ + 200, + "621 bytes" + ], + "stt": [ + 200, + "stub gateway transcript" + ] + } +} diff --git a/docs/proof/issue-792/probe-c-after-fix.json b/docs/proof/issue-792/probe-c-after-fix.json new file mode 100644 index 000000000..f8859c68e --- /dev/null +++ b/docs/proof/issue-792/probe-c-after-fix.json @@ -0,0 +1,81 @@ +{ + "scenario": "C. after fix, BYPASS_MODEL_ACCESS_CONTROL=true (as deployed)", + "owner (owui role=admin)": { + "picker": [ + "hive-auto", + "hive-default", + "hive-fast" + ], + "gateway_list": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "chat": [ + 200, + "ok" + ], + "rag_upload": [ + 200, + "ok" + ], + "rag_ingest_status": [ + 200, + "{'status': 'completed'}" + ], + "rag_query": [ + 200, + "1 hit(s)" + ], + "tts": [ + 200, + "621 bytes" + ], + "stt": [ + 200, + "stub gateway transcript" + ] + }, + "member (owui role=user)": { + "picker": [ + "hive-auto", + "hive-default", + "hive-fast" + ], + "gateway_list": [ + "hive-auto", + "hive-default", + "hive-embedding-default", + "hive-fast", + "hive-stt", + "hive-tts" + ], + "chat": [ + 200, + "ok" + ], + "rag_upload": [ + 200, + "ok" + ], + "rag_ingest_status": [ + 200, + "{'status': 'completed'}" + ], + "rag_query": [ + 200, + "1 hit(s)" + ], + "tts": [ + 200, + "621 bytes" + ], + "stt": [ + 200, + "stub gateway transcript" + ] + } +} diff --git a/docs/proof/pr-814-rebase-verification-2026-08-17/README.md b/docs/proof/pr-814-rebase-verification-2026-08-17/README.md new file mode 100644 index 000000000..a361d9338 --- /dev/null +++ b/docs/proof/pr-814-rebase-verification-2026-08-17/README.md @@ -0,0 +1,44 @@ +# PR #814 rebase-verification proof, 2026-08-17 + +The demo box runs `main`, not this branch, and the catalog/routing code has +moved since this PR was opened on 2026-08-09 (Phase 20 provider-catalog +waves, pricing corrections). Rather than trust the existing 2026-08-09 +`docs/proof/issue-792/*` captures against code that has since changed, this +directory is a fresh capture against the branch as rebased onto `main` at +`4e39de40`. + +## Method + +Same posture as the PR #909 demo-surface proof (`docs/proof/pr-909-demo-surface-2026-08-16/`): +two `deploy/docker/Dockerfile.open-webui` images, built from the same pinned +upstream digest, differing only in whether the `owui-patches/` tree carries +this PR's fix. Both run standalone (`WEBUI_AUTH=false`, no Postgres, no +Supabase) against a stub gateway on an isolated Docker network, answering +`GET /v1/models` with the real six-alias Hive catalog shape +(`hive-auto`, `hive-default`, `hive-fast`, `hive-embedding-default`, +`hive-stt`, `hive-tts`). `BYPASS_MODEL_ACCESS_CONTROL=true` matches the +deployed compose value. + +- `picker-main.png` / `picker-main.log` -- image built from `origin/main` + (no picker patch). Model selector dropdown, opened and screenshotted. +- `picker-branch.png` / `picker-branch.log` -- image built from this branch + (`HIVE_PICKER_HIDDEN_MODEL_IDS=hive-embedding-default,hive-stt,hive-tts`). + +## Result + +| | dropdown contents | +| --- | --- | +| `main` (unpatched) | `hive-auto`, `hive-default`, `hive-fast`, `hive-embedding-default`, `hive-stt`, `hive-tts` | +| this branch (patched) | `hive-auto`, `hive-default`, `hive-fast` | + +The `.log` files are the DOM read the capture script made (which ids from +the known six actually render in the opened dropdown), not just the +screenshot, same double-check pattern PR #909 used for its user-menu proof. + +## What this does not re-verify + +This is the picker only. `docs/proof/issue-792/probe-*.json` (unchanged by +this rebase) is still the evidence that RAG ingest/query, chat, TTS and STT +are unaffected by the fix and that the gateway's own `/v1/models`-shaped +list keeps serving all six aliases; nothing in this rebase touched that +code path. diff --git a/docs/proof/pr-814-rebase-verification-2026-08-17/picker-branch.log b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-branch.log new file mode 100644 index 000000000..e12e98777 --- /dev/null +++ b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-branch.log @@ -0,0 +1,10 @@ +{ + "label": "branch", + "baseUrl": "http://127.0.0.1:18081/", + "modelIdsShownInDropdown": [ + "hive-auto", + "hive-default", + "hive-fast" + ], + "capturedAt": "2026-08-17T08:35:26.731Z" +} \ No newline at end of file diff --git a/docs/proof/pr-814-rebase-verification-2026-08-17/picker-branch.png b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-branch.png new file mode 100644 index 000000000..1e16cf306 Binary files /dev/null and b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-branch.png differ diff --git a/docs/proof/pr-814-rebase-verification-2026-08-17/picker-main.log b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-main.log new file mode 100644 index 000000000..7571f48ad --- /dev/null +++ b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-main.log @@ -0,0 +1,13 @@ +{ + "label": "main", + "baseUrl": "http://127.0.0.1:18080/", + "modelIdsShownInDropdown": [ + "hive-auto", + "hive-default", + "hive-fast", + "hive-embedding-default", + "hive-stt", + "hive-tts" + ], + "capturedAt": "2026-08-17T08:35:46.346Z" +} \ No newline at end of file diff --git a/docs/proof/pr-814-rebase-verification-2026-08-17/picker-main.png b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-main.png new file mode 100644 index 000000000..633b61da1 Binary files /dev/null and b/docs/proof/pr-814-rebase-verification-2026-08-17/picker-main.png differ diff --git a/scripts/test_owui_model_picker_filter.py b/scripts/test_owui_model_picker_filter.py new file mode 100644 index 000000000..aba9deb42 --- /dev/null +++ b/scripts/test_owui_model_picker_filter.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Guard the chat model picker filter against the way its predecessor failed +(issues #772, #776, #792). + +#776 hid `hive-embedding-default`, `hive-stt` and `hive-tts` by writing Open +WebUI's per-model `access_control`. It was reviewed, tested, merged and +deployed, and it changed nothing, because `deploy/docker/docker-compose.yml` +sets `BYPASS_MODEL_ACCESS_CONTROL: "true"` and the pinned v0.10.2 image then +skips `get_filtered_models` for every role. Every test that shipped with it +still passed, because they all asserted that the right values were WRITTEN and +none asserted that anything was READ. + +So this file does not check that a value was written. It checks that the +filter is on the response path and that nothing can switch it off: + + 1. The filter itself, executed, against a realistic six-alias listing. + 2. The real patch, applied to a verbatim excerpt of the pinned image's own + `/api/models` handler, asserting the call lands after upstream's own + filtering, before the return, and at the handler's own indentation, so no + flag, role or access-control branch gates it. PR CI never builds + Dockerfile.open-webui, which is why the excerpt is checked in. + 3. The wiring: the Dockerfile must both stage and run the patch, and compose + must name the aliases. A patch that ships without its RUN line is + precisely as inert as #776 was. + 4. The coupling that caused #792: while compose sets the bypass, the repo + must carry a picker filter that does not depend on access control. + +`--live BASE --token TOKEN` runs the assertion that a unit test cannot make: +against a booted Open WebUI, the gateway's own list must still carry all three +aliases while the picker's list carries none of them and is not empty. That is +the shape of the bug, end to end, and it is the check that goes red the moment +a fix is inert again. + +No framework, no network, no Docker in the default mode. +Run: python3 scripts/test_owui_model_picker_filter.py +""" + +import argparse +import importlib.util +import json +import sys +import urllib.error +import urllib.request +from pathlib import Path + +REPO = Path(__file__).resolve().parents[1] +PATCHES = REPO / "deploy" / "docker" / "owui-patches" +COMPOSE = REPO / "deploy" / "docker" / "docker-compose.yml" +DOCKERFILE = REPO / "deploy" / "docker" / "Dockerfile.open-webui" +EXCERPTS = PATCHES / "pinned-main-excerpts.json" + +# The aliases #772 reported and #792 re-reported. Every one of them is a real +# row in the catalog and must stay on the gateway's own /v1/models. +NON_CHAT_ALIASES = ("hive-embedding-default", "hive-stt", "hive-tts") +CHAT_ALIASES = ("hive-auto", "hive-default", "hive-fast") + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, PATCHES / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +hive_model_picker = _load("hive_model_picker") +apply_model_picker_patch = _load("apply_model_picker_patch") + +# Reuse the Dockerfile instruction parser rather than writing a second one. It +# already folds backslash continuations, which matters: a substring check for +# the filename is satisfied by the COPY line alone and stays green with the RUN +# line neutered, which is the one arrangement that ships the aliases back. +spec = importlib.util.spec_from_file_location( + "test_caddy_owui_blocklist", REPO / "scripts" / "test_caddy_owui_blocklist.py" +) +_blocklist = importlib.util.module_from_spec(spec) +spec.loader.exec_module(_blocklist) +dockerfile_runs_patch = _blocklist._dockerfile_runs_patch + + +def listing(ids): + """A /api/models-shaped listing, as Open WebUI builds it.""" + return [{"id": alias, "name": alias, "owned_by": "openai"} for alias in ids] + + +# -------------------------------------------------------------------------- +# 1. The filter, executed +# -------------------------------------------------------------------------- + +def test_filter_drops_only_the_non_chat_aliases() -> None: + env = {"HIVE_PICKER_HIDDEN_MODEL_IDS": ",".join(NON_CHAT_ALIASES)} + kept = hive_model_picker.filter_models(listing(CHAT_ALIASES + NON_CHAT_ALIASES), env) + assert [m["id"] for m in kept] == list(CHAT_ALIASES), kept + + +def test_filter_is_a_no_op_when_nothing_is_configured() -> None: + """An unset variable must never empty the picker. + + An empty dropdown is the worse failure of the two: hiding three aliases + annoys, hiding all six looks identical to a gateway 403 and hid one for + four nights (#717). + """ + every = listing(CHAT_ALIASES + NON_CHAT_ALIASES) + assert hive_model_picker.filter_models(every, {}) == every + assert hive_model_picker.filter_models(every, {"HIVE_PICKER_HIDDEN_MODEL_IDS": " "}) == every + + +def test_filter_reads_open_webuis_own_modality_settings() -> None: + """The admin-selected embedding alias must not need a second mention. + + RAG_EMBEDDING_MODEL is admin-selectable and drives vector-store + provisioning (D-001), so a deployment that changes it must not silently + start listing the new alias in the chat picker. + """ + env = { + "RAG_EMBEDDING_MODEL": "hive-embedding-bge-m3", + "AUDIO_TTS_MODEL": "hive-tts", + "AUDIO_STT_MODEL": "hive-stt", + } + ids = CHAT_ALIASES + ("hive-embedding-bge-m3", "hive-stt", "hive-tts") + kept = hive_model_picker.filter_models(listing(ids), env) + assert [m["id"] for m in kept] == list(CHAT_ALIASES), kept + + +def test_filter_ignores_entries_without_an_id() -> None: + env = {"HIVE_PICKER_HIDDEN_MODEL_IDS": "hive-tts"} + kept = hive_model_picker.filter_models([{"name": "no id"}, {"id": "hive-tts"}], env) + assert kept == [{"name": "no id"}] + + +# -------------------------------------------------------------------------- +# 2. The patch, applied to the pinned image's own handler +# -------------------------------------------------------------------------- + +def test_patch_applies_to_the_pinned_api_models_handler() -> None: + fixture = json.loads(EXCERPTS.read_text()) + patched = apply_model_picker_patch.patch(fixture["api_models_handler"]) + body = apply_model_picker_patch.handler_body(patched) + + assert "hive_model_picker" in body, "the filter is not in the handler at all" + # patch() asserts ordering and non-gating itself; re-assert here so this + # file fails, and names why, if those assertions are ever weakened. + apply_model_picker_patch.assert_unconditional(body) + assert body.index(apply_model_picker_patch.CALL) < body.index( + apply_model_picker_patch.RETURN + ), "the filter runs after the handler returns, so the response is unfiltered" + + +def test_patch_refuses_a_handler_it_cannot_verify() -> None: + """A digest bump that moves the handler must break the build, not pass. + + sed exits 0 on a zero-match address and str.replace is silent on a miss; + this is what makes the difference between a patch and a no-op visible. + """ + fixture = json.loads(EXCERPTS.read_text()) + drifted = fixture["api_models_handler"].replace( + "models = await get_filtered_models(models, user)", "models = models" + ) + try: + apply_model_picker_patch.patch(drifted) + except AssertionError: + return + raise AssertionError("patch() accepted a handler whose anchor is gone") + + +def test_patch_rejects_a_gated_filter() -> None: + """The #776 failure mode, expressed as a test. + + A filter placed inside any conditional is a filter a deployment flag can + switch off. Open WebUI offers no conditional here that this deployment does + not already disable, so the only correct placement is unconditional. + """ + gated = ( + " if not BYPASS_MODEL_ACCESS_CONTROL:\n" + " models = _hive_filter_models(models, os.environ)\n" + ) + try: + apply_model_picker_patch.assert_unconditional(gated) + except AssertionError: + return + raise AssertionError( + "assert_unconditional accepted a filter behind BYPASS_MODEL_ACCESS_CONTROL, " + "which is exactly the arrangement that made #776 inert" + ) + + +# -------------------------------------------------------------------------- +# 3. Wiring: the patch has to actually reach the image, and compose has to +# name the aliases +# -------------------------------------------------------------------------- + +def test_dockerfile_stages_and_runs_the_patch() -> None: + staged, invoked = dockerfile_runs_patch(DOCKERFILE, "apply_model_picker_patch.py") + assert staged, f"{DOCKERFILE.name} no longer COPYs apply_model_picker_patch.py" + assert invoked, ( + f"{DOCKERFILE.name} no longer RUNs apply_model_picker_patch.py, so the " + "three non-chat aliases are back in the picker" + ) + text = DOCKERFILE.read_text(encoding="utf-8") + assert "owui-patches/hive_model_picker.py" in text, ( + f"{DOCKERFILE.name} no longer COPYs hive_model_picker.py into the image, " + "so the spliced import raises at startup" + ) + + +def test_compose_names_the_non_chat_aliases() -> None: + text = COMPOSE.read_text(encoding="utf-8") + assignments = [ + line for line in text.splitlines() if "HIVE_PICKER_HIDDEN_MODEL_IDS:" in line + ] + assert len(assignments) == 1, ( + "expected exactly one HIVE_PICKER_HIDDEN_MODEL_IDS assignment in " + f"{COMPOSE.name}, found {len(assignments)}" + ) + for alias in NON_CHAT_ALIASES: + assert alias in assignments[0], f"{alias} is not hidden from the chat picker" + for alias in CHAT_ALIASES: + assert alias not in assignments[0], ( + f"{alias} is a chat model and must stay selectable in the picker" + ) + + +def test_bypass_and_picker_filter_stay_coupled() -> None: + """The invariant #792 is about. + + While the bypass is on, Open WebUI's own access control cannot hide + anything from anyone, so the picker fix has to be something else. If a + future change turns the bypass off, this stops applying and whoever does it + has to come here and re-derive it deliberately rather than inherit a + silently dead mechanism. + """ + text = COMPOSE.read_text(encoding="utf-8") + bypass_on = 'BYPASS_MODEL_ACCESS_CONTROL: "true"' in text + if not bypass_on: + return + _, invoked = dockerfile_runs_patch(DOCKERFILE, "apply_model_picker_patch.py") + assert invoked, ( + "docker-compose.yml sets BYPASS_MODEL_ACCESS_CONTROL: \"true\", which " + "disables Open WebUI's per-model access control for every role, so an " + "access_control-shaped fix cannot hide the non-chat aliases from the " + "picker (this is #792). A picker filter that does not depend on access " + "control must be present and must run in the image." + ) + + +# -------------------------------------------------------------------------- +# 4. Live: the check a unit test cannot make +# -------------------------------------------------------------------------- + +def _get_json(base: str, path: str, token: str): + req = urllib.request.Request( + base.rstrip("/") + path, headers={"Authorization": f"Bearer {token}"} + ) + with urllib.request.urlopen(req, timeout=30) as response: + return json.loads(response.read()) + + +def run_live(base: str, token: str) -> None: + """Assert both halves against a booted Open WebUI. + + The gateway list is read through Open WebUI's own /openai/models, which is + the unfiltered upstream response, so this compares the two lists from one + running system in one moment rather than trusting either in isolation. + """ + gateway = {m["id"] for m in _get_json(base, "/openai/models", token)["data"]} + picker = {m["id"] for m in _get_json(base, "/api/models", token)["data"]} + + missing = [a for a in NON_CHAT_ALIASES if a not in gateway] + assert not missing, ( + f"the gateway stopped serving {missing}. The OpenAI-compatible model " + "list must keep every alias; only the chat picker hides them." + ) + leaked = sorted(a for a in NON_CHAT_ALIASES if a in picker) + assert not leaked, ( + f"the chat picker still lists {leaked} (issue #792). The picker filter " + "is not on the response path, or something is gating it." + ) + assert picker, ( + "the chat picker is empty, which is a worse failure than the one being " + "fixed and is indistinguishable from a gateway rejection (#717)." + ) + print( + f"live: gateway serves {len(gateway)} models including all " + f"{len(NON_CHAT_ALIASES)} non-chat aliases; picker lists " + f"{len(picker)} and none of them" + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--live", metavar="BASE_URL", help="booted Open WebUI base URL") + parser.add_argument("--token", help="a signed-in Open WebUI bearer token") + args = parser.parse_args() + + failures = [] + tests = [value for name, value in sorted(globals().items()) if name.startswith("test_")] + for test in tests: + try: + test() + except AssertionError as exc: + failures.append(f"{test.__name__}: {exc}") + + if args.live: + if not args.token: + parser.error("--live needs --token") + try: + run_live(args.live, args.token) + except (AssertionError, urllib.error.URLError, KeyError) as exc: + failures.append(f"run_live: {exc}") + + if failures: + print(f"OWUI chat model picker filter regression ({len(failures)} failure(s)):") + for line in failures: + print(f" {line}") + return 1 + + print(f"OWUI chat model picker filter: {len(tests)} checks passed") + return 0 + + +if __name__ == "__main__": + sys.exit(main())