Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions tests/e2e/coverage_registry/mgmt.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,4 @@
- {id: mgmt.config_override.hashicorp_vault.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "config_override_endpoints.py", rationale: "Vault integration (smoke)"}
- {id: mgmt.workflow.list.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "workflow_management_endpoints.py", rationale: "Workflow tracking (smoke)"}
- {id: mgmt.credential_migration.check.happy_path, module: mgmt, tier: P2, surface: api, assertions: [happy_path], source: "key_management_endpoints.py:4252", rationale: "Encryption migration (smoke)"}
- {id: mgmt.credential.new.serves_request, module: mgmt, tier: P1, surface: api, assertions: [serves_request], source: "credential_endpoints/endpoints.py:42", rationale: "Stored credential resolves into a deployment and serves a live /messages request"}

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 source field references production code, not the test file

Every other entry in this file sets source to a test-harness or test file (e.g. key_management_endpoints.py:4252, workflow_management_endpoints.py). This new entry sets source: "credential_endpoints/endpoints.py:42", which is the production endpoint. The test that covers this entry lives in llm_translation/test_credential_messages_e2e.py; pointing there would keep the registry consistent and make it easy to jump from the registry to the actual test.

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!

49 changes: 49 additions & 0 deletions tests/e2e/llm_translation/test_credential_messages_e2e.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
"""Live e2e: stored credentials resolve into a deployment serving /v1/messages."""

from __future__ import annotations

import os

import pytest

from e2e_config import unique_marker
from e2e_http import require_successful_call
from endpoints_client import EndpointsClient, MessagesResult
from lifecycle import ResourceManager
from models import CredentialCreateBody, LiteLLMParamsBody

pytestmark = pytest.mark.e2e


class TestCredentialBackedMessages:
@pytest.mark.covers("mgmt.credential.new.serves_request")
def test_credential_backed_messages(self, endpoints_client: EndpointsClient, resources: ResourceManager) -> None:
marker = unique_marker()
credential_name = f"e2e-cred-{marker}"
model = f"e2e-cred-messages-{marker}"
anthropic_api_key = os.getenv("ANTHROPIC_API_KEY")
assert anthropic_api_key, "ANTHROPIC_API_KEY must be set for this live e2e test"

endpoints_client.proxy.create_credential(
CredentialCreateBody(
credential_name=credential_name,
credential_values={"api_key": anthropic_api_key},
)
)
resources.defer(lambda: endpoints_client.proxy.delete_credential(credential_name))

model_id = endpoints_client.create_model(
model,
LiteLLMParamsBody(
model="anthropic/claude-haiku-4-5",
litellm_credential_name=credential_name,
),
)
resources.defer(lambda: endpoints_client.delete_model(model_id))

key = resources.key()
result = endpoints_client.messages(key, model, "reply with one word")
require_successful_call(result)
parsed = MessagesResult.model_validate_json(result.body)
assert parsed.role == "assistant", f"unexpected role: {result.body[:300]}"
assert parsed.text.strip(), f"/v1/messages returned no text: {result.body[:300]}"
11 changes: 11 additions & 0 deletions tests/e2e/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -492,6 +492,7 @@ class LiteLLMParamsBody(BaseModel):

model: str
api_key: str | None = None
litellm_credential_name: str | None = None
api_base: str | None = None
api_version: str | None = None
realtime_protocol: str | None = None
Expand Down Expand Up @@ -556,6 +557,16 @@ class ModelDeleteBody(BaseModel):
id: str


class CredentialCreateBody(BaseModel):
credential_name: str
credential_values: dict[str, str]
credential_info: dict[str, str] = {}
Comment on lines +560 to +563

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 credential_values and credential_info typed too narrowly

Both fields are typed as dict[str, str], but the actual API model (CreateCredentialItem) uses plain dict — values can be any JSON type. Pydantic will reject a harness call the moment a credential value is non-string (e.g. a boolean or numeric timeout), even though the server would accept it fine. The repo-preferred wide type is dict[str, object].



class CredentialCreateResponse(BaseModel):
success: bool


# ---------- key / team / user / organization management ----------


Expand Down
22 changes: 22 additions & 0 deletions tests/e2e/proxy_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
ChatResponse,
CountTokensBody,
CountTokensResponse,
CredentialCreateBody,
CredentialCreateResponse,
CustomerDeleteBody,
EmbedBody,
EmbedResponse,
Expand Down Expand Up @@ -219,6 +221,26 @@ def delete_model(self, model_id: str) -> None:
if not is_ok(result):
warnings.warn(f"delete_model({model_id!r}) failed: {result}", stacklevel=2)

def create_credential(self, body: CredentialCreateBody) -> None:
unwrap(
self.transport.post(
"/credentials",
headers=self.transport.master,
json=body,
response_type=CredentialCreateResponse,
)
)

def delete_credential(self, credential_name: str) -> None:
result = self.transport.delete(
f"/credentials/{credential_name}",
headers=self.transport.master,
json=NoBody(),
response_type=NoBody,
)
if not is_ok(result):
warnings.warn(f"delete_credential({credential_name!r}) failed: {result}", stacklevel=2)

# ---- LLM calls ------------------------------------------------------

def chat(self, key: str, body: ChatBody) -> Result[ChatResponse]:
Expand Down
Loading