Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions backend/services/llm_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from openai import AsyncOpenAI
from core.config import settings
from core.exceptions import LLMServiceError
from pydantic import BaseModel
from pydantic import BaseModel, Field
from services.llm_provider_urls import build_llm_provider_http_client

logger = logging.getLogger(__name__)
Expand All @@ -13,6 +13,12 @@ class ExtractionResult(BaseModel):
summary: str
todos: list[str]
provenance: str | None = None
confidence: int | None = Field(
default=None,
ge=0,
le=100,
description="Optional confidence score from 0 to 100",
)


async def extract_todos_and_summary(
Expand All @@ -36,7 +42,11 @@ async def extract_todos_and_summary(
messages=[
{
"role": "system",
"content": "You are a helpful assistant. Summarize the email and extract action items.",
"content": (
"You are a helpful assistant. Summarize the email, "
"extract action items, and include a confidence score "
"from 0 to 100 when enough evidence is available."
),
},
{"role": "user", "content": email_body},
],
Expand Down
7 changes: 4 additions & 3 deletions backend/tests/test_llm_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def test_llm_endpoints_exist(mock_draft, mock_extract, client):
from services.llm_service import ExtractionResult

mock_extract.return_value = ExtractionResult(
summary="Test summary", todos=["Task 1"]
summary="Test summary", todos=["Task 1"], provenance="OpenAI", confidence=90
)
mock_draft.return_value = "This is a draft reply."

Expand All @@ -66,15 +66,16 @@ def test_summarize_endpoint(mock_extract, client):
from services.llm_service import ExtractionResult

mock_extract.return_value = ExtractionResult(
summary="Test summary", todos=["Task 1"]
summary="Test summary", todos=["Task 1"], provenance="OpenAI", confidence=90
)

resp = client.post("/api/llm/summarize", json={"email_body": "test email"})
assert resp.status_code == 200
assert resp.json() == {
"summary": "Test summary",
"todos": ["Task 1"],
"provenance": None,
"provenance": "OpenAI",
"confidence": 90,
}


Expand Down
23 changes: 21 additions & 2 deletions backend/tests/test_llm_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,25 @@ def mock_openai():
# Removed reset_client fixture


def test_extraction_result_confidence_is_optional_and_bounded():
omitted = ExtractionResult(summary="Test summary", todos=[])
assert omitted.confidence is None

assert ExtractionResult(
summary="Test summary",
todos=[],
confidence=0,
).confidence == 0
assert ExtractionResult(
summary="Test summary",
todos=[],
confidence=100,
).confidence == 100

with pytest.raises(ValueError):
ExtractionResult(summary="Test summary", todos=[], confidence=101)


@pytest.mark.asyncio
async def test_llm_provider_pinned_backend_uses_validated_ip_address():
calls = []
Expand Down Expand Up @@ -176,7 +195,7 @@ async def test_extract_todos_and_summary_success(mock_openai):
# Setup mock response
mock_response = MagicMock()
mock_message = MagicMock()
mock_message.parsed = ExtractionResult(summary="Test summary", todos=["Task 1"])
mock_message.parsed = ExtractionResult(summary="Test summary", todos=["Task 1"], confidence=90)
mock_choice = MagicMock()
mock_choice.message = mock_message
mock_response.choices = [mock_choice]
Expand Down Expand Up @@ -225,7 +244,7 @@ def fake_getaddrinfo(host, port, type=0):
mock_client.close = AsyncMock()
mock_response = MagicMock()
mock_message = MagicMock()
mock_message.parsed = ExtractionResult(summary="Test summary", todos=["Task 1"])
mock_message.parsed = ExtractionResult(summary="Test summary", todos=["Task 1"], confidence=90)
mock_choice = MagicMock()
mock_choice.message = mock_message
mock_response.choices = [mock_choice]
Expand Down
3 changes: 3 additions & 0 deletions frontend/src/components/EmailDetail.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ describe("EmailDetail", () => {
return Promise.resolve(jsonResponse({
summary: "출시 메시지의 핵심 맥락입니다.",
todos: ["캘린더에 출시 리뷰 일정을 반영", "답장 초안 준비"],
confidence: 0.82,
}));
}
throw new Error(`Unexpected fetch: ${url}`);
Expand All @@ -300,7 +301,9 @@ describe("EmailDetail", () => {
);
expect(cards.find((card) => card.getAttribute("aria-label") === "답장 초안")?.querySelector('[role="heading"][aria-level="3"]')?.textContent).toContain("답장 초안");
expect(cards.find((card) => card.getAttribute("aria-label") === "맥락 종합")?.textContent).toContain("출시 메시지의 핵심 맥락입니다.");
expect(cards.find((card) => card.getAttribute("aria-label") === "맥락 종합")?.textContent).toContain("82%");
expect(cards.find((card) => card.getAttribute("aria-label") === "실행 항목")?.textContent).toContain("캘린더에 출시 리뷰 일정을 반영");
expect(cards.find((card) => card.getAttribute("aria-label") === "실행 항목")?.textContent).toContain("82%");
expect(cards.find((card) => card.getAttribute("aria-label") === "답장 초안")?.querySelector('textarea[aria-label="답장 초안"]')).not.toBeNull();
});

Expand Down
18 changes: 15 additions & 3 deletions frontend/src/components/EmailDetail.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,17 @@ type EmailData = ThreadEmailData & {
interface LlmData {
summary: string;
todos: string[];
provenance?: string;
confidence?: number;
}

function toConfidencePercent(confidence: number | undefined): number | undefined {
if (typeof confidence !== "number" || !Number.isFinite(confidence)) {
return undefined;
}

const percent = confidence >= 0 && confidence <= 1 ? confidence * 100 : confidence;
return Math.round(Math.min(100, Math.max(0, percent)));
}

interface CreateTasksFromEmailResponse {
Expand Down Expand Up @@ -316,6 +327,7 @@ export function EmailDetail({ emailId, actionCommand = null }: { emailId: number
const safeEmailSender = toMailDisplayText(email.sender, '보낸 사람');
const safeEmailSubject = toMailDisplayText(email.subject, '(제목 없음)');
const safeReplyTo = toMailDisplayText(email.reply_to || email.sender, '답장 주소 없음');
const confidencePercent = toConfidencePercent(llmData?.confidence);

return (
<div className="flex h-full flex-col bg-card">
Expand Down Expand Up @@ -355,8 +367,8 @@ export function EmailDetail({ emailId, actionCommand = null }: { emailId: number
icon={<span aria-hidden="true">✦</span>}
loading={!llmData && !llmError}
error={llmError}
provenance="AI 생성"
// TODO: API 연동 시 실제 llmData.confidence 값으로 대체
provenance={llmData?.provenance || "AI 생성"}
confidence={confidencePercent}
>
{llmData ? (
<div className="flex flex-col gap-2">
Expand All @@ -378,7 +390,7 @@ export function EmailDetail({ emailId, actionCommand = null }: { emailId: number
empty={Boolean(llmData && llmData.todos.length === 0)}
emptyMessage="실행 항목이 없습니다."
provenance={`${llmData?.todos.length || 0}개 실행 항목`}
// TODO: API 연동 시 실제 llmData.confidence 값으로 대체
confidence={confidencePercent}
footerActions={llmData && (llmData.todos.length > 0 || syncStatus || taskStatus) ? (
<>
{llmData.todos.length > 0 && (
Expand Down
Loading