From 97f36b9b8352b1293ec0a16ce868ef131f50659f Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 00:27:00 +0200 Subject: [PATCH 01/23] fix(attachments): @JsonIgnore base64 payload so it is never persisted The transient keyword alone does not stop Jackson's getter-based serialization (no PROPAGATE_TRANSIENT_MARKER configured), so raw attachment base64 payloads were serialized into Mongo conversation documents. Add @JsonIgnore to Attachment.getBase64Data() and prove via serialization tests that the payload never reaches persisted JSON while metadata (mimeType/fileName/storageRef) is preserved. Phase 0 of multimodal-attachments-completion-plan. --- .../eddi/engine/memory/model/Attachment.java | 10 +++- .../engine/memory/model/AttachmentTest.java | 49 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/main/java/ai/labs/eddi/engine/memory/model/Attachment.java b/src/main/java/ai/labs/eddi/engine/memory/model/Attachment.java index aa92260631..a5cc96ea68 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/model/Attachment.java +++ b/src/main/java/ai/labs/eddi/engine/memory/model/Attachment.java @@ -9,6 +9,7 @@ import java.util.Map; import java.util.UUID; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; /** @@ -35,8 +36,12 @@ public class Attachment { private String url; /** - * Inline base64-encoded data. Transient — used for context-based input only, - * never persisted to MongoDB. Consumers should decode this and pass to LLM. + * Inline base64-encoded data. Used for context-based input only; the payload + * lives in memory for the duration of the turn but is NEVER persisted to + * MongoDB — {@link #getBase64Data()} is {@link JsonIgnore}d so Jackson's + * getter-based serialization skips it. The {@code transient} keyword alone does + * not stop Jackson (no {@code PROPAGATE_TRANSIENT_MARKER} configured). + * Consumers should decode this and pass it to the LLM. */ @JsonInclude(JsonInclude.Include.NON_NULL) private transient String base64Data; @@ -122,6 +127,7 @@ public void setUrl(String url) { this.url = url; } + @JsonIgnore public String getBase64Data() { return base64Data; } diff --git a/src/test/java/ai/labs/eddi/engine/memory/model/AttachmentTest.java b/src/test/java/ai/labs/eddi/engine/memory/model/AttachmentTest.java index 49a9d36853..685709676e 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/model/AttachmentTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/model/AttachmentTest.java @@ -4,6 +4,8 @@ */ package ai.labs.eddi.engine.memory.model; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; import java.util.Map; @@ -109,4 +111,51 @@ void base64Data_getterSetter() { att.setBase64Data("iVBORw0KGgo="); assertEquals("iVBORw0KGgo=", att.getBase64Data()); } + + // ==================== Serialization (no payload persisted) + // ==================== + + @Test + void serialization_neverIncludesBase64Payload() throws Exception { + var mapper = new ObjectMapper().findAndRegisterModules(); + + var att = new Attachment("image/png", "photo.png", 1024, "gridfs://abc"); + var secretPayload = "SECRETiVBORw0KGgoAAAANSUhEUgAAPAYLOAD"; + att.setBase64Data(secretPayload); + + String json = mapper.writeValueAsString(att); + + // The raw base64 payload must NOT leak into persisted JSON — the transient + // keyword alone does not stop Jackson's getter-based serialization. + assertFalse(json.contains(secretPayload), + "base64 payload must not be serialized into persisted JSON: " + json); + assertFalse(json.contains("base64Data"), + "base64Data property must be absent from persisted JSON: " + json); + + // Metadata that behavior rules match on must still be present. + assertTrue(json.contains("image/png")); + assertTrue(json.contains("photo.png")); + assertTrue(json.contains("gridfs://abc")); + } + + @Test + void serialization_roundTripPreservesMetadataButNotPayload() throws Exception { + // Mirror EDDI's persistence mapper: attachments have round-tripped through + // Mongo since 6.0.0 despite the derived, setter-less contentSource getter, + // so the store mapper ignores unknown properties on read. + var mapper = new ObjectMapper().findAndRegisterModules() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + var att = new Attachment("application/pdf", "doc.pdf", 2048, "pg://xyz"); + att.setBase64Data("JVBERi0xLjQ="); + + String json = mapper.writeValueAsString(att); + var restored = mapper.readValue(json, Attachment.class); + + assertEquals("application/pdf", restored.getMimeType()); + assertEquals("doc.pdf", restored.getFileName()); + assertEquals(2048, restored.getSizeBytes()); + assertEquals("pg://xyz", restored.getStorageRef()); + assertNull(restored.getBase64Data(), "payload must not survive persistence round-trip"); + } } From 17d4ca8177d0154e5fd8a6e1ebaff02912519463 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 00:35:38 +0200 Subject: [PATCH 02/23] fix(attachments): scrub inline base64 from persisted context copies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The raw attachment_* context map (including its base64 `data` payload) was persisted into the Mongo conversation document via both the conversationOutput map and the stored context Data — ~1.33x file size per turn against the 16MB doc limit, and template-exposed via {context.attachment_*.data}. Add AttachmentContextExtractor.scrubInlinePayload(), which returns a metadata-only copy of an attachment_* context when it carries an inline payload, and have Conversation.createContextData() build the persisted copy through it. The live payload still rides ATTACHMENTS memory for the turn (extracted from the original context map), so LLM forwarding is unaffected. Mirrors the secret-input scrubbing pattern. Phase 0 of multimodal-attachments-completion-plan. --- .../memory/AttachmentContextExtractor.java | 47 ++++++++++- .../engine/runtime/internal/Conversation.java | 6 +- .../AttachmentContextExtractorTest.java | 77 +++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) diff --git a/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java b/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java index 1294bad15a..748d5aba82 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java +++ b/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -38,7 +39,15 @@ public final class AttachmentContextExtractor { private static final Logger LOGGER = Logger.getLogger(AttachmentContextExtractor.class); - private static final String ATTACHMENT_PREFIX = "attachment_"; + + /** + * Context keys with this prefix carry attachment references (attachment_0, + * attachment_1, …). + */ + public static final String ATTACHMENT_PREFIX = "attachment_"; + + /** Inline base64 payload field inside an attachment context value map. */ + public static final String FIELD_DATA = "data"; private AttachmentContextExtractor() { // non-instantiable utility @@ -112,7 +121,7 @@ private static Attachment parseAttachment(String contextKey, Context ctx) { } // Base64 inline path - String data = getStringField(attachMap, "data"); + String data = getStringField(attachMap, FIELD_DATA); if (data != null && !data.isBlank()) { attachment.setBase64Data(data); // Estimate size from base64 length (3/4 of encoded length) @@ -128,4 +137,38 @@ private static String getStringField(Map map, String key) { Object value = map.get(key); return value instanceof String s ? s : null; } + + /** + * Return a metadata-only copy of an {@code attachment_*} context whose value + * map carries an inline base64 {@link #FIELD_DATA} payload; every other context + * — and payload-free attachment contexts such as URL references — is returned + * unchanged. + *

+ * Callers use this to build the persisted copy of the context (step + * data and {@code context.*} conversation output) so the raw base64 never lands + * in the Mongo conversation document (~1.33× file size per turn against + * the 16 MB limit) and is never exposed via + * {@code {context.attachment_*.data}} templates. The live payload has already + * been captured into ATTACHMENTS memory for the turn by + * {@link #extractAttachments(Map)} reading the original context map, so LLM + * forwarding is unaffected. Mirrors the secret-input scrubbing pattern. + * + * @param contextKey + * the context key (only {@code attachment_*} keys are scrubbed) + * @param ctx + * the original context (may be null) + * @return a scrubbed copy when a payload is present, otherwise {@code ctx} + * unchanged + */ + public static Context scrubInlinePayload(String contextKey, Context ctx) { + if (contextKey == null || !contextKey.startsWith(ATTACHMENT_PREFIX) || ctx == null) { + return ctx; + } + if (!(ctx.getValue() instanceof Map value) || !value.containsKey(FIELD_DATA)) { + return ctx; + } + Map scrubbed = new LinkedHashMap<>(value); + scrubbed.remove(FIELD_DATA); + return new Context(ctx.getType(), scrubbed); + } } diff --git a/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java b/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java index fa0e686d7d..61ec5cd797 100644 --- a/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java +++ b/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java @@ -412,8 +412,10 @@ private List> createContextData(Map context) { List> contextData = new LinkedList<>(); if (context != null) { for (String key : context.keySet()) { - contextData.add(new Data<>(KEY_CONTEXT + ":" + key, context.get(key))); - + // Persisted copy is scrubbed of inline base64 payloads; the live payload + // has already been captured into ATTACHMENTS memory for this turn. + Context persistedCopy = AttachmentContextExtractor.scrubInlinePayload(key, context.get(key)); + contextData.add(new Data<>(KEY_CONTEXT + ":" + key, persistedCopy)); } } return contextData; diff --git a/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java b/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java index 98b3e86ac1..621c1075c9 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java @@ -169,6 +169,83 @@ void shouldPreferUrlOverBase64() { } } + // ==================== Payload Scrubbing (persistence) ==================== + + @Nested + class ScrubInlinePayload { + + @Test + void shouldRemoveBase64DataFromAttachmentContext() { + Map attachMap = new HashMap<>(); + attachMap.put("mimeType", "image/png"); + attachMap.put("fileName", "icon.png"); + attachMap.put("data", "iVBORw0KGgoAAAANSUhEUgAA"); + Context original = createContext(attachMap); + + Context scrubbed = AttachmentContextExtractor.scrubInlinePayload("attachment_0", original); + + assertNotSame(original, scrubbed, "a scrubbed copy must be returned, not the original"); + @SuppressWarnings("unchecked") + Map value = (Map) scrubbed.getValue(); + assertFalse(value.containsKey("data"), "base64 data must be scrubbed from the persisted copy"); + assertEquals("image/png", value.get("mimeType"), "metadata must be preserved"); + assertEquals("icon.png", value.get("fileName"), "metadata must be preserved"); + assertEquals(original.getType(), scrubbed.getType(), "context type must be preserved"); + } + + @Test + void shouldNotMutateOriginalContext() { + Map attachMap = new HashMap<>(); + attachMap.put("mimeType", "image/png"); + attachMap.put("data", "iVBORw0KGgo="); + Context original = createContext(attachMap); + + AttachmentContextExtractor.scrubInlinePayload("attachment_0", original); + + @SuppressWarnings("unchecked") + Map originalValue = (Map) original.getValue(); + assertTrue(originalValue.containsKey("data"), + "original context must keep its payload so the live turn can forward it"); + assertEquals("iVBORw0KGgo=", originalValue.get("data")); + } + + @Test + void shouldReturnSameInstanceForUrlAttachment() { + Context original = createContext(Map.of( + "mimeType", "image/png", "url", "https://example.com/x.png")); + + Context result = AttachmentContextExtractor.scrubInlinePayload("attachment_0", original); + + assertSame(original, result, "url-only attachments carry no payload — return unchanged"); + } + + @Test + void shouldReturnSameInstanceForNonAttachmentKey() { + Context original = createContext(Map.of("data", "somevalue")); + + Context result = AttachmentContextExtractor.scrubInlinePayload("language", original); + + assertSame(original, result, "non-attachment contexts must never be scrubbed"); + } + + @Test + void shouldHandleNullContext() { + assertNull(AttachmentContextExtractor.scrubInlinePayload("attachment_0", null)); + } + + @Test + void shouldHandleNullKey() { + Context original = createContext(Map.of("data", "x")); + assertSame(original, AttachmentContextExtractor.scrubInlinePayload(null, original)); + } + + @Test + void shouldReturnUnchangedWhenValueNotAMap() { + Context original = createContext("not a map"); + assertSame(original, AttachmentContextExtractor.scrubInlinePayload("attachment_0", original)); + } + } + // ==================== Helpers ==================== private static Context createContext(Object value) { From 961eb33d031a01b6aeee8068288d84ca65a003b9 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 00:46:10 +0200 Subject: [PATCH 03/23] refactor(attachments): extract shared AttachmentTextExtractor from PdfReaderTool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce AttachmentTextExtractor (modules/llm/tools/impl) owning the PDFBox machinery and a plain-text decode path behind a uniform, configurable character cap (eddi.attachments.extraction.max-chars, default 10k). It exposes extractText(bytes, mime[, maxChars]) with PDF + text-like (text/*, JSON, XML, CSV, YAML) dispatch, plus PDF-specific full/page-range/info methods and a canExtractText() capability check. PdfReaderTool now delegates all extraction to this service while keeping its download, SSRF validation and user-facing formatting. This is the shared extractor the Phase 2 forwarder and Phase 4 readAttachment tool will reuse. 22 new unit tests cover the extractor (PdfReaderToolTest remains CI-only — SafeHttpClient opens a loopback selector local JVMs may block). Phase 0 of multimodal-attachments-completion-plan. --- .../tools/impl/AttachmentTextExtractor.java | 231 +++++++++++++++ .../modules/llm/tools/impl/PdfReaderTool.java | 169 +++-------- .../impl/AttachmentTextExtractorTest.java | 269 ++++++++++++++++++ .../llm/tools/impl/PdfReaderToolTest.java | 6 +- 4 files changed, 545 insertions(+), 130 deletions(-) create mode 100644 src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java create mode 100644 src/test/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractorTest.java diff --git a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java new file mode 100644 index 0000000000..62bfa80866 --- /dev/null +++ b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java @@ -0,0 +1,231 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.tools.impl; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.apache.pdfbox.Loader; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.text.PDFTextStripper; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import java.nio.charset.StandardCharsets; +import java.util.Calendar; +import java.util.Locale; + +/** + * Shared text-extraction service for binary attachments. Owns the PDFBox + * machinery (previously embedded in {@link PdfReaderTool}) and the plain-text + * decode path, so the {@code PdfReaderTool}, the multimodal attachment + * forwarder, and the {@code readAttachment} recall tool all extract text + * through a single, uniformly-capped implementation. + *

+ * All operations work on {@code byte[]} so the caller controls sourcing (URL + * download, blob-store load, inline base64) and SSRF validation. + * + * @since 6.1.0 + */ +@ApplicationScoped +public class AttachmentTextExtractor { + + private static final Logger LOGGER = Logger.getLogger(AttachmentTextExtractor.class); + + /** Fallback cap when configuration yields a non-positive value. */ + public static final int DEFAULT_MAX_CHARS = 10_000; + + private static final String TRUNCATION_SUFFIX_FMT = "\n\n[Content truncated - showing first %d characters]"; + + private final int defaultMaxChars; + + @Inject + public AttachmentTextExtractor( + @ConfigProperty(name = "eddi.attachments.extraction.max-chars", + defaultValue = "10000") int defaultMaxChars) { + this.defaultMaxChars = defaultMaxChars > 0 ? defaultMaxChars : DEFAULT_MAX_CHARS; + } + + /** + * @return the configured default character cap for extraction + */ + public int getDefaultMaxChars() { + return defaultMaxChars; + } + + /** + * Whether this service can extract inline text for the given MIME type (PDF or + * any text-like type — {@code text/*}, JSON, XML, CSV, YAML). + */ + public boolean canExtractText(String mimeType) { + return isPdf(mimeType) || isTextLike(mimeType); + } + + /** + * Extract text from an attachment using the configured default cap. + * + * @see #extractText(byte[], String, int) + */ + public String extractText(byte[] bytes, String mimeType) throws AttachmentExtractionException { + return extractText(bytes, mimeType, defaultMaxChars); + } + + /** + * Extract text from an attachment, dispatching on MIME type. + *

    + *
  • {@code application/pdf} → PDFBox full-text extraction
    • + *
  • text-like ({@code text/*}, JSON, XML, CSV, YAML) → UTF-8 decode
    • + *
+ * The result is capped to {@code maxChars} characters with a truncation note. + * + * @param bytes + * the raw attachment bytes + * @param mimeType + * the attachment MIME type + * @param maxChars + * the character cap (non-positive falls back to the default) + * @return extracted text, capped (never null) + * @throws AttachmentExtractionException + * if the type is unsupported or extraction fails + */ + public String extractText(byte[] bytes, String mimeType, int maxChars) throws AttachmentExtractionException { + if (bytes == null || bytes.length == 0) { + return ""; + } + int cap = maxChars > 0 ? maxChars : defaultMaxChars; + if (isPdf(mimeType)) { + return extractPdfText(bytes, cap); + } + if (isTextLike(mimeType)) { + return cap(new String(bytes, StandardCharsets.UTF_8), cap); + } + throw new AttachmentExtractionException( + "Unsupported MIME type for text extraction: " + mimeType); + } + + /** + * Extract all text from a PDF, capped to the configured default. + */ + public String extractPdfText(byte[] pdfBytes) throws AttachmentExtractionException { + return extractPdfText(pdfBytes, defaultMaxChars); + } + + /** + * Extract all text from a PDF, capped to {@code maxChars}. + */ + public String extractPdfText(byte[] pdfBytes, int maxChars) throws AttachmentExtractionException { + int cap = maxChars > 0 ? maxChars : defaultMaxChars; + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + PDFTextStripper stripper = new PDFTextStripper(); + String text = stripper.getText(document); + LOGGER.infof("Extracted %d characters from PDF with %d pages", + text.length(), document.getNumberOfPages()); + return cap(text, cap); + } catch (Exception e) { + throw new AttachmentExtractionException("Failed to extract text from PDF: " + e.getMessage(), e); + } + } + + /** + * Extract text from a page range of a PDF, capped to {@code maxChars}. + * {@code endPage} beyond the last page is clamped; {@code startPage} out of + * range throws. + */ + public String extractPdfText(byte[] pdfBytes, int startPage, int endPage, int maxChars) + throws AttachmentExtractionException { + int cap = maxChars > 0 ? maxChars : defaultMaxChars; + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + int totalPages = document.getNumberOfPages(); + if (startPage < 1 || startPage > totalPages) { + throw new AttachmentExtractionException( + "Start page " + startPage + " is out of range (1-" + totalPages + ")"); + } + int effectiveEnd = (endPage < startPage || endPage > totalPages) ? totalPages : endPage; + + PDFTextStripper stripper = new PDFTextStripper(); + stripper.setStartPage(startPage); + stripper.setEndPage(effectiveEnd); + String text = stripper.getText(document); + LOGGER.infof("Extracted text from PDF pages %d-%d (%d characters)", + startPage, effectiveEnd, text.length()); + return cap(text, cap); + } catch (AttachmentExtractionException e) { + throw e; + } catch (Exception e) { + throw new AttachmentExtractionException("Failed to extract PDF pages: " + e.getMessage(), e); + } + } + + /** + * Extract structural metadata from a PDF (page count + document information). + */ + public PdfInfo extractPdfInfo(byte[] pdfBytes) throws AttachmentExtractionException { + try (PDDocument document = Loader.loadPDF(pdfBytes)) { + var info = document.getDocumentInformation(); + String title = info != null ? info.getTitle() : null; + String author = info != null ? info.getAuthor() : null; + String subject = info != null ? info.getSubject() : null; + String creator = info != null ? info.getCreator() : null; + Calendar creationDate = info != null ? info.getCreationDate() : null; + return new PdfInfo(document.getNumberOfPages(), title, author, subject, creator, creationDate); + } catch (Exception e) { + throw new AttachmentExtractionException("Failed to read PDF information: " + e.getMessage(), e); + } + } + + private String cap(String text, int maxChars) { + if (text == null) { + return ""; + } + if (text.length() > maxChars) { + return (text.substring(0, maxChars) + String.format(TRUNCATION_SUFFIX_FMT, maxChars)).trim(); + } + return text.trim(); + } + + private static boolean isPdf(String mimeType) { + return mimeType != null && mimeType.toLowerCase(Locale.ROOT).startsWith("application/pdf"); + } + + private static boolean isTextLike(String mimeType) { + if (mimeType == null) { + return false; + } + String mime = mimeType.toLowerCase(Locale.ROOT); + // Strip any parameters (e.g. "text/plain; charset=utf-8"). + int semi = mime.indexOf(';'); + if (semi >= 0) { + mime = mime.substring(0, semi).trim(); + } + if (mime.startsWith("text/")) { + return true; + } + return switch (mime) { + case "application/json", "application/xml", "application/csv", + "application/yaml", "application/x-yaml", "application/x-ndjson" -> + true; + default -> mime.endsWith("+json") || mime.endsWith("+xml"); + }; + } + + /** + * Structural metadata extracted from a PDF document. + */ + public record PdfInfo(int numberOfPages, String title, String author, + String subject, String creator, Calendar creationDate) { + } + + /** + * Thrown when text extraction fails or the MIME type is unsupported. + */ + public static class AttachmentExtractionException extends Exception { + public AttachmentExtractionException(String message) { + super(message); + } + + public AttachmentExtractionException(String message, Throwable cause) { + super(message, cause); + } + } +} diff --git a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderTool.java b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderTool.java index 683b1db78d..a848dcb52a 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderTool.java +++ b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderTool.java @@ -5,16 +5,13 @@ package ai.labs.eddi.modules.llm.tools.impl; import ai.labs.eddi.engine.httpclient.SafeHttpClient; +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor.PdfInfo; import dev.langchain4j.agent.tool.P; import dev.langchain4j.agent.tool.Tool; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; -import org.apache.pdfbox.Loader; -import org.apache.pdfbox.pdmodel.PDDocument; -import org.apache.pdfbox.text.PDFTextStripper; import org.jboss.logging.Logger; -import java.io.File; import java.io.IOException; import java.net.URI; import java.net.http.HttpRequest; @@ -26,17 +23,23 @@ import static ai.labs.eddi.modules.llm.tools.UrlValidationUtils.validateUrl; /** - * PDF reader tool for extracting text from PDF documents. Supports both local - * files and URLs. + * PDF reader tool for extracting text from PDF documents fetched by URL. + *

+ * Handles the download, SSRF validation and user-facing formatting; the actual + * PDFBox extraction is delegated to the shared {@link AttachmentTextExtractor} + * so the multimodal forwarder and {@code readAttachment} recall tool share one + * implementation. */ @ApplicationScoped public class PdfReaderTool { private static final Logger LOGGER = Logger.getLogger(PdfReaderTool.class); private final SafeHttpClient httpClient; + private final AttachmentTextExtractor textExtractor; @Inject - public PdfReaderTool(SafeHttpClient httpClient) { + public PdfReaderTool(SafeHttpClient httpClient, AttachmentTextExtractor textExtractor) { this.httpClient = httpClient; + this.textExtractor = textExtractor; } @Tool("Extracts all text content from a PDF file. Provide the URL to the PDF document.") @@ -46,19 +49,8 @@ public String extractTextFromPdf(@P("pdfLocation") String pdfLocation) { LOGGER.info("Extracting text from PDF: " + pdfLocation); validateUrl(pdfLocation); - Path tempFile = null; - try { - tempFile = downloadPdf(pdfLocation); - return extractTextFromFile(tempFile.toFile()); - } finally { - if (tempFile != null) { - try { - Files.deleteIfExists(tempFile); - } catch (IOException e) { - LOGGER.warn("Could not delete temp file: " + tempFile); - } - } - } + byte[] pdfBytes = downloadPdfBytes(pdfLocation); + return textExtractor.extractPdfText(pdfBytes); } catch (Exception e) { LOGGER.error("PDF extraction error for " + pdfLocation + ": " + e.getMessage()); @@ -73,19 +65,8 @@ public String extractTextFromPdfPages(@P("pdfLocation") String pdfLocation, @P(" LOGGER.info("Extracting text from PDF pages " + startPage + "-" + endPage + ": " + pdfLocation); validateUrl(pdfLocation); - Path tempFile = null; - try { - tempFile = downloadPdf(pdfLocation); - return extractTextFromFilePages(tempFile.toFile(), startPage, endPage); - } finally { - if (tempFile != null) { - try { - Files.deleteIfExists(tempFile); - } catch (IOException e) { - LOGGER.warn("Could not delete temp file: " + tempFile); - } - } - } + byte[] pdfBytes = downloadPdfBytes(pdfLocation); + return textExtractor.extractPdfText(pdfBytes, startPage, endPage, textExtractor.getDefaultMaxChars()); } catch (Exception e) { LOGGER.error("PDF page extraction error: " + e.getMessage()); @@ -96,55 +77,47 @@ public String extractTextFromPdfPages(@P("pdfLocation") String pdfLocation, @P(" @Tool("Gets metadata and information about a PDF file (number of pages, title, author, etc.)") public String getPdfInfo(@P("pdfLocation") String pdfLocation) { - PDDocument document = null; - Path tempFile = null; - try { LOGGER.info("Getting PDF info for: " + pdfLocation); validateUrl(pdfLocation); - tempFile = downloadPdf(pdfLocation); - File pdfFile = tempFile.toFile(); - - document = Loader.loadPDF(pdfFile); + byte[] pdfBytes = downloadPdfBytes(pdfLocation); + PdfInfo info = textExtractor.extractPdfInfo(pdfBytes); - StringBuilder info = new StringBuilder(); - info.append("PDF Information:\n\n"); - info.append("Number of pages: ").append(document.getNumberOfPages()).append("\n"); - - var metadata = document.getDocumentInformation(); - if (metadata != null) { - if (metadata.getTitle() != null) { - info.append("Title: ").append(metadata.getTitle()).append("\n"); - } - if (metadata.getAuthor() != null) { - info.append("Author: ").append(metadata.getAuthor()).append("\n"); - } - if (metadata.getSubject() != null) { - info.append("Subject: ").append(metadata.getSubject()).append("\n"); - } - if (metadata.getCreator() != null) { - info.append("Creator: ").append(metadata.getCreator()).append("\n"); - } - if (metadata.getCreationDate() != null) { - info.append("Creation date: ").append(metadata.getCreationDate().getTime()).append("\n"); - } + StringBuilder sb = new StringBuilder(); + sb.append("PDF Information:\n\n"); + sb.append("Number of pages: ").append(info.numberOfPages()).append("\n"); + if (info.title() != null) { + sb.append("Title: ").append(info.title()).append("\n"); + } + if (info.author() != null) { + sb.append("Author: ").append(info.author()).append("\n"); + } + if (info.subject() != null) { + sb.append("Subject: ").append(info.subject()).append("\n"); + } + if (info.creator() != null) { + sb.append("Creator: ").append(info.creator()).append("\n"); + } + if (info.creationDate() != null) { + sb.append("Creation date: ").append(info.creationDate().getTime()).append("\n"); } LOGGER.debug("PDF info extracted for " + pdfLocation); - return info.toString(); + return sb.toString(); } catch (Exception e) { LOGGER.error("PDF info extraction error: " + e.getMessage()); return "Error: Could not get PDF information - " + e.getMessage(); + } + } + + private byte[] downloadPdfBytes(String url) throws IOException, InterruptedException { + Path tempFile = null; + try { + tempFile = downloadPdf(url); + return Files.readAllBytes(tempFile); } finally { - if (document != null) { - try { - document.close(); - } catch (IOException e) { - LOGGER.warn("Error closing PDF document", e); - } - } if (tempFile != null) { try { Files.deleteIfExists(tempFile); @@ -155,64 +128,6 @@ public String getPdfInfo(@P("pdfLocation") String pdfLocation) { } } - private String extractTextFromFile(File pdfFile) throws IOException { - PDDocument document = null; - try { - document = Loader.loadPDF(pdfFile); - PDFTextStripper stripper = new PDFTextStripper(); - String text = stripper.getText(document); - - LOGGER.info("Extracted " + text.length() + " characters from PDF with " + document.getNumberOfPages() + " pages"); - - // Limit output size - if (text.length() > 10000) { - text = text.substring(0, 10000) + "\n\n[Content truncated - showing first 10000 characters]"; - } - - return text.trim(); - - } finally { - if (document != null) { - document.close(); - } - } - } - - private String extractTextFromFilePages(File pdfFile, int startPage, int endPage) throws IOException { - PDDocument document = null; - try { - document = Loader.loadPDF(pdfFile); - - int totalPages = document.getNumberOfPages(); - if (startPage < 1 || startPage > totalPages) { - return "Error: Start page " + startPage + " is out of range (1-" + totalPages + ")"; - } - if (endPage < startPage || endPage > totalPages) { - endPage = totalPages; - } - - PDFTextStripper stripper = new PDFTextStripper(); - stripper.setStartPage(startPage); - stripper.setEndPage(endPage); - - String text = stripper.getText(document); - - LOGGER.info("Extracted text from pages " + startPage + "-" + endPage + " (" + text.length() + " characters)"); - - // Limit output size - if (text.length() > 10000) { - text = text.substring(0, 10000) + "\n\n[Content truncated - showing first 10000 characters]"; - } - - return text.trim(); - - } finally { - if (document != null) { - document.close(); - } - } - } - private Path downloadPdf(String url) throws IOException, InterruptedException { LOGGER.debug("Downloading PDF from URL: " + url); diff --git a/src/test/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractorTest.java b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractorTest.java new file mode 100644 index 0000000000..c220ee6d5a --- /dev/null +++ b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractorTest.java @@ -0,0 +1,269 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.tools.impl; + +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor.AttachmentExtractionException; +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor.PdfInfo; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.GregorianCalendar; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for {@link AttachmentTextExtractor}. + */ +class AttachmentTextExtractorTest { + + private final AttachmentTextExtractor extractor = new AttachmentTextExtractor(10_000); + + // ==================== canExtractText ==================== + + @Nested + class CanExtractText { + + @ParameterizedTest + @ValueSource(strings = { + "application/pdf", "text/plain", "text/csv", "text/markdown", + "application/json", "application/xml", "application/csv", + "application/yaml", "application/x-yaml", "application/x-ndjson", + "application/ld+json", "image/svg+xml", "text/plain; charset=utf-8"}) + void shouldSupportTextLikeAndPdf(String mime) { + assertTrue(extractor.canExtractText(mime), "should extract text for " + mime); + } + + @ParameterizedTest + @ValueSource(strings = {"image/png", "image/jpeg", "audio/mpeg", "video/mp4", + "application/octet-stream", "application/zip"}) + void shouldNotSupportBinaryTypes(String mime) { + assertFalse(extractor.canExtractText(mime), "should not extract text for " + mime); + } + + @Test + void shouldHandleNullMime() { + assertFalse(extractor.canExtractText(null)); + } + } + + // ==================== Text-like extraction ==================== + + @Nested + class TextExtraction { + + @Test + void shouldDecodePlainText() throws Exception { + byte[] bytes = "Hello, EDDI!".getBytes(StandardCharsets.UTF_8); + assertEquals("Hello, EDDI!", extractor.extractText(bytes, "text/plain")); + } + + @Test + void shouldDecodeJson() throws Exception { + byte[] bytes = "{\"a\":1}".getBytes(StandardCharsets.UTF_8); + assertEquals("{\"a\":1}", extractor.extractText(bytes, "application/json")); + } + + @Test + void shouldDecodeTextWithCharsetParam() throws Exception { + byte[] bytes = "csv,data\n1,2".getBytes(StandardCharsets.UTF_8); + assertEquals("csv,data\n1,2", extractor.extractText(bytes, "text/csv; charset=utf-8")); + } + + @Test + void shouldReturnEmptyForNullBytes() throws Exception { + assertEquals("", extractor.extractText(null, "text/plain")); + } + + @Test + void shouldReturnEmptyForEmptyBytes() throws Exception { + assertEquals("", extractor.extractText(new byte[0], "text/plain")); + } + + @Test + void shouldThrowForUnsupportedType() { + byte[] bytes = "x".getBytes(StandardCharsets.UTF_8); + var ex = assertThrows(AttachmentExtractionException.class, + () -> extractor.extractText(bytes, "image/png")); + assertTrue(ex.getMessage().contains("image/png")); + } + + @Test + void shouldCapTextToMaxChars() throws Exception { + byte[] bytes = "ABCDEFGHIJ".repeat(50).getBytes(StandardCharsets.UTF_8); // 500 chars + String result = extractor.extractText(bytes, "text/plain", 100); + assertTrue(result.startsWith("ABCDEFGHIJ")); + assertTrue(result.contains("[Content truncated - showing first 100 characters]")); + } + + @Test + void shouldUseDefaultCapWhenMaxCharsNonPositive() throws Exception { + var smallExtractor = new AttachmentTextExtractor(20); + byte[] bytes = "ABCDEFGHIJ".repeat(10).getBytes(StandardCharsets.UTF_8); // 100 chars + String result = smallExtractor.extractText(bytes, "text/plain", 0); + assertTrue(result.contains("[Content truncated - showing first 20 characters]")); + } + } + + // ==================== Config cap ==================== + + @Test + void nonPositiveConfiguredCapFallsBackToDefault() { + var ex = new AttachmentTextExtractor(0); + assertEquals(AttachmentTextExtractor.DEFAULT_MAX_CHARS, ex.getDefaultMaxChars()); + } + + @Test + void positiveConfiguredCapIsHonored() { + var ex = new AttachmentTextExtractor(500); + assertEquals(500, ex.getDefaultMaxChars()); + } + + // ==================== PDF extraction ==================== + + @Nested + class PdfExtraction { + + @Test + void shouldExtractPdfTextViaGenericDispatch() throws Exception { + byte[] pdf = createPdf("Hello EDDI World"); + String result = extractor.extractText(pdf, "application/pdf"); + assertTrue(result.contains("Hello EDDI World"), "got: " + result); + } + + @Test + void shouldExtractFullPdfText() throws Exception { + byte[] pdf = createPdf("Some page content"); + String result = extractor.extractPdfText(pdf); + assertTrue(result.contains("Some page content")); + } + + @Test + void shouldTruncateLongPdfText() throws Exception { + var smallExtractor = new AttachmentTextExtractor(50); + byte[] pdf = createPdf("ABCDEFGHIJ ".repeat(20)); // ~220 chars + String result = smallExtractor.extractPdfText(pdf); + assertTrue(result.contains("[Content truncated - showing first 50 characters]"), "got: " + result); + } + + @Test + void shouldExtractPageRange() throws Exception { + byte[] pdf = createMultiPagePdf("First page", "Second page", "Third page"); + String result = extractor.extractPdfText(pdf, 2, 2, 10_000); + assertTrue(result.contains("Second page")); + assertFalse(result.contains("First page")); + assertFalse(result.contains("Third page")); + } + + @Test + void shouldClampEndPageBeyondTotal() throws Exception { + byte[] pdf = createMultiPagePdf("Page one", "Page two"); + String result = extractor.extractPdfText(pdf, 1, 99, 10_000); + assertTrue(result.contains("Page one")); + assertTrue(result.contains("Page two")); + } + + @Test + void shouldThrowWhenStartPageOutOfRange() throws Exception { + byte[] pdf = createPdf("only page"); + var ex = assertThrows(AttachmentExtractionException.class, + () -> extractor.extractPdfText(pdf, 5, 10, 10_000)); + assertTrue(ex.getMessage().contains("out of range")); + } + + @Test + void shouldExtractPdfInfo() throws Exception { + byte[] pdf = createPdfWithMetadata("My Title", "Jane Author", "The Subject", "The Creator"); + PdfInfo info = extractor.extractPdfInfo(pdf); + assertEquals(1, info.numberOfPages()); + assertEquals("My Title", info.title()); + assertEquals("Jane Author", info.author()); + assertEquals("The Subject", info.subject()); + assertEquals("The Creator", info.creator()); + assertNotNull(info.creationDate()); + } + + @Test + void shouldExtractPdfInfoWithoutMetadata() throws Exception { + byte[] pdf = createPdf("bare"); + PdfInfo info = extractor.extractPdfInfo(pdf); + assertEquals(1, info.numberOfPages()); + assertNull(info.title()); + assertNull(info.author()); + } + + @Test + void shouldThrowOnCorruptPdf() { + byte[] notAPdf = "this is not a pdf".getBytes(StandardCharsets.UTF_8); + assertThrows(AttachmentExtractionException.class, () -> extractor.extractPdfText(notAPdf)); + } + + @Test + void shouldThrowOnCorruptPdfInfo() { + byte[] notAPdf = "still not a pdf".getBytes(StandardCharsets.UTF_8); + assertThrows(AttachmentExtractionException.class, () -> extractor.extractPdfInfo(notAPdf)); + } + } + + // ==================== Helpers ==================== + + private static byte[] createPdf(String text) throws Exception { + return createMultiPagePdf(text); + } + + private static byte[] createMultiPagePdf(String... texts) throws Exception { + try (PDDocument doc = new PDDocument(); ByteArrayOutputStream out = new ByteArrayOutputStream()) { + for (String text : texts) { + var page = new PDPage(); + doc.addPage(page); + try (var cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 700); + cs.showText(text); + cs.endText(); + } + } + doc.save(out); + return out.toByteArray(); + } + } + + private static byte[] createPdfWithMetadata(String title, String author, + String subject, String creator) + throws Exception { + try (PDDocument doc = new PDDocument(); ByteArrayOutputStream out = new ByteArrayOutputStream()) { + var page = new PDPage(); + doc.addPage(page); + try (var cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 700); + cs.showText("content"); + cs.endText(); + } + var info = doc.getDocumentInformation(); + info.setTitle(title); + info.setAuthor(author); + info.setSubject(subject); + info.setCreator(creator); + info.setCreationDate(GregorianCalendar.from( + ZonedDateTime.of(2025, 1, 15, 10, 30, 0, 0, ZoneId.of("UTC")))); + doc.setDocumentInformation(info); + doc.save(out); + return out.toByteArray(); + } + } +} diff --git a/src/test/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderToolTest.java b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderToolTest.java index dcbe2fd985..ab568fae4c 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderToolTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/PdfReaderToolTest.java @@ -21,7 +21,7 @@ class PdfReaderToolTest { @BeforeEach void setUp() { - pdfReaderTool = new PdfReaderTool(new SafeHttpClient(10000)); + pdfReaderTool = new PdfReaderTool(new SafeHttpClient(10000), new AttachmentTextExtractor(10000)); } // === SSRF Protection Tests === @@ -152,7 +152,7 @@ class HttpErrorPathTests { @BeforeEach void setUpMocked() { mockedHttpClient = org.mockito.Mockito.mock(SafeHttpClient.class); - mockedTool = new PdfReaderTool(mockedHttpClient); + mockedTool = new PdfReaderTool(mockedHttpClient, new AttachmentTextExtractor(10000)); } @Test @@ -267,7 +267,7 @@ class PdfContentExtractionTests { @BeforeEach void setUpMocked() { mockedHttpClient = org.mockito.Mockito.mock(SafeHttpClient.class); - mockedTool = new PdfReaderTool(mockedHttpClient); + mockedTool = new PdfReaderTool(mockedHttpClient, new AttachmentTextExtractor(10000)); } private java.nio.file.Path createTinyPdf(String text) throws Exception { From 21ae5f369a2137d339621a4eda76d1dbffaa9fe6 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 00:51:38 +0200 Subject: [PATCH 04/23] feat(attachments): add ModelCapabilityService for multimodal gating MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New ModelCapabilityService (modules/llm/capability) resolves whether a (provider, model) pair supports vision / native documents / audio / image-by-URL before the forwarder sends content. Resolution precedence: per-task override (Support.ON/OFF/AUTO) > deployment override (eddi.multimodal.. then eddi.multimodal.) > conservative model-aware built-in defaults (plan §5). Unknown provider/model => unsupported => fallback, so we never send content that errors the provider. Injectable via MicroProfile Config; a Function-based constructor keeps it fully unit-testable. 74 tests cover the default matrix across all 11 providers plus override precedence and token parsing. Phase 0 of multimodal-attachments-completion-plan. --- .../capability/ModelCapabilityService.java | 275 ++++++++++++++++++ .../ModelCapabilityServiceTest.java | 264 +++++++++++++++++ 2 files changed, 539 insertions(+) create mode 100644 src/main/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityService.java create mode 100644 src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java diff --git a/src/main/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityService.java b/src/main/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityService.java new file mode 100644 index 0000000000..4a0936d896 --- /dev/null +++ b/src/main/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityService.java @@ -0,0 +1,275 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.capability; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.Config; + +import java.util.Locale; +import java.util.Optional; +import java.util.function.Function; + +import static ai.labs.eddi.modules.llm.bootstrap.LlmModule.*; + +/** + * Resolves whether a given {@code (provider, model)} pair supports a multimodal + * capability (vision, native documents, audio, image-by-URL) before the + * attachment forwarder hands content to the provider. + *

+ * Resolution precedence, highest first: + *

    + *
  1. Per-task override — {@link Support#ON}/{@link Support#OFF} from + * the agent's {@code LlmConfiguration.Task.multimodal} block; + * {@link Support#AUTO} falls through.
    2. + *
  2. Deployment override — + * {@code eddi.multimodal..} then the global + * {@code eddi.multimodal.} (each {@code on|off|auto}).
    3. + *
  3. Built-in defaults — the conservative, model-aware table + * below.
    4. + *
+ * Unknown providers/models resolve to unsupported, so the forwarder + * falls back to text extraction or a metadata note and never sends content that + * would error the provider. Capability is model-level; the defaults are + * deliberately cautious and should be verified against the langchain4j release + * in use. + * + * @since 6.1.0 + */ +@ApplicationScoped +public class ModelCapabilityService { + + /** A multimodal capability that a provider/model may or may not support. */ + public enum Capability { + VISION("vision"), DOCUMENTS("documents"), AUDIO("audio"), IMAGE_URL("image-url"); + + private final String configSuffix; + + Capability(String configSuffix) { + this.configSuffix = configSuffix; + } + + public String configSuffix() { + return configSuffix; + } + } + + /** Tri-state override for a capability. */ + public enum Support { + AUTO, ON, OFF; + + /** + * Parse a config/override token. {@code on|true|yes|enabled} → ON, + * {@code off|false|no|disabled} → OFF, everything else (incl. null) → AUTO. + */ + public static Support parse(String token) { + if (token == null) { + return AUTO; + } + return switch (token.trim().toLowerCase(Locale.ROOT)) { + case "on", "true", "yes", "enabled", "enable" -> ON; + case "off", "false", "no", "disabled", "disable" -> OFF; + default -> AUTO; + }; + } + } + + private static final String CONFIG_PREFIX = "eddi.multimodal."; + + private final Function> configLookup; + + @Inject + public ModelCapabilityService(Config config) { + this(key -> config.getOptionalValue(key, String.class)); + } + + /** + * Programmatic constructor for tests / non-CDI callers. + * + * @param configLookup + * resolves a config key to its value (empty when unset) + */ + public ModelCapabilityService(Function> configLookup) { + this.configLookup = configLookup; + } + + public boolean supportsVision(String provider, String model) { + return supports(Capability.VISION, provider, model, Support.AUTO); + } + + public boolean supportsVision(String provider, String model, Support taskOverride) { + return supports(Capability.VISION, provider, model, taskOverride); + } + + public boolean supportsDocuments(String provider, String model) { + return supports(Capability.DOCUMENTS, provider, model, Support.AUTO); + } + + public boolean supportsDocuments(String provider, String model, Support taskOverride) { + return supports(Capability.DOCUMENTS, provider, model, taskOverride); + } + + public boolean supportsAudio(String provider, String model) { + return supports(Capability.AUDIO, provider, model, Support.AUTO); + } + + public boolean supportsAudio(String provider, String model, Support taskOverride) { + return supports(Capability.AUDIO, provider, model, taskOverride); + } + + public boolean supportsImageUrl(String provider, String model) { + return supports(Capability.IMAGE_URL, provider, model, Support.AUTO); + } + + public boolean supportsImageUrl(String provider, String model, Support taskOverride) { + return supports(Capability.IMAGE_URL, provider, model, taskOverride); + } + + /** + * Resolve a capability applying the full precedence chain. + * + * @param capability + * the capability in question + * @param provider + * the LLM provider type (e.g. {@code openai}, {@code anthropic}) + * @param model + * the resolved model name (may be null/blank) + * @param taskOverride + * the per-task override ({@link Support#AUTO} to defer) + * @return {@code true} if the capability is supported + */ + public boolean supports(Capability capability, String provider, String model, Support taskOverride) { + if (taskOverride == Support.ON) { + return true; + } + if (taskOverride == Support.OFF) { + return false; + } + Support deployment = deploymentOverride(capability, provider); + if (deployment == Support.ON) { + return true; + } + if (deployment == Support.OFF) { + return false; + } + return builtInDefault(capability, normalize(provider), normalize(model)); + } + + private Support deploymentOverride(Capability capability, String provider) { + String p = normalize(provider); + if (!p.isEmpty()) { + Support providerSpecific = Support.parse( + configLookup.apply(CONFIG_PREFIX + p + "." + capability.configSuffix()).orElse(null)); + if (providerSpecific != Support.AUTO) { + return providerSpecific; + } + } + return Support.parse(configLookup.apply(CONFIG_PREFIX + capability.configSuffix()).orElse(null)); + } + + private boolean builtInDefault(Capability capability, String provider, String model) { + if (provider.isEmpty()) { + return false; + } + return switch (capability) { + case VISION -> defaultVision(provider, model); + case DOCUMENTS -> defaultDocuments(provider, model); + case AUDIO -> defaultAudio(provider); + case IMAGE_URL -> defaultImageUrl(provider); + }; + } + + // ----- Vision ------------------------------------------------------------- + + private boolean defaultVision(String provider, String model) { + return switch (provider) { + // Vision-first providers: on by default, downgraded for known text-only models. + case LLM_TYPE_OPENAI, LLM_TYPE_AZURE_OPENAI, LLM_TYPE_ANTHROPIC, + LLM_TYPE_GEMINI, LLM_TYPE_GEMINI_VERTEX, LLM_TYPE_MISTRAL -> + !isKnownTextOnlyModel(model); + // Model-dependent providers: off by default, upgraded for known vision models. + case LLM_TYPE_OLLAMA, LLM_TYPE_BEDROCK, LLM_TYPE_ORACLE_GENAI -> isKnownVisionModel(model); + // No vision support. + default -> false; + }; + } + + private static boolean isKnownTextOnlyModel(String model) { + if (model.isEmpty()) { + return false; + } + return model.contains("gpt-3.5") + || model.contains("text-davinci") + || model.contains("davinci") + || model.contains("babbage") + || model.contains("text-embedding") + || model.contains("-embed") + || model.contains("embed-") + || model.contains("text-moderation") + || model.contains("mistral-embed") + || model.contains("mistral-7b") + || model.contains("mixtral"); + } + + private static boolean isKnownVisionModel(String model) { + if (model.isEmpty()) { + return false; + } + return model.contains("llava") + || model.contains("bakllava") + || model.contains("vision") + || model.contains("pixtral") + || model.contains("claude-3") + || model.contains("claude-sonnet") || model.contains("claude-opus") || model.contains("claude-haiku") + || model.contains("nova-lite") || model.contains("nova-pro") || model.contains("nova-premier") + || model.contains("llama3.2") || model.contains("llama-3.2") || model.contains("llama3-2") + || model.contains("gemma3") || model.contains("gemma-3") + || model.contains("qwen2-vl") || model.contains("qwen2.5-vl") + || model.contains("minicpm-v") || model.contains("moondream"); + } + + // ----- Documents (native PDF) -------------------------------------------- + + private boolean defaultDocuments(String provider, String model) { + return switch (provider) { + case LLM_TYPE_ANTHROPIC -> !isLegacyAnthropicModel(model); + case LLM_TYPE_GEMINI, LLM_TYPE_GEMINI_VERTEX -> true; + // OpenAI/Azure native PDF is model-dependent and inconsistent → conservative + // off + // (falls back to text extraction). Everything else: no native documents. + default -> false; + }; + } + + private static boolean isLegacyAnthropicModel(String model) { + // Native PDF/document support arrived with Claude 3; Claude 2 / instant lack + // it. + return model.contains("claude-2") || model.contains("claude-instant") || model.contains("claude-1"); + } + + // ----- Audio -------------------------------------------------------------- + + private boolean defaultAudio(String provider) { + return switch (provider) { + case LLM_TYPE_GEMINI, LLM_TYPE_GEMINI_VERTEX -> true; + default -> false; + }; + } + + // ----- Image by URL ------------------------------------------------------- + + private boolean defaultImageUrl(String provider) { + // Only OpenAI/Azure reliably fetch images by URL. Every other provider needs + // the bytes inlined, so the forwarder downloads and base64-encodes instead. + return switch (provider) { + case LLM_TYPE_OPENAI, LLM_TYPE_AZURE_OPENAI -> true; + default -> false; + }; + } + + private static String normalize(String value) { + return value == null ? "" : value.trim().toLowerCase(Locale.ROOT); + } +} diff --git a/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java b/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java new file mode 100644 index 0000000000..80576df2d9 --- /dev/null +++ b/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java @@ -0,0 +1,264 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.capability; + +import ai.labs.eddi.modules.llm.capability.ModelCapabilityService.Capability; +import ai.labs.eddi.modules.llm.capability.ModelCapabilityService.Support; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for {@link ModelCapabilityService}. + */ +class ModelCapabilityServiceTest { + + private final Map config = new HashMap<>(); + private final Function> lookup = key -> Optional.ofNullable(config.get(key)); + private final ModelCapabilityService service = new ModelCapabilityService(lookup); + + // ==================== Vision defaults ==================== + + @Nested + class VisionDefaults { + + @ParameterizedTest + @CsvSource({ + "openai,gpt-4o,true", + "openai,gpt-4.1,true", + "openai,gpt-3.5-turbo,false", + "openai,text-embedding-3-small,false", + "azure-openai,gpt-4o,true", + "anthropic,claude-sonnet-4,true", + "gemini,gemini-2.0-flash,true", + "gemini-vertex,gemini-1.5-pro,true", + "mistral,pixtral-12b,true", + "mistral,mistral-embed,false", + "mistral,mixtral-8x7b,false", + "ollama,llava,true", + "ollama,llama3.2-vision,true", + "ollama,llama3,false", + "bedrock,amazon.nova-pro-v1,true", + "bedrock,anthropic.claude-3-sonnet,true", + "bedrock,amazon.titan-text,false", + "oracle-genai,meta.llama-3.2-90b-vision,true", + "jlama,tjake/llama,false", + "huggingface,any-model,false", + "unknown-provider,some-model,false"}) + void visionDefaults(String provider, String model, boolean expected) { + assertEquals(expected, service.supportsVision(provider, model), + provider + "/" + model + " vision should be " + expected); + } + + @Test + void blankProviderIsUnsupported() { + assertFalse(service.supportsVision("", "gpt-4o")); + assertFalse(service.supportsVision(null, "gpt-4o")); + } + + @Test + void blankModelUsesProviderDefault() { + assertTrue(service.supportsVision("openai", "")); + assertTrue(service.supportsVision("openai", null)); + assertFalse(service.supportsVision("ollama", "")); // model-dependent → off without a known vision model + } + } + + // ==================== Documents defaults ==================== + + @Nested + class DocumentDefaults { + + @ParameterizedTest + @CsvSource({ + "anthropic,claude-sonnet-4,true", + "anthropic,claude-3-opus,true", + "anthropic,claude-2.1,false", + "anthropic,claude-instant-1,false", + "gemini,gemini-2.0-flash,true", + "gemini-vertex,gemini-1.5-pro,true", + "openai,gpt-4o,false", + "azure-openai,gpt-4o,false", + "mistral,pixtral-12b,false", + "ollama,llava,false", + "bedrock,anthropic.claude-3,false", + "jlama,x,false"}) + void documentDefaults(String provider, String model, boolean expected) { + assertEquals(expected, service.supportsDocuments(provider, model), + provider + "/" + model + " documents should be " + expected); + } + } + + // ==================== Audio defaults ==================== + + @Nested + class AudioDefaults { + + @ParameterizedTest + @CsvSource({ + "gemini,gemini-2.0-flash,true", + "gemini-vertex,gemini-1.5-pro,true", + "openai,gpt-4o,false", + "anthropic,claude-sonnet-4,false", + "ollama,llava,false"}) + void audioDefaults(String provider, String model, boolean expected) { + assertEquals(expected, service.supportsAudio(provider, model)); + } + } + + // ==================== Image-by-URL defaults ==================== + + @Nested + class ImageUrlDefaults { + + @ParameterizedTest + @CsvSource({ + "openai,gpt-4o,true", + "azure-openai,gpt-4o,true", + "anthropic,claude-sonnet-4,false", + "gemini,gemini-2.0-flash,false", + "mistral,pixtral-12b,false", + "ollama,llava,false"}) + void imageUrlDefaults(String provider, String model, boolean expected) { + assertEquals(expected, service.supportsImageUrl(provider, model)); + } + } + + // ==================== Task overrides ==================== + + @Nested + class TaskOverrides { + + @Test + void onForcesTrueEvenWhenDefaultFalse() { + assertFalse(service.supportsVision("jlama", "x")); + assertTrue(service.supportsVision("jlama", "x", Support.ON)); + } + + @Test + void offForcesFalseEvenWhenDefaultTrue() { + assertTrue(service.supportsVision("openai", "gpt-4o")); + assertFalse(service.supportsVision("openai", "gpt-4o", Support.OFF)); + } + + @Test + void autoFallsThroughToDefault() { + assertTrue(service.supportsVision("openai", "gpt-4o", Support.AUTO)); + assertFalse(service.supportsVision("jlama", "x", Support.AUTO)); + } + } + + // ==================== Deployment overrides ==================== + + @Nested + class DeploymentOverrides { + + @Test + void providerSpecificOverrideEnables() { + assertFalse(service.supportsVision("ollama", "llama3")); // default off + config.put("eddi.multimodal.ollama.vision", "on"); + assertTrue(service.supportsVision("ollama", "llama3")); + } + + @Test + void providerSpecificOverrideDisables() { + assertTrue(service.supportsVision("openai", "gpt-4o")); // default on + config.put("eddi.multimodal.openai.vision", "off"); + assertFalse(service.supportsVision("openai", "gpt-4o")); + } + + @Test + void globalOverrideAppliesToAllProviders() { + config.put("eddi.multimodal.vision", "off"); + assertFalse(service.supportsVision("openai", "gpt-4o")); + assertFalse(service.supportsVision("anthropic", "claude-sonnet-4")); + } + + @Test + void providerSpecificTakesPrecedenceOverGlobal() { + config.put("eddi.multimodal.vision", "off"); + config.put("eddi.multimodal.openai.vision", "on"); + assertTrue(service.supportsVision("openai", "gpt-4o")); + assertFalse(service.supportsVision("anthropic", "claude-sonnet-4")); + } + + @Test + void taskOverrideBeatsDeploymentOverride() { + config.put("eddi.multimodal.openai.vision", "off"); + assertTrue(service.supportsVision("openai", "gpt-4o", Support.ON)); + } + + @Test + void autoValuedOverrideFallsThrough() { + config.put("eddi.multimodal.openai.vision", "auto"); + assertTrue(service.supportsVision("openai", "gpt-4o")); + } + + @Test + void documentsAndAudioOverridable() { + config.put("eddi.multimodal.openai.documents", "on"); + assertTrue(service.supportsDocuments("openai", "gpt-4o")); + config.put("eddi.multimodal.audio", "on"); + assertTrue(service.supportsAudio("anthropic", "claude-sonnet-4")); + } + } + + // ==================== Support.parse ==================== + + @Nested + class SupportParsing { + + @ParameterizedTest + @ValueSource(strings = {"on", "true", "yes", "enabled", "ON", "True"}) + void parsesOn(String token) { + assertEquals(Support.ON, Support.parse(token)); + } + + @ParameterizedTest + @ValueSource(strings = {"off", "false", "no", "disabled", "OFF"}) + void parsesOff(String token) { + assertEquals(Support.OFF, Support.parse(token)); + } + + @ParameterizedTest + @ValueSource(strings = {"auto", "maybe", "", "garbage"}) + void parsesAuto(String token) { + assertEquals(Support.AUTO, Support.parse(token)); + } + + @Test + void parsesNullAsAuto() { + assertEquals(Support.AUTO, Support.parse(null)); + } + } + + // ==================== Generic supports() + capability suffixes + // ==================== + + @Test + void genericSupportsMatchesConvenienceMethods() { + assertEquals(service.supportsVision("openai", "gpt-4o"), + service.supports(Capability.VISION, "openai", "gpt-4o", Support.AUTO)); + assertEquals(service.supportsImageUrl("openai", "gpt-4o"), + service.supports(Capability.IMAGE_URL, "openai", "gpt-4o", Support.AUTO)); + } + + @Test + void capabilityConfigSuffixes() { + assertEquals("vision", Capability.VISION.configSuffix()); + assertEquals("documents", Capability.DOCUMENTS.configSuffix()); + assertEquals("audio", Capability.AUDIO.configSuffix()); + assertEquals("image-url", Capability.IMAGE_URL.configSuffix()); + } +} From 5a6e540daaa98cb9c20a0ff9666955b7e2df2413 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 00:55:27 +0200 Subject: [PATCH 05/23] chore(attachments): align max-body-size with attachment cap + Phase 0 changelog Set quarkus.http.limits.max-body-size=25M so 10-20MB uploads are not rejected with a bare HTTP 413 by Quarkus' 10MB default before the attachment layer sees them. Document eddi.attachments.max-size-bytes and eddi.attachments.extraction.max-chars alongside it. Record Phase 0 in the changelog. Phase 0 of multimodal-attachments-completion-plan (complete). --- docs/changelog.md | 29 +++++++++++++++++++++++ src/main/resources/application.properties | 11 +++++++++ 2 files changed, 40 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index 5c3dd94a44..7621f5a52b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,35 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Phase 0: Foundations & bug fixes (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) +**Plan:** `planning/multimodal-attachments-completion-plan.md` (Phase 0 of 6). Low-risk foundations that ship alone. + +### What changed + +1. **`@JsonIgnore` on `Attachment.getBase64Data()`** (`engine/memory/model/Attachment.java`) — the `transient` keyword did **not** stop Jackson (getter-based serialization, no `PROPAGATE_TRANSIENT_MARKER`), so inline base64 payloads were being serialized into Mongo conversation documents. Now excluded; metadata still persists. Serialization tests prove the payload never reaches persisted JSON. +2. **Scrub inline base64 from persisted context copies** (`engine/memory/AttachmentContextExtractor.java` + `engine/runtime/internal/Conversation.java`) — new `AttachmentContextExtractor.scrubInlinePayload()` returns a metadata-only copy of an `attachment_*` context when it carries a `data` payload. `Conversation.createContextData()` builds the persisted copy (step data + `context.*` conversation output) through it, so the raw base64 (~1.33× file size/turn against the 16 MB doc limit) never lands in Mongo and is never exposed via `{context.attachment_*.data}`. The live payload still rides ATTACHMENTS memory for the turn. Mirrors secret-input scrubbing. +3. **`AttachmentTextExtractor`** (`modules/llm/tools/impl/`, new) — shared PDFBox + plain-text extraction behind a uniform, configurable cap (`eddi.attachments.extraction.max-chars`, default 10k). `extractText(bytes, mime[, maxChars])` dispatches PDF + text-like (text/*, JSON, XML, CSV, YAML); PDF full/page-range/info methods; `canExtractText()`. `PdfReaderTool` now delegates all extraction to it (download/SSRF/formatting unchanged). Reused by the Phase 2 forwarder and Phase 4 readAttachment tool. +4. **`ModelCapabilityService`** (`modules/llm/capability/`, new) — resolves vision/documents/audio/image-by-URL support for a `(provider, model)` pair. Precedence: per-task override > deployment override (`eddi.multimodal..` then `eddi.multimodal.`) > conservative model-aware defaults (plan §5). Unknown ⇒ unsupported ⇒ fallback. Injectable via MicroProfile Config; Function-based constructor keeps it unit-testable. +5. **Body-size alignment** (`application.properties`) — added `quarkus.http.limits.max-body-size=25M` (was Quarkus' 10 MB default, below the 20 MB attachment cap → 10–20 MB uploads died with a bare 413), plus documented `eddi.attachments.max-size-bytes` and `eddi.attachments.extraction.max-chars`. + +### Design decisions + +- **Scrub is a copy, not a mutation** — the original context map keeps its payload so the current turn's extraction/forwarding is unaffected; only the persisted derivative is stripped. +- **Extractor owns extraction, tool owns presentation** — `PdfReaderTool.getPdfInfo` still formats the human-readable string; the extractor returns a structured `PdfInfo`, so the shared service stays presentation-free and reusable by the forwarder. +- **Capability defaults are conservative and model-aware** — vision-first providers (OpenAI/Anthropic/Gemini/Mistral) default on but downgrade for known text-only models; model-dependent providers (Ollama/Bedrock/Oracle) default off but upgrade for known vision models; image-by-URL only for OpenAI/Azure (everything else inlines). + +### Tests + +146 new/covered unit tests: `AttachmentTest` (serialization no-payload), `AttachmentContextExtractorTest` (scrub matrix), `AttachmentTextExtractorTest` (PDF/text/caps/corrupt), `ModelCapabilityServiceTest` (74 — default matrix across 11 providers + override precedence). `PdfReaderToolTest` remains CI-only (SafeHttpClient opens a loopback selector local JVMs may block). + +### What's next + +Phase 1 — storage unification (collapse `IAttachmentStorage` into `IAttachmentStore`, port conversation-delete + GDPR cascades), grants (`grantAccess`/grant-aware `load`), authenticated upload/list/download/delete, quotas, `storageRef` extraction branch, UUID ref hardening. + --- ## 🐛 Fix: PostgreSQL group conversations broken — JDBC `?|` operator escape (2026-07-02) diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 04a5be1c25..5e8150f306 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -124,6 +124,17 @@ quarkus.http.cors.origins=http://localhost:3000,http://localhost:7070,https://lo quarkus.http.cors.headers=accept, origin, authorization, content-type, x-requested-with quarkus.http.cors.exposed-headers=Location quarkus.http.cors.methods=OPTIONS,HEAD,GET,PUT,POST,DELETE,PATCH +# Multimodal Attachments — file upload + LLM forwarding +# Max size of a single uploaded attachment, in bytes (default 20 MB). +eddi.attachments.max-size-bytes=20971520 +# Raise the HTTP request body limit above the attachment cap: Quarkus' default is +# 10 MB, so 10–20 MB uploads would otherwise die with a bare HTTP 413 before the +# attachment layer ever sees them. Keep this >= eddi.attachments.max-size-bytes +# plus multipart/form-data overhead. +quarkus.http.limits.max-body-size=25M +# Max characters of text extracted from a PDF / text attachment before truncation +# (shared by PdfReaderTool, the attachment forwarder and the readAttachment tool). +eddi.attachments.extraction.max-chars=10000 # Jackson JSON json.prettyPrint=false quarkus.jackson.write-dates-as-timestamps=true From 754184950ef5cc2a2b1524b2b0c1512d8b0f3284 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 01:12:23 +0200 Subject: [PATCH 06/23] refactor(attachments): unify on IAttachmentStore with grants + quotas MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Collapse the duplicate blob-store abstractions onto a single IAttachmentStore. Previously uploads wrote to IAttachmentStore (GridFsAttachmentStore / PostgresAttachmentStore) while conversation-deletion and GDPR erasure cascaded through a *different* store (IAttachmentStorage → Mongo/PostgresAttachmentStorage), so uploaded blobs were never actually deleted. - Extend IAttachmentStore with getMetadata() (server-validated metadata, no bytes), grantAccess() (trusted-caller-only cross-conversation read grant), and single-item delete() (owner-only). load()/getMetadata() authz is now owner-OR-grant; grants die with the blob. - GridFS: switch the public storageRef to an unguessable random UUID kept in metadata (legacy ObjectId-hex refs still resolve), store grants as a metadata.grants array, enforce per-conversation count + total-byte quotas. - Postgres: add a grants TEXT[] column (additive migration), same quota enforcement; already used UUID refs. - Port the two IAttachmentStorage consumers (RestConversationStore delete cascade, GdprComplianceService erasure) to IAttachmentStore, then delete IAttachmentStorage + MongoAttachmentStorage + PostgresAttachmentStorage and their tests (verified write-dead — only the delete cascades referenced them). New config: eddi.attachments.max-per-conversation (50), eddi.attachments.max-total-bytes-per-conversation (100MB). GridFsAttachmentStoreTest rewritten for UUID refs + grants + quota (26 tests); consumer tests re-typed. Postgres store IT stays CI-only. Phase 1 of multimodal-attachments-completion-plan (part 1/2). --- .../mongo/GridFsAttachmentStore.java | 181 +++++-- .../postgres/PostgresAttachmentStorage.java | 176 ------- .../postgres/PostgresAttachmentStore.java | 157 +++++- .../engine/attachments/IAttachmentStore.java | 80 +++- .../engine/gdpr/GdprComplianceService.java | 6 +- .../engine/memory/IAttachmentStorage.java | 66 --- .../memory/mongo/MongoAttachmentStorage.java | 124 ----- .../memory/rest/RestConversationStore.java | 6 +- .../mongo/GridFsAttachmentStoreTest.java | 449 ++++++++++-------- .../mongo/MongoAttachmentStorageTest.java | 111 ----- .../PostgresAttachmentStorageTest.java | 119 ----- .../PostgresAttachmentStorageUnitTest.java | 195 -------- .../gdpr/GdprComplianceServiceTest.java | 12 +- .../mongo/MongoAttachmentStorageTest.java | 164 ------- .../rest/RestConversationStoreFilterTest.java | 4 +- .../rest/RestConversationStoreTest.java | 8 +- 16 files changed, 619 insertions(+), 1239 deletions(-) delete mode 100644 src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorage.java delete mode 100644 src/main/java/ai/labs/eddi/engine/memory/IAttachmentStorage.java delete mode 100644 src/main/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorage.java delete mode 100644 src/test/java/ai/labs/eddi/datastore/mongo/MongoAttachmentStorageTest.java delete mode 100644 src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageTest.java delete mode 100644 src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageUnitTest.java delete mode 100644 src/test/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorageTest.java diff --git a/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java b/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java index a9f8ee695a..290ecf1185 100644 --- a/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java @@ -6,16 +6,19 @@ import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.attachments.MimeValidator; +import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoDatabase; import com.mongodb.client.gridfs.GridFSBucket; import com.mongodb.client.gridfs.GridFSBuckets; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.gridfs.model.GridFSUploadOptions; import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Updates; import io.quarkus.arc.DefaultBean; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.bson.Document; +import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.eclipse.microprofile.config.inject.ConfigProperty; import org.jboss.logging.Logger; @@ -24,14 +27,18 @@ import java.io.ByteArrayOutputStream; import java.util.ArrayList; import java.util.List; +import java.util.UUID; import static ai.labs.eddi.utils.LogSanitizer.sanitize; /** * MongoDB GridFS implementation of {@link IAttachmentStore}. *

- * Uses GridFS for storing binary attachments with metadata. Files are stored in - * the {@code attachments} GridFS bucket. + * Files are stored in the {@code attachments} GridFS bucket. The public + * {@code storageRef} is a random UUID kept in the file metadata (not the raw + * ObjectId), so references are unguessable; legacy blobs referenced by their + * plain ObjectId hex still resolve. Access grants are stored as a + * {@code metadata.grants} array and die with the blob. * * @since 6.0.0 */ @@ -41,15 +48,27 @@ public class GridFsAttachmentStore implements IAttachmentStore { private static final Logger LOGGER = Logger.getLogger(GridFsAttachmentStore.class); private static final String BUCKET_NAME = "attachments"; + private static final String META_CONVERSATION_ID = "conversationId"; + private static final String META_STORAGE_REF = "storageRef"; + private static final String META_MIME_TYPE = "mimeType"; + private static final String META_GRANTS = "grants"; private final GridFSBucket gridFSBucket; + private final MongoCollection filesCollection; @ConfigProperty(name = "eddi.attachments.max-size-bytes", defaultValue = "20971520") // 20 MB long maxSizeBytes; + @ConfigProperty(name = "eddi.attachments.max-per-conversation", defaultValue = "50") + long maxPerConversation; + + @ConfigProperty(name = "eddi.attachments.max-total-bytes-per-conversation", defaultValue = "104857600") // 100 MB + long maxTotalBytesPerConversation; + @Inject public GridFsAttachmentStore(MongoDatabase database) { this.gridFSBucket = GridFSBuckets.create(database, BUCKET_NAME); + this.filesCollection = database.getCollection(BUCKET_NAME + ".files"); } @Override @@ -72,23 +91,26 @@ public Attachment store(byte[] bytes, String declaredMime, String filename, "MIME type mismatch: declared='%s', detected='%s'".formatted(declaredMime, detectedMime)); } + enforceQuota(conversationId, bytes.length); + String resolvedMime = MimeValidator.normalize(declaredMime != null ? declaredMime : detectedMime); + String storageRef = UUID.randomUUID().toString(); Document metadata = new Document() - .append("conversationId", conversationId) + .append(META_CONVERSATION_ID, conversationId) .append("tenantId", tenantId) - .append("mimeType", resolvedMime) - .append("sizeBytes", (long) bytes.length); + .append(META_MIME_TYPE, resolvedMime) + .append("sizeBytes", (long) bytes.length) + .append(META_STORAGE_REF, storageRef) + .append(META_GRANTS, new ArrayList()); - GridFSUploadOptions options = new GridFSUploadOptions() - .metadata(metadata); + GridFSUploadOptions options = new GridFSUploadOptions().metadata(metadata); - ObjectId fileId = gridFSBucket.uploadFromStream( + gridFSBucket.uploadFromStream( filename != null ? filename : "unnamed", new ByteArrayInputStream(bytes), options); - String storageRef = fileId.toHexString(); LOGGER.debugf("Stored attachment '%s' (%s, %d bytes) for conversation '%s' → GridFS %s", sanitize(filename), resolvedMime, bytes.length, sanitize(conversationId), storageRef); @@ -97,37 +119,67 @@ public Attachment store(byte[] bytes, String declaredMime, String filename, @Override public byte[] load(String storageRef, String requestingConversationId) throws AttachmentStoreException { - try { - ObjectId fileId = new ObjectId(storageRef); - GridFSFile file = gridFSBucket.find(Filters.eq("_id", fileId)).first(); + GridFSFile file = findFileByRef(storageRef); + if (file == null) { + throw new AttachmentStoreException("Attachment not found: " + storageRef); + } + authorize(file, requestingConversationId); - if (file == null) { - throw new AttachmentStoreException("Attachment not found: " + storageRef); - } + ByteArrayOutputStream out = new ByteArrayOutputStream(); + gridFSBucket.downloadToStream(file.getObjectId(), out); + return out.toByteArray(); + } - // Check conversation ownership - Document metadata = file.getMetadata(); - if (metadata != null) { - String ownerConv = metadata.getString("conversationId"); - if (ownerConv != null && !ownerConv.equals(requestingConversationId)) { - throw new AttachmentStoreException( - "Cross-conversation access denied: attachment belongs to '%s', requested from '%s'" - .formatted(ownerConv, requestingConversationId)); - } - } - - ByteArrayOutputStream out = new ByteArrayOutputStream(); - gridFSBucket.downloadToStream(fileId, out); - return out.toByteArray(); - } catch (IllegalArgumentException e) { - throw new AttachmentStoreException("Invalid storage reference: " + storageRef, e); + @Override + public Attachment getMetadata(String storageRef, String requestingConversationId) throws AttachmentStoreException { + GridFSFile file = findFileByRef(storageRef); + if (file == null) { + throw new AttachmentStoreException("Attachment not found: " + storageRef); } + authorize(file, requestingConversationId); + + Document metadata = file.getMetadata(); + String mime = metadata != null ? metadata.getString(META_MIME_TYPE) : null; + String owner = metadata != null ? metadata.getString(META_CONVERSATION_ID) : null; + String ref = metadata != null && metadata.getString(META_STORAGE_REF) != null + ? metadata.getString(META_STORAGE_REF) + : storageRef; + return new Attachment(ref, file.getFilename(), + mime != null ? mime : "application/octet-stream", file.getLength(), owner); + } + + @Override + public void grantAccess(String storageRef, String conversationId) throws AttachmentStoreException { + var result = filesCollection.updateOne(refFilter(storageRef), + Updates.addToSet("metadata." + META_GRANTS, conversationId)); + if (result.getMatchedCount() == 0) { + throw new AttachmentStoreException("Attachment not found: " + storageRef); + } + LOGGER.debugf("Granted conversation '%s' access to attachment %s", + sanitize(conversationId), storageRef); + } + + @Override + public boolean delete(String storageRef, String requestingConversationId) throws AttachmentStoreException { + GridFSFile file = findFileByRef(storageRef); + if (file == null) { + return false; + } + Document metadata = file.getMetadata(); + String owner = metadata != null ? metadata.getString(META_CONVERSATION_ID) : null; + if (owner != null && !owner.equals(requestingConversationId)) { + throw new AttachmentStoreException( + "Delete denied: attachment belongs to '%s', requested from '%s'" + .formatted(owner, requestingConversationId)); + } + gridFSBucket.delete(file.getObjectId()); + return true; } @Override public long deleteByConversation(String conversationId) { long count = 0; - for (GridFSFile file : gridFSBucket.find(Filters.eq("metadata.conversationId", conversationId))) { + for (GridFSFile file : gridFSBucket.find(Filters.eq("metadata." + META_CONVERSATION_ID, conversationId))) { gridFSBucket.delete(file.getObjectId()); count++; } @@ -140,15 +192,74 @@ public long deleteByConversation(String conversationId) { @Override public List listByConversation(String conversationId) { List results = new ArrayList<>(); - for (GridFSFile file : gridFSBucket.find(Filters.eq("metadata.conversationId", conversationId))) { + for (GridFSFile file : gridFSBucket.find(Filters.eq("metadata." + META_CONVERSATION_ID, conversationId))) { Document metadata = file.getMetadata(); + String ref = metadata != null && metadata.getString(META_STORAGE_REF) != null + ? metadata.getString(META_STORAGE_REF) + : file.getObjectId().toHexString(); results.add(new Attachment( - file.getObjectId().toHexString(), + ref, file.getFilename(), - metadata != null ? metadata.getString("mimeType") : "application/octet-stream", + metadata != null && metadata.getString(META_MIME_TYPE) != null + ? metadata.getString(META_MIME_TYPE) + : "application/octet-stream", file.getLength(), conversationId)); } return results; } + + private void enforceQuota(String conversationId, long incomingBytes) throws AttachmentStoreException { + if (maxPerConversation <= 0 && maxTotalBytesPerConversation <= 0) { + return; + } + long count = 0; + long totalBytes = 0; + for (GridFSFile file : gridFSBucket.find(Filters.eq("metadata." + META_CONVERSATION_ID, conversationId))) { + count++; + totalBytes += file.getLength(); + } + if (maxPerConversation > 0 && count >= maxPerConversation) { + throw new AttachmentStoreException( + "Attachment quota exceeded for conversation: %d/%d files. Delete some attachments first." + .formatted(count, maxPerConversation)); + } + if (maxTotalBytesPerConversation > 0 && totalBytes + incomingBytes > maxTotalBytesPerConversation) { + throw new AttachmentStoreException( + "Attachment storage quota exceeded for conversation: %d + %d bytes exceeds limit of %d. Delete some attachments first." + .formatted(totalBytes, incomingBytes, maxTotalBytesPerConversation)); + } + } + + private GridFSFile findFileByRef(String storageRef) { + return gridFSBucket.find(refFilter(storageRef)).first(); + } + + private Bson refFilter(String storageRef) { + if (ObjectId.isValid(storageRef)) { + // Match the modern UUID metadata ref or a legacy plain-ObjectId ref. + return Filters.or( + Filters.eq("metadata." + META_STORAGE_REF, storageRef), + Filters.eq("_id", new ObjectId(storageRef))); + } + return Filters.eq("metadata." + META_STORAGE_REF, storageRef); + } + + private void authorize(GridFSFile file, String requester) throws AttachmentStoreException { + Document metadata = file.getMetadata(); + if (metadata == null) { + return; // legacy blobs without metadata remain accessible + } + String owner = metadata.getString(META_CONVERSATION_ID); + if (owner == null || owner.equals(requester)) { + return; + } + List grants = metadata.getList(META_GRANTS, String.class); + if (grants != null && grants.contains(requester)) { + return; + } + throw new AttachmentStoreException( + "Cross-conversation access denied: attachment belongs to '%s', requested from '%s'" + .formatted(owner, requester)); + } } diff --git a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorage.java b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorage.java deleted file mode 100644 index 3b0836bd20..0000000000 --- a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorage.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.datastore.postgres; - -import ai.labs.eddi.engine.memory.IAttachmentStorage; -import io.quarkus.arc.DefaultBean; -import jakarta.annotation.PostConstruct; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.inject.Instance; -import jakarta.inject.Inject; -import org.jboss.logging.Logger; - -import javax.sql.DataSource; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.sql.*; -import java.util.UUID; - -/** - * PostgreSQL implementation of {@link IAttachmentStorage}. - *

- * Stores binary attachment payloads in a dedicated {@code attachments} table - * using {@code BYTEA} columns. Each row is linked to a conversation for GDPR - * cleanup. - *

- * Activated via {@code @DefaultBean} — yields to MongoDB GridFS when both are - * available. - * - * @since 6.0.0 - */ -@ApplicationScoped -@DefaultBean -public class PostgresAttachmentStorage implements IAttachmentStorage { - - private static final Logger LOGGER = Logger.getLogger(PostgresAttachmentStorage.class); - - private static final String CREATE_TABLE = """ - CREATE TABLE IF NOT EXISTS attachments ( - id UUID PRIMARY KEY, - conversation_id TEXT NOT NULL, - file_name TEXT, - mime_type TEXT NOT NULL, - size_bytes BIGINT NOT NULL DEFAULT 0, - data BYTEA NOT NULL, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() - ) - """; - - private static final String CREATE_INDEX = """ - CREATE INDEX IF NOT EXISTS idx_attachments_conversation - ON attachments (conversation_id) - """; - - private static final String INSERT = """ - INSERT INTO attachments (id, conversation_id, file_name, mime_type, size_bytes, data) - VALUES (?, ?, ?, ?, ?, ?) - """; - - private static final String SELECT_BY_ID = """ - SELECT data FROM attachments WHERE id = ? - """; - - private static final String DELETE_BY_CONVERSATION = """ - DELETE FROM attachments WHERE conversation_id = ? - """; - - private final Instance dataSourceInstance; - - @Inject - public PostgresAttachmentStorage(Instance dataSourceInstance) { - this.dataSourceInstance = dataSourceInstance; - } - - @PostConstruct - void createTable() { - if (!dataSourceInstance.isResolvable()) { - LOGGER.debug("DataSource not available — attachment table creation skipped"); - return; - } - try (Connection conn = dataSourceInstance.get().getConnection(); - Statement stmt = conn.createStatement()) { - stmt.execute(CREATE_TABLE); - stmt.execute(CREATE_INDEX); - } catch (SQLException e) { - LOGGER.warnf("Failed to create attachments table: %s", e.getMessage()); - } - } - - @Override - public String store(String conversationId, String fileName, String mimeType, InputStream data, long sizeBytes) { - String id = UUID.randomUUID().toString(); - - try (Connection conn = dataSourceInstance.get().getConnection(); - PreparedStatement ps = conn.prepareStatement(INSERT)) { - - long storedSizeBytes = Math.max(sizeBytes, 0L); - ps.setObject(1, UUID.fromString(id)); - ps.setString(2, conversationId); - ps.setString(3, fileName); - ps.setString(4, mimeType); - ps.setLong(5, storedSizeBytes); - if (sizeBytes > 0) { - ps.setBinaryStream(6, data, sizeBytes); - } else { - ps.setBinaryStream(6, data); - } - ps.executeUpdate(); - - String storageRef = "pg://" + id; - LOGGER.debugf("Stored attachment '%s' (%s, %d bytes) → %s", - fileName, mimeType, storedSizeBytes, storageRef); - return storageRef; - - } catch (SQLException e) { - throw new RuntimeException("Failed to store attachment: " + e.getMessage(), e); - } - } - - @Override - public InputStream load(String storageRef) throws AttachmentNotFoundException { - UUID id = parseStorageRef(storageRef); - - try (Connection conn = dataSourceInstance.get().getConnection(); - PreparedStatement ps = conn.prepareStatement(SELECT_BY_ID)) { - - ps.setObject(1, id); - try (ResultSet rs = ps.executeQuery()) { - if (rs.next()) { - byte[] data = rs.getBytes("data"); - return new ByteArrayInputStream(data); - } - throw new AttachmentNotFoundException("No attachment found for: " + storageRef); - } - - } catch (SQLException e) { - throw new RuntimeException("Failed to load attachment: " + e.getMessage(), e); - } - } - - @Override - public long deleteByConversation(String conversationId) { - try (Connection conn = dataSourceInstance.get().getConnection(); - PreparedStatement ps = conn.prepareStatement(DELETE_BY_CONVERSATION)) { - - ps.setString(1, conversationId); - int deleted = ps.executeUpdate(); - - if (deleted > 0) { - LOGGER.debugf("Deleted %d attachments for conversation '%s'", deleted, conversationId); - } - return deleted; - - } catch (SQLException e) { - throw new RuntimeException("Failed to delete attachments: " + e.getMessage(), e); - } - } - - /** - * Parse a storage reference back to a UUID. - * - * @param storageRef - * format: {@code pg://} - */ - private static UUID parseStorageRef(String storageRef) throws AttachmentNotFoundException { - if (storageRef == null || !storageRef.startsWith("pg://")) { - throw new AttachmentNotFoundException("Invalid PostgreSQL storage ref: " + storageRef); - } - try { - return UUID.fromString(storageRef.substring("pg://".length())); - } catch (IllegalArgumentException e) { - throw new AttachmentNotFoundException("Invalid UUID in storage ref: " + storageRef); - } - } -} diff --git a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java index 91f2da0b28..27ee961ce6 100644 --- a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java @@ -24,9 +24,9 @@ /** * PostgreSQL implementation of {@link IAttachmentStore}. *

- * Stores attachment data in a {@code attachments} table using {@code BYTEA} - * columns. For large files, PostgreSQL large objects could be used, but BYTEA - * is simpler and sufficient for the 20MB cap. + * Stores attachment data in an {@code attachments} table using {@code BYTEA} + * columns. The {@code storage_ref} is a random UUID (unguessable). Access + * grants are held in a {@code grants TEXT[]} column and die with the row. * * @since 6.0.0 */ @@ -45,9 +45,11 @@ CREATE TABLE IF NOT EXISTS attachments ( mime_type TEXT NOT NULL, size_bytes BIGINT NOT NULL, data BYTEA NOT NULL, + grants TEXT[] NOT NULL DEFAULT '{}', created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ) """; + private static final String ADD_GRANTS_COLUMN = "ALTER TABLE attachments ADD COLUMN IF NOT EXISTS grants TEXT[] NOT NULL DEFAULT '{}'"; private static final String CREATE_INDEX_CONV = "CREATE INDEX IF NOT EXISTS idx_attach_conv ON attachments (conversation_id)"; private static final String CREATE_INDEX_TENANT = "CREATE INDEX IF NOT EXISTS idx_attach_tenant ON attachments (tenant_id)"; @@ -57,6 +59,12 @@ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() @ConfigProperty(name = "eddi.attachments.max-size-bytes", defaultValue = "20971520") // 20 MB long maxSizeBytes; + @ConfigProperty(name = "eddi.attachments.max-per-conversation", defaultValue = "50") + long maxPerConversation; + + @ConfigProperty(name = "eddi.attachments.max-total-bytes-per-conversation", defaultValue = "104857600") // 100 MB + long maxTotalBytesPerConversation; + @Inject public PostgresAttachmentStore(Instance dataSourceInstance) { this.dataSourceInstance = dataSourceInstance; @@ -67,6 +75,7 @@ private synchronized void ensureSchema() { return; try (Connection conn = dataSourceInstance.get().getConnection(); Statement stmt = conn.createStatement()) { stmt.execute(CREATE_TABLE); + stmt.execute(ADD_GRANTS_COLUMN); stmt.execute(CREATE_INDEX_CONV); stmt.execute(CREATE_INDEX_TENANT); schemaInitialized = true; @@ -99,6 +108,8 @@ public Attachment store(byte[] bytes, String declaredMime, String filename, String storageRef = UUID.randomUUID().toString(); ensureSchema(); + enforceQuota(conversationId, bytes.length); + String sql = "INSERT INTO attachments " + "(storage_ref, conversation_id, tenant_id, filename, mime_type, size_bytes, data) " + "VALUES (?, ?, ?, ?, ?, ?, ?)"; @@ -125,19 +136,14 @@ public Attachment store(byte[] bytes, String declaredMime, String filename, @Override public byte[] load(String storageRef, String requestingConversationId) throws AttachmentStoreException { ensureSchema(); - String sql = "SELECT data, conversation_id FROM attachments WHERE storage_ref = ?"; + String sql = "SELECT data, conversation_id, grants FROM attachments WHERE storage_ref = ?"; try (Connection conn = dataSourceInstance.get().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { ps.setString(1, storageRef); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) { throw new AttachmentStoreException("Attachment not found: " + storageRef); } - String ownerConv = rs.getString("conversation_id"); - if (!ownerConv.equals(requestingConversationId)) { - throw new AttachmentStoreException( - "Cross-conversation access denied: attachment belongs to '%s', requested from '%s'" - .formatted(ownerConv, requestingConversationId)); - } + authorize(rs.getString("conversation_id"), rs, requestingConversationId); return rs.getBytes("data"); } } catch (SQLException e) { @@ -145,6 +151,82 @@ public byte[] load(String storageRef, String requestingConversationId) throws At } } + @Override + public Attachment getMetadata(String storageRef, String requestingConversationId) throws AttachmentStoreException { + ensureSchema(); + String sql = "SELECT storage_ref, filename, mime_type, size_bytes, conversation_id, grants " + + "FROM attachments WHERE storage_ref = ?"; + try (Connection conn = dataSourceInstance.get().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, storageRef); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + throw new AttachmentStoreException("Attachment not found: " + storageRef); + } + authorize(rs.getString("conversation_id"), rs, requestingConversationId); + return new Attachment( + rs.getString("storage_ref"), + rs.getString("filename"), + rs.getString("mime_type"), + rs.getLong("size_bytes"), + rs.getString("conversation_id")); + } + } catch (SQLException e) { + throw new AttachmentStoreException("Failed to read attachment metadata", e); + } + } + + @Override + public void grantAccess(String storageRef, String conversationId) throws AttachmentStoreException { + ensureSchema(); + String sql = "UPDATE attachments SET grants = " + + "CASE WHEN ? = ANY(COALESCE(grants, '{}')) THEN grants " + + "ELSE array_append(COALESCE(grants, '{}'), ?) END " + + "WHERE storage_ref = ?"; + try (Connection conn = dataSourceInstance.get().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, conversationId); + ps.setString(2, conversationId); + ps.setString(3, storageRef); + int updated = ps.executeUpdate(); + if (updated == 0) { + throw new AttachmentStoreException("Attachment not found: " + storageRef); + } + LOGGER.debugf("Granted conversation '%s' access to attachment %s", + sanitize(conversationId), storageRef); + } catch (SQLException e) { + throw new AttachmentStoreException("Failed to grant attachment access", e); + } + } + + @Override + public boolean delete(String storageRef, String requestingConversationId) throws AttachmentStoreException { + ensureSchema(); + try (Connection conn = dataSourceInstance.get().getConnection()) { + String owner; + try (PreparedStatement ps = conn.prepareStatement( + "SELECT conversation_id FROM attachments WHERE storage_ref = ?")) { + ps.setString(1, storageRef); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + return false; + } + owner = rs.getString("conversation_id"); + } + } + if (owner != null && !owner.equals(requestingConversationId)) { + throw new AttachmentStoreException( + "Delete denied: attachment belongs to '%s', requested from '%s'" + .formatted(owner, requestingConversationId)); + } + try (PreparedStatement ps = conn.prepareStatement( + "DELETE FROM attachments WHERE storage_ref = ?")) { + ps.setString(1, storageRef); + return ps.executeUpdate() > 0; + } + } catch (SQLException e) { + throw new AttachmentStoreException("Failed to delete attachment", e); + } + } + @Override public long deleteByConversation(String conversationId) { ensureSchema(); @@ -185,4 +267,59 @@ public List listByConversation(String conversationId) { throw new RuntimeException("Failed to list attachments", e); } } + + private void enforceQuota(String conversationId, long incomingBytes) throws AttachmentStoreException { + if (maxPerConversation <= 0 && maxTotalBytesPerConversation <= 0) { + return; + } + String sql = "SELECT COUNT(*), COALESCE(SUM(size_bytes), 0) FROM attachments WHERE conversation_id = ?"; + try (Connection conn = dataSourceInstance.get().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { + ps.setString(1, conversationId); + try (ResultSet rs = ps.executeQuery()) { + rs.next(); + long count = rs.getLong(1); + long totalBytes = rs.getLong(2); + if (maxPerConversation > 0 && count >= maxPerConversation) { + throw new AttachmentStoreException( + "Attachment quota exceeded for conversation: %d/%d files. Delete some attachments first." + .formatted(count, maxPerConversation)); + } + if (maxTotalBytesPerConversation > 0 && totalBytes + incomingBytes > maxTotalBytesPerConversation) { + throw new AttachmentStoreException( + "Attachment storage quota exceeded for conversation: %d + %d bytes exceeds limit of %d. Delete some attachments first." + .formatted(totalBytes, incomingBytes, maxTotalBytesPerConversation)); + } + } + } catch (SQLException e) { + throw new AttachmentStoreException("Failed to check attachment quota", e); + } + } + + private void authorize(String owner, ResultSet rs, String requester) throws SQLException, AttachmentStoreException { + if (owner != null && owner.equals(requester)) { + return; + } + if (grantsContain(rs, requester)) { + return; + } + throw new AttachmentStoreException( + "Cross-conversation access denied: attachment belongs to '%s', requested from '%s'" + .formatted(owner, requester)); + } + + private static boolean grantsContain(ResultSet rs, String value) throws SQLException { + Array arr = rs.getArray("grants"); + if (arr == null) { + return false; + } + Object raw = arr.getArray(); + if (raw instanceof Object[] elements) { + for (Object element : elements) { + if (value != null && value.equals(String.valueOf(element))) { + return true; + } + } + } + return false; + } } diff --git a/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java b/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java index 9425151bb0..f2cd79a6be 100644 --- a/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java @@ -11,9 +11,19 @@ * Supports MongoDB GridFS and PostgreSQL backends via the DB-agnostic pattern * in {@code DataStoreProducers}. *

- * Security: All access is scoped to the owning conversation. - * Cross-conversation access is rejected. GDPR erasure cascades via - * {@link #deleteByConversation(String)}. + * Ownership & access: every blob is owned by the + * conversation it was uploaded to. Read access ({@link #load} / + * {@link #getMetadata}) is granted to the owning conversation or any + * conversation that has been given an explicit {@linkplain #grantAccess grant}. + * Grants are written only by trusted server code (e.g. group fan-out) — never + * derived from client-supplied context. Deletion of a single blob + * ({@link #delete}) is restricted to the owner. GDPR/conversation erasure + * cascades via {@link #deleteByConversation(String)}, which also removes the + * blob's grants. + *

+ * This is the single blob-store abstraction for EDDI; the former + * {@code IAttachmentStorage} was folded into it so uploads, LLM forwarding, + * conversation deletion and GDPR erasure all operate on the same store. * * @since 6.0.0 */ @@ -31,31 +41,81 @@ public interface IAttachmentStore { * @param conversationId * owning conversation * @param tenantId - * the tenant (for quota enforcement) + * the tenant (advisory metadata only; not an access boundary) * @return the stored attachment metadata * @throws AttachmentStoreException - * if storage fails, MIME validation fails, or size limit exceeded + * if storage fails, MIME validation fails, the size limit is + * exceeded, or a per-conversation quota is exceeded */ Attachment store(byte[] bytes, String declaredMime, String filename, String conversationId, String tenantId) throws AttachmentStoreException; /** - * Load an attachment by storage reference. + * Load an attachment's bytes. * * @param storageRef * the storage reference from {@link Attachment#storageRef()} * @param requestingConversationId - * the conversation requesting access (must match owning - * conversation) + * the conversation requesting access (must own the blob or hold a + * grant) * @return the raw bytes * @throws AttachmentStoreException - * if not found or cross-conversation access attempted + * if not found or access is denied (not owner and not granted) */ byte[] load(String storageRef, String requestingConversationId) throws AttachmentStoreException; /** - * Delete all attachments for a conversation (GDPR erasure). + * Resolve an attachment's server-validated metadata (MIME, filename, size, + * owner) without transferring the bytes. Same owner-or-grant authorization as + * {@link #load}. Used at extraction time so behavior rules and the forwarder + * see the truth for {@code storageRef}-only references. + * + * @param storageRef + * the storage reference + * @param requestingConversationId + * the conversation requesting access (must own the blob or hold a + * grant) + * @return the attachment metadata + * @throws AttachmentStoreException + * if not found or access is denied + */ + Attachment getMetadata(String storageRef, String requestingConversationId) throws AttachmentStoreException; + + /** + * Grant a conversation read access to a blob it does not own. Idempotent. + *

+ * Trusted callers only. This must be invoked exclusively by + * server-side orchestration (e.g. {@code GroupConversationService} at member + * fan-out) — never from client-supplied context — since it widens the access + * boundary of the blob. The grant lives with the blob and dies when the blob is + * deleted. + * + * @param storageRef + * the storage reference of the blob to share + * @param conversationId + * the conversation to grant read access to + * @throws AttachmentStoreException + * if the blob does not exist + */ + void grantAccess(String storageRef, String conversationId) throws AttachmentStoreException; + + /** + * Delete a single attachment. Restricted to the owning conversation — a grantee + * cannot delete another conversation's blob. + * + * @param storageRef + * the storage reference + * @param requestingConversationId + * the conversation requesting deletion (must be the owner) + * @return {@code true} if a blob was deleted, {@code false} if none matched + * @throws AttachmentStoreException + * if the blob exists but is owned by another conversation + */ + boolean delete(String storageRef, String requestingConversationId) throws AttachmentStoreException; + + /** + * Delete all attachments for a conversation (GDPR/conversation erasure). * * @param conversationId * the conversation ID diff --git a/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java b/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java index 280b8aa607..2f6697cd31 100644 --- a/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java +++ b/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java @@ -10,7 +10,7 @@ import ai.labs.eddi.engine.audit.AuditLedgerService; import ai.labs.eddi.engine.audit.IAuditStore; import ai.labs.eddi.engine.audit.model.AuditEntry; -import ai.labs.eddi.engine.memory.IAttachmentStorage; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemoryStore; import ai.labs.eddi.engine.runtime.IDatabaseLogs; import ai.labs.eddi.engine.triggermanagement.IUserConversationStore; @@ -53,7 +53,7 @@ public class GdprComplianceService { private final IDatabaseLogs databaseLogs; private final IAuditStore auditStore; private final AuditLedgerService auditLedgerService; - private final Instance attachmentStorageInstance; + private final Instance attachmentStorageInstance; @Inject public GdprComplianceService(IUserMemoryStore userMemoryStore, @@ -62,7 +62,7 @@ public GdprComplianceService(IUserMemoryStore userMemoryStore, IDatabaseLogs databaseLogs, IAuditStore auditStore, AuditLedgerService auditLedgerService, - Instance attachmentStorageInstance) { + Instance attachmentStorageInstance) { this.userMemoryStore = userMemoryStore; this.conversationMemoryStore = conversationMemoryStore; this.userConversationStore = userConversationStore; diff --git a/src/main/java/ai/labs/eddi/engine/memory/IAttachmentStorage.java b/src/main/java/ai/labs/eddi/engine/memory/IAttachmentStorage.java deleted file mode 100644 index 2a69977392..0000000000 --- a/src/main/java/ai/labs/eddi/engine/memory/IAttachmentStorage.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.engine.memory; - -import java.io.InputStream; - -/** - * Service Provider Interface for binary attachment storage. - *

- * Attachments are stored externally (GridFS, PostgreSQL bytea, or object - * storage) and referenced by a storage key. This SPI is DB-agnostic from day 1. - *

- * Implementations must be idempotent for {@code deleteByConversation}. - * - * @since 6.0.0 - */ -public interface IAttachmentStorage { - - /** - * Store binary data and return a storage reference key. - * - * @param conversationId - * owning conversation - * @param fileName - * original file name (e.g., "screenshot.png") - * @param mimeType - * MIME type (e.g., "image/png") - * @param data - * binary input stream (caller closes) - * @param sizeBytes - * size in bytes (-1 if unknown) - * @return storage reference key (opaque string, used to load/delete) - */ - String store(String conversationId, String fileName, String mimeType, InputStream data, long sizeBytes); - - /** - * Load binary data by storage reference. - * - * @param storageRef - * key returned by {@link #store} - * @return input stream of the binary data (caller must close) - * @throws AttachmentNotFoundException - * if the reference does not exist - */ - InputStream load(String storageRef) throws AttachmentNotFoundException; - - /** - * Delete all attachments for a conversation (GDPR cleanup). - * - * @param conversationId - * the conversation whose attachments to delete - * @return number of attachments deleted - */ - long deleteByConversation(String conversationId); - - /** - * Thrown when a storage reference does not resolve to an attachment. - */ - class AttachmentNotFoundException extends Exception { - public AttachmentNotFoundException(String message) { - super(message); - } - } -} diff --git a/src/main/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorage.java b/src/main/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorage.java deleted file mode 100644 index 8d47e03930..0000000000 --- a/src/main/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorage.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.engine.memory.mongo; - -import ai.labs.eddi.engine.memory.IAttachmentStorage; -import com.mongodb.client.MongoDatabase; -import com.mongodb.client.gridfs.GridFSBucket; -import com.mongodb.client.gridfs.GridFSBuckets; -import com.mongodb.client.gridfs.model.GridFSFile; -import com.mongodb.client.gridfs.model.GridFSUploadOptions; -import com.mongodb.client.model.Filters; -import io.quarkus.arc.DefaultBean; -import jakarta.enterprise.context.ApplicationScoped; -import jakarta.inject.Inject; -import org.bson.Document; -import org.bson.types.ObjectId; -import org.jboss.logging.Logger; - -import java.io.InputStream; - -/** - * MongoDB GridFS implementation of {@link IAttachmentStorage}. - *

- * Stores binary attachment payloads in a GridFS bucket named - * {@code eddi_attachments}. Each file has metadata linking it to the owning - * conversation for efficient GDPR cleanup via - * {@link #deleteByConversation(String)}. - *

- * Annotated {@code @DefaultBean} so it yields to other implementations when - * alternative profiles are active (e.g., PostgreSQL). - * - * @since 6.0.0 - */ -@ApplicationScoped -@DefaultBean -public class MongoAttachmentStorage implements IAttachmentStorage { - - private static final Logger LOGGER = Logger.getLogger(MongoAttachmentStorage.class); - private static final String BUCKET_NAME = "eddi_attachments"; - private static final String META_CONVERSATION_ID = "conversationId"; - private static final String META_MIME_TYPE = "mimeType"; - - private final GridFSBucket gridFSBucket; - - @Inject - public MongoAttachmentStorage(MongoDatabase database) { - this.gridFSBucket = GridFSBuckets.create(database, BUCKET_NAME); - } - - @Override - public String store(String conversationId, String fileName, String mimeType, InputStream data, long sizeBytes) { - Document metadata = new Document() - .append(META_CONVERSATION_ID, conversationId) - .append(META_MIME_TYPE, mimeType); - - if (sizeBytes > 0) { - metadata.append("sizeBytes", sizeBytes); - } - - GridFSUploadOptions options = new GridFSUploadOptions() - .metadata(metadata); - - ObjectId fileId = gridFSBucket.uploadFromStream( - fileName != null ? fileName : "unnamed", - data, - options); - - String storageRef = "gridfs://" + fileId.toHexString(); - LOGGER.debugf("Stored attachment '%s' (%s, %d bytes) → %s", - fileName, mimeType, sizeBytes, storageRef); - return storageRef; - } - - @Override - public InputStream load(String storageRef) throws AttachmentNotFoundException { - ObjectId fileId = parseStorageRef(storageRef); - - // Verify the file exists first - GridFSFile file = gridFSBucket.find(Filters.eq("_id", fileId)).first(); - if (file == null) { - throw new AttachmentNotFoundException("No attachment found for storage ref: " + storageRef); - } - - return gridFSBucket.openDownloadStream(fileId); - } - - @Override - public long deleteByConversation(String conversationId) { - long deleted = 0; - - // Find all files belonging to this conversation - for (GridFSFile file : gridFSBucket.find( - Filters.eq("metadata." + META_CONVERSATION_ID, conversationId))) { - gridFSBucket.delete(file.getObjectId()); - deleted++; - } - - if (deleted > 0) { - LOGGER.debugf("Deleted %d attachments for conversation '%s'", deleted, conversationId); - } - return deleted; - } - - /** - * Parse a storage reference back to a GridFS ObjectId. - * - * @param storageRef - * format: {@code gridfs://} - * @throws AttachmentNotFoundException - * if the format is invalid - */ - private static ObjectId parseStorageRef(String storageRef) throws AttachmentNotFoundException { - if (storageRef == null || !storageRef.startsWith("gridfs://")) { - throw new AttachmentNotFoundException("Invalid GridFS storage ref: " + storageRef); - } - try { - return new ObjectId(storageRef.substring("gridfs://".length())); - } catch (IllegalArgumentException e) { - throw new AttachmentNotFoundException("Invalid ObjectId in storage ref: " + storageRef); - } - } -} diff --git a/src/main/java/ai/labs/eddi/engine/memory/rest/RestConversationStore.java b/src/main/java/ai/labs/eddi/engine/memory/rest/RestConversationStore.java index 7edafdaec1..dc7b35ea5a 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/rest/RestConversationStore.java +++ b/src/main/java/ai/labs/eddi/engine/memory/rest/RestConversationStore.java @@ -7,7 +7,7 @@ import ai.labs.eddi.configs.descriptors.IDocumentDescriptorStore; import ai.labs.eddi.configs.properties.IUserMemoryStore; import ai.labs.eddi.datastore.IResourceStore; -import ai.labs.eddi.engine.memory.IAttachmentStorage; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemoryStore; import ai.labs.eddi.engine.memory.descriptor.IConversationDescriptorStore; import ai.labs.eddi.engine.memory.descriptor.model.ConversationDescriptor; @@ -55,7 +55,7 @@ public class RestConversationStore implements IRestConversationStore { private final IRuntime runtime; private final Integer deleteEndedConversationsOnceOlderThanDays; private final Integer deleteMemoriesOlderThanDays; - private final Instance attachmentStorageInstance; + private final Instance attachmentStorageInstance; private static final Logger log = Logger.getLogger(RestConversationStore.class); @@ -71,7 +71,7 @@ public RestConversationStore( Integer deleteEndedConversationsOnceOlderThanDays, @ConfigProperty(name = "eddi.usermemories.deleteOlderThanDays") Integer deleteMemoriesOlderThanDays, - Instance attachmentStorageInstance) { + Instance attachmentStorageInstance) { // @formatter:on this.documentDescriptorStore = documentDescriptorStore; diff --git a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java index b66b87f345..54e7b85e0e 100644 --- a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java +++ b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java @@ -6,18 +6,18 @@ import ai.labs.eddi.engine.attachments.IAttachmentStore.Attachment; import ai.labs.eddi.engine.attachments.IAttachmentStore.AttachmentStoreException; -import com.mongodb.client.FindIterable; +import com.mongodb.client.MongoCollection; import com.mongodb.client.MongoCursor; import com.mongodb.client.gridfs.GridFSBucket; import com.mongodb.client.gridfs.GridFSFindIterable; import com.mongodb.client.gridfs.model.GridFSFile; import com.mongodb.client.gridfs.model.GridFSUploadOptions; +import com.mongodb.client.result.UpdateResult; import org.bson.Document; import org.bson.conversions.Bson; import org.bson.types.ObjectId; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; import java.io.ByteArrayInputStream; import java.io.OutputStream; @@ -34,72 +34,113 @@ class GridFsAttachmentStoreTest { private GridFSBucket gridFSBucket; + private MongoCollection filesCollection; private GridFsAttachmentStore sut; @BeforeEach void setUp() throws Exception { gridFSBucket = mock(GridFSBucket.class); + filesCollection = mock(MongoCollection.class); + sut = createWithMocks(gridFSBucket, filesCollection); - // Create the store with a mocked MongoDatabase, then replace the bucket via - // reflection - // since the constructor calls the static GridFSBuckets.create() - sut = createWithMockedBucket(gridFSBucket); + setLong("maxSizeBytes", 20_971_520L); + // Unlimited quotas by default so store() skips the usage query. + setLong("maxPerConversation", -1L); + setLong("maxTotalBytesPerConversation", -1L); + } - // Set maxSizeBytes via reflection - Field maxSizeField = GridFsAttachmentStore.class.getDeclaredField("maxSizeBytes"); - maxSizeField.setAccessible(true); - maxSizeField.set(sut, 20_971_520L); + private void setLong(String field, long value) throws Exception { + Field f = GridFsAttachmentStore.class.getDeclaredField(field); + f.setAccessible(true); + f.setLong(sut, value); } - private static GridFsAttachmentStore createWithMockedBucket(GridFSBucket bucket) throws Exception { - // Allocate instance without calling constructor (which would call - // GridFSBuckets.create()) + private static GridFsAttachmentStore createWithMocks(GridFSBucket bucket, MongoCollection files) + throws Exception { var objenesis = new org.objenesis.ObjenesisStd(); GridFsAttachmentStore store = objenesis.newInstance(GridFsAttachmentStore.class); Field bucketField = GridFsAttachmentStore.class.getDeclaredField("gridFSBucket"); bucketField.setAccessible(true); bucketField.set(store, bucket); + Field filesField = GridFsAttachmentStore.class.getDeclaredField("filesCollection"); + filesField.setAccessible(true); + filesField.set(store, files); return store; } + /** Mock a single GridFSFile with the given owner / grants metadata. */ + private static GridFSFile mockFile(ObjectId id, String owner, String mime, List grants, String ref) { + GridFSFile f = mock(GridFSFile.class); + when(f.getObjectId()).thenReturn(id); + when(f.getFilename()).thenReturn("file.bin"); + when(f.getLength()).thenReturn(123L); + Document md = new Document(); + if (owner != null) + md.append("conversationId", owner); + if (mime != null) + md.append("mimeType", mime); + if (ref != null) + md.append("storageRef", ref); + if (grants != null) + md.append("grants", grants); + when(f.getMetadata()).thenReturn(md); + return f; + } + + /** + * Make gridFSBucket.find(any) resolve to a single-first() iterable returning + * {@code file}. + */ + private void whenFindFirst(GridFSFile file) { + GridFSFindIterable it = mock(GridFSFindIterable.class); + when(gridFSBucket.find(any(Bson.class))).thenReturn(it); + when(it.first()).thenReturn(file); + } + + /** + * Make gridFSBucket.find(any) iterate over the given files (for + * delete/list/quota). + */ + private void whenFindIterate(GridFSFile... files) { + GridFSFindIterable it = mock(GridFSFindIterable.class); + when(gridFSBucket.find(any(Bson.class))).thenReturn(it); + @SuppressWarnings("unchecked") + MongoCursor cursor = mock(MongoCursor.class); + doReturn(cursor).when(it).iterator(); + Boolean[] hasNext = new Boolean[files.length + 1]; + for (int i = 0; i < files.length; i++) + hasNext[i] = true; + hasNext[files.length] = false; + if (files.length == 0) { + when(cursor.hasNext()).thenReturn(false); + } else if (files.length == 1) { + when(cursor.hasNext()).thenReturn(true, false); + when(cursor.next()).thenReturn(files[0]); + } else { + when(cursor.hasNext()).thenReturn(true, true, false); + when(cursor.next()).thenReturn(files[0], files[1]); + } + } + // ─── store() ──────────────────────────────────────────────── @Test - void store_validData_returnsAttachment() throws Exception { - // given + void store_validData_returnsUuidRef() throws Exception { byte[] data = "Hello, World!".getBytes(); - ObjectId fileId = new ObjectId(); when(gridFSBucket.uploadFromStream(anyString(), any(ByteArrayInputStream.class), any(GridFSUploadOptions.class))) - .thenReturn(fileId); + .thenReturn(new ObjectId()); - // when Attachment result = sut.store(data, "application/octet-stream", "test.txt", "conv-1", "tenant-1"); - // then assertNotNull(result); - assertEquals(fileId.toHexString(), result.storageRef()); + // storageRef is a random UUID, not the ObjectId hex + assertDoesNotThrow(() -> java.util.UUID.fromString(result.storageRef())); assertEquals("test.txt", result.filename()); assertEquals("application/octet-stream", result.mimeType()); assertEquals(data.length, result.sizeBytes()); assertEquals("conv-1", result.conversationId()); } - @Test - void store_nullFilename_usesUnnamed() throws Exception { - // given - byte[] data = "content".getBytes(); - ObjectId fileId = new ObjectId(); - ArgumentCaptor filenameCaptor = ArgumentCaptor.forClass(String.class); - when(gridFSBucket.uploadFromStream(filenameCaptor.capture(), any(ByteArrayInputStream.class), - any(GridFSUploadOptions.class))).thenReturn(fileId); - - // when - sut.store(data, "application/octet-stream", null, "conv-1", "tenant-1"); - - // then - assertEquals("unnamed", filenameCaptor.getValue()); - } - @Test void store_nullBytes_throwsException() { var ex = assertThrows(AttachmentStoreException.class, @@ -116,11 +157,7 @@ void store_emptyBytes_throwsException() { @Test void store_exceedsMaxSize_throwsException() throws Exception { - // given — set small max - Field maxField = GridFsAttachmentStore.class.getDeclaredField("maxSizeBytes"); - maxField.setAccessible(true); - maxField.set(sut, 5L); - + setLong("maxSizeBytes", 5L); var ex = assertThrows(AttachmentStoreException.class, () -> sut.store(new byte[6], "application/octet-stream", "big.bin", "conv", "t")); assertTrue(ex.getMessage().contains("exceeds max size")); @@ -128,205 +165,218 @@ void store_exceedsMaxSize_throwsException() throws Exception { @Test void store_mimeMismatch_throwsException() { - // given — PNG magic bytes, declared as JPEG byte[] pngBytes = new byte[]{(byte) 0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A}; - var ex = assertThrows(AttachmentStoreException.class, () -> sut.store(pngBytes, "image/jpeg", "fake.jpg", "conv", "t")); assertTrue(ex.getMessage().contains("MIME type mismatch")); } + @Test + void store_countQuotaExceeded_throwsException() throws Exception { + setLong("maxPerConversation", 1L); + // one existing file → adding a second exceeds the count quota + whenFindIterate(mockFile(new ObjectId(), "conv", "text/plain", List.of(), "r1")); + + var ex = assertThrows(AttachmentStoreException.class, + () -> sut.store("data".getBytes(), "text/plain", "f.txt", "conv", "t")); + assertTrue(ex.getMessage().contains("quota exceeded")); + } + + @Test + void store_byteQuotaExceeded_throwsException() throws Exception { + setLong("maxTotalBytesPerConversation", 100L); + GridFSFile existing = mock(GridFSFile.class); + when(existing.getLength()).thenReturn(60L); + whenFindIterate(existing); // 60 existing + 50 incoming > 100 + + var ex = assertThrows(AttachmentStoreException.class, + () -> sut.store(new byte[50], "text/plain", "f.txt", "conv", "t")); + assertTrue(ex.getMessage().contains("storage quota exceeded")); + } + // ─── load() ───────────────────────────────────────────────── @Test - void load_validRef_returnsBytes() throws Exception { - // given - ObjectId fileId = new ObjectId(); - GridFSFile gridFSFile = mock(GridFSFile.class); - Document metadata = new Document().append("conversationId", "conv-1"); - - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - when(findIterable.first()).thenReturn(gridFSFile); - when(gridFSFile.getMetadata()).thenReturn(metadata); - - doAnswer(invocation -> { - OutputStream out = invocation.getArgument(1); + void load_owner_returnsBytes() throws Exception { + ObjectId id = new ObjectId(); + whenFindFirst(mockFile(id, "conv-1", "text/plain", List.of(), "uuid-1")); + doAnswer(inv -> { + OutputStream out = inv.getArgument(1); out.write("file content".getBytes()); return null; - }).when(gridFSBucket).downloadToStream(eq(fileId), any(OutputStream.class)); - - // when - byte[] result = sut.load(fileId.toHexString(), "conv-1"); + }).when(gridFSBucket).downloadToStream(eq(id), any(OutputStream.class)); - // then + byte[] result = sut.load("uuid-1", "conv-1"); assertArrayEquals("file content".getBytes(), result); } @Test - void load_notFound_throwsException() throws Exception { - // given - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - when(findIterable.first()).thenReturn(null); + void load_grantedConversation_returnsBytes() throws Exception { + ObjectId id = new ObjectId(); + whenFindFirst(mockFile(id, "conv-owner", "text/plain", List.of("conv-guest"), "uuid-1")); + doAnswer(inv -> { + OutputStream out = inv.getArgument(1); + out.write("granted".getBytes()); + return null; + }).when(gridFSBucket).downloadToStream(eq(id), any(OutputStream.class)); - // when/then - var ex = assertThrows(AttachmentStoreException.class, - () -> sut.load(new ObjectId().toHexString(), "conv-1")); + byte[] result = sut.load("uuid-1", "conv-guest"); + assertArrayEquals("granted".getBytes(), result); + } + + @Test + void load_notFound_throwsException() { + whenFindFirst(null); + var ex = assertThrows(AttachmentStoreException.class, () -> sut.load("missing", "conv-1")); assertTrue(ex.getMessage().contains("not found")); } @Test - void load_crossConversation_throwsException() throws Exception { - // given - ObjectId fileId = new ObjectId(); - GridFSFile gridFSFile = mock(GridFSFile.class); - Document metadata = new Document().append("conversationId", "conv-owner"); - - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - when(findIterable.first()).thenReturn(gridFSFile); - when(gridFSFile.getMetadata()).thenReturn(metadata); - - // when/then - var ex = assertThrows(AttachmentStoreException.class, - () -> sut.load(fileId.toHexString(), "conv-other")); + void load_crossConversationNoGrant_throwsException() { + whenFindFirst(mockFile(new ObjectId(), "conv-owner", "text/plain", List.of(), "uuid-1")); + var ex = assertThrows(AttachmentStoreException.class, () -> sut.load("uuid-1", "conv-other")); assertTrue(ex.getMessage().contains("Cross-conversation access denied")); } @Test void load_nullMetadata_allowsAccess() throws Exception { - // given — metadata is null, owner check is skipped - ObjectId fileId = new ObjectId(); - GridFSFile gridFSFile = mock(GridFSFile.class); + ObjectId id = new ObjectId(); + GridFSFile f = mock(GridFSFile.class); + when(f.getObjectId()).thenReturn(id); + when(f.getMetadata()).thenReturn(null); + whenFindFirst(f); + doAnswer(inv -> { + OutputStream out = inv.getArgument(1); + out.write("data".getBytes()); + return null; + }).when(gridFSBucket).downloadToStream(eq(id), any(OutputStream.class)); - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - when(findIterable.first()).thenReturn(gridFSFile); - when(gridFSFile.getMetadata()).thenReturn(null); + assertArrayEquals("data".getBytes(), sut.load("any", "any-conv")); + } - doAnswer(invocation -> { - OutputStream out = invocation.getArgument(1); - out.write("data".getBytes()); + @Test + void load_legacyObjectIdRef_resolves() throws Exception { + ObjectId id = new ObjectId(); + whenFindFirst(mockFile(id, "conv-1", "text/plain", List.of(), null)); + doAnswer(inv -> { + OutputStream out = inv.getArgument(1); + out.write("legacy".getBytes()); return null; - }).when(gridFSBucket).downloadToStream(eq(fileId), any(OutputStream.class)); + }).when(gridFSBucket).downloadToStream(eq(id), any(OutputStream.class)); - // when - byte[] result = sut.load(fileId.toHexString(), "any-conv"); + // a valid ObjectId hex still resolves (legacy blobs) + byte[] result = sut.load(id.toHexString(), "conv-1"); + assertArrayEquals("legacy".getBytes(), result); + } - // then - assertArrayEquals("data".getBytes(), result); + // ─── getMetadata() ────────────────────────────────────────── + + @Test + void getMetadata_owner_returnsMetadata() throws Exception { + whenFindFirst(mockFile(new ObjectId(), "conv-1", "image/png", List.of(), "uuid-1")); + Attachment meta = sut.getMetadata("uuid-1", "conv-1"); + assertEquals("uuid-1", meta.storageRef()); + assertEquals("image/png", meta.mimeType()); + assertEquals("conv-1", meta.conversationId()); + assertEquals(123L, meta.sizeBytes()); } @Test - void load_metadataNullConversationId_allowsAccess() throws Exception { - // given — metadata exists but conversationId is null - ObjectId fileId = new ObjectId(); - GridFSFile gridFSFile = mock(GridFSFile.class); - Document metadata = new Document(); // no conversationId key - - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - when(findIterable.first()).thenReturn(gridFSFile); - when(gridFSFile.getMetadata()).thenReturn(metadata); - - doAnswer(invocation -> { - OutputStream out = invocation.getArgument(1); - out.write("bytes".getBytes()); - return null; - }).when(gridFSBucket).downloadToStream(eq(fileId), any(OutputStream.class)); + void getMetadata_denied_throws() { + whenFindFirst(mockFile(new ObjectId(), "conv-1", "image/png", List.of(), "uuid-1")); + assertThrows(AttachmentStoreException.class, () -> sut.getMetadata("uuid-1", "conv-other")); + } + + @Test + void getMetadata_notFound_throws() { + whenFindFirst(null); + assertThrows(AttachmentStoreException.class, () -> sut.getMetadata("missing", "conv-1")); + } - // when - byte[] result = sut.load(fileId.toHexString(), "conv-1"); + // ─── grantAccess() ────────────────────────────────────────── - // then - assertNotNull(result); + @Test + void grantAccess_updatesMetadata() throws Exception { + UpdateResult result = mock(UpdateResult.class); + when(result.getMatchedCount()).thenReturn(1L); + when(filesCollection.updateOne(any(Bson.class), any(Bson.class))).thenReturn(result); + + assertDoesNotThrow(() -> sut.grantAccess("uuid-1", "conv-guest")); + verify(filesCollection).updateOne(any(Bson.class), any(Bson.class)); } @Test - void load_invalidStorageRef_throwsException() { - // given — not a valid ObjectId - var ex = assertThrows(AttachmentStoreException.class, - () -> sut.load("not-a-valid-object-id", "conv-1")); - assertTrue(ex.getMessage().contains("Invalid storage reference")); + void grantAccess_notFound_throws() { + UpdateResult result = mock(UpdateResult.class); + when(result.getMatchedCount()).thenReturn(0L); + when(filesCollection.updateOne(any(Bson.class), any(Bson.class))).thenReturn(result); + + assertThrows(AttachmentStoreException.class, () -> sut.grantAccess("missing", "conv-guest")); + } + + // ─── delete() ─────────────────────────────────────────────── + + @Test + void delete_owner_deletesAndReturnsTrue() throws Exception { + ObjectId id = new ObjectId(); + whenFindFirst(mockFile(id, "conv-1", "text/plain", List.of(), "uuid-1")); + assertTrue(sut.delete("uuid-1", "conv-1")); + verify(gridFSBucket).delete(id); + } + + @Test + void delete_notFound_returnsFalse() throws Exception { + whenFindFirst(null); + assertFalse(sut.delete("missing", "conv-1")); + } + + @Test + void delete_nonOwner_throws() { + whenFindFirst(mockFile(new ObjectId(), "conv-owner", "text/plain", List.of("conv-guest"), "uuid-1")); + // even a grantee cannot delete + assertThrows(AttachmentStoreException.class, () -> sut.delete("uuid-1", "conv-guest")); } // ─── deleteByConversation() ───────────────────────────────── @Test - void deleteByConversation_deletesMatchingFiles() throws Exception { - // given - GridFSFile file1 = mock(GridFSFile.class); - GridFSFile file2 = mock(GridFSFile.class); + void deleteByConversation_deletesMatchingFiles() { ObjectId id1 = new ObjectId(); ObjectId id2 = new ObjectId(); - when(file1.getObjectId()).thenReturn(id1); - when(file2.getObjectId()).thenReturn(id2); + GridFSFile f1 = mock(GridFSFile.class); + GridFSFile f2 = mock(GridFSFile.class); + when(f1.getObjectId()).thenReturn(id1); + when(f2.getObjectId()).thenReturn(id2); + whenFindIterate(f1, f2); - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - - @SuppressWarnings("unchecked") - MongoCursor cursor = mock(MongoCursor.class); - doReturn(cursor).when(findIterable).iterator(); - when(cursor.hasNext()).thenReturn(true, true, false); - when(cursor.next()).thenReturn(file1, file2); - - // when long count = sut.deleteByConversation("conv-1"); - - // then assertEquals(2, count); verify(gridFSBucket).delete(id1); verify(gridFSBucket).delete(id2); } @Test - void deleteByConversation_noFiles_returnsZero() throws Exception { - // given - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - - @SuppressWarnings("unchecked") - MongoCursor cursor = mock(MongoCursor.class); - doReturn(cursor).when(findIterable).iterator(); - when(cursor.hasNext()).thenReturn(false); - - // when - long count = sut.deleteByConversation("conv-empty"); - - // then - assertEquals(0, count); + void deleteByConversation_noFiles_returnsZero() { + whenFindIterate(); + assertEquals(0, sut.deleteByConversation("conv-empty")); } // ─── listByConversation() ─────────────────────────────────── @Test - void listByConversation_returnsAttachments() throws Exception { - // given + void listByConversation_returnsAttachmentsWithUuidRef() { ObjectId id1 = new ObjectId(); - GridFSFile file1 = mock(GridFSFile.class); - when(file1.getObjectId()).thenReturn(id1); - when(file1.getFilename()).thenReturn("image.png"); - when(file1.getLength()).thenReturn(1024L); - Document metadata1 = new Document().append("mimeType", "image/png"); - when(file1.getMetadata()).thenReturn(metadata1); - - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); + GridFSFile f1 = mock(GridFSFile.class); + when(f1.getObjectId()).thenReturn(id1); + when(f1.getFilename()).thenReturn("image.png"); + when(f1.getLength()).thenReturn(1024L); + when(f1.getMetadata()).thenReturn(new Document() + .append("mimeType", "image/png").append("storageRef", "uuid-1")); + whenFindIterate(f1); - @SuppressWarnings("unchecked") - MongoCursor cursor = mock(MongoCursor.class); - doReturn(cursor).when(findIterable).iterator(); - when(cursor.hasNext()).thenReturn(true, false); - when(cursor.next()).thenReturn(file1); - - // when List results = sut.listByConversation("conv-1"); - - // then assertEquals(1, results.size()); - assertEquals(id1.toHexString(), results.getFirst().storageRef()); + assertEquals("uuid-1", results.getFirst().storageRef()); assertEquals("image.png", results.getFirst().filename()); assertEquals("image/png", results.getFirst().mimeType()); assertEquals(1024L, results.getFirst().sizeBytes()); @@ -334,47 +384,24 @@ void listByConversation_returnsAttachments() throws Exception { } @Test - void listByConversation_nullMetadata_usesDefaultMime() throws Exception { - // given + void listByConversation_nullMetadata_usesDefaultsAndObjectIdRef() { ObjectId id1 = new ObjectId(); - GridFSFile file1 = mock(GridFSFile.class); - when(file1.getObjectId()).thenReturn(id1); - when(file1.getFilename()).thenReturn("unknown.bin"); - when(file1.getLength()).thenReturn(512L); - when(file1.getMetadata()).thenReturn(null); + GridFSFile f1 = mock(GridFSFile.class); + when(f1.getObjectId()).thenReturn(id1); + when(f1.getFilename()).thenReturn("unknown.bin"); + when(f1.getLength()).thenReturn(512L); + when(f1.getMetadata()).thenReturn(null); + whenFindIterate(f1); - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - - @SuppressWarnings("unchecked") - MongoCursor cursor = mock(MongoCursor.class); - doReturn(cursor).when(findIterable).iterator(); - when(cursor.hasNext()).thenReturn(true, false); - when(cursor.next()).thenReturn(file1); - - // when List results = sut.listByConversation("conv-1"); - - // then assertEquals(1, results.size()); assertEquals("application/octet-stream", results.getFirst().mimeType()); + assertEquals(id1.toHexString(), results.getFirst().storageRef()); } @Test - void listByConversation_emptyResults() throws Exception { - // given - GridFSFindIterable findIterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(findIterable); - - @SuppressWarnings("unchecked") - MongoCursor cursor = mock(MongoCursor.class); - doReturn(cursor).when(findIterable).iterator(); - when(cursor.hasNext()).thenReturn(false); - - // when - List results = sut.listByConversation("conv-empty"); - - // then - assertTrue(results.isEmpty()); + void listByConversation_emptyResults() { + whenFindIterate(); + assertTrue(sut.listByConversation("conv-empty").isEmpty()); } } diff --git a/src/test/java/ai/labs/eddi/datastore/mongo/MongoAttachmentStorageTest.java b/src/test/java/ai/labs/eddi/datastore/mongo/MongoAttachmentStorageTest.java deleted file mode 100644 index 28620f5164..0000000000 --- a/src/test/java/ai/labs/eddi/datastore/mongo/MongoAttachmentStorageTest.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.datastore.mongo; - -import ai.labs.eddi.engine.memory.IAttachmentStorage.AttachmentNotFoundException; -import ai.labs.eddi.engine.memory.mongo.MongoAttachmentStorage; -import org.junit.jupiter.api.*; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Integration tests for {@link MongoAttachmentStorage} (GridFS) using - * Testcontainers. - * - * @since 6.0.0 - */ -@DisplayName("MongoAttachmentStorage IT") -class MongoAttachmentStorageTest extends MongoTestBase { - - private static MongoAttachmentStorage storage; - - @BeforeAll - static void init() { - storage = new MongoAttachmentStorage(getDatabase()); - } - - @BeforeEach - void clean() { - // Drop GridFS collections - dropCollections("eddi_attachments.files", "eddi_attachments.chunks"); - } - - @Test - @DisplayName("store + load — binary round-trip") - void storeAndLoad() throws AttachmentNotFoundException, IOException { - byte[] content = "Hello GridFS!".getBytes(StandardCharsets.UTF_8); - String ref = storage.store("conv-1", "test.txt", "text/plain", - new ByteArrayInputStream(content), content.length); - - assertNotNull(ref); - assertTrue(ref.startsWith("gridfs://")); - - try (InputStream loaded = storage.load(ref)) { - assertArrayEquals(content, loaded.readAllBytes()); - } - } - - @Test - @DisplayName("store with null fileName — uses 'unnamed'") - void storeNullFilename() throws AttachmentNotFoundException, IOException { - byte[] content = "data".getBytes(StandardCharsets.UTF_8); - String ref = storage.store("conv-1", null, "application/octet-stream", - new ByteArrayInputStream(content), content.length); - - try (InputStream loaded = storage.load(ref)) { - assertArrayEquals(content, loaded.readAllBytes()); - } - } - - @Test - @DisplayName("load non-existent — throws AttachmentNotFoundException") - void loadNonExistent() { - assertThrows(AttachmentNotFoundException.class, - () -> storage.load("gridfs://aaaaaaaaaaaaaaaaaaaaaaaa")); - } - - @Test - @DisplayName("load invalid ref — throws AttachmentNotFoundException") - void loadInvalidRef() { - assertThrows(AttachmentNotFoundException.class, - () -> storage.load("invalid-ref")); - } - - @Test - @DisplayName("load null ref — throws AttachmentNotFoundException") - void loadNullRef() { - assertThrows(AttachmentNotFoundException.class, - () -> storage.load(null)); - } - - @Test - @DisplayName("deleteByConversation — removes all for conversation") - void deleteByConversation() throws AttachmentNotFoundException { - byte[] data = "x".getBytes(StandardCharsets.UTF_8); - String ref1 = storage.store("conv-del", "a.txt", "text/plain", - new ByteArrayInputStream(data), data.length); - String ref2 = storage.store("conv-del", "b.txt", "text/plain", - new ByteArrayInputStream(data), data.length); - storage.store("conv-keep", "c.txt", "text/plain", - new ByteArrayInputStream(data), data.length); - - long deleted = storage.deleteByConversation("conv-del"); - assertEquals(2, deleted); - - assertThrows(AttachmentNotFoundException.class, () -> storage.load(ref1)); - assertThrows(AttachmentNotFoundException.class, () -> storage.load(ref2)); - } - - @Test - @DisplayName("deleteByConversation non-existent — returns 0") - void deleteNonExistent() { - assertEquals(0, storage.deleteByConversation("no-such-conv")); - } -} diff --git a/src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageTest.java b/src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageTest.java deleted file mode 100644 index b7d54f3d65..0000000000 --- a/src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.datastore.postgres; - -import ai.labs.eddi.engine.memory.IAttachmentStorage.AttachmentNotFoundException; -import org.junit.jupiter.api.*; - -import javax.sql.DataSource; -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.sql.SQLException; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Integration tests for {@link PostgresAttachmentStorage} using Testcontainers. - * - * @since 6.0.0 - */ -@DisplayName("PostgresAttachmentStorage IT") -class PostgresAttachmentStorageTest extends PostgresTestBase { - - private static PostgresAttachmentStorage storage; - private static DataSource ds; - - @BeforeAll - static void init() { - var dsInstance = createDataSourceInstance(); - ds = dsInstance.get(); - storage = new PostgresAttachmentStorage(dsInstance); - // Manually trigger schema creation (normally @PostConstruct) - storage.createTable(); - } - - @BeforeEach - void clean() { - try { - truncateTables(ds, "attachments"); - } catch (SQLException ignored) { - } - } - - @Test - @DisplayName("store + load — binary round-trip") - void storeAndLoad() throws AttachmentNotFoundException, IOException { - byte[] content = "Hello, World!".getBytes(StandardCharsets.UTF_8); - var input = new ByteArrayInputStream(content); - - String ref = storage.store("conv-1", "test.txt", "text/plain", input, content.length); - assertNotNull(ref); - assertTrue(ref.startsWith("pg://")); - - try (InputStream loaded = storage.load(ref)) { - assertArrayEquals(content, loaded.readAllBytes()); - } - } - - @Test - @DisplayName("store with zero sizeBytes — still persists") - void storeZeroSize() throws AttachmentNotFoundException, IOException { - byte[] content = "data".getBytes(StandardCharsets.UTF_8); - String ref = storage.store("conv-2", "zero.bin", "application/octet-stream", - new ByteArrayInputStream(content), 0); - - try (InputStream loaded = storage.load(ref)) { - assertArrayEquals(content, loaded.readAllBytes()); - } - } - - @Test - @DisplayName("load non-existent — throws AttachmentNotFoundException") - void loadNonExistent() { - assertThrows(AttachmentNotFoundException.class, - () -> storage.load("pg://00000000-0000-0000-0000-000000000000")); - } - - @Test - @DisplayName("load invalid ref — throws AttachmentNotFoundException") - void loadInvalidRef() { - assertThrows(AttachmentNotFoundException.class, - () -> storage.load("invalid-ref")); - } - - @Test - @DisplayName("load null ref — throws AttachmentNotFoundException") - void loadNullRef() { - assertThrows(AttachmentNotFoundException.class, - () -> storage.load(null)); - } - - @Test - @DisplayName("deleteByConversation — removes all attachments for conversation") - void deleteByConversation() throws AttachmentNotFoundException, IOException { - byte[] data = "x".getBytes(StandardCharsets.UTF_8); - String ref1 = storage.store("conv-del", "a.txt", "text/plain", - new ByteArrayInputStream(data), data.length); - String ref2 = storage.store("conv-del", "b.txt", "text/plain", - new ByteArrayInputStream(data), data.length); - storage.store("conv-keep", "c.txt", "text/plain", - new ByteArrayInputStream(data), data.length); - - long deleted = storage.deleteByConversation("conv-del"); - assertEquals(2, deleted); - - // Deleted attachments should not be loadable - assertThrows(AttachmentNotFoundException.class, () -> storage.load(ref1)); - assertThrows(AttachmentNotFoundException.class, () -> storage.load(ref2)); - } - - @Test - @DisplayName("deleteByConversation non-existent — returns 0") - void deleteNonExistent() { - assertEquals(0, storage.deleteByConversation("no-such-conv")); - } -} diff --git a/src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageUnitTest.java b/src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageUnitTest.java deleted file mode 100644 index ea12eddea8..0000000000 --- a/src/test/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStorageUnitTest.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.datastore.postgres; - -import ai.labs.eddi.engine.memory.IAttachmentStorage; -import ai.labs.eddi.engine.memory.IAttachmentStorage.AttachmentNotFoundException; -import jakarta.enterprise.inject.Instance; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.mockito.MockitoAnnotations; - -import javax.sql.DataSource; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.sql.*; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.*; - -class PostgresAttachmentStorageUnitTest { - - @Mock - private Instance dataSourceInstance; - @Mock - private DataSource dataSource; - @Mock - private Connection connection; - @Mock - private Statement statement; - @Mock - private PreparedStatement preparedStatement; - @Mock - private ResultSet resultSet; - - private PostgresAttachmentStorage storage; - - @BeforeEach - void setUp() throws Exception { - MockitoAnnotations.openMocks(this); - lenient().when(dataSourceInstance.get()).thenReturn(dataSource); - lenient().when(dataSource.getConnection()).thenReturn(connection); - lenient().when(connection.createStatement()).thenReturn(statement); - lenient().when(connection.prepareStatement(anyString())).thenReturn(preparedStatement); - lenient().when(dataSourceInstance.isResolvable()).thenReturn(true); - - storage = new PostgresAttachmentStorage(dataSourceInstance); - } - - // ─── createTable (@PostConstruct) ─── - - @Test - void createTable_whenDataSourceResolvable_createsTable() throws Exception { - storage.createTable(); - - verify(statement, times(2)).execute(anyString()); - } - - @Test - void createTable_whenDataSourceNotResolvable_skips() throws Exception { - when(dataSourceInstance.isResolvable()).thenReturn(false); - - storage.createTable(); - - verify(connection, never()).createStatement(); - } - - @Test - void createTable_sqlException_logsWarning() throws Exception { - when(statement.execute(anyString())).thenThrow(new SQLException("error")); - - assertDoesNotThrow(() -> storage.createTable()); - } - - // ─── store ─── - - @Test - void store_withPositiveSize() throws Exception { - when(preparedStatement.executeUpdate()).thenReturn(1); - InputStream data = new ByteArrayInputStream("test data".getBytes()); - - String ref = storage.store("conv-1", "test.txt", "text/plain", data, 9); - - assertNotNull(ref); - assertTrue(ref.startsWith("pg://")); - verify(preparedStatement).setBinaryStream(eq(6), eq(data), eq(9L)); - } - - @Test - void store_withZeroSize() throws Exception { - when(preparedStatement.executeUpdate()).thenReturn(1); - InputStream data = new ByteArrayInputStream("data".getBytes()); - - String ref = storage.store("conv-1", "test.txt", "text/plain", data, 0); - - assertNotNull(ref); - assertTrue(ref.startsWith("pg://")); - verify(preparedStatement).setBinaryStream(eq(6), eq(data)); - } - - @Test - void store_withNegativeSize() throws Exception { - when(preparedStatement.executeUpdate()).thenReturn(1); - InputStream data = new ByteArrayInputStream("data".getBytes()); - - String ref = storage.store("conv-1", "test.txt", "text/plain", data, -1); - - assertNotNull(ref); - verify(preparedStatement).setLong(5, 0L); // Math.max(-1, 0) = 0 - verify(preparedStatement).setBinaryStream(eq(6), eq(data)); - } - - @Test - void store_sqlException_throwsRuntimeException() throws Exception { - when(preparedStatement.executeUpdate()).thenThrow(new SQLException("DB error")); - InputStream data = new ByteArrayInputStream("data".getBytes()); - - assertThrows(RuntimeException.class, - () -> storage.store("conv-1", "test.txt", "text/plain", data, 4)); - } - - // ─── load ─── - - @Test - void load_found() throws Exception { - when(preparedStatement.executeQuery()).thenReturn(resultSet); - when(resultSet.next()).thenReturn(true); - when(resultSet.getBytes("data")).thenReturn("test content".getBytes()); - - String validUuid = "pg://" + java.util.UUID.randomUUID(); - InputStream result = storage.load(validUuid); - - assertNotNull(result); - assertTrue(result.available() > 0); - } - - @Test - void load_notFound_throwsAttachmentNotFoundException() throws Exception { - when(preparedStatement.executeQuery()).thenReturn(resultSet); - when(resultSet.next()).thenReturn(false); - - String validUuid = "pg://" + java.util.UUID.randomUUID(); - assertThrows(AttachmentNotFoundException.class, () -> storage.load(validUuid)); - } - - @Test - void load_invalidRef_nullRef_throwsAttachmentNotFoundException() { - assertThrows(AttachmentNotFoundException.class, () -> storage.load(null)); - } - - @Test - void load_invalidRef_wrongPrefix_throwsAttachmentNotFoundException() { - assertThrows(AttachmentNotFoundException.class, () -> storage.load("mongo://abc")); - } - - @Test - void load_invalidRef_badUuid_throwsAttachmentNotFoundException() { - assertThrows(AttachmentNotFoundException.class, () -> storage.load("pg://not-a-uuid")); - } - - @Test - void load_sqlException_throwsRuntimeException() throws Exception { - when(preparedStatement.executeQuery()).thenThrow(new SQLException("DB error")); - - String validUuid = "pg://" + java.util.UUID.randomUUID(); - assertThrows(RuntimeException.class, () -> storage.load(validUuid)); - } - - // ─── deleteByConversation ─── - - @Test - void deleteByConversation_deletesAndReturnsCount() throws Exception { - when(preparedStatement.executeUpdate()).thenReturn(3); - - assertEquals(3, storage.deleteByConversation("conv-1")); - verify(preparedStatement).setString(1, "conv-1"); - } - - @Test - void deleteByConversation_noneDeleted() throws Exception { - when(preparedStatement.executeUpdate()).thenReturn(0); - - assertEquals(0, storage.deleteByConversation("conv-empty")); - } - - @Test - void deleteByConversation_sqlException_throwsRuntimeException() throws Exception { - when(preparedStatement.executeUpdate()).thenThrow(new SQLException("DB error")); - - assertThrows(RuntimeException.class, () -> storage.deleteByConversation("conv-1")); - } -} diff --git a/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java b/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java index fd964ad90b..7705ea0363 100644 --- a/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java +++ b/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java @@ -8,7 +8,7 @@ import ai.labs.eddi.configs.properties.model.UserMemoryEntry; import ai.labs.eddi.engine.audit.AuditLedgerService; import ai.labs.eddi.engine.audit.IAuditStore; -import ai.labs.eddi.engine.memory.IAttachmentStorage; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemoryStore; import ai.labs.eddi.engine.memory.model.ConversationMemorySnapshot; import ai.labs.eddi.engine.memory.model.ConversationState; @@ -54,7 +54,7 @@ void setUp() { auditLedgerService = mock(AuditLedgerService.class); @SuppressWarnings("unchecked") - Instance attachmentStorageInstance = mock(Instance.class); + Instance attachmentStorageInstance = mock(Instance.class); when(attachmentStorageInstance.isResolvable()).thenReturn(false); service = new GdprComplianceService( @@ -154,9 +154,9 @@ void deleteUserData_continuesOnPartialFailure() throws Exception { @SuppressWarnings("unchecked") void deleteUserData_deletesAttachmentsWhenStorageAvailable() throws Exception { // Given — attachment storage is resolvable - Instance attachInstance = mock(Instance.class); + Instance attachInstance = mock(Instance.class); when(attachInstance.isResolvable()).thenReturn(true); - var attachmentStorage = mock(IAttachmentStorage.class); + var attachmentStorage = mock(IAttachmentStore.class); when(attachInstance.get()).thenReturn(attachmentStorage); when(attachmentStorage.deleteByConversation("conv-1")).thenReturn(2L); when(attachmentStorage.deleteByConversation("conv-2")).thenReturn(3L); @@ -187,9 +187,9 @@ void deleteUserData_deletesAttachmentsWhenStorageAvailable() throws Exception { @SuppressWarnings("unchecked") void deleteUserData_handlesAttachmentFailureGracefully() throws Exception { // Given — attachment storage throws - Instance attachInstance = mock(Instance.class); + Instance attachInstance = mock(Instance.class); when(attachInstance.isResolvable()).thenReturn(true); - var attachmentStorage = mock(IAttachmentStorage.class); + var attachmentStorage = mock(IAttachmentStore.class); when(attachInstance.get()).thenReturn(attachmentStorage); var serviceWithAttachments = new GdprComplianceService( diff --git a/src/test/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorageTest.java b/src/test/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorageTest.java deleted file mode 100644 index 6144944805..0000000000 --- a/src/test/java/ai/labs/eddi/engine/memory/mongo/MongoAttachmentStorageTest.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.engine.memory.mongo; - -import ai.labs.eddi.engine.memory.IAttachmentStorage; -import com.mongodb.client.gridfs.GridFSBucket; -import com.mongodb.client.gridfs.GridFSBuckets; -import com.mongodb.client.gridfs.GridFSFindIterable; -import com.mongodb.client.gridfs.model.GridFSFile; -import com.mongodb.client.gridfs.model.GridFSUploadOptions; -import com.mongodb.client.MongoDatabase; -import org.bson.BsonObjectId; -import org.bson.conversions.Bson; -import org.bson.types.ObjectId; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; - -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.util.function.Consumer; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - -@SuppressWarnings("unchecked") -class MongoAttachmentStorageTest { - - private static final String VALID_ID = "aabbccddeeff112233445566"; - private static final ObjectId TEST_OID = new ObjectId(VALID_ID); - - private GridFSBucket gridFSBucket; - private MongoAttachmentStorage storage; - - @BeforeEach - void setUp() { - gridFSBucket = mock(GridFSBucket.class); - MongoDatabase database = mock(MongoDatabase.class); - - try (MockedStatic mocked = mockStatic(GridFSBuckets.class)) { - mocked.when(() -> GridFSBuckets.create(database, "eddi_attachments")).thenReturn(gridFSBucket); - storage = new MongoAttachmentStorage(database); - } - } - - // ==================== store ==================== - - @Test - @DisplayName("store — uploads and returns gridfs:// reference") - void storeAttachment() { - when(gridFSBucket.uploadFromStream(anyString(), any(InputStream.class), any(GridFSUploadOptions.class))) - .thenReturn(TEST_OID); - - InputStream data = new ByteArrayInputStream("hello".getBytes()); - String ref = storage.store("conv-1", "file.txt", "text/plain", data, 5); - - assertEquals("gridfs://" + VALID_ID, ref); - verify(gridFSBucket).uploadFromStream(eq("file.txt"), eq(data), any(GridFSUploadOptions.class)); - } - - @Test - @DisplayName("store — uses 'unnamed' when fileName is null") - void storeNullFileName() { - when(gridFSBucket.uploadFromStream(anyString(), any(InputStream.class), any(GridFSUploadOptions.class))) - .thenReturn(TEST_OID); - - InputStream data = new ByteArrayInputStream("data".getBytes()); - storage.store("conv-1", null, "application/octet-stream", data, 0); - - verify(gridFSBucket).uploadFromStream(eq("unnamed"), any(InputStream.class), any(GridFSUploadOptions.class)); - } - - // ==================== load ==================== - - @Test - @DisplayName("load — returns input stream when file exists") - void loadFound() throws Exception { - GridFSFile gridFSFile = mock(GridFSFile.class); - GridFSFindIterable iterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(iterable); - when(iterable.first()).thenReturn(gridFSFile); - - var downloadStream = mock(com.mongodb.client.gridfs.GridFSDownloadStream.class); - when(gridFSBucket.openDownloadStream(any(ObjectId.class))).thenReturn(downloadStream); - - InputStream result = storage.load("gridfs://" + VALID_ID); - assertNotNull(result); - } - - @Test - @DisplayName("load — throws AttachmentNotFoundException when file not found") - void loadNotFound() { - GridFSFindIterable iterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(iterable); - when(iterable.first()).thenReturn(null); - - assertThrows(IAttachmentStorage.AttachmentNotFoundException.class, - () -> storage.load("gridfs://" + VALID_ID)); - } - - @Test - @DisplayName("load — throws on null storageRef") - void loadNullRef() { - assertThrows(IAttachmentStorage.AttachmentNotFoundException.class, - () -> storage.load(null)); - } - - @Test - @DisplayName("load — throws on invalid prefix") - void loadInvalidPrefix() { - assertThrows(IAttachmentStorage.AttachmentNotFoundException.class, - () -> storage.load("s3://bucket/key")); - } - - @Test - @DisplayName("load — throws on invalid ObjectId") - void loadInvalidObjectId() { - assertThrows(IAttachmentStorage.AttachmentNotFoundException.class, - () -> storage.load("gridfs://invalid-id")); - } - - // ==================== deleteByConversation ==================== - - @Test - @DisplayName("deleteByConversation — deletes matching files") - void deleteByConversation() { - GridFSFile file1 = mock(GridFSFile.class); - when(file1.getObjectId()).thenReturn(new ObjectId()); - GridFSFile file2 = mock(GridFSFile.class); - when(file2.getObjectId()).thenReturn(new ObjectId()); - - GridFSFindIterable iterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(iterable); - - @SuppressWarnings("unchecked") - com.mongodb.client.MongoCursor cursor = mock(com.mongodb.client.MongoCursor.class); - doReturn(cursor).when(iterable).iterator(); - when(cursor.hasNext()).thenReturn(true, true, false); - when(cursor.next()).thenReturn(file1, file2); - - long deleted = storage.deleteByConversation("conv-1"); - assertEquals(2, deleted); - verify(gridFSBucket, times(2)).delete(any(ObjectId.class)); - } - - @Test - @DisplayName("deleteByConversation — returns 0 when no files") - void deleteByConversationEmpty() { - GridFSFindIterable iterable = mock(GridFSFindIterable.class); - when(gridFSBucket.find(any(Bson.class))).thenReturn(iterable); - - @SuppressWarnings("unchecked") - com.mongodb.client.MongoCursor cursor = mock(com.mongodb.client.MongoCursor.class); - doReturn(cursor).when(iterable).iterator(); - when(cursor.hasNext()).thenReturn(false); - - long deleted = storage.deleteByConversation("conv-1"); - assertEquals(0, deleted); - } -} diff --git a/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreFilterTest.java b/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreFilterTest.java index 6127a47ada..aa78d51ccc 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreFilterTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreFilterTest.java @@ -7,7 +7,7 @@ import ai.labs.eddi.configs.descriptors.IDocumentDescriptorStore; import ai.labs.eddi.configs.descriptors.model.DocumentDescriptor; import ai.labs.eddi.configs.properties.IUserMemoryStore; -import ai.labs.eddi.engine.memory.IAttachmentStorage; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemoryStore; import ai.labs.eddi.engine.memory.descriptor.IConversationDescriptorStore; import ai.labs.eddi.engine.memory.descriptor.model.ConversationDescriptor; @@ -53,7 +53,7 @@ void setUp() { conversationMemoryStore = mock(IConversationMemoryStore.class); IUserMemoryStore userMemoryStore = mock(IUserMemoryStore.class); IRuntime runtime = mock(IRuntime.class); - Instance attachmentStorageInstance = mock(Instance.class); + Instance attachmentStorageInstance = mock(Instance.class); when(attachmentStorageInstance.isResolvable()).thenReturn(false); restConversationStore = new RestConversationStore( diff --git a/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreTest.java b/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreTest.java index f54165b812..225419e214 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/rest/RestConversationStoreTest.java @@ -8,7 +8,7 @@ import ai.labs.eddi.configs.descriptors.model.DocumentDescriptor; import ai.labs.eddi.configs.properties.IUserMemoryStore; import ai.labs.eddi.datastore.IResourceStore; -import ai.labs.eddi.engine.memory.IAttachmentStorage; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemoryStore; import ai.labs.eddi.engine.memory.descriptor.IConversationDescriptorStore; import ai.labs.eddi.engine.memory.descriptor.model.ConversationDescriptor; @@ -41,8 +41,8 @@ class RestConversationStoreTest { private IConversationMemoryStore conversationMemoryStore; private IUserMemoryStore userMemoryStore; private IRuntime runtime; - private Instance attachmentStorageInstance; - private IAttachmentStorage attachmentStorage; + private Instance attachmentStorageInstance; + private IAttachmentStore attachmentStorage; private RestConversationStore restConversationStore; @SuppressWarnings("unchecked") @@ -54,7 +54,7 @@ void setUp() { userMemoryStore = mock(IUserMemoryStore.class); runtime = mock(IRuntime.class); attachmentStorageInstance = mock(Instance.class); - attachmentStorage = mock(IAttachmentStorage.class); + attachmentStorage = mock(IAttachmentStore.class); when(attachmentStorageInstance.isResolvable()).thenReturn(false); restConversationStore = new RestConversationStore( From 136e6cdb97c8c604687aee46daa515699feba11c Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 01:35:14 +0200 Subject: [PATCH 07/23] feat(attachments): storageRef extraction branch + secure REST surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete Phase 1 by wiring uploads through to the pipeline and hardening the REST surface. - AttachmentContextExtractor: parse {storageRef} (precedence storageRef > url > data) and add resolveAndGuard(), which resolves each stored ref's authoritative MIME/size via IAttachmentStore.getMetadata (owner/grant authorized) before behavior rules run, enforces the per-turn cap, and records every drop/failure to attachments:errors — never silent. Fixes the "upload is orphaned" defect (STORED source was never produced). - Conversation resolves stored metadata at init via new IPropertiesHandler.getAttachmentStore()/getMaxAttachmentsPerTurn(), populated by ConversationService (field-injected to avoid touching the many direct-construction unit tests). New MemoryKeys.ATTACHMENT_ERRORS. - RestAttachmentUpload: forwardableInline hint on upload (upload cap 20MB > forward cap 10MB), single-item download endpoint (owner/grant-checked, Content-Disposition sanitized) and single-item DELETE. Auth model matches EDDI's anonymous-capable conversations: store-level owner-or-grant authz + unguessable UUID refs rather than an OIDC role gate (no other conversation endpoint uses @RolesAllowed). New config: eddi.attachments.max-per-turn (5), max-forward-bytes (10MB). +48 unit tests (extractor storageRef/resolveAndGuard, download/delete-one/ forwardableInline). Phase 1 complete. --- docs/changelog.md | 29 ++++ .../engine/internal/ConversationService.java | 18 ++ .../memory/AttachmentContextExtractor.java | 99 +++++++++++ .../engine/memory/IPropertiesHandler.java | 18 ++ .../labs/eddi/engine/memory/MemoryKeys.java | 10 ++ .../memory/rest/RestAttachmentUpload.java | 107 +++++++++++- .../engine/runtime/internal/Conversation.java | 26 ++- src/main/resources/application.properties | 6 + .../AttachmentContextExtractorTest.java | 156 ++++++++++++++++++ .../memory/rest/RestAttachmentUploadTest.java | 135 ++++++++++++++- 10 files changed, 594 insertions(+), 10 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 7621f5a52b..897cf5454c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,35 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Phase 1: Storage unification + secure upload (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) +**Plan:** `planning/multimodal-attachments-completion-plan.md` (Phase 1 of 6). + +### What changed + +1. **One blob store.** Collapsed the two parallel abstractions onto `IAttachmentStore`. Uploads already wrote to it (GridFS / Postgres `*Store`), but conversation-deletion and GDPR erasure cascaded through a *different* store (`IAttachmentStorage` → `Mongo`/`PostgresAttachmentStorage`), so uploaded blobs were never actually deleted. Ported both consumers (`RestConversationStore` delete cascade, `GdprComplianceService` erasure) to `IAttachmentStore`, then deleted `IAttachmentStorage` + both impls + their 4 tests (verified write-dead — only the delete cascades referenced them). +2. **Grants + owner-or-grant authz.** New `IAttachmentStore.getMetadata()` (server-validated metadata, no bytes), `grantAccess()` (trusted-caller-only cross-conversation read grant), single-item `delete()` (owner-only). `load()`/`getMetadata()` authorize owner **OR** an explicit grant; grants die with the blob. This is what lets group members read a blob uploaded to the group conversation (Phase 3) without opening cross-conversation access generally. +3. **UUID ref hardening (open decision #4).** GridFS now returns an unguessable random-UUID `storageRef` held in file metadata (legacy ObjectId-hex refs still resolve); Postgres already used UUIDs. Both backends unified on one opaque ref format. +4. **Quotas.** Per-conversation count + total-byte caps enforced in `store()` (`eddi.attachments.max-per-conversation` = 50, `eddi.attachments.max-total-bytes-per-conversation` = 100 MB; non-positive disables). +5. **`storageRef` extraction branch (defect #2 — upload was orphaned).** `AttachmentContextExtractor` now parses `{storageRef}` (precedence storageRef > url > data) and `resolveAndGuard()` resolves each stored ref's authoritative MIME/size via `getMetadata` (owner/grant authorized) **before** behavior rules run, enforces the per-turn cap (`eddi.attachments.max-per-turn` = 5), and records every drop/failure to `attachments:errors` — never silent. Wired into `Conversation` init via `IPropertiesHandler.getAttachmentStore()`/`getMaxAttachmentsPerTurn()` (populated by `ConversationService`). +6. **Secure REST surface.** `RestAttachmentUpload` gains a `forwardableInline` hint on upload (upload cap 20 MB > forward cap 10 MB — warn at upload, not silently at forward), a single-item download endpoint (`GET /conversations/{id}/attachments/{storageRef}`, owner/grant-checked, Content-Disposition sanitized) and single-item `DELETE`. + +### Design decisions + +- **Auth model fits EDDI's anonymous-capable conversations.** No other conversation endpoint uses `@RolesAllowed` (only admin endpoints do), and anonymous deployments must keep working (D2). Enforcement is therefore store-level owner-or-grant authorization on every `load`/`getMetadata`/`delete`, plus unguessable UUID refs — not an OIDC role gate. `@RolesAllowed` can be layered on when a deployment makes OIDC mandatory. `tenantId` stays advisory (sanitized, not an access boundary). +- **Field injection for the two new `ConversationService` deps** (attachment store + per-turn cap) so the numerous direct-construction unit tests need no change. + +### Tests + +161 unit tests across the affected classes: `GridFsAttachmentStoreTest` rewritten for UUID refs + grants + quota (26), `AttachmentContextExtractorTest` +storageRef/resolveAndGuard (27), `RestAttachmentUploadTest` +download/delete-one/forwardableInline/CD-sanitization (21), re-typed consumer tests. Postgres store IT and full ITs stay CI-only. + +### What's next + +Phase 2 — the unified `AttachmentForwarder` (replaces `MultimodalMessageEnhancer` + `convertMessage`): hybrid PDF (native `PdfFileContent` vs PDFBox text), universal text inline, uniform per-file/aggregate caps across all sources, provider image-URL normalization, capability gating via `ModelCapabilityService`, and extracts-in-history stitching. + --- ## 📎 Multimodal Attachments Completion — Phase 0: Foundations & bug fixes (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java b/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java index 1adb1d62f9..0bcd338aaa 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java @@ -85,6 +85,14 @@ public class ConversationService implements IConversationService { private final IConversationSetup conversationSetup; private final ICache conversationStateCache; + // Field-injected so the numerous direct-construction unit tests need no change; + // used only to resolve stored-attachment metadata at conversation init. + @Inject + ai.labs.eddi.engine.attachments.IAttachmentStore attachmentStore; + + @ConfigProperty(name = "eddi.attachments.max-per-turn", defaultValue = "5") + int maxAttachmentsPerTurn; + // Metrics private final Timer timerConversationStart; private final Timer timerConversationEnd; @@ -672,6 +680,16 @@ public AgentConfiguration.UserMemoryConfig getUserMemoryConfig() { public String getUserId() { return userId; } + + @Override + public ai.labs.eddi.engine.attachments.IAttachmentStore getAttachmentStore() { + return attachmentStore; + } + + @Override + public int getMaxAttachmentsPerTurn() { + return maxAttachmentsPerTurn; + } }; } diff --git a/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java b/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java index 748d5aba82..a4381b3ff8 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java +++ b/src/main/java/ai/labs/eddi/engine/memory/AttachmentContextExtractor.java @@ -4,6 +4,7 @@ */ package ai.labs.eddi.engine.memory; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.model.Context; import org.jboss.logging.Logger; @@ -49,6 +50,12 @@ public final class AttachmentContextExtractor { /** Inline base64 payload field inside an attachment context value map. */ public static final String FIELD_DATA = "data"; + /** Uploaded-blob reference field inside an attachment context value map. */ + public static final String FIELD_STORAGE_REF = "storageRef"; + + /** Default per-turn cap on the number of attachments forwarded. */ + public static final int DEFAULT_MAX_ATTACHMENTS_PER_TURN = 5; + private AttachmentContextExtractor() { // non-instantiable utility } @@ -103,6 +110,19 @@ private static Attachment parseAttachment(String contextKey, Context ctx) { Map attachMap = (Map) map; + // Stored-reference path (highest precedence). The client sends only + // {storageRef} (+ an optional fileName display hint); the authoritative + // MIME type and size are resolved from validated store metadata later + // (see resolveAndGuard), so no client-supplied MIME is trusted for + // stored blobs. + String storageRef = getStringField(attachMap, FIELD_STORAGE_REF); + if (storageRef != null && !storageRef.isBlank()) { + Attachment attachment = new Attachment(); + attachment.setStorageRef(storageRef); + attachment.setFileName(getStringField(attachMap, "fileName")); + return attachment; + } + String mimeType = getStringField(attachMap, "mimeType"); if (mimeType == null || mimeType.isBlank()) { LOGGER.warnv("Attachment context '{0}' missing required 'mimeType' field", contextKey); @@ -138,6 +158,85 @@ private static String getStringField(Map map, String key) { return value instanceof String s ? s : null; } + /** + * Result of resolving parsed attachments against the store and per-turn cap. + * + * @param attachments + * the forwardable attachments (stored refs resolved) + * @param errors + * human-readable notes for dropped/failed attachments (never silent) + */ + public record ExtractionResult(List attachments, List errors) { + } + + /** + * Resolve server-side metadata for {@link Attachment.ContentSource#STORED} + * attachments and enforce the per-turn count cap. + *

+ * For each stored reference, {@link IAttachmentStore#getMetadata} supplies the + * authoritative MIME type / size (owner-or-grant authorized), so behavior rules + * and the forwarder see the truth rather than client-declared values. URL and + * inline attachments pass through unchanged. Anything dropped — an unresolvable + * reference, a missing store, or an attachment beyond the per-turn cap — is + * reported in {@link ExtractionResult#errors()} and never silently discarded. + * + * @param parsed + * attachments from {@link #extractAttachments(Map)} + * @param store + * the attachment store (may be null if none configured) + * @param conversationId + * the requesting conversation (authorization boundary) + * @param maxPerTurn + * per-turn cap; non-positive means unlimited + * @return resolved attachments plus error notes + */ + public static ExtractionResult resolveAndGuard(List parsed, IAttachmentStore store, + String conversationId, int maxPerTurn) { + List out = new ArrayList<>(); + List errors = new ArrayList<>(); + int cap = maxPerTurn > 0 ? maxPerTurn : Integer.MAX_VALUE; + + for (Attachment att : parsed) { + if (out.size() >= cap) { + errors.add("Attachment '" + displayName(att) + "' skipped: per-turn limit of " + + maxPerTurn + " attachment(s) reached."); + continue; + } + if (att.getContentSource() == Attachment.ContentSource.STORED) { + if (store == null) { + errors.add("Stored attachment '" + att.getStorageRef() + + "' could not be resolved: no attachment store is configured."); + continue; + } + try { + IAttachmentStore.Attachment meta = store.getMetadata(att.getStorageRef(), conversationId); + att.setMimeType(meta.mimeType()); + if (att.getFileName() == null) { + att.setFileName(meta.filename()); + } + att.setSizeBytes(meta.sizeBytes()); + out.add(att); + } catch (IAttachmentStore.AttachmentStoreException e) { + errors.add("Stored attachment '" + att.getStorageRef() + + "' could not be resolved: " + e.getMessage()); + } + } else { + out.add(att); + } + } + return new ExtractionResult(out, errors); + } + + private static String displayName(Attachment att) { + if (att.getFileName() != null) { + return att.getFileName(); + } + if (att.getStorageRef() != null) { + return att.getStorageRef(); + } + return att.getMimeType() != null ? att.getMimeType() : "attachment"; + } + /** * Return a metadata-only copy of an {@code attachment_*} context whose value * map carries an inline base64 {@link #FIELD_DATA} payload; every other context diff --git a/src/main/java/ai/labs/eddi/engine/memory/IPropertiesHandler.java b/src/main/java/ai/labs/eddi/engine/memory/IPropertiesHandler.java index 3e2d55778f..7f735e0a78 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/IPropertiesHandler.java +++ b/src/main/java/ai/labs/eddi/engine/memory/IPropertiesHandler.java @@ -6,6 +6,7 @@ import ai.labs.eddi.configs.agents.model.AgentConfiguration; import ai.labs.eddi.configs.properties.IUserMemoryStore; +import ai.labs.eddi.engine.attachments.IAttachmentStore; /** * Bridge between the conversation engine and user-scoped storage. Provides the @@ -29,4 +30,21 @@ default AgentConfiguration.UserMemoryConfig getUserMemoryConfig() { /** The userId this handler is scoped to. */ String getUserId(); + + /** + * Attachment blob store, used at conversation init to resolve server-side + * metadata for {@code storageRef}-only attachment references. {@code null} when + * no store is configured. + */ + default IAttachmentStore getAttachmentStore() { + return null; + } + + /** + * Per-turn cap on the number of attachments forwarded to the LLM. Defaults to + * {@link ai.labs.eddi.engine.memory.AttachmentContextExtractor#DEFAULT_MAX_ATTACHMENTS_PER_TURN}. + */ + default int getMaxAttachmentsPerTurn() { + return AttachmentContextExtractor.DEFAULT_MAX_ATTACHMENTS_PER_TURN; + } } diff --git a/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java b/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java index f1a1638cfb..182aaefb7d 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java +++ b/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java @@ -104,4 +104,14 @@ private MemoryKeys() { * @since 6.0.0 */ public static final MemoryKey> ATTACHMENTS = MemoryKey.ofPublic("attachments"); + + /** + * Human-readable notes for attachments that were dropped, skipped, or failed to + * resolve/forward this turn (unresolvable stored ref, per-turn cap reached, + * capability gate, oversize). Non-public — surfaced to the LLM as a note and + * available for audit, never silently discarded. + * + * @since 6.1.0 + */ + public static final MemoryKey> ATTACHMENT_ERRORS = MemoryKey.of("attachments:errors"); } diff --git a/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java b/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java index d89a05e918..a0d31c3fb2 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java +++ b/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java @@ -56,15 +56,19 @@ public class RestAttachmentUpload { private final IAttachmentStore attachmentStore; private final ManagedExecutor managedExecutor; private final long maxUploadBytes; + private final long maxForwardBytes; @Inject public RestAttachmentUpload(IAttachmentStore attachmentStore, ManagedExecutor managedExecutor, @ConfigProperty(name = "eddi.attachments.max-size-bytes", - defaultValue = "20971520") long maxUploadBytes) { + defaultValue = "20971520") long maxUploadBytes, + @ConfigProperty(name = "eddi.attachments.max-forward-bytes", + defaultValue = "10485760") long maxForwardBytes) { this.attachmentStore = attachmentStore; this.managedExecutor = managedExecutor; this.maxUploadBytes = maxUploadBytes; + this.maxForwardBytes = maxForwardBytes; } @POST @@ -136,7 +140,11 @@ public void uploadAttachment( "fileName", attachment.filename() != null ? attachment.filename() : "", "mimeType", attachment.mimeType(), "sizeBytes", attachment.sizeBytes(), - "conversationId", attachment.conversationId())) + "conversationId", attachment.conversationId(), + // Uploads may be larger than what is inlined to the LLM: warn now, + // not silently at forward time. Oversize files remain retrievable + // via the readAttachment tool / download endpoint. + "forwardableInline", attachment.sizeBytes() <= maxForwardBytes)) .build()); } catch (IAttachmentStore.AttachmentStoreException e) { @@ -191,6 +199,91 @@ public void listAttachments( }, managedExecutor); } + @GET + @Path("/{conversationId}/attachments/{storageRef}") + @Operation( + operationId = "downloadAttachment", + summary = "Download a single attachment", + description = "Streams the raw bytes of one attachment. Access is checked against " + + "the owning conversation (owner or explicit grant); references are " + + "unguessable.") + @APIResponse(responseCode = "200", description = "Attachment bytes with Content-Type and Content-Disposition.") + @APIResponse(responseCode = "403", description = "The conversation is not permitted to access this attachment.") + @APIResponse(responseCode = "404", description = "Attachment not found.") + public void downloadAttachment( + @Parameter(description = "Owning conversation ID.") + @PathParam("conversationId") String conversationId, + @Parameter(description = "Storage reference of the attachment.") + @PathParam("storageRef") String storageRef, + @Suspended AsyncResponse asyncResponse) { + CompletableFuture.runAsync(() -> { + try { + Attachment meta = attachmentStore.getMetadata(storageRef, conversationId); + byte[] bytes = attachmentStore.load(storageRef, conversationId); + String downloadName = sanitizeContentDisposition( + meta.filename() != null ? meta.filename() : "attachment"); + asyncResponse.resume(Response.ok(bytes) + .header("Content-Type", meta.mimeType() != null ? meta.mimeType() : "application/octet-stream") + .header("Content-Disposition", "attachment; filename=\"" + downloadName + "\"") + .build()); + } catch (IAttachmentStore.AttachmentStoreException e) { + boolean denied = e.getMessage() != null && e.getMessage().contains("denied"); + var status = denied ? Response.Status.FORBIDDEN : Response.Status.NOT_FOUND; + LOGGER.debugf("Attachment download %s for conversation '%s': %s", + denied ? "denied" : "not found", sanitize(conversationId), e.getMessage()); + asyncResponse.resume(Response.status(status) + .entity(Map.of("error", e.getMessage(), "code", + denied ? "ATTACHMENT_ACCESS_DENIED" : "ATTACHMENT_NOT_FOUND")) + .build()); + } catch (Exception e) { + LOGGER.errorf(e, "Failed to download attachment for conversation '%s'", sanitize(conversationId)); + asyncResponse.resume(Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Map.of("error", "Failed to download attachment")).build()); + } + }, managedExecutor); + } + + @DELETE + @Path("/{conversationId}/attachments/{storageRef}") + @Produces(MediaType.APPLICATION_JSON) + @Operation( + operationId = "deleteAttachment", + summary = "Delete a single attachment", + description = "Removes one attachment. Restricted to the owning conversation.") + @APIResponse(responseCode = "200", description = "Attachment deleted.") + @APIResponse(responseCode = "403", description = "The conversation does not own this attachment.") + @APIResponse(responseCode = "404", description = "Attachment not found.") + public void deleteAttachment( + @Parameter(description = "Owning conversation ID.") + @PathParam("conversationId") String conversationId, + @Parameter(description = "Storage reference of the attachment.") + @PathParam("storageRef") String storageRef, + @Suspended AsyncResponse asyncResponse) { + CompletableFuture.runAsync(() -> { + try { + boolean deleted = attachmentStore.delete(storageRef, conversationId); + if (deleted) { + asyncResponse.resume(Response.ok(Map.of( + "storageRef", storageRef, "deleted", true)).build()); + } else { + asyncResponse.resume(Response.status(Response.Status.NOT_FOUND) + .entity(Map.of("error", "Attachment not found", "code", "ATTACHMENT_NOT_FOUND")) + .build()); + } + } catch (IAttachmentStore.AttachmentStoreException e) { + LOGGER.debugf("Attachment delete denied for conversation '%s': %s", + sanitize(conversationId), e.getMessage()); + asyncResponse.resume(Response.status(Response.Status.FORBIDDEN) + .entity(Map.of("error", e.getMessage(), "code", "ATTACHMENT_ACCESS_DENIED")) + .build()); + } catch (Exception e) { + LOGGER.errorf(e, "Failed to delete attachment for conversation '%s'", sanitize(conversationId)); + asyncResponse.resume(Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Map.of("error", "Failed to delete attachment")).build()); + } + }, managedExecutor); + } + @DELETE @Path("/{conversationId}/attachments") @Produces(MediaType.APPLICATION_JSON) @@ -231,4 +324,14 @@ private static String sanitizeTenantId(String tenantId) { } return tenantId; } + + /** + * Strip characters that could break out of the quoted + * {@code Content-Disposition} filename or inject headers (quotes, backslashes, + * control characters). + */ + private static String sanitizeContentDisposition(String filename) { + String cleaned = filename.replaceAll("[\"\\\\\\r\\n]", "_").trim(); + return cleaned.isEmpty() ? "attachment" : cleaned; + } } diff --git a/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java b/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java index 61ec5cd797..7cb2f0666b 100644 --- a/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java +++ b/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java @@ -233,12 +233,26 @@ private List> prepareLifecycleData(String message, Map addContextToConversationOutput(currentStep, contextData); removedTaskTypeResultsFromPreviousRuns(currentStep, taskTypeResultsToBeRemoved); - // Extract attachments from context (attachment_0, attachment_1, etc.) - var attachments = AttachmentContextExtractor.extractAttachments(contexts); - if (!attachments.isEmpty()) { - var data = new Data<>(MemoryKeys.ATTACHMENTS.key(), attachments); - data.setPublic(true); - currentStep.storeData(data); + // Extract attachments from context (attachment_0, attachment_1, etc.), + // resolve stored-blob metadata (owner/grant authorized) and enforce the + // per-turn cap. Failures surface as attachments:errors — never silent. + var parsedAttachments = AttachmentContextExtractor.extractAttachments(contexts); + if (!parsedAttachments.isEmpty()) { + var extraction = AttachmentContextExtractor.resolveAndGuard( + parsedAttachments, propertiesHandler.getAttachmentStore(), + conversationMemory.getConversationId(), propertiesHandler.getMaxAttachmentsPerTurn()); + + if (!extraction.errors().isEmpty()) { + extraction.errors().forEach(err -> LOGGER.warnv("Attachment issue: {0}", err)); + var errorData = new Data<>(MemoryKeys.ATTACHMENT_ERRORS.key(), extraction.errors()); + errorData.setPublic(false); + currentStep.storeData(errorData); + } + if (!extraction.attachments().isEmpty()) { + var data = new Data<>(MemoryKeys.ATTACHMENTS.key(), extraction.attachments()); + data.setPublic(true); + currentStep.storeData(data); + } } boolean isSecretInput = isSecretInputFlagged(contexts); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 5e8150f306..40a4fd3fd0 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -135,6 +135,12 @@ quarkus.http.limits.max-body-size=25M # Max characters of text extracted from a PDF / text attachment before truncation # (shared by PdfReaderTool, the attachment forwarder and the readAttachment tool). eddi.attachments.extraction.max-chars=10000 +# Per-turn cap on how many attachments are forwarded to the LLM; extras are dropped +# with an attachments:errors note (never silently). Non-positive means unlimited. +eddi.attachments.max-per-turn=5 +# Per-conversation storage quotas (0 or negative disables that limit). +eddi.attachments.max-per-conversation=50 +eddi.attachments.max-total-bytes-per-conversation=104857600 # Jackson JSON json.prettyPrint=false quarkus.jackson.write-dates-as-timestamps=true diff --git a/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java b/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java index 621c1075c9..3498032f93 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/AttachmentContextExtractorTest.java @@ -4,6 +4,7 @@ */ package ai.labs.eddi.engine.memory; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.model.Context; import org.junit.jupiter.api.Nested; @@ -14,6 +15,10 @@ import java.util.Map; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; /** * Unit tests for {@link AttachmentContextExtractor}. @@ -169,6 +174,157 @@ void shouldPreferUrlOverBase64() { } } + // ==================== Stored References ==================== + + @Nested + class StoredReferences { + + @Test + void shouldExtractStorageRefAttachment() { + Map contexts = new HashMap<>(); + contexts.put("attachment_0", createContext(Map.of( + "storageRef", "abc-123", + "fileName", "report.pdf"))); + + List result = AttachmentContextExtractor.extractAttachments(contexts); + + assertEquals(1, result.size()); + Attachment att = result.get(0); + assertEquals("abc-123", att.getStorageRef()); + assertEquals("report.pdf", att.getFileName()); + assertEquals(Attachment.ContentSource.STORED, att.getContentSource()); + // client MIME is never trusted for stored refs — resolved from store later + assertNull(att.getMimeType()); + } + + @Test + void shouldPreferStorageRefOverUrlAndData() { + Map map = new HashMap<>(); + map.put("storageRef", "ref-1"); + map.put("url", "https://example.com/x.png"); + map.put("data", "base64garbage"); + map.put("mimeType", "image/png"); + Map contexts = new HashMap<>(); + contexts.put("attachment_0", createContext(map)); + + List result = AttachmentContextExtractor.extractAttachments(contexts); + + assertEquals(1, result.size()); + assertEquals(Attachment.ContentSource.STORED, result.get(0).getContentSource()); + assertEquals("ref-1", result.get(0).getStorageRef()); + assertNull(result.get(0).getUrl()); + assertNull(result.get(0).getBase64Data()); + } + } + + // ==================== resolveAndGuard ==================== + + @Nested + class ResolveAndGuard { + + @Test + void shouldResolveStoredMetadataFromStore() throws Exception { + var stored = new Attachment(); + stored.setStorageRef("ref-1"); + var store = mock(IAttachmentStore.class); + when(store.getMetadata("ref-1", "conv-1")).thenReturn( + new IAttachmentStore.Attachment("ref-1", "doc.pdf", "application/pdf", 2048, "conv-1")); + + var result = AttachmentContextExtractor.resolveAndGuard(List.of(stored), store, "conv-1", 5); + + assertTrue(result.errors().isEmpty()); + assertEquals(1, result.attachments().size()); + Attachment resolved = result.attachments().get(0); + assertEquals("application/pdf", resolved.getMimeType()); + assertEquals("doc.pdf", resolved.getFileName()); + assertEquals(2048, resolved.getSizeBytes()); + } + + @Test + void shouldKeepClientFileNameHint() throws Exception { + var stored = new Attachment(); + stored.setStorageRef("ref-1"); + stored.setFileName("client-name.pdf"); + var store = mock(IAttachmentStore.class); + when(store.getMetadata(eq("ref-1"), any())).thenReturn( + new IAttachmentStore.Attachment("ref-1", "server-name.pdf", "application/pdf", 10, "conv-1")); + + var result = AttachmentContextExtractor.resolveAndGuard(List.of(stored), store, "conv-1", 5); + + assertEquals("client-name.pdf", result.attachments().get(0).getFileName()); + } + + @Test + void shouldRecordErrorWhenNoStoreConfigured() { + var stored = new Attachment(); + stored.setStorageRef("ref-1"); + + var result = AttachmentContextExtractor.resolveAndGuard(List.of(stored), null, "conv-1", 5); + + assertTrue(result.attachments().isEmpty()); + assertEquals(1, result.errors().size()); + assertTrue(result.errors().get(0).contains("no attachment store")); + } + + @Test + void shouldRecordErrorWhenResolutionFails() throws Exception { + var stored = new Attachment(); + stored.setStorageRef("bad-ref"); + var store = mock(IAttachmentStore.class); + when(store.getMetadata(any(), any())) + .thenThrow(new IAttachmentStore.AttachmentStoreException("Attachment not found: bad-ref")); + + var result = AttachmentContextExtractor.resolveAndGuard(List.of(stored), store, "conv-1", 5); + + assertTrue(result.attachments().isEmpty()); + assertEquals(1, result.errors().size()); + assertTrue(result.errors().get(0).contains("could not be resolved")); + } + + @Test + void shouldPassThroughUrlAndBase64Unchanged() { + var url = new Attachment(); + url.setMimeType("image/png"); + url.setUrl("https://example.com/x.png"); + var b64 = new Attachment(); + b64.setMimeType("image/png"); + b64.setBase64Data("iVBOR"); + + var result = AttachmentContextExtractor.resolveAndGuard(List.of(url, b64), null, "conv-1", 5); + + assertTrue(result.errors().isEmpty()); + assertEquals(2, result.attachments().size()); + } + + @Test + void shouldEnforcePerTurnCap() { + var a = urlAttachment(); + var b = urlAttachment(); + var c = urlAttachment(); + + var result = AttachmentContextExtractor.resolveAndGuard(List.of(a, b, c), null, "conv-1", 2); + + assertEquals(2, result.attachments().size()); + assertEquals(1, result.errors().size()); + assertTrue(result.errors().get(0).contains("per-turn limit")); + } + + @Test + void shouldTreatNonPositiveCapAsUnlimited() { + var list = List.of(urlAttachment(), urlAttachment(), urlAttachment()); + var result = AttachmentContextExtractor.resolveAndGuard(list, null, "conv-1", 0); + assertEquals(3, result.attachments().size()); + assertTrue(result.errors().isEmpty()); + } + + private Attachment urlAttachment() { + var a = new Attachment(); + a.setMimeType("image/png"); + a.setUrl("https://example.com/x.png"); + return a; + } + } + // ==================== Payload Scrubbing (persistence) ==================== @Nested diff --git a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java index 8c06d2942a..d9659f034c 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java @@ -33,6 +33,7 @@ class RestAttachmentUploadTest { private static final long MAX_UPLOAD_BYTES = 20 * 1024 * 1024; // 20MB + private static final long MAX_FORWARD_BYTES = 10 * 1024 * 1024; // 10MB private IAttachmentStore attachmentStore; private ManagedExecutor managedExecutor; @@ -42,7 +43,7 @@ class RestAttachmentUploadTest { void setUp() { attachmentStore = mock(IAttachmentStore.class); managedExecutor = ManagedExecutor.builder().build(); - endpoint = new RestAttachmentUpload(attachmentStore, managedExecutor, MAX_UPLOAD_BYTES); + endpoint = new RestAttachmentUpload(attachmentStore, managedExecutor, MAX_UPLOAD_BYTES, MAX_FORWARD_BYTES); } /** @@ -114,6 +115,35 @@ void shouldReturn201OnSuccessfulUpload() throws Exception { assertEquals("image/png", body.get("mimeType")); assertEquals(42L, body.get("sizeBytes")); assertEquals("conv-1", body.get("conversationId")); + assertEquals(true, body.get("forwardableInline")); + + Files.deleteIfExists(tempFile); + } + + @Test + void shouldMarkOversizeUploadNotForwardableInline() throws Exception { + // store reports an 11 MB blob (> 10 MB forward cap) though the temp file is + // tiny + var attachment = new Attachment( + "ref-big", "huge.png", "image/png", 11L * 1024 * 1024, "conv-1"); + when(attachmentStore.store(any(byte[].class), eq("image/png"), + eq("huge.png"), eq("conv-1"), isNull())) + .thenReturn(attachment); + + Path tempFile = Files.createTempFile("test-upload", ".png"); + Files.write(tempFile, new byte[10]); + + FileUpload file = mock(FileUpload.class); + when(file.fileName()).thenReturn("huge.png"); + when(file.contentType()).thenReturn("image/png"); + when(file.uploadedFile()).thenReturn(tempFile); + + Response response = captureAsync(ar -> endpoint.uploadAttachment("conv-1", file, null, ar)); + + assertEquals(201, response.getStatus()); + @SuppressWarnings("unchecked") + var body = (Map) response.getEntity(); + assertEquals(false, body.get("forwardableInline")); Files.deleteIfExists(tempFile); } @@ -251,7 +281,7 @@ void shouldReturn500WhenFileReadFails() throws Exception { @Test void shouldReturn400WhenFileTooLarge() throws Exception { // Create endpoint with very small max size - var smallEndpoint = new RestAttachmentUpload(attachmentStore, managedExecutor, 100); + var smallEndpoint = new RestAttachmentUpload(attachmentStore, managedExecutor, 100, MAX_FORWARD_BYTES); Path tempFile = Files.createTempFile("test-large", ".bin"); Files.write(tempFile, new byte[200]); // Exceeds 100 byte limit @@ -343,4 +373,105 @@ void shouldReturnZeroWhenNoAttachmentsToDelete() throws Exception { assertEquals(0L, body.get("deletedCount")); } } + + // ==================== Download Tests ==================== + + @Nested + class DownloadTests { + + @Test + void shouldStreamBytesWithHeaders() throws Exception { + var meta = new Attachment("ref-1", "doc.pdf", "application/pdf", 4, "conv-1"); + when(attachmentStore.getMetadata("ref-1", "conv-1")).thenReturn(meta); + when(attachmentStore.load("ref-1", "conv-1")).thenReturn(new byte[]{1, 2, 3, 4}); + + Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-1", "ref-1", ar)); + + assertEquals(200, response.getStatus()); + assertInstanceOf(byte[].class, response.getEntity()); + assertEquals("application/pdf", response.getHeaderString("Content-Type")); + assertTrue(response.getHeaderString("Content-Disposition").contains("doc.pdf")); + } + + @Test + void shouldReturn404WhenNotFound() throws Exception { + when(attachmentStore.getMetadata("missing", "conv-1")) + .thenThrow(new AttachmentStoreException("Attachment not found: missing")); + + Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-1", "missing", ar)); + + assertEquals(404, response.getStatus()); + @SuppressWarnings("unchecked") + var body = (Map) response.getEntity(); + assertEquals("ATTACHMENT_NOT_FOUND", body.get("code")); + } + + @Test + void shouldReturn403WhenDenied() throws Exception { + when(attachmentStore.getMetadata("ref-1", "conv-other")) + .thenThrow(new AttachmentStoreException( + "Cross-conversation access denied: attachment belongs to 'conv-1', requested from 'conv-other'")); + + Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-other", "ref-1", ar)); + + assertEquals(403, response.getStatus()); + @SuppressWarnings("unchecked") + var body = (Map) response.getEntity(); + assertEquals("ATTACHMENT_ACCESS_DENIED", body.get("code")); + } + + @Test + void shouldSanitizeContentDispositionFilename() throws Exception { + var meta = new Attachment("ref-1", "bad\"name\r\n.png", "image/png", 2, "conv-1"); + when(attachmentStore.getMetadata("ref-1", "conv-1")).thenReturn(meta); + when(attachmentStore.load("ref-1", "conv-1")).thenReturn(new byte[]{1, 2}); + + Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-1", "ref-1", ar)); + + String cd = response.getHeaderString("Content-Disposition"); + assertFalse(cd.contains("\"" + "name"), "quotes must be stripped from filename"); + assertFalse(cd.contains("\r") || cd.contains("\n"), "CR/LF must be stripped"); + } + } + + // ==================== Delete-One Tests ==================== + + @Nested + class DeleteOneTests { + + @Test + void shouldDeleteAndReturnTrue() throws Exception { + when(attachmentStore.delete("ref-1", "conv-1")).thenReturn(true); + + Response response = captureAsync(ar -> endpoint.deleteAttachment("conv-1", "ref-1", ar)); + + assertEquals(200, response.getStatus()); + @SuppressWarnings("unchecked") + var body = (Map) response.getEntity(); + assertEquals(true, body.get("deleted")); + } + + @Test + void shouldReturn404WhenNotFound() throws Exception { + when(attachmentStore.delete("missing", "conv-1")).thenReturn(false); + + Response response = captureAsync(ar -> endpoint.deleteAttachment("conv-1", "missing", ar)); + + assertEquals(404, response.getStatus()); + } + + @Test + void shouldReturn403WhenNotOwner() throws Exception { + when(attachmentStore.delete("ref-1", "conv-other")) + .thenThrow(new AttachmentStoreException( + "Delete denied: attachment belongs to 'conv-1', requested from 'conv-other'")); + + Response response = captureAsync(ar -> endpoint.deleteAttachment("conv-other", "ref-1", ar)); + + assertEquals(403, response.getStatus()); + @SuppressWarnings("unchecked") + var body = (Map) response.getEntity(); + assertEquals("ATTACHMENT_ACCESS_DENIED", body.get("code")); + } + } } From b6c90f74a5c364b76bdc5d80dbe574c6bed9489a Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 01:49:41 +0200 Subject: [PATCH 08/23] feat(attachments): unified AttachmentForwarder (hybrid PDF, text inline, caps) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the image-only MultimodalMessageEnhancer with AttachmentForwarder — the single place attachments become langchain4j Content on the outgoing user message. Per attachment it resolves bytes from any source (stored blob, URL download via SafeHttpClient, base64 decode) under uniform per-file (10MB) + aggregate (20MB) caps across ALL sources (base64 was previously unguarded), gates on ModelCapabilityService(provider, model), and emits: - image/* -> ImageContent when vision-capable (URL passthrough when the provider fetches URLs, else download-and-inline normalization); else a note - application/pdf -> hybrid: native PdfFileContent when documents supported, else PDFBox text extraction inlined as TextContent - text/*, JSON, XML, CSV, YAML -> decoded + inlined (no capability required) - audio/* -> AudioContent when supported, else a note - else -> metadata note pointing at the readAttachment tool Extracted text -> attachments:extracts (for history stitching); every drop/skip/gate -> attachments:errors AND a relayable note, never silent. LlmTask calls the forwarder with the resolved (provider, model), field-injected + null-guarded so the six direct-construction LlmTask tests are untouched. MultimodalMessageEnhancer + its tests deleted; 18 forwarder tests cover the full branch matrix. Phase 2 (forwarder core) of multimodal-attachments-completion-plan. --- docs/changelog.md | 31 ++ .../labs/eddi/engine/memory/MemoryKeys.java | 10 + .../modules/llm/impl/AttachmentForwarder.java | 339 +++++++++++++++ .../labs/eddi/modules/llm/impl/LlmTask.java | 13 +- .../llm/impl/MultimodalMessageEnhancer.java | 224 ---------- .../llm/impl/AttachmentForwarderTest.java | 388 ++++++++++++++++++ ...MultimodalMessageEnhancerExtendedTest.java | 228 ---------- .../impl/MultimodalMessageEnhancerTest.java | 222 ---------- 8 files changed, 778 insertions(+), 677 deletions(-) create mode 100644 src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java delete mode 100644 src/main/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancer.java create mode 100644 src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java delete mode 100644 src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerExtendedTest.java delete mode 100644 src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerTest.java diff --git a/docs/changelog.md b/docs/changelog.md index 897cf5454c..22d2def839 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,37 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Phase 2: Unified forwarder (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) +**Plan:** `planning/multimodal-attachments-completion-plan.md` (Phase 2 of 6). Forwarder core. + +### What changed + +- **`AttachmentForwarder`** (`modules/llm/impl`, new) — the single place attachments become langchain4j `Content` on the outgoing user message. Replaces the image-only `MultimodalMessageEnhancer` (deleted, with its tests). Per attachment it resolves bytes from any source (stored blob → `store.load`, URL → `SafeHttpClient` download, base64 → decode) under **uniform per-file (10 MB) and aggregate (20 MB) byte caps across all sources** (the base64 path was previously unguarded), gates on `ModelCapabilityService(provider, model)`, and emits: + - `image/*` → `ImageContent` when vision-capable (URL passed through when the provider fetches URLs, else **downloaded and inlined** — provider URL normalization, D7), else a note; + - `application/pdf` → **hybrid**: native `PdfFileContent` when the model supports documents, else PDFBox text extraction inlined as `TextContent`; + - text-like (`text/*`, JSON, XML, CSV, YAML) → decoded + inlined, **no capability required** (always works); + - `audio/*` → `AudioContent` when supported, else a note; + - everything else → a metadata note pointing at the (Phase 4) `readAttachment` tool. +- Extracted text is persisted to `attachments:extracts` (for Phase-2 history stitching) and every drop/skip/gate is appended to `attachments:errors` — **never silent**; each also leaves a note the LLM can relay. +- **`LlmTask`** now calls the forwarder with the resolved `(provider, model)` instead of the static enhancer (field-injected + null-guarded so the six direct-construction `LlmTask` tests are untouched). + +### Design decisions + +- **Capability service uses the real defaults, not mocks, in tests** — the forwarder test drives the true `ModelCapabilityService` matrix (OpenAI URL-image fast path, Gemini download-and-inline, Anthropic native PDF, OpenAI PDF text-fallback, jlama no-vision note). +- **Skip ≠ silence** — a per-file/aggregate cap hit, store-load failure, or download failure records to `attachments:errors` *and* emits a `TextContent` note so the model can tell the user, rather than dropping the attachment invisibly. + +### Tests + +`AttachmentForwarderTest` (18) covers the full branch matrix incl. URL-passthrough vs download-inline, base64/stored images, PDF native vs text-fallback (with extract persistence), text inline, audio on/off, unsupported note, per-file cap, store-load failure, and no-source skip. Enhancer tests removed. + +### What's next (remaining Phase 2, then 3–6) + +Still open in Phase 2: history stitching (inject `attachments:extracts` into the rebuilt turn's user message in `ConversationHistoryBuilder`) and per-task config (`LlmConfiguration.Task.multimodal` override + `reattachTurns`). Then Phase 3 (group parity), 4 (`readAttachment` tool), 5 (UX), 6 (ops). + --- ## 📎 Multimodal Attachments Completion — Phase 1: Storage unification + secure upload (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java b/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java index 182aaefb7d..e8968345a7 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java +++ b/src/main/java/ai/labs/eddi/engine/memory/MemoryKeys.java @@ -114,4 +114,14 @@ private MemoryKeys() { * @since 6.1.0 */ public static final MemoryKey> ATTACHMENT_ERRORS = MemoryKey.of("attachments:errors"); + + /** + * Text extracted from attachments this turn (PDF text-fallback, inlined text + * documents), one entry per attachment as {@code "fileName: "}. + * Non-public — stitched into that turn's user message when history is rebuilt + * so later turns retain the content, while the visible transcript stays clean. + * + * @since 6.1.0 + */ + public static final MemoryKey> ATTACHMENT_EXTRACTS = MemoryKey.of("attachments:extracts"); } diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java new file mode 100644 index 0000000000..eb1cf002a7 --- /dev/null +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java @@ -0,0 +1,339 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.impl; + +import ai.labs.eddi.engine.attachments.IAttachmentStore; +import ai.labs.eddi.engine.httpclient.SafeHttpClient; +import ai.labs.eddi.engine.memory.IConversationMemory; +import ai.labs.eddi.engine.memory.IConversationMemory.IWritableConversationStep; +import ai.labs.eddi.engine.memory.IData; +import ai.labs.eddi.engine.memory.model.Attachment; +import ai.labs.eddi.engine.memory.model.Data; +import ai.labs.eddi.modules.llm.capability.ModelCapabilityService; +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor; +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor.AttachmentExtractionException; +import dev.langchain4j.data.message.AudioContent; +import dev.langchain4j.data.message.ChatMessage; +import dev.langchain4j.data.message.Content; +import dev.langchain4j.data.message.ImageContent; +import dev.langchain4j.data.message.PdfFileContent; +import dev.langchain4j.data.message.TextContent; +import dev.langchain4j.data.message.UserMessage; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.jboss.logging.Logger; + +import java.net.URI; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Locale; + +import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENTS; +import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENT_ERRORS; +import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENT_EXTRACTS; +import static ai.labs.eddi.modules.llm.tools.UrlValidationUtils.validateUrl; + +/** + * The single place attachments become langchain4j {@link Content} on the + * outgoing user message. Replaces the image-only + * {@code MultimodalMessageEnhancer} and the divergent handling in + * {@code convertMessage}. + *

+ * Per attachment it resolves bytes from any source (stored blob, URL, inline + * base64) under uniform per-file and aggregate byte caps, gates on + * {@link ModelCapabilityService}, and emits the right content: + *

    + *
  • {@code image/*} → {@link ImageContent} when the model has vision (URL + * kept as-is when the provider fetches URLs, otherwise downloaded and inlined), + * else a metadata note.
    • + *
  • {@code application/pdf} → native {@link PdfFileContent} when the model + * supports documents, else PDFBox text extraction inlined as + * {@link TextContent}.
    • + *
  • text-like ({@code text/*}, JSON, XML, CSV, YAML) → decoded and inlined + * (no capability required).
    • + *
  • {@code audio/*} → {@link AudioContent} when supported, else a note.
    • + *
  • anything else → a metadata note pointing at the {@code readAttachment} + * tool.
    • + *
+ * Extracted text is persisted to {@code attachments:extracts} (for history + * stitching) and every drop/skip/gate is appended to {@code attachments:errors} + * — never silent. + * + * @since 6.1.0 + */ +@ApplicationScoped +public class AttachmentForwarder { + + private static final Logger LOGGER = Logger.getLogger(AttachmentForwarder.class); + + private final IAttachmentStore attachmentStore; + private final ModelCapabilityService capabilityService; + private final AttachmentTextExtractor textExtractor; + private final SafeHttpClient httpClient; + private final long maxForwardBytes; + private final long maxAggregateBytes; + + @Inject + public AttachmentForwarder(IAttachmentStore attachmentStore, + ModelCapabilityService capabilityService, + AttachmentTextExtractor textExtractor, + SafeHttpClient httpClient, + @ConfigProperty(name = "eddi.attachments.max-forward-bytes", + defaultValue = "10485760") long maxForwardBytes, + @ConfigProperty(name = "eddi.attachments.max-forward-aggregate-bytes", + defaultValue = "20971520") long maxAggregateBytes) { + this.attachmentStore = attachmentStore; + this.capabilityService = capabilityService; + this.textExtractor = textExtractor; + this.httpClient = httpClient; + this.maxForwardBytes = maxForwardBytes; + this.maxAggregateBytes = maxAggregateBytes; + } + + /** + * Enhance the last {@link UserMessage} in {@code messages} (modified in place) + * with content for the current step's attachments, gated on the model's + * capabilities. Extracted text and error notes are persisted to step data. + * + * @param messages + * the outgoing chat messages (modified in place) + * @param memory + * conversation memory for the current turn + * @param provider + * the resolved LLM provider (e.g. {@code openai}) + * @param model + * the resolved model name + */ + public void forward(List messages, IConversationMemory memory, String provider, String model) { + if (messages == null || messages.isEmpty()) { + return; + } + List attachments = readAttachments(memory); + if (attachments.isEmpty()) { + return; + } + int lastUserIdx = lastUserMessageIndex(messages); + if (lastUserIdx < 0) { + return; + } + + UserMessage original = (UserMessage) messages.get(lastUserIdx); + List contents = new ArrayList<>(original.contents()); + + List extracts = new ArrayList<>(); + List errors = new ArrayList<>(); + long[] aggregate = {0L}; + int added = 0; + + for (Attachment att : attachments) { + Content content = process(att, memory.getConversationId(), provider, model, aggregate, extracts, errors); + if (content != null) { + contents.add(content); + added++; + } + } + + if (added > 0) { + messages.set(lastUserIdx, UserMessage.from(contents)); + LOGGER.debugf("Forwarded %d attachment content item(s) to the LLM", added); + } + persist(memory.getCurrentStep(), extracts, errors); + } + + private Content process(Attachment att, String conversationId, String provider, String model, + long[] aggregate, List extracts, List errors) { + String mime = att.getMimeType() == null ? "" : att.getMimeType().toLowerCase(Locale.ROOT); + String name = att.getFileName() != null ? att.getFileName() : "unnamed"; + + if (att.getContentSource() == Attachment.ContentSource.NONE) { + errors.add("Attachment '" + name + "' has no content and was not forwarded."); + return null; + } + + // Fast path: an image the provider can fetch by URL — no download needed. + boolean isImage = mime.startsWith("image/"); + boolean isAudio = mime.startsWith("audio/"); + boolean isPdf = mime.startsWith("application/pdf"); + + if (isImage && att.getContentSource() == Attachment.ContentSource.URL + && capabilityService.supportsVision(provider, model) + && capabilityService.supportsImageUrl(provider, model)) { + try { + return ImageContent.from(URI.create(att.getUrl())); + } catch (Exception e) { + errors.add("Image '" + name + "' URL could not be attached: " + e.getMessage()); + return null; + } + } + + // Everything else needs the bytes in hand. A skip (cap/load/fetch failure) + // still leaves a note the LLM can relay, in addition to the error record. + byte[] bytes; + try { + bytes = resolveBytes(att, conversationId, aggregate); + } catch (ForwardSkipException e) { + errors.add(e.getMessage()); + return TextContent.from(e.getMessage()); + } + + if (isImage) { + if (!capabilityService.supportsVision(provider, model)) { + return note(errors, name, mime, att.getSizeBytes(), + "model does not support images"); + } + return ImageContent.from(Base64.getEncoder().encodeToString(bytes), att.getMimeType()); + } + + if (isPdf) { + if (capabilityService.supportsDocuments(provider, model)) { + return PdfFileContent.from(Base64.getEncoder().encodeToString(bytes), att.getMimeType()); + } + return extractInline(bytes, att.getMimeType(), name, extracts, errors, + "PDF text-extracted (model has no native document support)"); + } + + if (textExtractor.canExtractText(att.getMimeType())) { + return extractInline(bytes, att.getMimeType(), name, extracts, errors, "inlined as text"); + } + + if (isAudio) { + if (!capabilityService.supportsAudio(provider, model)) { + return note(errors, name, mime, att.getSizeBytes(), "model does not support audio"); + } + return AudioContent.from(Base64.getEncoder().encodeToString(bytes), att.getMimeType()); + } + + return note(errors, name, mime, att.getSizeBytes(), "unsupported type"); + } + + /** + * Resolve the attachment bytes from its source, enforcing the per-file and + * running aggregate caps. + */ + private byte[] resolveBytes(Attachment att, String conversationId, long[] aggregate) throws ForwardSkipException { + String name = att.getFileName() != null ? att.getFileName() : "unnamed"; + byte[] bytes; + switch (att.getContentSource()) { + case STORED -> { + try { + bytes = attachmentStore.load(att.getStorageRef(), conversationId); + } catch (IAttachmentStore.AttachmentStoreException e) { + throw new ForwardSkipException("Stored attachment '" + name + "' could not be loaded: " + e.getMessage()); + } + } + case URL -> bytes = download(att.getUrl(), name); + case BASE64 -> { + try { + bytes = Base64.getDecoder().decode(att.getBase64Data()); + } catch (IllegalArgumentException e) { + throw new ForwardSkipException("Attachment '" + name + "' has invalid base64 data."); + } + } + default -> throw new ForwardSkipException("Attachment '" + name + "' has no content."); + } + + if (bytes.length > maxForwardBytes) { + throw new ForwardSkipException(("Attachment '%s' (%d bytes) exceeds the per-file forward limit of %d bytes " + + "and was not sent. Use the readAttachment tool to access it.") + .formatted(name, bytes.length, maxForwardBytes)); + } + if (aggregate[0] + bytes.length > maxAggregateBytes) { + throw new ForwardSkipException(("Attachment '%s' skipped: the per-request attachment budget of %d bytes " + + "was reached.").formatted(name, maxAggregateBytes)); + } + aggregate[0] += bytes.length; + return bytes; + } + + private byte[] download(String url, String name) throws ForwardSkipException { + try { + validateUrl(url); + HttpRequest request = HttpRequest.newBuilder().uri(URI.create(url)).GET().build(); + HttpResponse response = httpClient.sendValidated(request, HttpResponse.BodyHandlers.ofByteArray()); + if (response.statusCode() != 200) { + throw new ForwardSkipException("Attachment '" + name + "' download failed: HTTP " + response.statusCode()); + } + return response.body(); + } catch (ForwardSkipException e) { + throw e; + } catch (Exception e) { + throw new ForwardSkipException("Attachment '" + name + "' could not be fetched: " + e.getMessage()); + } + } + + private Content extractInline(byte[] bytes, String mime, String name, + List extracts, List errors, String outcome) { + try { + String text = textExtractor.extractText(bytes, mime); + if (text == null || text.isBlank()) { + return note(errors, name, mime, bytes.length, "no extractable text"); + } + extracts.add(name + ": " + text); + return TextContent.from("[Attachment " + name + " (" + mime + ") — " + outcome + "]\n" + text); + } catch (AttachmentExtractionException e) { + return note(errors, name, mime, bytes.length, "text extraction failed: " + e.getMessage()); + } + } + + private Content note(List errors, String name, String mime, long sizeBytes, String reason) { + String msg = "[Attachment: %s (%s, %d bytes) — not forwarded: %s. Use the readAttachment tool to read it.]" + .formatted(name, mime, sizeBytes, reason); + errors.add(msg); + return TextContent.from(msg); + } + + private List readAttachments(IConversationMemory memory) { + IData> data = memory.getCurrentStep().getLatestData(ATTACHMENTS); + if (data == null || data.getResult() == null) { + return List.of(); + } + List attachments = new ArrayList<>(); + for (Object o : data.getResult()) { + if (o instanceof Attachment a) { + attachments.add(a); + } + } + return attachments; + } + + private static int lastUserMessageIndex(List messages) { + for (int i = messages.size() - 1; i >= 0; i--) { + if (messages.get(i) instanceof UserMessage) { + return i; + } + } + return -1; + } + + private void persist(IWritableConversationStep step, List extracts, List errors) { + if (!extracts.isEmpty()) { + Data> data = new Data<>(ATTACHMENT_EXTRACTS.key(), extracts); + data.setPublic(false); + step.storeData(data); + } + if (!errors.isEmpty()) { + List merged = new ArrayList<>(); + IData> existing = step.getLatestData(ATTACHMENT_ERRORS.key()); + if (existing != null && existing.getResult() != null) { + existing.getResult().forEach(e -> merged.add(String.valueOf(e))); + } + merged.addAll(errors); + Data> data = new Data<>(ATTACHMENT_ERRORS.key(), merged); + data.setPublic(false); + step.storeData(data); + } + } + + /** Internal signal that a single attachment should be skipped with a note. */ + private static final class ForwardSkipException extends Exception { + ForwardSkipException(String message) { + super(message); + } + } +} diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java index 75a79af1fa..05791b45a0 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java @@ -106,6 +106,11 @@ public class LlmTask implements ILifecycleTask { private final IdentityMaskingService identityMaskingService; private final IAttachmentStore attachmentStore; + // Field-injected so the many direct-construction unit tests are unaffected; + // null-guarded at the call site. + @jakarta.inject.Inject + AttachmentForwarder attachmentForwarder; + // Retained for httpCall RAG discovery + execution (Phase 8c-0) private final IApiCallExecutor apiCallExecutor; private final IRestAgentStore restAgentStore; @@ -334,9 +339,11 @@ private void executeTask(IConversationMemory memory, Task task, IWritableConvers includeFirstAgentMessage, summaryPrefix, skipSteps); } - // Enhance the last user message with multimodal attachment content (images, - // etc.) - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, attachmentStore); + // Forward the current step's attachments to the LLM as multimodal content, + // gated on the resolved (provider, model) capabilities. + if (attachmentForwarder != null) { + attachmentForwarder.forward(messages, memory, resolvedType, resolveModelName(processedParams)); + } if (messages.isEmpty()) { return; diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancer.java b/src/main/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancer.java deleted file mode 100644 index 1b14657372..0000000000 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancer.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.modules.llm.impl; - -import ai.labs.eddi.engine.attachments.IAttachmentStore; -import ai.labs.eddi.engine.memory.IConversationMemory; -import ai.labs.eddi.engine.memory.IData; -import ai.labs.eddi.engine.memory.model.Attachment; -import dev.langchain4j.data.message.ChatMessage; -import dev.langchain4j.data.message.Content; -import dev.langchain4j.data.message.ImageContent; -import dev.langchain4j.data.message.TextContent; -import dev.langchain4j.data.message.UserMessage; -import org.jboss.logging.Logger; - -import java.net.URI; -import java.util.ArrayList; -import java.util.List; - -import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENTS; - -/** - * Enhances the last user message in a ChatMessage list with multimodal content - * from conversation memory attachments. - *

- * This bridges the gap between the attachment pipeline (which stores - * {@link Attachment} objects in memory) and langchain4j's multimodal content - * types ({@link ImageContent}, etc.). - *

- * Currently supports: - *

    - *
  • {@code image/*} → {@link ImageContent} (via URL or Base64)
    • - *
- *

- * Future content types (PDF, audio, video) can be added as langchain4j - * providers expand their multimodal support. - * - * @since 6.0.0 - */ -final class MultimodalMessageEnhancer { - - private static final Logger LOGGER = Logger.getLogger(MultimodalMessageEnhancer.class); - - /** - * Maximum byte size for stored images forwarded to the LLM as base64. Larger - * files get a text placeholder instead of being inlined. - *

- * 10 MB raw → ~13 MB base64 — keeps LLM requests within typical provider limits - * while being generous enough for high-res images. - */ - static final long MAX_MULTIMODAL_FORWARD_BYTES = 10L * 1024 * 1024; // 10 MB - - private MultimodalMessageEnhancer() { - // non-instantiable utility - } - - /** - * If the current conversation step contains attachments, replace the last - * {@link UserMessage} in the messages list with a multimodal version that - * includes both the original text and the attachment content. - *

- * Messages list is modified in-place. If there are no attachments or no - * UserMessage in the list, nothing happens. - * - * @param messages - * the chat message list to enhance (modified in-place) - * @param memory - * conversation memory to read attachments from - */ - static void enhanceLastUserMessage(List messages, - IConversationMemory memory, - IAttachmentStore attachmentStore) { - if (messages == null || messages.isEmpty()) { - return; - } - - // Find attachments in the current step - IData> attachmentData = memory.getCurrentStep().getLatestData(ATTACHMENTS); - if (attachmentData == null || attachmentData.getResult() == null || attachmentData.getResult().isEmpty()) { - return; - } - - List rawAttachments = attachmentData.getResult(); - List attachments = new ArrayList<>(); - for (Object obj : rawAttachments) { - if (obj instanceof Attachment att) { - attachments.add(att); - } - } - - if (attachments.isEmpty()) { - return; - } - - // Find the last UserMessage in the list - int lastUserIdx = -1; - for (int i = messages.size() - 1; i >= 0; i--) { - if (messages.get(i) instanceof UserMessage) { - lastUserIdx = i; - break; - } - } - - if (lastUserIdx < 0) { - return; - } - - UserMessage originalMessage = (UserMessage) messages.get(lastUserIdx); - - // Build multimodal content list: original text content + attachment content - List contentList = new ArrayList<>(originalMessage.contents()); - - int imagesAdded = 0; - for (Attachment att : attachments) { - Content content = convertToContent(att, attachmentStore, - memory.getConversationId()); - if (content != null) { - contentList.add(content); - imagesAdded++; - } - } - - if (imagesAdded > 0) { - messages.set(lastUserIdx, UserMessage.from(contentList)); - LOGGER.debugf("Enhanced user message with %d multimodal attachment(s)", imagesAdded); - } - } - - /** - * Convert an Attachment to a langchain4j Content object based on MIME type and - * content source. - * - * @return the Content object, or null if the attachment type is not supported - */ - private static Content convertToContent(Attachment attachment, - IAttachmentStore attachmentStore, - String conversationId) { - String mimeType = attachment.getMimeType(); - if (mimeType == null) { - return null; - } - - // Image attachments → ImageContent - if (mimeType.startsWith("image/")) { - return convertImageAttachment(attachment, attachmentStore, conversationId); - } - - // For non-image types, add a text description so the LLM knows an attachment - // was present (metadata-only forwarding) - return TextContent.from(String.format("[Attachment: %s (%s, %d bytes)]", - attachment.getFileName() != null ? attachment.getFileName() : "unnamed", - mimeType, - attachment.getSizeBytes())); - } - - /** - * Convert an image attachment to ImageContent based on its content source. - */ - private static Content convertImageAttachment(Attachment attachment, - IAttachmentStore attachmentStore, - String conversationId) { - return switch (attachment.getContentSource()) { - case URL -> { - try { - yield ImageContent.from(URI.create(attachment.getUrl())); - } catch (Exception e) { - LOGGER.warnf("Failed to create ImageContent from URL '%s': %s", - attachment.getUrl(), e.getMessage()); - yield null; - } - } - case BASE64 -> { - try { - String dataUri = "data:" + attachment.getMimeType() + ";base64," + attachment.getBase64Data(); - yield ImageContent.from(dataUri); - } catch (Exception e) { - LOGGER.warnf("Failed to create ImageContent from base64 attachment '%s': %s", - attachment.getFileName(), e.getMessage()); - yield null; - } - } - case STORED -> { - if (attachmentStore == null) { - LOGGER.warnf("Cannot load stored attachment '%s' — no IAttachmentStore available", - attachment.getFileName()); - yield TextContent.from(String.format("[Stored attachment: %s (%s) — no attachment store configured]", - attachment.getFileName() != null ? attachment.getFileName() : "unnamed", - attachment.getMimeType())); - } - try { - byte[] bytes = attachmentStore.load( - attachment.getStorageRef(), conversationId); - - // Size guard — prevent blowing up the LLM request with - // very large base64 payloads (raw 10 MB → ~13 MB base64) - if (bytes.length > MAX_MULTIMODAL_FORWARD_BYTES) { - LOGGER.warnf("Stored attachment '%s' too large for multimodal forwarding " + - "(%d bytes exceeds %d byte limit)", - attachment.getFileName(), bytes.length, MAX_MULTIMODAL_FORWARD_BYTES); - yield TextContent.from(String.format( - "[Stored attachment: %s (%s, %d bytes) — too large for multimodal forwarding (limit: %d bytes)]", - attachment.getFileName() != null ? attachment.getFileName() : "unnamed", - attachment.getMimeType(), bytes.length, MAX_MULTIMODAL_FORWARD_BYTES)); - } - - String base64 = java.util.Base64.getEncoder().encodeToString(bytes); - String dataUri = "data:" + attachment.getMimeType() + ";base64," + base64; - LOGGER.debugf("Loaded stored attachment '%s' (%d bytes) for multimodal forwarding", - attachment.getFileName(), bytes.length); - yield ImageContent.from(dataUri); - } catch (IAttachmentStore.AttachmentStoreException e) { - LOGGER.warnf("Failed to load stored attachment '%s': %s", - attachment.getFileName(), e.getMessage()); - yield TextContent.from(String.format("[Stored attachment: %s (%s) — load failed: %s]", - attachment.getFileName() != null ? attachment.getFileName() : "unnamed", - attachment.getMimeType(), e.getMessage())); - } - } - case NONE -> null; - }; - } -} diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java new file mode 100644 index 0000000000..91552c6c0c --- /dev/null +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java @@ -0,0 +1,388 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.impl; + +import ai.labs.eddi.engine.attachments.IAttachmentStore; +import ai.labs.eddi.engine.httpclient.SafeHttpClient; +import ai.labs.eddi.engine.memory.IConversationMemory; +import ai.labs.eddi.engine.memory.IConversationMemory.IWritableConversationStep; +import ai.labs.eddi.engine.memory.IData; +import ai.labs.eddi.engine.memory.model.Attachment; +import ai.labs.eddi.modules.llm.capability.ModelCapabilityService; +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor; +import dev.langchain4j.data.message.*; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +import java.io.ByteArrayOutputStream; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Base64; +import java.util.List; +import java.util.Optional; + +import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENTS; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +/** + * Unit tests for {@link AttachmentForwarder}. + */ +class AttachmentForwarderTest { + + private IAttachmentStore store; + private SafeHttpClient httpClient; + private IConversationMemory memory; + private IWritableConversationStep currentStep; + private AttachmentForwarder forwarder; + + @BeforeEach + void setUp() { + store = mock(IAttachmentStore.class); + httpClient = mock(SafeHttpClient.class); + memory = mock(IConversationMemory.class); + currentStep = mock(IWritableConversationStep.class); + when(memory.getCurrentStep()).thenReturn(currentStep); + when(memory.getConversationId()).thenReturn("conv-1"); + forwarder = newForwarder(10L * 1024 * 1024, 20L * 1024 * 1024); + } + + private AttachmentForwarder newForwarder(long perFile, long aggregate) { + var capability = new ModelCapabilityService(k -> Optional.empty()); + var extractor = new AttachmentTextExtractor(10_000); + return new AttachmentForwarder(store, capability, extractor, httpClient, perFile, aggregate); + } + + // ==================== No-op cases ==================== + + @Nested + class NoOpCases { + + @Test + void nullMessages() { + forwarder.forward(null, memory, "openai", "gpt-4o"); + } + + @Test + void emptyMessages() { + List messages = new ArrayList<>(); + forwarder.forward(messages, memory, "openai", "gpt-4o"); + assertTrue(messages.isEmpty()); + } + + @Test + void noAttachments() { + when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(null); + List messages = messages(UserMessage.from("Hi")); + forwarder.forward(messages, memory, "openai", "gpt-4o"); + assertEquals(1, messages.size()); + } + + @Test + void noUserMessage() { + mockAttachments(urlImage()); + List messages = messages(new SystemMessage("s"), AiMessage.from("a")); + forwarder.forward(messages, memory, "openai", "gpt-4o"); + assertEquals(2, messages.size()); + } + } + + // ==================== Images ==================== + + @Nested + class Images { + + @Test + void imageUrlWithVisionAndUrlSupport_usesImageContentUrl() { + mockAttachments(urlImage()); + List messages = messages(UserMessage.from("Describe")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + UserMessage enhanced = (UserMessage) messages.get(0); + assertEquals(2, enhanced.contents().size()); + assertInstanceOf(ImageContent.class, enhanced.contents().get(1)); + verifyNoInteractions(httpClient); // URL not downloaded + } + + @Test + void imageUrlWithoutUrlSupport_downloadsAndInlines() throws Exception { + mockAttachments(urlImage()); + mockDownload("imgbytes".getBytes()); + List messages = messages(UserMessage.from("Describe")); + + // gemini supports vision but not image-by-URL → download + inline + forwarder.forward(messages, memory, "gemini", "gemini-2.0-flash"); + + UserMessage enhanced = (UserMessage) messages.get(0); + assertInstanceOf(ImageContent.class, enhanced.contents().get(1)); + verify(httpClient).sendValidated(any(), any()); + } + + @Test + void base64Image_inlines() { + Attachment att = new Attachment(); + att.setMimeType("image/jpeg"); + att.setBase64Data(Base64.getEncoder().encodeToString("img".getBytes())); + mockAttachments(att); + List messages = messages(UserMessage.from("What is this")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + assertInstanceOf(ImageContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + + @Test + void storedImage_loadsAndInlines() throws Exception { + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setStorageRef("ref-1"); + mockAttachments(att); + when(store.load("ref-1", "conv-1")).thenReturn("png".getBytes()); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + assertInstanceOf(ImageContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + + @Test + void imageWithoutVision_addsNote() { + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setBase64Data(Base64.getEncoder().encodeToString("png".getBytes())); + mockAttachments(att); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "jlama", "any"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertInstanceOf(TextContent.class, c); + assertTrue(((TextContent) c).text().contains("not forwarded")); + } + } + + // ==================== PDF ==================== + + @Nested + class Pdfs { + + @Test + void pdfWithDocumentSupport_usesPdfFileContent() { + Attachment att = new Attachment(); + att.setMimeType("application/pdf"); + att.setBase64Data(Base64.getEncoder().encodeToString("%PDF-1.4 fake".getBytes())); + mockAttachments(att); + List messages = messages(UserMessage.from("summarize")); + + forwarder.forward(messages, memory, "anthropic", "claude-sonnet-4"); + + assertInstanceOf(PdfFileContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + + @Test + void pdfWithoutDocumentSupport_extractsText() throws Exception { + Attachment att = new Attachment(); + att.setMimeType("application/pdf"); + att.setBase64Data(Base64.getEncoder().encodeToString(tinyPdf("Hello from PDF"))); + att.setFileName("doc.pdf"); + mockAttachments(att); + List messages = messages(UserMessage.from("summarize")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertInstanceOf(TextContent.class, c); + assertTrue(((TextContent) c).text().contains("Hello from PDF")); + // extracted text persisted for history stitching + assertTrue(capturePersisted("attachments:extracts").stream() + .anyMatch(s -> s.contains("Hello from PDF"))); + } + } + + // ==================== Text ==================== + + @Test + void textDocument_inlined() { + Attachment att = new Attachment(); + att.setMimeType("text/plain"); + att.setFileName("notes.txt"); + att.setBase64Data(Base64.getEncoder().encodeToString("plain text body".getBytes(StandardCharsets.UTF_8))); + mockAttachments(att); + List messages = messages(UserMessage.from("read")); + + forwarder.forward(messages, memory, "jlama", "any"); // no capability needed + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertInstanceOf(TextContent.class, c); + assertTrue(((TextContent) c).text().contains("plain text body")); + } + + // ==================== Audio ==================== + + @Test + void audioWithSupport_usesAudioContent() { + Attachment att = audio(); + mockAttachments(att); + List messages = messages(UserMessage.from("transcribe")); + + forwarder.forward(messages, memory, "gemini", "gemini-2.0-flash"); + + assertInstanceOf(AudioContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + + @Test + void audioWithoutSupport_addsNote() { + mockAttachments(audio()); + List messages = messages(UserMessage.from("transcribe")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); // audio unsupported by default + + assertInstanceOf(TextContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + + // ==================== Unsupported + caps ==================== + + @Test + void unsupportedType_addsNote() { + Attachment att = new Attachment(); + att.setMimeType("application/zip"); + att.setBase64Data(Base64.getEncoder().encodeToString("zip".getBytes())); + att.setFileName("a.zip"); + mockAttachments(att); + List messages = messages(UserMessage.from("open")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertTrue(((TextContent) c).text().contains("unsupported type")); + } + + @Test + void perFileCapExceeded_skipsWithNote() { + var small = newForwarder(4, 20L * 1024 * 1024); // 4-byte per-file cap + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setBase64Data(Base64.getEncoder().encodeToString("way-too-big".getBytes())); + att.setFileName("big.png"); + mockAttachments(att); + List messages = messages(UserMessage.from("look")); + + small.forward(messages, memory, "openai", "gpt-4o"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertInstanceOf(TextContent.class, c); + assertTrue(((TextContent) c).text().contains("per-file forward limit")); + assertTrue(capturePersisted("attachments:errors").stream() + .anyMatch(s -> s.contains("per-file forward limit"))); + } + + @Test + void storeLoadFailure_addsErrorNote() throws Exception { + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setStorageRef("missing"); + mockAttachments(att); + when(store.load("missing", "conv-1")) + .thenThrow(new IAttachmentStore.AttachmentStoreException("not found")); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertTrue(((TextContent) c).text().contains("could not be loaded")); + } + + @Test + void noContentSource_skipped() { + Attachment att = new Attachment(); // NONE + att.setMimeType("image/png"); + mockAttachments(att); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + // nothing added — only original text remains + assertEquals(1, ((UserMessage) messages.get(0)).contents().size()); + } + + // ==================== Helpers ==================== + + private List capturePersisted(String key) { + ArgumentCaptor captor = ArgumentCaptor.forClass(IData.class); + verify(currentStep, atLeast(0)).storeData(captor.capture()); + for (IData d : captor.getAllValues()) { + if (key.equals(d.getKey()) && d.getResult() instanceof List list) { + List out = new ArrayList<>(); + list.forEach(o -> out.add(String.valueOf(o))); + return out; + } + } + return List.of(); + } + + @SuppressWarnings("unchecked") + private void mockDownload(byte[] bytes) throws Exception { + HttpResponse resp = mock(HttpResponse.class); + when(resp.statusCode()).thenReturn(200); + when(resp.body()).thenReturn(bytes); + doReturn(resp).when(httpClient).sendValidated(any(), any()); + } + + private static Attachment urlImage() { + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setUrl("https://example.com/photo.png"); + att.setFileName("photo.png"); + return att; + } + + private static Attachment audio() { + Attachment att = new Attachment(); + att.setMimeType("audio/mpeg"); + att.setFileName("clip.mp3"); + att.setBase64Data(Base64.getEncoder().encodeToString("audio".getBytes())); + return att; + } + + private static List messages(ChatMessage... m) { + List list = new ArrayList<>(); + for (ChatMessage cm : m) + list.add(cm); + return list; + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private void mockAttachments(Attachment... attachments) { + IData data = mock(IData.class); + when(data.getResult()).thenReturn(List.of(attachments)); + when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); + } + + private static byte[] tinyPdf(String text) throws Exception { + try (PDDocument doc = new PDDocument(); ByteArrayOutputStream out = new ByteArrayOutputStream()) { + var page = new PDPage(); + doc.addPage(page); + try (var cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 700); + cs.showText(text); + cs.endText(); + } + doc.save(out); + return out.toByteArray(); + } + } +} diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerExtendedTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerExtendedTest.java deleted file mode 100644 index 1e86512658..0000000000 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerExtendedTest.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.modules.llm.impl; - -import ai.labs.eddi.engine.memory.IConversationMemory; -import ai.labs.eddi.engine.memory.IConversationMemory.IWritableConversationStep; -import ai.labs.eddi.engine.memory.IData; -import ai.labs.eddi.engine.memory.model.Attachment; -import dev.langchain4j.data.message.*; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENTS; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -/** - * Extended tests for {@link MultimodalMessageEnhancer} — covers STORED path, - * invalid URL handling, null mimeType, null attachment result, and - * non-Attachment objects. - */ -class MultimodalMessageEnhancerExtendedTest { - - private IConversationMemory memory; - private IWritableConversationStep currentStep; - - @BeforeEach - void setUp() { - memory = mock(IConversationMemory.class); - currentStep = mock(IWritableConversationStep.class); - when(memory.getCurrentStep()).thenReturn(currentStep); - } - - @Nested - @DisplayName("STORED content source") - class StoredContentSource { - - @Test - @DisplayName("should produce text description for STORED image when no store available") - void storedImageProducesTextFallback() { - Attachment att = new Attachment(); - att.setMimeType("image/png"); - att.setFileName("stored-img.png"); - att.setStorageRef("store://abc-123"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("What is this?")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(2, enhanced.contents().size()); - TextContent fallback = (TextContent) enhanced.contents().get(1); - assertTrue(fallback.text().contains("stored-img.png")); - assertTrue(fallback.text().contains("no attachment store configured")); - } - - @Test - @DisplayName("should handle null fileName for STORED attachment") - void storedWithNullFileName() { - Attachment att = new Attachment(); - att.setMimeType("image/jpeg"); - att.setFileName(null); - att.setStorageRef("store://xyz"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Describe")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(2, enhanced.contents().size()); - TextContent fallback = (TextContent) enhanced.contents().get(1); - assertTrue(fallback.text().contains("unnamed")); - } - - @Test - @DisplayName("should reject oversized stored image with text placeholder") - void oversizedStoredImageProducesTextFallback() throws Exception { - Attachment att = new Attachment(); - att.setMimeType("image/png"); - att.setFileName("huge-photo.png"); - att.setStorageRef("store://large-ref"); - mockAttachments(att); - - // Create a mock store that returns bytes exceeding the forwarding limit - var mockStore = mock(ai.labs.eddi.engine.attachments.IAttachmentStore.class); - byte[] oversizedBytes = new byte[(int) (MultimodalMessageEnhancer.MAX_MULTIMODAL_FORWARD_BYTES + 1)]; - when(mockStore.load("store://large-ref", "conv-test")).thenReturn(oversizedBytes); - when(memory.getConversationId()).thenReturn("conv-test"); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("What is in this image?")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, mockStore); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(2, enhanced.contents().size()); - TextContent fallback = (TextContent) enhanced.contents().get(1); - assertTrue(fallback.text().contains("too large for multimodal forwarding")); - assertTrue(fallback.text().contains("huge-photo.png")); - } - } - - @Nested - @DisplayName("Error handling") - class ErrorHandling { - - @Test - @DisplayName("should handle invalid URL gracefully") - void invalidUrlReturnsNull() { - Attachment att = new Attachment(); - att.setMimeType("image/png"); - att.setUrl("not a valid url !!!"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Describe")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - // Invalid URL → null → not added → original text only - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(1, enhanced.contents().size()); - } - - @Test - @DisplayName("should skip attachment with null mimeType") - void nullMimeTypeSkipped() { - Attachment att = new Attachment(); - att.setMimeType(null); - att.setUrl("https://example.com/img.png"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Hello")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(1, enhanced.contents().size()); - } - - @Test - @DisplayName("should handle null result in attachment data") - void nullResultInAttachmentData() { - @SuppressWarnings("unchecked") - IData> data = mock(IData.class); - when(data.getResult()).thenReturn(null); - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Hello")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - assertEquals(1, messages.size()); - } - } - - @Nested - @DisplayName("Non-Attachment objects in list") - class NonAttachmentObjects { - - @Test - @DisplayName("should skip non-Attachment objects in raw attachments list") - void skipsNonAttachmentObjects() { - @SuppressWarnings({"unchecked", "rawtypes"}) - IData data = mock(IData.class); - List mixed = new ArrayList<>(); - mixed.add("just a string"); - mixed.add(42); - when(data.getResult()).thenReturn(mixed); - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Hello")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - // No valid Attachment → message unchanged - UserMessage msg = (UserMessage) messages.get(0); - assertEquals(1, msg.contents().size()); - } - } - - @Nested - @DisplayName("Non-image attachment with null fileName") - class NonImageNullFileName { - - @Test - @DisplayName("should use 'unnamed' for non-image attachment with null fileName") - void usesUnnamedFallback() { - Attachment att = new Attachment(); - att.setMimeType("application/octet-stream"); - att.setFileName(null); - att.setSizeBytes(1024); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Process")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(2, enhanced.contents().size()); - TextContent meta = (TextContent) enhanced.contents().get(1); - assertTrue(meta.text().contains("unnamed")); - assertTrue(meta.text().contains("1024 bytes")); - } - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - private void mockAttachments(Attachment... attachments) { - IData data = mock(IData.class); - when(data.getResult()).thenReturn(List.of(attachments)); - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); - } -} diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerTest.java deleted file mode 100644 index 39533f979d..0000000000 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/MultimodalMessageEnhancerTest.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Copyright EDDI contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package ai.labs.eddi.modules.llm.impl; - -import ai.labs.eddi.engine.memory.IConversationMemory; -import ai.labs.eddi.engine.memory.IConversationMemory.IWritableConversationStep; -import ai.labs.eddi.engine.memory.IData; -import ai.labs.eddi.engine.memory.model.Attachment; -import dev.langchain4j.data.message.*; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENTS; -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -/** - * Unit tests for {@link MultimodalMessageEnhancer}. - */ -class MultimodalMessageEnhancerTest { - - private IConversationMemory memory; - private IWritableConversationStep currentStep; - - @BeforeEach - void setUp() { - memory = mock(IConversationMemory.class); - currentStep = mock(IWritableConversationStep.class); - when(memory.getCurrentStep()).thenReturn(currentStep); - } - - // ==================== No-Op Cases ==================== - - @Nested - class NoOpCases { - - @Test - void shouldDoNothingWhenMessagesNull() { - MultimodalMessageEnhancer.enhanceLastUserMessage(null, memory, null); - // No exception = pass - } - - @Test - void shouldDoNothingWhenMessagesEmpty() { - List messages = new ArrayList<>(); - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - assertTrue(messages.isEmpty()); - } - - @Test - void shouldDoNothingWhenNoAttachments() { - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(null); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Hello")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - assertEquals(1, messages.size()); - assertInstanceOf(UserMessage.class, messages.get(0)); - } - - @Test - void shouldDoNothingWhenAttachmentDataEmpty() { - @SuppressWarnings("unchecked") - IData> data = mock(IData.class); - when(data.getResult()).thenReturn(List.of()); - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Hello")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - assertEquals(1, messages.size()); - } - - @Test - void shouldDoNothingWhenNoUserMessage() { - Attachment att = new Attachment(); - att.setMimeType("image/png"); - att.setUrl("https://example.com/img.png"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(new SystemMessage("System")); - messages.add(AiMessage.from("Response")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - // Messages unchanged - assertEquals(2, messages.size()); - assertInstanceOf(SystemMessage.class, messages.get(0)); - assertInstanceOf(AiMessage.class, messages.get(1)); - } - } - - // ==================== Image Attachments ==================== - - @Nested - class ImageAttachments { - - @Test - void shouldEnhanceWithUrlImage() { - Attachment att = new Attachment(); - att.setMimeType("image/png"); - att.setUrl("https://example.com/photo.png"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(new SystemMessage("system")); - messages.add(UserMessage.from("Describe this image")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - assertEquals(2, messages.size()); - UserMessage enhanced = (UserMessage) messages.get(1); - // Should have original text + image content - assertEquals(2, enhanced.contents().size()); - assertInstanceOf(TextContent.class, enhanced.contents().get(0)); - assertInstanceOf(ImageContent.class, enhanced.contents().get(1)); - } - - @Test - void shouldEnhanceWithBase64Image() { - Attachment att = new Attachment(); - att.setMimeType("image/jpeg"); - att.setBase64Data("iVBORw0KGgo="); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("What is this?")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(2, enhanced.contents().size()); - assertInstanceOf(TextContent.class, enhanced.contents().get(0)); - assertInstanceOf(ImageContent.class, enhanced.contents().get(1)); - } - - @Test - void shouldEnhanceWithMultipleImages() { - Attachment att1 = new Attachment(); - att1.setMimeType("image/png"); - att1.setUrl("https://example.com/1.png"); - - Attachment att2 = new Attachment(); - att2.setMimeType("image/jpeg"); - att2.setUrl("https://example.com/2.jpg"); - - mockAttachments(att1, att2); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Compare these")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - // 1 text + 2 images - assertEquals(3, enhanced.contents().size()); - } - } - - // ==================== Non-Image Attachments ==================== - - @Nested - class NonImageAttachments { - - @Test - void shouldAddMetadataTextForNonImageTypes() { - Attachment att = new Attachment(); - att.setMimeType("application/pdf"); - att.setFileName("report.pdf"); - att.setSizeBytes(15240); - att.setUrl("https://example.com/report.pdf"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Summarize this")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(2, enhanced.contents().size()); - TextContent metadata = (TextContent) enhanced.contents().get(1); - assertTrue(metadata.text().contains("report.pdf")); - assertTrue(metadata.text().contains("application/pdf")); - } - - @Test - void shouldHandleAttachmentWithNoContentSource() { - Attachment att = new Attachment(); // no url, no base64, no storageRef - att.setMimeType("image/png"); - mockAttachments(att); - - List messages = new ArrayList<>(); - messages.add(UserMessage.from("Hello")); - - MultimodalMessageEnhancer.enhanceLastUserMessage(messages, memory, null); - - // NONE content source → null → not added - UserMessage enhanced = (UserMessage) messages.get(0); - assertEquals(1, enhanced.contents().size()); // only original text - } - } - - // ==================== Helpers ==================== - - @SuppressWarnings({"unchecked", "rawtypes"}) - private void mockAttachments(Attachment... attachments) { - IData data = mock(IData.class); - when(data.getResult()).thenReturn(List.of(attachments)); - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); - } -} From c72b7f050f65854ada31b531224593b05dd502d3 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 01:56:45 +0200 Subject: [PATCH 09/23] test(attachments): raise coverage above 90% instr / 80% branch gate Add targeted tests for previously-uncovered branches so every delivered attachment class clears the >90% instruction / >80% branch bar: - AttachmentForwarder: aggregate-cap skip, download non-200, download exception, empty-text note, invalid-base64 note (85->93% instr, 88% branch) - RestAttachmentUpload: list/delete-all/download 500 error paths (84->93%) - GridFsAttachmentStore: null-owner allow, getMetadata grant + null-metadata defaults (79->86% branch) - ModelCapabilityService: full vision-model / text-only substring matrices (77->95% branch) PostgresAttachmentStore mirrors GridFs and is covered by its CI-only Testcontainers IT. --- .../mongo/GridFsAttachmentStoreTest.java | 35 ++++++++ .../memory/rest/RestAttachmentUploadTest.java | 28 +++++++ .../ModelCapabilityServiceTest.java | 26 ++++++ .../llm/impl/AttachmentForwarderTest.java | 79 +++++++++++++++++++ 4 files changed, 168 insertions(+) diff --git a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java index 54e7b85e0e..30acbc1ac1 100644 --- a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java +++ b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java @@ -269,6 +269,20 @@ void load_legacyObjectIdRef_resolves() throws Exception { assertArrayEquals("legacy".getBytes(), result); } + @Test + void load_nullOwnerMetadata_allowsAccess() throws Exception { + // metadata present but conversationId absent → owner check skipped + ObjectId id = new ObjectId(); + whenFindFirst(mockFile(id, null, "text/plain", List.of(), "uuid-1")); + doAnswer(inv -> { + OutputStream out = inv.getArgument(1); + out.write("ok".getBytes()); + return null; + }).when(gridFSBucket).downloadToStream(eq(id), any(OutputStream.class)); + + assertArrayEquals("ok".getBytes(), sut.load("uuid-1", "any-conv")); + } + // ─── getMetadata() ────────────────────────────────────────── @Test @@ -293,6 +307,27 @@ void getMetadata_notFound_throws() { assertThrows(AttachmentStoreException.class, () -> sut.getMetadata("missing", "conv-1")); } + @Test + void getMetadata_grantedConversation_returnsMetadata() throws Exception { + whenFindFirst(mockFile(new ObjectId(), "conv-owner", "application/pdf", List.of("conv-guest"), "uuid-1")); + Attachment meta = sut.getMetadata("uuid-1", "conv-guest"); + assertEquals("application/pdf", meta.mimeType()); + } + + @Test + void getMetadata_nullMetadataDefaults() throws Exception { + ObjectId id = new ObjectId(); + GridFSFile f = mock(GridFSFile.class); + when(f.getFilename()).thenReturn("x.bin"); + when(f.getLength()).thenReturn(9L); + when(f.getMetadata()).thenReturn(null); + whenFindFirst(f); + + Attachment meta = sut.getMetadata("uuid-1", "conv-1"); + assertEquals("application/octet-stream", meta.mimeType()); + assertEquals("uuid-1", meta.storageRef()); + } + // ─── grantAccess() ────────────────────────────────────────── @Test diff --git a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java index d9659f034c..a3b394ac2b 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java @@ -339,6 +339,15 @@ void shouldReturnEmptyListWhenNoAttachments() throws Exception { var list = (List) response.getEntity(); assertTrue(list.isEmpty()); } + + @Test + void shouldReturn500WhenListFails() throws Exception { + when(attachmentStore.listByConversation("conv-err")).thenThrow(new RuntimeException("db down")); + + Response response = captureAsync(ar -> endpoint.listAttachments("conv-err", ar)); + + assertEquals(500, response.getStatus()); + } } // ==================== Delete Tests ==================== @@ -372,6 +381,15 @@ void shouldReturnZeroWhenNoAttachmentsToDelete() throws Exception { var body = (Map) response.getEntity(); assertEquals(0L, body.get("deletedCount")); } + + @Test + void shouldReturn500WhenDeleteAllFails() throws Exception { + when(attachmentStore.deleteByConversation("conv-err")).thenThrow(new RuntimeException("db down")); + + Response response = captureAsync(ar -> endpoint.deleteAttachments("conv-err", ar)); + + assertEquals(500, response.getStatus()); + } } // ==================== Download Tests ==================== @@ -420,6 +438,16 @@ void shouldReturn403WhenDenied() throws Exception { assertEquals("ATTACHMENT_ACCESS_DENIED", body.get("code")); } + @Test + void shouldReturn500OnUnexpectedError() throws Exception { + when(attachmentStore.getMetadata("ref-1", "conv-1")) + .thenThrow(new RuntimeException("unexpected")); + + Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-1", "ref-1", ar)); + + assertEquals(500, response.getStatus()); + } + @Test void shouldSanitizeContentDispositionFilename() throws Exception { var meta = new Attachment("ref-1", "bad\"name\r\n.png", "image/png", 2, "conv-1"); diff --git a/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java b/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java index 80576df2d9..c9b434c8d5 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/capability/ModelCapabilityServiceTest.java @@ -61,6 +61,32 @@ void visionDefaults(String provider, String model, boolean expected) { provider + "/" + model + " vision should be " + expected); } + @ParameterizedTest + @ValueSource(strings = { + "llava", "bakllava", "some-vision-model", "pixtral-12b", + "anthropic.claude-3-haiku", "anthropic.claude-opus", "anthropic.claude-sonnet", + "amazon.nova-lite", "amazon.nova-pro", "amazon.nova-premier", + "meta.llama3.2-vision", "meta.llama-3.2-90b", "meta.llama3-2-11b", + "gemma3-27b", "gemma-3-12b", "qwen2-vl-7b", "qwen2.5-vl-7b", + "minicpm-v-2.6", "moondream2"}) + void modelDependentProviderUpgradesForKnownVisionModels(String model) { + // bedrock is model-dependent → these known vision models flip it on + assertTrue(service.supportsVision("bedrock", model), + "bedrock/" + model + " should be vision-capable"); + } + + @ParameterizedTest + @ValueSource(strings = { + "gpt-3.5-turbo", "text-davinci-003", "davinci-002", "babbage-002", + "text-embedding-3-large", "some-embed-model", "embed-english-v3", + "text-moderation-latest", "mistral-embed", "mistral-7b-instruct", "mixtral-8x22b"}) + void visionFirstProviderDowngradesForKnownTextOnlyModels(String model) { + // openai/mistral are vision-first → these text-only models flip vision off + String provider = model.startsWith("mistral") || model.startsWith("mixtral") ? "mistral" : "openai"; + assertFalse(service.supportsVision(provider, model), + provider + "/" + model + " should not be vision-capable"); + } + @Test void blankProviderIsUnsupported() { assertFalse(service.supportsVision("", "gpt-4o")); diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java index 91552c6c0c..2ead72ad1a 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java @@ -304,6 +304,85 @@ void storeLoadFailure_addsErrorNote() throws Exception { assertTrue(((TextContent) c).text().contains("could not be loaded")); } + @Test + void aggregateCapExceeded_skipsSecondWithNote() { + var small = newForwarder(10L * 1024 * 1024, 4); // 4-byte aggregate budget + Attachment a = new Attachment(); + a.setMimeType("image/png"); + a.setFileName("a.png"); + a.setBase64Data(Base64.getEncoder().encodeToString("abc".getBytes())); // 3 bytes + Attachment b = new Attachment(); + b.setMimeType("image/png"); + b.setFileName("b.png"); + b.setBase64Data(Base64.getEncoder().encodeToString("de".getBytes())); // 2 bytes → 3+2 > 4 + mockAttachments(a, b); + List messages = messages(UserMessage.from("look")); + + small.forward(messages, memory, "openai", "gpt-4o"); + + // a inlined (ImageContent), b noted (aggregate budget) + UserMessage enhanced = (UserMessage) messages.get(0); + assertInstanceOf(ImageContent.class, enhanced.contents().get(1)); + assertTrue(((TextContent) enhanced.contents().get(2)).text().contains("attachment budget")); + } + + @Test + void downloadNon200_addsNote() throws Exception { + mockAttachments(urlImage()); + @SuppressWarnings("unchecked") + HttpResponse resp = mock(HttpResponse.class); + when(resp.statusCode()).thenReturn(500); + doReturn(resp).when(httpClient).sendValidated(any(), any()); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "gemini", "gemini-2.0-flash"); // needs download + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertTrue(((TextContent) c).text().contains("download failed")); + } + + @Test + void downloadException_addsNote() throws Exception { + mockAttachments(urlImage()); + doThrow(new java.io.IOException("boom")).when(httpClient).sendValidated(any(), any()); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "gemini", "gemini-2.0-flash"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertTrue(((TextContent) c).text().contains("could not be fetched")); + } + + @Test + void textWithNoExtractableContent_addsNote() { + Attachment att = new Attachment(); + att.setMimeType("text/plain"); + att.setFileName("empty.txt"); + att.setBase64Data(""); // decodes to empty + mockAttachments(att); + List messages = messages(UserMessage.from("read")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertTrue(((TextContent) c).text().contains("no extractable text")); + } + + @Test + void invalidBase64_addsNote() { + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setFileName("bad.png"); + att.setBase64Data("!!!not-base64!!!"); + mockAttachments(att); + List messages = messages(UserMessage.from("look")); + + forwarder.forward(messages, memory, "openai", "gpt-4o"); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertTrue(((TextContent) c).text().contains("invalid base64")); + } + @Test void noContentSource_skipped() { Attachment att = new Attachment(); // NONE From fe7601fe5e523260383df310772e5378c581837b Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 02:15:24 +0200 Subject: [PATCH 10/23] feat(attachments): per-task multimodal overrides + history extract stitching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the Phase 2 tail. Per-task config: LlmConfiguration.Task gains optional multimodal {vision|documents|audio: auto|on|off} + reattachTurns (default 0). Old JSON deserializes cleanly. AttachmentForwarder.forward gains a Support-parameterized overload; LlmTask parses the task block and applies overrides (per-task > deployment > default). History stitching: ConversationLogGenerator.generate gains an opt-in stitchAttachmentExtracts flag — only the LLM-facing ConversationHistoryBuilder paths (normal + skipSteps windowing) pass true, so the visible transcript stays clean. Each past turn's attachments:extracts is appended to its rebuilt user message. Verified: outputs align 1:1 with getAllSteps(), and non-public step data survives snapshot persist/reload — so a turn-2 follow-up sees turn-1's PDF/text extracts. reattachTurns is schema-ready; extracts + the readAttachment tool are the continuity mechanisms. Tests: forwarder override on/off, Task config fields, 3 stitching tests; existing history/log tests unchanged (stitching is inert without extract data). --- docs/changelog.md | 9 ++- .../memory/ConversationLogGenerator.java | 44 ++++++++++- .../modules/llm/impl/AttachmentForwarder.java | 31 ++++++-- .../llm/impl/ConversationHistoryBuilder.java | 11 +-- .../labs/eddi/modules/llm/impl/LlmTask.java | 17 ++++- .../modules/llm/model/LlmConfiguration.java | 73 +++++++++++++++++++ .../memory/ConversationLogGeneratorTest.java | 56 ++++++++++++++ .../llm/impl/AttachmentForwarderTest.java | 38 ++++++++++ .../llm/model/LlmConfigurationTaskTest.java | 61 ++++++++++++++++ 9 files changed, 324 insertions(+), 16 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 22d2def839..c956fe4025 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -30,9 +30,14 @@ `AttachmentForwarderTest` (18) covers the full branch matrix incl. URL-passthrough vs download-inline, base64/stored images, PDF native vs text-fallback (with extract persistence), text inline, audio on/off, unsupported note, per-file cap, store-load failure, and no-source skip. Enhancer tests removed. -### What's next (remaining Phase 2, then 3–6) +### Phase 2 tail (completed same branch) -Still open in Phase 2: history stitching (inject `attachments:extracts` into the rebuilt turn's user message in `ConversationHistoryBuilder`) and per-task config (`LlmConfiguration.Task.multimodal` override + `reattachTurns`). Then Phase 3 (group parity), 4 (`readAttachment` tool), 5 (UX), 6 (ops). +- **Per-task multimodal override + reattachTurns** — `LlmConfiguration.Task` gains an optional `multimodal { vision|documents|audio: auto|on|off }` block and `reattachTurns` (default 0). Old JSON configs deserialize cleanly (`FAIL_ON_UNKNOWN_PROPERTIES=false`). `AttachmentForwarder.forward` gains a `Support`-parameterized overload; `LlmTask` parses the task block and passes the overrides (per-task > deployment > default precedence). +- **History stitching** — `ConversationLogGenerator.generate` gains an opt-in `stitchAttachmentExtracts` flag (only the LLM-facing `ConversationHistoryBuilder` path passes `true`, so the visible transcript stays clean). Per turn it appends that step's `attachments:extracts` to the rebuilt user message; verified aligned 1:1 with conversation outputs and that non-public step data survives snapshot persistence/reload, so a turn-2 follow-up sees turn-1's PDF/text extracts. `reattachTurns` is schema-ready; extract-stitching + the `readAttachment` tool (Phase 4) are the primary multi-turn continuity mechanisms. + +### What's next (Phases 3–6) + +Phase 3 (group parity), 4 (`readAttachment` tool), 5 (UX), 6 (ops). --- diff --git a/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java b/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java index 22bf2e6a5e..0b5a65d58a 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java +++ b/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java @@ -51,6 +51,18 @@ public ConversationLog generate(int logSize) { } public ConversationLog generate(int logSize, boolean includeFirstAgentMessage) { + return generate(logSize, includeFirstAgentMessage, false); + } + + /** + * @param stitchAttachmentExtracts + * when {@code true} and backed by a live + * {@link IConversationMemory}, the per-step attachment text extracts + * ({@link MemoryKeys#ATTACHMENT_EXTRACTS}) are appended to that + * turn's user input. Used only for the LLM-facing message build so + * the visible transcript stays clean. + */ + public ConversationLog generate(int logSize, boolean includeFirstAgentMessage, boolean stitchAttachmentExtracts) { if (conversationMemory == null && memorySnapshot == null) { throw new IllegalStateException( "ConversationMemory was null. " + "You need to either set IConversationMemory or ConversationMemorySnapshot"); @@ -62,6 +74,10 @@ public ConversationLog generate(int logSize, boolean includeFirstAgentMessage) { ? conversationMemory.getConversationOutputs() : memorySnapshot.getConversationOutputs(); + var allSteps = (stitchAttachmentExtracts && conversationMemory != null) + ? conversationMemory.getAllSteps() + : null; + var startIndex = 0; if (logSize > 0) { startIndex = conversationOutputs.size() > logSize ? conversationOutputs.size() - logSize : 0; @@ -89,7 +105,7 @@ public ConversationLog generate(int logSize, boolean includeFirstAgentMessage) { if (input != null) { var inputText = new Content(); inputText.setType(text); - inputText.setValue(input); + inputText.setValue(withAttachmentExtracts(allSteps, index, input)); var inputs = new ArrayList<>(contentList); inputs.add(inputText); conversationLog.getMessages().add(new ConversationPart(KEY_ROLE_USER, inputs)); @@ -129,6 +145,32 @@ public ConversationLog generate(int logSize, boolean includeFirstAgentMessage) { return conversationLog; } + /** + * Append the step's attachment text extracts (if any) to a turn's user input. + * Returns {@code input} unchanged when there is no step stack (snapshot mode), + * the index is out of range, or the step carries no extracts. + * + * @param allSteps + * the memory's step stack aligned 1:1 with conversation outputs (may + * be null) + * @param stepIndex + * the output/step index for this turn + * @param input + * the raw user input text + * @return the input, with extracts appended when present + */ + public static String withAttachmentExtracts(IConversationMemory.IConversationStepStack allSteps, + int stepIndex, String input) { + if (allSteps == null || input == null || stepIndex < 0 || stepIndex >= allSteps.size()) { + return input; + } + IData> data = allSteps.get(stepIndex).getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS); + if (data == null || data.getResult() == null || data.getResult().isEmpty()) { + return input; + } + return input + "\n\n" + String.join("\n\n", data.getResult()); + } + private static ContentType getContentType(String type) { return switch (type) { case "pdf" -> pdf; diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java index eb1cf002a7..a458b7ea8f 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java @@ -111,6 +111,21 @@ public AttachmentForwarder(IAttachmentStore attachmentStore, * the resolved model name */ public void forward(List messages, IConversationMemory memory, String provider, String model) { + forward(messages, memory, provider, model, + ModelCapabilityService.Support.AUTO, + ModelCapabilityService.Support.AUTO, + ModelCapabilityService.Support.AUTO); + } + + /** + * Overload honoring per-task multimodal overrides (from + * {@code LlmConfiguration.Task.multimodal}). {@code AUTO} defers to the + * capability service's deployment/built-in defaults. + */ + public void forward(List messages, IConversationMemory memory, String provider, String model, + ModelCapabilityService.Support visionOverride, + ModelCapabilityService.Support documentsOverride, + ModelCapabilityService.Support audioOverride) { if (messages == null || messages.isEmpty()) { return; } @@ -132,7 +147,8 @@ public void forward(List messages, IConversationMemory memory, Stri int added = 0; for (Attachment att : attachments) { - Content content = process(att, memory.getConversationId(), provider, model, aggregate, extracts, errors); + Content content = process(att, memory.getConversationId(), provider, model, aggregate, extracts, errors, + visionOverride, documentsOverride, audioOverride); if (content != null) { contents.add(content); added++; @@ -147,7 +163,10 @@ public void forward(List messages, IConversationMemory memory, Stri } private Content process(Attachment att, String conversationId, String provider, String model, - long[] aggregate, List extracts, List errors) { + long[] aggregate, List extracts, List errors, + ModelCapabilityService.Support visionOverride, + ModelCapabilityService.Support documentsOverride, + ModelCapabilityService.Support audioOverride) { String mime = att.getMimeType() == null ? "" : att.getMimeType().toLowerCase(Locale.ROOT); String name = att.getFileName() != null ? att.getFileName() : "unnamed"; @@ -162,7 +181,7 @@ private Content process(Attachment att, String conversationId, String provider, boolean isPdf = mime.startsWith("application/pdf"); if (isImage && att.getContentSource() == Attachment.ContentSource.URL - && capabilityService.supportsVision(provider, model) + && capabilityService.supportsVision(provider, model, visionOverride) && capabilityService.supportsImageUrl(provider, model)) { try { return ImageContent.from(URI.create(att.getUrl())); @@ -183,7 +202,7 @@ private Content process(Attachment att, String conversationId, String provider, } if (isImage) { - if (!capabilityService.supportsVision(provider, model)) { + if (!capabilityService.supportsVision(provider, model, visionOverride)) { return note(errors, name, mime, att.getSizeBytes(), "model does not support images"); } @@ -191,7 +210,7 @@ private Content process(Attachment att, String conversationId, String provider, } if (isPdf) { - if (capabilityService.supportsDocuments(provider, model)) { + if (capabilityService.supportsDocuments(provider, model, documentsOverride)) { return PdfFileContent.from(Base64.getEncoder().encodeToString(bytes), att.getMimeType()); } return extractInline(bytes, att.getMimeType(), name, extracts, errors, @@ -203,7 +222,7 @@ private Content process(Attachment att, String conversationId, String provider, } if (isAudio) { - if (!capabilityService.supportsAudio(provider, model)) { + if (!capabilityService.supportsAudio(provider, model, audioOverride)) { return note(errors, name, mime, att.getSizeBytes(), "model does not support audio"); } return AudioContent.from(Base64.getEncoder().encodeToString(bytes), att.getMimeType()); diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/ConversationHistoryBuilder.java b/src/main/java/ai/labs/eddi/modules/llm/impl/ConversationHistoryBuilder.java index a2c3fec446..45a455192f 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/ConversationHistoryBuilder.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/ConversationHistoryBuilder.java @@ -91,8 +91,8 @@ List buildMessages(IConversationMemory memory, String systemMessage if (skipSteps > 0) { chatMessages = generateMessagesFromOutputs(memory, skipSteps, logSizeLimit, includeFirstAgentMessage); } else { - chatMessages = new ArrayList<>(new ConversationLogGenerator(memory).generate(logSizeLimit, includeFirstAgentMessage).getMessages() - .stream().map(this::convertMessage).toList()); + chatMessages = new ArrayList<>(new ConversationLogGenerator(memory).generate(logSizeLimit, includeFirstAgentMessage, true) + .getMessages().stream().map(this::convertMessage).toList()); } // If a custom prompt is defined, replace the last user input with it @@ -192,8 +192,8 @@ List buildTokenAwareMessages(IConversationMemory memory, String sys if (skipSteps > 0) { allMessages = generateMessagesFromOutputs(memory, skipSteps, -1, includeFirstAgentMessage); } else { - allMessages = new ArrayList<>(new ConversationLogGenerator(memory).generate(-1, includeFirstAgentMessage).getMessages().stream() - .map(this::convertMessage).toList()); + allMessages = new ArrayList<>(new ConversationLogGenerator(memory).generate(-1, includeFirstAgentMessage, true).getMessages() + .stream().map(this::convertMessage).toList()); } // If a custom prompt is defined, replace the last user input with it @@ -316,12 +316,13 @@ private ArrayList generateMessagesFromOutputs(IConversationMemory m startIndex = Math.max(startIndex, windowStart); } + var allSteps = memory.getAllSteps(); var result = new ArrayList(); for (int i = startIndex; i < outputs.size(); i++) { var output = outputs.get(i); var input = output.get("input", String.class); if (input != null) { - result.add(UserMessage.from(input)); + result.add(UserMessage.from(ConversationLogGenerator.withAttachmentExtracts(allSteps, i, input))); } Object outputObj = output.get("output"); diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java index 05791b45a0..b0b4b716eb 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java @@ -29,6 +29,7 @@ import ai.labs.eddi.engine.setup.AgentSetupService; import ai.labs.eddi.engine.tenancy.TenantQuotaService; import ai.labs.eddi.modules.apicalls.impl.PrePostUtils; +import ai.labs.eddi.modules.llm.capability.ModelCapabilityService; import ai.labs.eddi.modules.llm.model.LlmConfiguration; import ai.labs.eddi.modules.llm.model.LlmConfiguration.Task; import ai.labs.eddi.modules.apicalls.impl.IApiCallExecutor; @@ -340,9 +341,21 @@ private void executeTask(IConversationMemory memory, Task task, IWritableConvers } // Forward the current step's attachments to the LLM as multimodal content, - // gated on the resolved (provider, model) capabilities. + // gated on the resolved (provider, model) capabilities, honoring any + // per-task multimodal overrides. if (attachmentForwarder != null) { - attachmentForwarder.forward(messages, memory, resolvedType, resolveModelName(processedParams)); + var mm = task.getMultimodal(); + var vision = mm != null + ? ModelCapabilityService.Support.parse(mm.getVision()) + : ModelCapabilityService.Support.AUTO; + var documents = mm != null + ? ModelCapabilityService.Support.parse(mm.getDocuments()) + : ModelCapabilityService.Support.AUTO; + var audio = mm != null + ? ModelCapabilityService.Support.parse(mm.getAudio()) + : ModelCapabilityService.Support.AUTO; + attachmentForwarder.forward(messages, memory, resolvedType, resolveModelName(processedParams), + vision, documents, audio); } if (messages.isEmpty()) { diff --git a/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java b/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java index 7a623c2fff..5f02be0921 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java +++ b/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java @@ -298,6 +298,25 @@ public static class Task { */ private IdentityMaskingConfig identityMasking; + /** + * Per-task multimodal capability overrides. Each of {@code vision}, + * {@code documents}, {@code audio} may be {@code "auto"} (defer to + * deployment/built-in defaults), {@code "on"} (force enabled) or {@code "off"} + * (force disabled). {@code null} means all defer. + * + * @since 6.1.0 + */ + private MultimodalOverride multimodal; + + /** + * Number of past turns whose attachments are natively re-attached to the LLM on + * later turns (in addition to the always-on text-extract stitching). {@code 0} + * (default) means attachments attach only on their own turn. + * + * @since 6.1.0 + */ + private Integer reattachTurns = 0; + // === Helper Methods === /** @@ -630,6 +649,60 @@ public void setIdentityMasking(IdentityMaskingConfig identityMasking) { this.identityMasking = identityMasking; } + public MultimodalOverride getMultimodal() { + return multimodal; + } + + public void setMultimodal(MultimodalOverride multimodal) { + this.multimodal = multimodal; + } + + public Integer getReattachTurns() { + return reattachTurns != null ? reattachTurns : 0; + } + + public void setReattachTurns(Integer reattachTurns) { + this.reattachTurns = reattachTurns; + } + + } + + /** + * Per-task multimodal capability overrides. Each field is a tri-state token + * {@code "auto"|"on"|"off"} parsed by + * {@link ai.labs.eddi.modules.llm.capability.ModelCapabilityService.Support#parse}. + * Unset fields default to {@code "auto"}. + * + * @since 6.1.0 + */ + public static class MultimodalOverride { + private String vision = "auto"; + private String documents = "auto"; + private String audio = "auto"; + + public String getVision() { + return vision; + } + + public void setVision(String vision) { + this.vision = vision; + } + + public String getDocuments() { + return documents; + } + + public void setDocuments(String documents) { + this.documents = documents; + } + + public String getAudio() { + return audio; + } + + public void setAudio(String audio) { + this.audio = audio; + } } /** diff --git a/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java b/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java index ab951b61b9..fd0d0312d4 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java @@ -38,6 +38,62 @@ void nullEverything() { } } + // ─── Attachment extract stitching ─────────────────────────── + + @Nested + @DisplayName("Attachment extract stitching") + class ExtractStitching { + + private IConversationMemory memoryWithExtract(String input, List extracts) { + var output = new ConversationOutput(); + output.put("input", input); + var memory = mock(IConversationMemory.class); + when(memory.getConversationOutputs()).thenReturn(new ArrayList<>(List.of(output))); + + var step = mock(IConversationMemory.IConversationStep.class); + @SuppressWarnings("unchecked") + IData> data = mock(IData.class); + when(data.getResult()).thenReturn(extracts); + when(step.getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS)).thenReturn(data); + + var stack = mock(IConversationMemory.IConversationStepStack.class); + when(stack.size()).thenReturn(1); + when(stack.get(0)).thenReturn(step); + when(memory.getAllSteps()).thenReturn(stack); + return memory; + } + + @Test + @DisplayName("stitchExtracts=true appends extracts to the user turn") + void stitchesWhenEnabled() { + var memory = memoryWithExtract("Summarize this", List.of("report.pdf: quarterly numbers")); + var log = new ConversationLogGenerator(memory).generate(-1, true, true); + + String userText = log.getMessages().getFirst().getContent().getLast().getValue(); + assertTrue(userText.contains("Summarize this")); + assertTrue(userText.contains("quarterly numbers"), "extracts should be stitched: " + userText); + } + + @Test + @DisplayName("stitchExtracts=false leaves the transcript clean") + void noStitchWhenDisabled() { + var memory = memoryWithExtract("Summarize this", List.of("report.pdf: quarterly numbers")); + var log = new ConversationLogGenerator(memory).generate(-1, true, false); + + String userText = log.getMessages().getFirst().getContent().getLast().getValue(); + assertEquals("Summarize this", userText); + verify(memory, never()).getAllSteps(); + } + + @Test + @DisplayName("no extracts on the step leaves input unchanged") + void noExtractsUnchanged() { + var memory = memoryWithExtract("Hi", List.of()); + var log = new ConversationLogGenerator(memory).generate(-1, true, true); + assertEquals("Hi", log.getMessages().getFirst().getContent().getLast().getValue()); + } + } + // ─── Basic generation ─────────────────────────────────────── @Nested diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java index 2ead72ad1a..2563e9e29e 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java @@ -252,6 +252,44 @@ void audioWithoutSupport_addsNote() { assertInstanceOf(TextContent.class, ((UserMessage) messages.get(0)).contents().get(1)); } + // ==================== Per-task overrides ==================== + + @Test + void visionOverrideOff_forcesImageNoteOnCapableModel() { + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setBase64Data(Base64.getEncoder().encodeToString("png".getBytes())); + mockAttachments(att); + List messages = messages(UserMessage.from("look")); + + // openai/gpt-4o has vision, but per-task OFF suppresses it + forwarder.forward(messages, memory, "openai", "gpt-4o", + ModelCapabilityService.Support.OFF, + ModelCapabilityService.Support.AUTO, + ModelCapabilityService.Support.AUTO); + + Content c = ((UserMessage) messages.get(0)).contents().get(1); + assertInstanceOf(TextContent.class, c); + assertTrue(((TextContent) c).text().contains("not forwarded")); + } + + @Test + void documentsOverrideOn_forcesNativePdfOnNonDocModel() { + Attachment att = new Attachment(); + att.setMimeType("application/pdf"); + att.setBase64Data(Base64.getEncoder().encodeToString("%PDF-1.4".getBytes())); + mockAttachments(att); + List messages = messages(UserMessage.from("summarize")); + + // openai defaults documents=off, but per-task ON forces native PdfFileContent + forwarder.forward(messages, memory, "openai", "gpt-4o", + ModelCapabilityService.Support.AUTO, + ModelCapabilityService.Support.ON, + ModelCapabilityService.Support.AUTO); + + assertInstanceOf(PdfFileContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + // ==================== Unsupported + caps ==================== @Test diff --git a/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java b/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java index 8953342e35..606b73e007 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java @@ -19,6 +19,67 @@ */ class LlmConfigurationTaskTest { + @Nested + @DisplayName("Multimodal override") + class MultimodalOverrideTests { + + @Test + @DisplayName("defaults to null") + void defaultsNull() { + assertNull(new LlmConfiguration.Task().getMultimodal()); + } + + @Test + @DisplayName("MultimodalOverride fields default to auto") + void overrideDefaultsAuto() { + var mm = new LlmConfiguration.MultimodalOverride(); + assertEquals("auto", mm.getVision()); + assertEquals("auto", mm.getDocuments()); + assertEquals("auto", mm.getAudio()); + } + + @Test + @DisplayName("getter/setter round-trip") + void setAndGet() { + var task = new LlmConfiguration.Task(); + var mm = new LlmConfiguration.MultimodalOverride(); + mm.setVision("on"); + mm.setDocuments("off"); + mm.setAudio("auto"); + task.setMultimodal(mm); + assertEquals("on", task.getMultimodal().getVision()); + assertEquals("off", task.getMultimodal().getDocuments()); + assertEquals("auto", task.getMultimodal().getAudio()); + } + } + + @Nested + @DisplayName("reattachTurns") + class ReattachTurns { + + @Test + @DisplayName("defaults to 0") + void defaultsZero() { + assertEquals(0, new LlmConfiguration.Task().getReattachTurns()); + } + + @Test + @DisplayName("null coalesces to 0") + void nullCoalescesZero() { + var task = new LlmConfiguration.Task(); + task.setReattachTurns(null); + assertEquals(0, task.getReattachTurns()); + } + + @Test + @DisplayName("getter/setter round-trip") + void setAndGet() { + var task = new LlmConfiguration.Task(); + task.setReattachTurns(3); + assertEquals(3, task.getReattachTurns()); + } + } + @Nested @DisplayName("isAgentMode") class IsAgentMode { From 3ea20074a288ef7ee931bd30acaf363e7f59a2ff Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 09:00:21 +0200 Subject: [PATCH 11/23] feat(attachments): readAttachment tool for multi-turn recall (Phase 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ReadAttachmentTool (@Vetoed) gives the LLM on-demand access to the conversation's attachments: listAttachments() and readAttachment(nameOrRef, page) — 1-based PDF page or 0 for the whole doc, else a no-extractable-text note. Conversation id is implicit (constructor-injected), so the LLM never supplies userId/conversationId and can only reach its own or granted attachments, enforced by IAttachmentStore. AgentOrchestrator gains setAttachmentServices(store, extractor), wired by LlmTask in a new @PostConstruct after CDI injection (long constructor + its six direct-construction tests untouched). Auto-added in the no-whitelist branch when the turn has attachments, and under whitelist key "readattachment"; skipped when services are unset or no attachments. Forwarder fallback notes already reference the tool. Tests: ReadAttachmentToolTest (11) + 5 orchestrator auto-add branch tests. Phase 4 of multimodal-attachments-completion-plan. --- docs/changelog.md | 16 ++ .../modules/llm/impl/AgentOrchestrator.java | 42 +++++ .../labs/eddi/modules/llm/impl/LlmTask.java | 16 ++ .../llm/tools/impl/ReadAttachmentTool.java | 130 +++++++++++++++ .../llm/impl/AgentOrchestratorBranchTest.java | 86 ++++++++++ .../tools/impl/ReadAttachmentToolTest.java | 154 ++++++++++++++++++ 6 files changed, 444 insertions(+) create mode 100644 src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java create mode 100644 src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java diff --git a/docs/changelog.md b/docs/changelog.md index c956fe4025..9116eb5220 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,22 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Phase 4: readAttachment tool (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) +**Plan:** `planning/multimodal-attachments-completion-plan.md` (Phase 4 of 6). + +### What changed + +- **`ReadAttachmentTool`** (`modules/llm/tools/impl`, `@Vetoed`) — the multi-turn recall path. Two `@Tool`s: `listAttachments()` (name/type/size/ref of every attachment in the conversation) and `readAttachment(nameOrRef, page)` (loads one attachment, extracts text — 1-based PDF page or 0 for whole doc — else a "no extractable text" note). The conversation id is implicit (constructor-injected), so the LLM never supplies a userId/conversationId and can only reach its own (or granted) attachments — enforced by `IAttachmentStore`. +- **Auto-add wiring** — `AgentOrchestrator` gains `setAttachmentServices(store, extractor)` (wired by `LlmTask` in a new `@PostConstruct`, after CDI injection, so the long constructor + its six direct-construction tests are untouched). `addReadAttachmentToolIfEnabled` adds the tool in the no-whitelist branch when the turn has attachments, and in the whitelist branch under key `readattachment`; skipped when the services are unset (isolated tests) or the turn has no attachments. The forwarder's fallback notes already point the model at this tool. + +### Tests + +`ReadAttachmentToolTest` (11 — list/read by name & ref, case-insensitive, PDF page, not-found, non-extractable, denied load, empty text, blank ref) + 5 orchestrator auto-add branch tests (no-whitelist, whitelisted, whitelist-excluded, services-unset, no-attachments). Existing orchestrator/LlmTask tests unchanged. + --- ## 📎 Multimodal Attachments Completion — Phase 2: Unified forwarder (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java index da4807c638..d13e2a4190 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java @@ -18,8 +18,11 @@ import ai.labs.eddi.configs.properties.model.Property; import ai.labs.eddi.engine.api.IConversationService; import ai.labs.eddi.engine.lifecycle.exceptions.LifecycleException; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemory; +import ai.labs.eddi.engine.memory.IData; import ai.labs.eddi.engine.memory.IMemoryItemConverter; +import ai.labs.eddi.engine.memory.MemoryKeys; import ai.labs.eddi.engine.memory.MemorySnapshotService; import ai.labs.eddi.engine.runtime.IAgentFactory; import ai.labs.eddi.engine.runtime.client.configuration.IResourceClientLibrary; @@ -114,6 +117,21 @@ class AgentOrchestrator { private final IAgentFactory agentFactory; private final IAgentStore agentStore; + // Wired post-construction by LlmTask (see setAttachmentServices) so the long + // constructor and its many direct-construction unit tests stay unchanged. + private IAttachmentStore attachmentStore; + private AttachmentTextExtractor attachmentTextExtractor; + + /** + * Provide the attachment services used to build the {@code readAttachment} + * tool. Called by {@code LlmTask} after CDI injection completes; when unset + * (e.g. in isolated unit tests) the tool is simply never added. + */ + void setAttachmentServices(IAttachmentStore attachmentStore, AttachmentTextExtractor attachmentTextExtractor) { + this.attachmentStore = attachmentStore; + this.attachmentTextExtractor = attachmentTextExtractor; + } + AgentOrchestrator(CalculatorTool calculatorTool, DateTimeTool dateTimeTool, WebSearchTool webSearchTool, DataFormatterTool dataFormatterTool, WebScraperTool webScraperTool, TextSummarizerTool textSummarizerTool, PdfReaderTool pdfReaderTool, WeatherTool weatherTool, FetchToolResponsePageTool fetchToolResponsePageTool, @@ -563,6 +581,8 @@ private List collectAllBuiltInTools(LlmConfiguration.Task task, IConvers addUserMemoryToolIfEnabled(tools, memory); if (whitelist.contains("conversationRecall")) addConversationRecallToolIfEnabled(tools, task, memory); + if (whitelist.contains("readattachment")) + addReadAttachmentToolIfEnabled(tools, memory); // Dynamic agent tools (whitelist-gated, shared tracking lists) { List sharedCreatedIds = new java.util.concurrent.CopyOnWriteArrayList<>(); @@ -620,6 +640,8 @@ private List collectAllBuiltInTools(LlmConfiguration.Task task, IConvers addUserMemoryToolIfEnabled(tools, memory); // Auto-add conversation recall tool if rolling summary is active addConversationRecallToolIfEnabled(tools, task, memory); + // Auto-add the readAttachment tool when this turn has attachments + addReadAttachmentToolIfEnabled(tools, memory); } return tools; @@ -674,6 +696,26 @@ private void addConversationRecallToolIfEnabled(List tools, LlmConfigura summaryConfig.getMaxRecallTurns()); } + /** + * Constructs and adds a {@link ReadAttachmentTool} when this turn carries + * attachments, giving the LLM on-demand access to attachment text (recall of an + * earlier turn's file, oversize files not inlined, page-targeted PDF reads). + * The conversation id is implicit — the tool never takes it as a parameter. + */ + private void addReadAttachmentToolIfEnabled(List tools, IConversationMemory memory) { + if (attachmentStore == null || attachmentTextExtractor == null) { + return; + } + IData> attachmentData = memory.getCurrentStep().getLatestData(MemoryKeys.ATTACHMENTS); + if (attachmentData == null || attachmentData.getResult() == null || attachmentData.getResult().isEmpty()) { + return; + } + var tool = new ReadAttachmentTool(attachmentStore, attachmentTextExtractor, memory.getConversationId()); + tools.add(tool); + LOGGER.infof("[ATTACHMENTS] ReadAttachmentTool enabled for conversation='%s' with %d attachment(s)", + sanitize(memory.getConversationId()), attachmentData.getResult().size()); + } + /** * Resolves the DynamicAgentConfig for the current conversation. If the agent is * participating in a group discussion, the group's {@link DynamicAgentConfig} diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java index b0b4b716eb..483c690263 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java @@ -112,6 +112,22 @@ public class LlmTask implements ILifecycleTask { @jakarta.inject.Inject AttachmentForwarder attachmentForwarder; + @jakarta.inject.Inject + AttachmentTextExtractor attachmentTextExtractor; + + /** + * Wire the attachment services into the (constructor-built) AgentOrchestrator + * after CDI field injection completes, so the {@code readAttachment} tool can + * be offered. Skipped in direct-construction unit tests (no CDI) — the tool is + * simply never added there. + */ + @jakarta.annotation.PostConstruct + void wireAttachmentServices() { + if (agentOrchestrator != null) { + agentOrchestrator.setAttachmentServices(attachmentStore, attachmentTextExtractor); + } + } + // Retained for httpCall RAG discovery + execution (Phase 8c-0) private final IApiCallExecutor apiCallExecutor; private final IRestAgentStore restAgentStore; diff --git a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java new file mode 100644 index 0000000000..4ad485e929 --- /dev/null +++ b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java @@ -0,0 +1,130 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.tools.impl; + +import ai.labs.eddi.engine.attachments.IAttachmentStore; +import ai.labs.eddi.engine.attachments.IAttachmentStore.Attachment; +import ai.labs.eddi.modules.llm.tools.impl.AttachmentTextExtractor.AttachmentExtractionException; +import dev.langchain4j.agent.tool.P; +import dev.langchain4j.agent.tool.Tool; +import jakarta.enterprise.inject.Vetoed; +import org.jboss.logging.Logger; + +import java.util.List; +import java.util.Locale; + +/** + * Built-in LLM tool for reading the current conversation's uploaded attachments + * on demand — the multi-turn recall path for content that was forwarded on an + * earlier turn, is too large to inline, or needs a specific PDF page. + *

+ * The conversation is implicit: the tool is constructed with the conversation + * id by {@code AgentOrchestrator}, so the LLM never supplies a + * userId/conversationId — it can only see attachments belonging to (or granted + * to) its own conversation, enforced by {@link IAttachmentStore}. + *

+ * Constructed per-invocation — NOT a CDI bean. + * + * @since 6.1.0 + */ +@Vetoed +public class ReadAttachmentTool { + + private static final Logger LOGGER = Logger.getLogger(ReadAttachmentTool.class); + + private final IAttachmentStore attachmentStore; + private final AttachmentTextExtractor textExtractor; + private final String conversationId; + + public ReadAttachmentTool(IAttachmentStore attachmentStore, AttachmentTextExtractor textExtractor, + String conversationId) { + this.attachmentStore = attachmentStore; + this.textExtractor = textExtractor; + this.conversationId = conversationId; + } + + @Tool("Lists the files attached to this conversation. Returns each attachment's name, type and size. " + + "Use this to discover what is available before calling readAttachment.") + public String listAttachments() { + List attachments = attachmentStore.listByConversation(conversationId); + if (attachments.isEmpty()) { + return "No attachments are available in this conversation."; + } + StringBuilder sb = new StringBuilder("Attachments in this conversation:\n"); + for (Attachment a : attachments) { + sb.append("- ").append(a.filename() != null ? a.filename() : "(unnamed)") + .append(" [").append(a.mimeType()).append(", ").append(a.sizeBytes()).append(" bytes]") + .append(" ref=").append(a.storageRef()).append("\n"); + } + return sb.toString(); + } + + @Tool("Reads the text content of one attachment in this conversation. Identify it by file name or " + + "reference. For PDFs, pass a 1-based page number to read just that page, or 0 for the whole " + + "document. Returns extracted text, or a note if the attachment has no extractable text.") + public String readAttachment( + @P("The attachment's file name or storage reference") String nameOrRef, + @P("For PDFs: 1-based page to read, or 0 for the whole document. Ignored for other types.") int page) { + + Attachment match = resolve(nameOrRef); + if (match == null) { + return "No attachment named '" + nameOrRef + "' was found in this conversation. " + + "Call listAttachments to see what is available."; + } + try { + byte[] bytes = attachmentStore.load(match.storageRef(), conversationId); + String mime = match.mimeType() == null ? "" : match.mimeType().toLowerCase(Locale.ROOT); + + String text; + if (mime.startsWith("application/pdf") && page > 0) { + text = textExtractor.extractPdfText(bytes, page, page, textExtractor.getDefaultMaxChars()); + } else if (textExtractor.canExtractText(match.mimeType())) { + text = textExtractor.extractText(bytes, match.mimeType()); + } else { + return "Attachment '" + display(match) + "' is a " + match.mimeType() + + " and has no extractable text."; + } + if (text == null || text.isBlank()) { + return "Attachment '" + display(match) + "' contains no extractable text."; + } + return "Content of '" + display(match) + "' (" + match.mimeType() + "):\n" + text; + } catch (IAttachmentStore.AttachmentStoreException e) { + LOGGER.debugf("readAttachment load failed for '%s': %s", nameOrRef, e.getMessage()); + return "Could not read attachment '" + nameOrRef + "': " + e.getMessage(); + } catch (AttachmentExtractionException e) { + LOGGER.debugf("readAttachment extraction failed for '%s': %s", nameOrRef, e.getMessage()); + return "Could not extract text from '" + display(match) + "': " + e.getMessage(); + } + } + + private Attachment resolve(String nameOrRef) { + if (nameOrRef == null || nameOrRef.isBlank()) { + return null; + } + List attachments = attachmentStore.listByConversation(conversationId); + // Prefer an exact storageRef match, then an exact file-name match, then + // case-insensitive name. + for (Attachment a : attachments) { + if (nameOrRef.equals(a.storageRef())) { + return a; + } + } + for (Attachment a : attachments) { + if (nameOrRef.equals(a.filename())) { + return a; + } + } + for (Attachment a : attachments) { + if (a.filename() != null && a.filename().equalsIgnoreCase(nameOrRef)) { + return a; + } + } + return null; + } + + private static String display(Attachment a) { + return a.filename() != null ? a.filename() : a.storageRef(); + } +} diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java index 255b4fd897..d141c3f4af 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java @@ -10,8 +10,11 @@ import ai.labs.eddi.configs.properties.model.Property; import ai.labs.eddi.configs.workflows.IRestWorkflowStore; import ai.labs.eddi.datastore.serialization.IJsonSerialization; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.IConversationMemory; +import ai.labs.eddi.engine.memory.IData; import ai.labs.eddi.engine.memory.IMemoryItemConverter; +import ai.labs.eddi.engine.memory.MemoryKeys; import ai.labs.eddi.engine.memory.MemorySnapshotService; import ai.labs.eddi.engine.runtime.client.configuration.IResourceClientLibrary; import ai.labs.eddi.engine.tenancy.TenantQuotaService; @@ -144,6 +147,89 @@ void trueNoWhitelist() { } } + // ========================================================= + // collectEnabledTools — readAttachment auto-add + // ========================================================= + + @Nested + @DisplayName("collectEnabledTools — readAttachment tool") + class ReadAttachmentAutoAdd { + + @SuppressWarnings({"rawtypes", "unchecked"}) + private void withAttachments(boolean present) { + var step = mock(IConversationMemory.IWritableConversationStep.class); + when(memory.getCurrentStep()).thenReturn(step); + when(memory.getConversationId()).thenReturn("conv-1"); + if (present) { + IData data = mock(IData.class); + doReturn(List.of(new Object())).when(data).getResult(); + doReturn(data).when(step).getLatestData(MemoryKeys.ATTACHMENTS); + } else { + doReturn(null).when(step).getLatestData(MemoryKeys.ATTACHMENTS); + } + } + + private boolean hasReadAttachmentTool(List tools) { + return tools.stream().anyMatch(t -> t instanceof ReadAttachmentTool); + } + + @Test + @DisplayName("auto-added (no whitelist) when services set and attachments present") + void autoAdded() { + orchestrator.setAttachmentServices(mock(IAttachmentStore.class), new AttachmentTextExtractor(10_000)); + withAttachments(true); + var task = new LlmConfiguration.Task(); + task.setEnableBuiltInTools(true); + + assertTrue(hasReadAttachmentTool(orchestrator.collectEnabledTools(task, memory))); + } + + @Test + @DisplayName("added when whitelisted by 'readattachment'") + void addedViaWhitelist() { + orchestrator.setAttachmentServices(mock(IAttachmentStore.class), new AttachmentTextExtractor(10_000)); + withAttachments(true); + var task = new LlmConfiguration.Task(); + task.setEnableBuiltInTools(true); + task.setBuiltInToolsWhitelist(List.of("readattachment")); + + assertTrue(hasReadAttachmentTool(orchestrator.collectEnabledTools(task, memory))); + } + + @Test + @DisplayName("NOT added when whitelist excludes it") + void notAddedWhenWhitelistExcludes() { + orchestrator.setAttachmentServices(mock(IAttachmentStore.class), new AttachmentTextExtractor(10_000)); + withAttachments(true); + var task = new LlmConfiguration.Task(); + task.setEnableBuiltInTools(true); + task.setBuiltInToolsWhitelist(List.of("calculator")); + + assertFalse(hasReadAttachmentTool(orchestrator.collectEnabledTools(task, memory))); + } + + @Test + @DisplayName("NOT added when attachment services are unset") + void notAddedWithoutServices() { + withAttachments(true); // services never set + var task = new LlmConfiguration.Task(); + task.setEnableBuiltInTools(true); + + assertFalse(hasReadAttachmentTool(orchestrator.collectEnabledTools(task, memory))); + } + + @Test + @DisplayName("NOT added when the turn has no attachments") + void notAddedWithoutAttachments() { + orchestrator.setAttachmentServices(mock(IAttachmentStore.class), new AttachmentTextExtractor(10_000)); + withAttachments(false); + var task = new LlmConfiguration.Task(); + task.setEnableBuiltInTools(true); + + assertFalse(hasReadAttachmentTool(orchestrator.collectEnabledTools(task, memory))); + } + } + // ========================================================= // collectEnabledTools — whitelist filtering // ========================================================= diff --git a/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java new file mode 100644 index 0000000000..1a32ddba88 --- /dev/null +++ b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java @@ -0,0 +1,154 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.modules.llm.tools.impl; + +import ai.labs.eddi.engine.attachments.IAttachmentStore; +import ai.labs.eddi.engine.attachments.IAttachmentStore.Attachment; +import ai.labs.eddi.engine.attachments.IAttachmentStore.AttachmentStoreException; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDPage; +import org.apache.pdfbox.pdmodel.PDPageContentStream; +import org.apache.pdfbox.pdmodel.font.PDType1Font; +import org.apache.pdfbox.pdmodel.font.Standard14Fonts; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +/** + * Unit tests for {@link ReadAttachmentTool}. + */ +class ReadAttachmentToolTest { + + private static final String CONV = "conv-1"; + + private IAttachmentStore store; + private ReadAttachmentTool tool; + + @BeforeEach + void setUp() { + store = mock(IAttachmentStore.class); + tool = new ReadAttachmentTool(store, new AttachmentTextExtractor(10_000), CONV); + } + + private static Attachment att(String ref, String name, String mime, long size) { + return new Attachment(ref, name, mime, size, CONV); + } + + // ==================== listAttachments ==================== + + @Test + void list_empty() { + when(store.listByConversation(CONV)).thenReturn(List.of()); + assertTrue(tool.listAttachments().contains("No attachments")); + } + + @Test + void list_formatsEntries() { + when(store.listByConversation(CONV)).thenReturn(List.of( + att("r1", "report.pdf", "application/pdf", 2048), + att("r2", "notes.txt", "text/plain", 12))); + String out = tool.listAttachments(); + assertTrue(out.contains("report.pdf")); + assertTrue(out.contains("application/pdf")); + assertTrue(out.contains("notes.txt")); + assertTrue(out.contains("r2")); + } + + // ==================== readAttachment ==================== + + @Test + void read_byFileName_extractsText() throws Exception { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "notes.txt", "text/plain", 5))); + when(store.load("r1", CONV)).thenReturn("hello world".getBytes(StandardCharsets.UTF_8)); + + String out = tool.readAttachment("notes.txt", 0); + assertTrue(out.contains("hello world")); + } + + @Test + void read_byStorageRef_extractsText() throws Exception { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "notes.txt", "text/plain", 5))); + when(store.load("r1", CONV)).thenReturn("body".getBytes(StandardCharsets.UTF_8)); + + assertTrue(tool.readAttachment("r1", 0).contains("body")); + } + + @Test + void read_caseInsensitiveFileName() throws Exception { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "Notes.TXT", "text/plain", 5))); + when(store.load("r1", CONV)).thenReturn("x".getBytes(StandardCharsets.UTF_8)); + assertTrue(tool.readAttachment("notes.txt", 0).contains("x")); + } + + @Test + void read_pdfPage() throws Exception { + byte[] pdf = multiPagePdf("Alpha page", "Beta page"); + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "doc.pdf", "application/pdf", pdf.length))); + when(store.load("r1", CONV)).thenReturn(pdf); + + String out = tool.readAttachment("doc.pdf", 2); + assertTrue(out.contains("Beta page")); + assertFalse(out.contains("Alpha page")); + } + + @Test + void read_notFound() { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); + assertTrue(tool.readAttachment("missing.txt", 0).contains("No attachment named")); + } + + @Test + void read_nonExtractableType_note() throws Exception { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "pic.png", "image/png", 100))); + when(store.load("r1", CONV)).thenReturn(new byte[]{1, 2, 3}); + assertTrue(tool.readAttachment("pic.png", 0).contains("no extractable text")); + } + + @Test + void read_loadDenied_error() throws Exception { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); + when(store.load("r1", CONV)).thenThrow(new AttachmentStoreException("access denied")); + assertTrue(tool.readAttachment("a.txt", 0).contains("Could not read attachment")); + } + + @Test + void read_emptyText_note() throws Exception { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "empty.txt", "text/plain", 0))); + when(store.load("r1", CONV)).thenReturn(new byte[0]); + assertTrue(tool.readAttachment("empty.txt", 0).contains("no extractable text")); + } + + @Test + void read_blankRef_notFound() { + when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); + assertTrue(tool.readAttachment(" ", 0).contains("No attachment named")); + } + + // ==================== helper ==================== + + private static byte[] multiPagePdf(String... pages) throws Exception { + try (PDDocument doc = new PDDocument(); ByteArrayOutputStream out = new ByteArrayOutputStream()) { + for (String text : pages) { + var page = new PDPage(); + doc.addPage(page); + try (var cs = new PDPageContentStream(doc, page)) { + cs.beginText(); + cs.setFont(new PDType1Font(Standard14Fonts.FontName.HELVETICA), 12); + cs.newLineAtOffset(50, 700); + cs.showText(text); + cs.endText(); + } + } + doc.save(out); + return out.toByteArray(); + } + } +} From 2ce8e94adebfcbb09b393dab03f2d08544b09af2 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 09:19:03 +0200 Subject: [PATCH 12/23] =?UTF-8?q?feat(attachments):=20group=20parity=20?= =?UTF-8?q?=E2=80=94=20fan-out=20grants=20+=20member=20injection=20(Phase?= =?UTF-8?q?=203)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Share discussion attachments with every group member. - IRestGroupConversation.DiscussRequest gains optional attachments (AttachmentRef = {mimeType,data,url,fileName}) + a 2-arg compat constructor; IGroupConversationService.discuss/startAndDiscussAsync gain attachment-carrying overloads. - GroupConversationService.materializeAttachments stores inline base64 files in IAttachmentStore bound to the group conversation id (grantable + reapable with it) and passes url refs through, stashing them on the transient GroupConversation.attachments. - On each member's FIRST turn, grantAndInjectAttachments grants the member conversation read access (the sole grant-minting site — trusted server code) and injects attachment_* context into its InputData; later phases use extract-stitching + the readAttachment tool. Nested groups re-grant down the chain. - RestGroupConversation converts AttachmentRef -> Attachment and routes through the attachment overload only when attachments are present, leaving the no-attachment path (and its tests) untouched. Transport is JSON inline; a multipart file-part endpoint variant is a thin follow-up. Tests: 7 service (materialize/grant/inject) + 2 REST routing. Phase 3 of multimodal-attachments-completion-plan. --- docs/changelog.md | 22 ++++ .../groups/model/GroupConversation.java | 19 +++ .../engine/api/IGroupConversationService.java | 22 ++++ .../engine/api/IRestGroupConversation.java | 17 ++- .../internal/GroupConversationService.java | 112 +++++++++++++++- .../internal/RestGroupConversation.java | 42 +++++- .../GroupConversationServiceTest.java | 124 ++++++++++++++++++ .../internal/RestGroupConversationTest.java | 37 ++++++ 8 files changed, 391 insertions(+), 4 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 9116eb5220..653ae1665e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,28 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Phase 3: Group parity (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) +**Plan:** `planning/multimodal-attachments-completion-plan.md` (Phase 3 of 6). + +### What changed + +- **`DiscussRequest` carries attachments** — `IRestGroupConversation.DiscussRequest` gains an optional `List attachments` (`AttachmentRef = {mimeType, data, url, fileName}`) plus a two-argument compat constructor, so existing JSON clients and call sites are unaffected. `IGroupConversationService.discuss(...)` and `startAndDiscussAsync(...)` gain attachment-carrying overloads (default methods → real impl overrides). +- **Materialize + bind at fan-out** — `GroupConversationService.materializeAttachments` stores inline base64 files in `IAttachmentStore` **bound to the group conversation id** (so they can be granted and reaped with it) and passes hosted `url` refs through, stashing the result on the (transient) `GroupConversation.attachments`. +- **Grant + inject per member** — on each member's **first** turn, `grantAndInjectAttachments` calls `IAttachmentStore.grantAccess(storageRef, memberConversationId)` (the only place grants are minted — trusted server code, D2) and injects `attachment_*` context into the member's `InputData`. Stored refs are granted; URL refs are forwarded without a grant. Later phases rely on the Phase-2 extract-stitching and the Phase-4 `readAttachment` tool. Nested groups receive the parent's attachments and re-grant down the chain. +- **REST routing** — `RestGroupConversation` converts `AttachmentRef → Attachment` and routes through the attachment overload only when attachments are present (so the no-attachment path — and its existing mock-based tests — is untouched). + +### Design note + +Group members run in their **own** conversations, so strict per-conversation ownership would block them from reading a group-uploaded blob — grants are exactly the primitive that makes this safe without opening cross-conversation access generally. Transport is JSON inline (base64/url); a multipart file-part variant of the endpoint is a thin follow-up (the capability and service path are complete). + +### Tests + +7 service tests (materialize base64/url/no-store/empty; grant+inject stored-ref/url/grant-failure/none) + 2 REST routing tests (attachment overload vs plain). Group ITs (member observes content, grant-before-turn, nested) stay CI-only. + --- ## 📎 Multimodal Attachments Completion — Phase 4: readAttachment tool (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java b/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java index 225dc5deb5..b0239fa0c2 100644 --- a/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java +++ b/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java @@ -51,6 +51,16 @@ public class GroupConversation { @JsonIgnore private transient AgentGroupConfiguration.DynamicAgentConfig dynamicAgentConfig; + /** + * Transient attachments for this discussion. Set at fan-out by + * {@code GroupConversationService} — inline files are materialized into the + * blob store (owned by this group conversation) and each member conversation is + * granted access. Not persisted to the transcript document; the blobs live in + * {@code IAttachmentStore} bound to this conversation's id. + */ + @JsonIgnore + private transient List attachments; + /** * A single entry in the discussion transcript. Each entry records one agent's * contribution during a specific phase. @@ -296,4 +306,13 @@ public AgentGroupConfiguration.DynamicAgentConfig getDynamicAgentConfig() { public void setDynamicAgentConfig(AgentGroupConfiguration.DynamicAgentConfig dynamicAgentConfig) { this.dynamicAgentConfig = dynamicAgentConfig; } + + @JsonIgnore + public List getAttachments() { + return attachments; + } + + public void setAttachments(List attachments) { + this.attachments = attachments; + } } diff --git a/src/main/java/ai/labs/eddi/engine/api/IGroupConversationService.java b/src/main/java/ai/labs/eddi/engine/api/IGroupConversationService.java index 76d8f3a8f6..973f2070f5 100644 --- a/src/main/java/ai/labs/eddi/engine/api/IGroupConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/api/IGroupConversationService.java @@ -7,6 +7,7 @@ import ai.labs.eddi.configs.groups.model.GroupConversation; import ai.labs.eddi.datastore.IResourceStore; import ai.labs.eddi.engine.lifecycle.GroupConversationEventSink; +import ai.labs.eddi.engine.memory.model.Attachment; import java.util.List; @@ -40,6 +41,17 @@ GroupConversation discuss(String groupId, String question, String userId, int de GroupConversation discuss(String groupId, String question, String userId, int depth, GroupDiscussionEventListener listener) throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException; + /** + * Start a group discussion with event callbacks, sharing {@code attachments} + * with every member agent (stored bound to the group conversation, members + * granted access). {@code attachments} may be null/empty. + */ + default GroupConversation discuss(String groupId, String question, String userId, int depth, + GroupDiscussionEventListener listener, List attachments) + throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException { + return discuss(groupId, question, userId, depth, listener); + } + /** * Start a group discussion asynchronously. Creates the GroupConversation record * synchronously (so the caller gets the ID), then runs phases in a background @@ -50,6 +62,16 @@ GroupConversation discuss(String groupId, String question, String userId, int de GroupConversation startAndDiscussAsync(String groupId, String question, String userId, GroupDiscussionEventListener listener) throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException; + /** + * Async variant sharing {@code attachments} with every member agent. + * {@code attachments} may be null/empty. + */ + default GroupConversation startAndDiscussAsync(String groupId, String question, String userId, + GroupDiscussionEventListener listener, List attachments) + throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException { + return startAndDiscussAsync(groupId, question, userId, listener); + } + /** * Read a group conversation transcript. */ diff --git a/src/main/java/ai/labs/eddi/engine/api/IRestGroupConversation.java b/src/main/java/ai/labs/eddi/engine/api/IRestGroupConversation.java index 045cb5f79e..680b23d92c 100644 --- a/src/main/java/ai/labs/eddi/engine/api/IRestGroupConversation.java +++ b/src/main/java/ai/labs/eddi/engine/api/IRestGroupConversation.java @@ -75,7 +75,22 @@ List listGroupConversations(@PathParam("groupId") String grou /** * Request body for starting a group discussion. + *

+ * Optional {@code attachments} are shared with every member agent: inline files + * ({@code data}) are stored server-side bound to the group conversation and + * each member is granted access; {@code url} references are forwarded as-is. + * Old two-argument clients remain compatible. */ - record DiscussRequest(String question, String userId) { + record DiscussRequest(String question, String userId, List attachments) { + public DiscussRequest(String question, String userId) { + this(question, userId, null); + } + } + + /** + * A single attachment reference on a group discussion request. Provide either + * inline base64 {@code data} (+ {@code mimeType}) or a hosted {@code url}. + */ + record AttachmentRef(String mimeType, String data, String url, String fileName) { } } diff --git a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java index 02e36b4d48..b53f8d2274 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java @@ -38,6 +38,7 @@ import ai.labs.eddi.engine.api.IGroupConversationService; import ai.labs.eddi.engine.memory.ConversationOutputExtractor; import ai.labs.eddi.engine.memory.model.ConversationState; +import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.model.Context; import ai.labs.eddi.engine.model.Deployment.Environment; import ai.labs.eddi.engine.model.InputData; @@ -89,6 +90,11 @@ public class GroupConversationService implements IGroupConversationService { private final NonceCacheService nonceCacheService; private final String defaultTenantId; + // Field-injected so the direct-construction unit tests stay unchanged; used to + // materialize and share discussion attachments with member conversations. + @jakarta.inject.Inject + ai.labs.eddi.engine.attachments.IAttachmentStore attachmentStore; + // Incremental peer verification: tracks the last verified transcript index // per group conversation ID, so we only verify new entries each turn (O(N) // amortized instead of O(N²)). Cleaned up when conversations complete. @@ -147,6 +153,13 @@ public GroupConversation discuss(String groupId, String question, String userId, @Override public GroupConversation discuss(String groupId, String question, String userId, int depth, GroupDiscussionEventListener listener) throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException { + return discuss(groupId, question, userId, depth, listener, null); + } + + @Override + public GroupConversation discuss(String groupId, String question, String userId, int depth, + GroupDiscussionEventListener listener, List attachments) + throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException { if (depth > maxDepth) { throw new GroupDepthExceededException("Maximum group discussion depth (%d) exceeded".formatted(maxDepth)); @@ -173,12 +186,20 @@ public GroupConversation discuss(String groupId, String question, String userId, } GroupConversation gc = createGroupConversation(groupId, question, userId, depth); + materializeAttachments(gc, attachments); return executeDiscussion(gc, config, phases, question, listener); } @Override public GroupConversation startAndDiscussAsync(String groupId, String question, String userId, GroupDiscussionEventListener listener) throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException { + return startAndDiscussAsync(groupId, question, userId, listener, null); + } + + @Override + public GroupConversation startAndDiscussAsync(String groupId, String question, String userId, + GroupDiscussionEventListener listener, List attachments) + throws GroupDiscussionException, IResourceStore.ResourceStoreException, IResourceStore.ResourceNotFoundException { if (groupId == null) { throw new IllegalArgumentException("groupId must not be null"); @@ -201,6 +222,7 @@ public GroupConversation startAndDiscussAsync(String groupId, String question, S // Create the conversation synchronously so we can return its ID GroupConversation gc = createGroupConversation(groupId, question, userId, 0); + materializeAttachments(gc, attachments); // Run the discussion in a virtual thread — reuse the same gc (no duplicate // creation) @@ -218,6 +240,84 @@ public GroupConversation startAndDiscussAsync(String groupId, String question, S return gc; } + /** + * Materialize discussion attachments and bind them to the group conversation. + * Inline base64 files are stored in the blob store owned by {@code gc.getId()} + * (so they can be granted to members and reaped with the conversation); hosted + * {@code url} references and pre-stored {@code storageRef}s pass through. The + * resulting list is stashed on the {@link GroupConversation} for fan-out. + */ + void materializeAttachments(GroupConversation gc, List incoming) { + if (incoming == null || incoming.isEmpty()) { + return; + } + if (attachmentStore == null) { + LOGGER.warn("Group attachments were provided but no attachment store is configured; ignoring them."); + return; + } + List materialized = new ArrayList<>(); + for (Attachment a : incoming) { + try { + if (a.getBase64Data() != null && !a.getBase64Data().isBlank()) { + byte[] bytes = Base64.getDecoder().decode(a.getBase64Data()); + var stored = attachmentStore.store(bytes, a.getMimeType(), a.getFileName(), gc.getId(), defaultTenantId); + materialized.add(new Attachment(stored.mimeType(), stored.filename(), stored.sizeBytes(), stored.storageRef())); + } else if (a.getUrl() != null && !a.getUrl().isBlank()) { + materialized.add(a); + } else if (a.getStorageRef() != null && !a.getStorageRef().isBlank()) { + materialized.add(a); + } + } catch (Exception e) { + LOGGER.warnf("Failed to materialize group attachment '%s': %s", a.getFileName(), e.getMessage()); + } + } + if (!materialized.isEmpty()) { + gc.setAttachments(materialized); + LOGGER.infof("Group conversation '%s' has %d shared attachment(s)", gc.getId(), materialized.size()); + } + } + + /** + * Grant a member conversation access to the group's stored attachments and + * inject them as {@code attachment_*} context on the member's first turn. URL + * references are forwarded as-is (no grant needed). + */ + void grantAndInjectAttachments(GroupConversation gc, String memberConvId, Map context) { + List atts = gc.getAttachments(); + if (atts == null || atts.isEmpty()) { + return; + } + int index = 0; + for (Attachment a : atts) { + Map value = new LinkedHashMap<>(); + if (a.getStorageRef() != null) { + if (attachmentStore != null) { + try { + attachmentStore.grantAccess(a.getStorageRef(), memberConvId); + } catch (Exception e) { + LOGGER.warnf("Failed to grant attachment '%s' to member conversation '%s': %s", + a.getStorageRef(), memberConvId, e.getMessage()); + continue; + } + } + value.put("storageRef", a.getStorageRef()); + if (a.getFileName() != null) { + value.put("fileName", a.getFileName()); + } + } else if (a.getUrl() != null) { + value.put("mimeType", a.getMimeType()); + value.put("url", a.getUrl()); + if (a.getFileName() != null) { + value.put("fileName", a.getFileName()); + } + } else { + continue; + } + context.put("attachment_" + index, new Context(Context.ContextType.object, value)); + index++; + } + } + /** * Core discussion execution loop. Shared by both synchronous {@link #discuss} * and asynchronous {@link #startAndDiscussAsync} to avoid duplicate @@ -1267,6 +1367,7 @@ private TranscriptEntry executeAgentTurn(GroupMember member, GroupConversation g // Get or create private conversation String privateConvId = gc.getMemberConversationIds().get(member.agentId()); + boolean firstMemberTurn = privateConvId == null; if (privateConvId == null) { try { Map groupContext = new LinkedHashMap<>(); @@ -1298,6 +1399,13 @@ private TranscriptEntry executeAgentTurn(GroupMember member, GroupConversation g context.put("dynamicAgentConfig", new Context(Context.ContextType.object, gc.getDynamicAgentConfig())); } + // Share discussion attachments with this member on its first turn: grant the + // member conversation access to group-owned blobs and inject attachment_*. + // Later phases rely on extraction-in-history and the readAttachment tool. + if (firstMemberTurn) { + grantAndInjectAttachments(gc, privateConvId, context); + } + // Wave 6: Peer verification — if the receiving agent requires it, // verify all signed entries from prior speakers before sending context verifyPriorEntriesIfRequired(member.agentId(), gc); @@ -1674,7 +1782,9 @@ private TranscriptEntry executeGroupMemberTurn(GroupMember member, GroupConversa LOGGER.infof("Executing sub-group '%s' (depth %d) as member of parent group '%s'", subGroupId, nextDepth, gc.getGroupId()); - GroupConversation subConversation = discuss(subGroupId, input, gc.getUserId(), nextDepth); + // Propagate the parent's attachments to the nested group so its members + // receive them too (each nested member conversation is granted in turn). + GroupConversation subConversation = discuss(subGroupId, input, gc.getUserId(), nextDepth, null, gc.getAttachments()); // Extract the synthesized answer, or concatenate all responses String response = subConversation.getSynthesizedAnswer(); diff --git a/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java b/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java index aaa0f89d96..0a331e7607 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java +++ b/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java @@ -11,6 +11,7 @@ import ai.labs.eddi.engine.api.IGroupConversationService.GroupDiscussionEventListener; import ai.labs.eddi.engine.api.IRestGroupConversation; import ai.labs.eddi.engine.lifecycle.GroupConversationEventSink; +import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.security.OwnershipValidator; import io.quarkus.security.ForbiddenException; import io.quarkus.security.identity.SecurityIdentity; @@ -22,6 +23,7 @@ import org.jboss.logging.Logger; import java.net.URI; +import java.util.ArrayList; import java.util.List; import static ai.labs.eddi.engine.exception.SneakyThrow.sneakyThrow; @@ -58,7 +60,10 @@ public Response discuss(String groupId, DiscussRequest request) { String userId = ownershipValidator.validateAndResolveUserId(identity, request.userId()); if (userId == null || userId.isBlank()) userId = "anonymous"; - GroupConversation gc = groupConversationService.discuss(groupId, request.question(), userId, 0); + List attachments = toAttachments(request.attachments()); + GroupConversation gc = attachments == null + ? groupConversationService.discuss(groupId, request.question(), userId, 0) + : groupConversationService.discuss(groupId, request.question(), userId, 0, null, attachments); URI location = URI.create("/groups/" + groupId + "/conversations/" + gc.getId()); return Response.created(location).entity(gc).build(); } catch (IGroupConversationService.GroupDepthExceededException e) { @@ -132,7 +137,12 @@ public void onTaskVerified(GroupConversationEventSink.TaskVerifiedEvent event) { } }; - groupConversationService.startAndDiscussAsync(groupId, request.question(), userId, listener); + List attachments = toAttachments(request.attachments()); + if (attachments == null) { + groupConversationService.startAndDiscussAsync(groupId, request.question(), userId, listener); + } else { + groupConversationService.startAndDiscussAsync(groupId, request.question(), userId, listener, attachments); + } } catch (IResourceStore.ResourceNotFoundException e) { sendEvent(eventSink, sse, GroupConversationEventSink.EVENT_GROUP_ERROR, @@ -146,6 +156,34 @@ public void onTaskVerified(GroupConversationEventSink.TaskVerifiedEvent event) { } } + /** + * Convert request-level attachment refs into the memory model carrier the + * service materializes. Refs without inline data or a url are skipped. + */ + private static List toAttachments(List refs) { + if (refs == null || refs.isEmpty()) { + return null; + } + List out = new ArrayList<>(); + for (AttachmentRef r : refs) { + if (r == null) { + continue; + } + Attachment a = new Attachment(); + a.setMimeType(r.mimeType()); + a.setFileName(r.fileName()); + if (r.data() != null && !r.data().isBlank()) { + a.setBase64Data(r.data()); + } else if (r.url() != null && !r.url().isBlank()) { + a.setUrl(r.url()); + } else { + continue; + } + out.add(a); + } + return out.isEmpty() ? null : out; + } + @Override public GroupConversation readGroupConversation(String groupId, String groupConversationId) { try { diff --git a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java index 43713f4733..3dc3f91af8 100644 --- a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java +++ b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java @@ -23,6 +23,8 @@ import ai.labs.eddi.engine.api.IConversationService; import ai.labs.eddi.engine.api.IGroupConversationService.GroupDepthExceededException; import ai.labs.eddi.engine.api.IGroupConversationService.GroupDiscussionException; +import ai.labs.eddi.engine.attachments.IAttachmentStore; +import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.memory.model.ConversationOutput; import ai.labs.eddi.engine.memory.model.SimpleConversationMemorySnapshot; import ai.labs.eddi.engine.runtime.IAgentFactory; @@ -87,6 +89,128 @@ void setUp() { nonceCacheService, DEFAULT_TENANT, MAX_DEPTH); } + // ================================================================= + // attachment materialize / grant / inject + // ================================================================= + + @Nested + class Attachments { + + private GroupConversation gc(String id) { + var gc = new GroupConversation(); + gc.setId(id); + return gc; + } + + @Test + void materialize_base64_storesAndBinds() throws Exception { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + when(store.store(any(), eq("image/png"), eq("a.png"), eq("gc-1"), eq(DEFAULT_TENANT))) + .thenReturn(new IAttachmentStore.Attachment("ref-1", "a.png", "image/png", 3, "gc-1")); + + var inline = new Attachment(); + inline.setMimeType("image/png"); + inline.setFileName("a.png"); + inline.setBase64Data(java.util.Base64.getEncoder().encodeToString("png".getBytes())); + var gc = gc("gc-1"); + + service.materializeAttachments(gc, List.of(inline)); + + assertEquals(1, gc.getAttachments().size()); + assertEquals("ref-1", gc.getAttachments().get(0).getStorageRef()); + } + + @Test + void materialize_url_passesThrough() { + service.attachmentStore = mock(IAttachmentStore.class); + var url = new Attachment(); + url.setMimeType("image/png"); + url.setUrl("https://example.com/y.png"); + var gc = gc("gc-1"); + + service.materializeAttachments(gc, List.of(url)); + + assertEquals("https://example.com/y.png", gc.getAttachments().get(0).getUrl()); + } + + @Test + void materialize_noStore_ignored() { + service.attachmentStore = null; + var inline = new Attachment(); + inline.setBase64Data("x"); + var gc = gc("gc-1"); + + service.materializeAttachments(gc, List.of(inline)); + assertNull(gc.getAttachments()); + } + + @Test + void materialize_nullOrEmpty_noop() { + service.attachmentStore = mock(IAttachmentStore.class); + var gc = gc("gc-1"); + service.materializeAttachments(gc, null); + service.materializeAttachments(gc, List.of()); + assertNull(gc.getAttachments()); + } + + @Test + void grantAndInject_storedRef_grantsAndInjects() throws Exception { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + var gc = gc("gc-1"); + gc.setAttachments(List.of(new Attachment("application/pdf", "doc.pdf", 10, "ref-1"))); + Map context = new LinkedHashMap<>(); + + service.grantAndInjectAttachments(gc, "member-conv", context); + + verify(store).grantAccess("ref-1", "member-conv"); + assertTrue(context.containsKey("attachment_0")); + var value = (Map) context.get("attachment_0").getValue(); + assertEquals("ref-1", value.get("storageRef")); + assertEquals("doc.pdf", value.get("fileName")); + } + + @Test + void grantAndInject_url_injectsWithoutGrant() { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + var gc = gc("gc-1"); + var url = new Attachment(); + url.setMimeType("image/png"); + url.setUrl("https://example.com/y.png"); + gc.setAttachments(List.of(url)); + Map context = new LinkedHashMap<>(); + + service.grantAndInjectAttachments(gc, "member-conv", context); + + verifyNoInteractions(store); + var value = (Map) context.get("attachment_0").getValue(); + assertEquals("https://example.com/y.png", value.get("url")); + } + + @Test + void grantAndInject_grantFailure_skipsEntry() throws Exception { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + doThrow(new IAttachmentStore.AttachmentStoreException("nope")).when(store).grantAccess(any(), any()); + var gc = gc("gc-1"); + gc.setAttachments(List.of(new Attachment("application/pdf", "d.pdf", 1, "ref-1"))); + Map context = new LinkedHashMap<>(); + + service.grantAndInjectAttachments(gc, "m", context); + assertFalse(context.containsKey("attachment_0")); + } + + @Test + void grantAndInject_noAttachments_noop() { + service.attachmentStore = mock(IAttachmentStore.class); + Map context = new LinkedHashMap<>(); + service.grantAndInjectAttachments(gc("gc-1"), "m", context); + assertTrue(context.isEmpty()); + } + } + // ================================================================= // discuss() tests // ================================================================= diff --git a/src/test/java/ai/labs/eddi/engine/internal/RestGroupConversationTest.java b/src/test/java/ai/labs/eddi/engine/internal/RestGroupConversationTest.java index f2ceea3d3e..6e9e089072 100644 --- a/src/test/java/ai/labs/eddi/engine/internal/RestGroupConversationTest.java +++ b/src/test/java/ai/labs/eddi/engine/internal/RestGroupConversationTest.java @@ -8,7 +8,9 @@ import ai.labs.eddi.datastore.IResourceStore; import ai.labs.eddi.datastore.serialization.IJsonSerialization; import ai.labs.eddi.engine.api.IGroupConversationService; +import ai.labs.eddi.engine.api.IRestGroupConversation.AttachmentRef; import ai.labs.eddi.engine.api.IRestGroupConversation.DiscussRequest; +import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.security.OwnershipValidator; import io.quarkus.security.ForbiddenException; import io.quarkus.security.identity.SecurityIdentity; @@ -17,6 +19,7 @@ import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import java.util.ArrayList; import java.util.List; @@ -78,6 +81,40 @@ void anonymousUser() throws Exception { verify(groupService).discuss("group-1", "Hello", "anonymous", 0); } + @Test + @DisplayName("routes inline attachments through the attachment-aware overload") + @SuppressWarnings("unchecked") + void discussWithAttachments() throws Exception { + var gc = new GroupConversation(); + gc.setId("gc-3"); + when(groupService.discuss(eq("group-1"), eq("Q"), eq("user-1"), eq(0), isNull(), anyList())) + .thenReturn(gc); + + var req = new DiscussRequest("Q", "user-1", + List.of(new AttachmentRef("image/png", "aGVsbG8=", null, "a.png"))); + Response response = restGroupConversation.discuss("group-1", req); + + assertEquals(201, response.getStatus()); + ArgumentCaptor> captor = ArgumentCaptor.forClass(List.class); + verify(groupService).discuss(eq("group-1"), eq("Q"), eq("user-1"), eq(0), isNull(), captor.capture()); + assertEquals(1, captor.getValue().size()); + assertEquals("aGVsbG8=", captor.getValue().get(0).getBase64Data()); + assertEquals("image/png", captor.getValue().get(0).getMimeType()); + verify(groupService, never()).discuss("group-1", "Q", "user-1", 0); + } + + @Test + @DisplayName("empty attachments use the plain overload") + void discussWithEmptyAttachments() throws Exception { + var gc = new GroupConversation(); + gc.setId("gc-4"); + when(groupService.discuss("group-1", "Q", "user-1", 0)).thenReturn(gc); + + restGroupConversation.discuss("group-1", new DiscussRequest("Q", "user-1", List.of())); + + verify(groupService).discuss("group-1", "Q", "user-1", 0); + } + @Test @DisplayName("should return 400 for GroupDepthExceededException") void depthExceeded() throws Exception { From 834945998250a331119ae01100b43dadbc0007d2 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 09:27:46 +0200 Subject: [PATCH 13/23] test(attachments): branch coverage for history-stitching generator Direct branch tests for ConversationLogGenerator.withAttachmentExtracts (null stack/input, out-of-range index, null/empty extracts, present) plus a few reachable generate() compound-condition branches, lifting the class from 71.7% to 83.3% branch. All new/changed attachment classes now clear the >90% instruction / >80% branch gate. --- .../memory/ConversationLogGeneratorTest.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java b/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java index fd0d0312d4..f17d5089b3 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java @@ -38,6 +38,116 @@ void nullEverything() { } } + // ─── generate() branch coverage (compound conditions) ────── + + @Nested + @DisplayName("generate branch coverage") + class GenerateBranches { + + private IConversationMemory memoryWith(ConversationOutput output) { + var memory = mock(IConversationMemory.class); + when(memory.getConversationOutputs()).thenReturn(new ArrayList<>(List.of(output))); + return memory; + } + + @Test + @DisplayName("inputFiles first element not a Map → only text content") + void inputFilesFirstNotMap() { + var output = new ConversationOutput(); + output.put("input", "hi"); + output.put("context", Map.of("inputFiles", List.of("not-a-map"))); + var log = new ConversationLogGenerator(memoryWith(output)).generate(-1, true); + assertEquals("hi", log.getMessages().getFirst().getContent().getLast().getValue()); + assertEquals(1, log.getMessages().getFirst().getContent().size()); + } + + @Test + @DisplayName("context without inputFiles → only text content") + void contextWithoutInputFiles() { + var output = new ConversationOutput(); + output.put("input", "hi"); + output.put("context", Map.of("language", "en")); + var log = new ConversationLogGenerator(memoryWith(output)).generate(-1, true); + assertEquals(1, log.getMessages().getFirst().getContent().size()); + } + + @Test + @DisplayName("empty output list → no assistant message") + void emptyOutputList() { + var output = new ConversationOutput(); + output.put("input", "hi"); + output.put("output", new ArrayList<>()); + var log = new ConversationLogGenerator(memoryWith(output)).generate(-1, true); + assertEquals(1, log.getMessages().size()); + } + } + + // ─── withAttachmentExtracts (direct branch coverage) ──────── + + @Nested + @DisplayName("withAttachmentExtracts helper") + class WithAttachmentExtracts { + + private IConversationMemory.IConversationStepStack stackWith(List extracts) { + var step = mock(IConversationMemory.IConversationStep.class); + @SuppressWarnings("unchecked") + IData> data = mock(IData.class); + when(data.getResult()).thenReturn(extracts); + when(step.getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS)).thenReturn(data); + var stack = mock(IConversationMemory.IConversationStepStack.class); + when(stack.size()).thenReturn(1); + when(stack.get(0)).thenReturn(step); + return stack; + } + + @Test + void nullStack_returnsInput() { + assertEquals("hi", ConversationLogGenerator.withAttachmentExtracts(null, 0, "hi")); + } + + @Test + void nullInput_returnsNull() { + assertNull(ConversationLogGenerator.withAttachmentExtracts(stackWith(List.of("x")), 0, null)); + } + + @Test + void negativeIndex_returnsInput() { + assertEquals("hi", ConversationLogGenerator.withAttachmentExtracts(stackWith(List.of("x")), -1, "hi")); + } + + @Test + void indexOutOfRange_returnsInput() { + assertEquals("hi", ConversationLogGenerator.withAttachmentExtracts(stackWith(List.of("x")), 5, "hi")); + } + + @Test + void nullData_returnsInput() { + var step = mock(IConversationMemory.IConversationStep.class); + when(step.getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS)).thenReturn(null); + var stack = mock(IConversationMemory.IConversationStepStack.class); + when(stack.size()).thenReturn(1); + when(stack.get(0)).thenReturn(step); + assertEquals("hi", ConversationLogGenerator.withAttachmentExtracts(stack, 0, "hi")); + } + + @Test + void nullResult_returnsInput() { + assertEquals("hi", ConversationLogGenerator.withAttachmentExtracts(stackWith(null), 0, "hi")); + } + + @Test + void emptyResult_returnsInput() { + assertEquals("hi", ConversationLogGenerator.withAttachmentExtracts(stackWith(List.of()), 0, "hi")); + } + + @Test + void presentResult_appends() { + String out = ConversationLogGenerator.withAttachmentExtracts(stackWith(List.of("doc: text")), 0, "hi"); + assertTrue(out.startsWith("hi")); + assertTrue(out.contains("doc: text")); + } + } + // ─── Attachment extract stitching ─────────────────────────── @Nested From 282e89968571544b1da519c24987f58e398c95e1 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 09:37:24 +0200 Subject: [PATCH 14/23] feat(attachments): forwarder metrics + GDPR portability metadata (Phase 6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - AttachmentForwarder records eddi.attachment.forwarded and eddi.attachment.errors via MeterRegistry (AGENTS.md metrics mandate for the multimodal hot path). - UserDataExport gains an attachments list (AttachmentExportEntry — metadata only, never bytes) + a backward-compatible constructor; GdprComplianceService.exportUserData collects attachment metadata across the user's conversations and records attachmentsExported in the compliance audit event. Deferred follow-ups: nightly reaper (orphan blobs / stale grants), CostTracker multimodal estimates, attachmentsForwarded audit entry, and Phase 5 (multipart say + frontend). The two-step upload->say flow already works end-to-end. Partial Phase 6 of multimodal-attachments-completion-plan. --- docs/changelog.md | 20 ++++++++++++ .../engine/gdpr/GdprComplianceService.java | 26 +++++++++++++--- .../labs/eddi/engine/gdpr/UserDataExport.java | 24 +++++++++++++- .../modules/llm/impl/AttachmentForwarder.java | 16 ++++++++++ .../gdpr/GdprComplianceServiceTest.java | 31 +++++++++++++++++-- .../llm/impl/AttachmentForwarderTest.java | 22 ++++++++++++- 6 files changed, 131 insertions(+), 8 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 653ae1665e..b69b9fbf65 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,26 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Phase 6 (partial): Metrics + GDPR portability (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) +**Plan:** `planning/multimodal-attachments-completion-plan.md` (Phase 6 of 6, partial). + +### What changed + +- **Forwarder metrics** — `AttachmentForwarder` now takes a `MeterRegistry` and records `eddi.attachment.forwarded` (content items sent to the LLM) and `eddi.attachment.errors` (dropped/gated/failed) per turn, satisfying AGENTS.md's "always add metrics" rule for the multimodal hot path. +- **GDPR portability** — `UserDataExport` gains an `attachments` list (`AttachmentExportEntry` = conversationId/storageRef/fileName/mimeType/sizeBytes, **metadata only, never bytes**) plus a backward-compatible constructor. `GdprComplianceService.exportUserData` collects attachment metadata across the user's conversations via `IAttachmentStore.listByConversation`, and the compliance audit event records `attachmentsExported`. + +### Deferred (documented follow-ups) + +Still open in Phase 6: nightly reaper (orphaned blobs / stale grants via `ScheduleFireExecutor`), `CostTracker` multimodal token estimates, and an `attachmentsForwarded` audit-ledger entry. Phase 5 (multipart 1:1 `say`, SSE/output chips, and the EDDI-Manager / eddi-chat-ui frontend in their own repos) is likewise a follow-up — the two-step upload→say flow already works end-to-end. + +### Tests + +Forwarder metrics assertion + GDPR attachment-metadata export test. Both green. + --- ## 📎 Multimodal Attachments Completion — Phase 3: Group parity (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java b/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java index 2f6697cd31..7595a07e04 100644 --- a/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java +++ b/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java @@ -258,20 +258,38 @@ public UserDataExport exportUserData(String userId) { pseudonym); } + // 5. Attachment metadata (no bytes — the payload is fetched via the + // download API; portability requires the metadata, not the blobs). + var attachmentEntries = new ArrayList(); + try { + if (attachmentStorageInstance.isResolvable()) { + var store = attachmentStorageInstance.get(); + for (var convId : conversationMemoryStore.getConversationIdsByUserId(userId)) { + for (var a : store.listByConversation(convId)) { + attachmentEntries.add(new UserDataExport.AttachmentExportEntry( + convId, a.storageRef(), a.filename(), a.mimeType(), a.sizeBytes())); + } + } + } + } catch (Exception e) { + LOGGER.errorf(e, "[GDPR] Failed to export attachment metadata [%s]", pseudonym); + } + LOGGER.infof("[GDPR] Export complete [%s]: memories=%d, " - + "conversations=%d, managedConversations=%d, auditEntries=%d", + + "conversations=%d, managedConversations=%d, auditEntries=%d, attachments=%d", pseudonym, memories.size(), conversations.size(), - managedConversations.size(), auditExportEntries.size()); + managedConversations.size(), auditExportEntries.size(), attachmentEntries.size()); // Write compliance event to immutable audit ledger submitComplianceAuditEntry("GDPR_EXPORT", pseudonym, Map.of( "memoriesExported", memories.size(), "conversationsExported", conversations.size(), "managedConversationsExported", managedConversations.size(), - "auditEntriesExported", auditExportEntries.size())); + "auditEntriesExported", auditExportEntries.size(), + "attachmentsExported", attachmentEntries.size())); return new UserDataExport(userId, Instant.now(), memories, - conversations, managedConversations, auditExportEntries); + conversations, managedConversations, auditExportEntries, attachmentEntries); } // === Right to Restriction of Processing (GDPR Art. 18) === diff --git a/src/main/java/ai/labs/eddi/engine/gdpr/UserDataExport.java b/src/main/java/ai/labs/eddi/engine/gdpr/UserDataExport.java index ca6a91166b..ead8c88fb7 100644 --- a/src/main/java/ai/labs/eddi/engine/gdpr/UserDataExport.java +++ b/src/main/java/ai/labs/eddi/engine/gdpr/UserDataExport.java @@ -39,7 +39,29 @@ public record UserDataExport( List memories, List conversations, List managedConversations, - List auditEntries) { + List auditEntries, + List attachments) { + + /** + * Backward-compatible constructor without attachment metadata. + */ + public UserDataExport(String userId, Instant exportedAt, List memories, + List conversations, List managedConversations, + List auditEntries) { + this(userId, exportedAt, memories, conversations, managedConversations, auditEntries, List.of()); + } + + /** + * Attachment metadata for export — never includes the binary payload + * (portability is metadata; the bytes can be fetched via the download API). + */ + public record AttachmentExportEntry( + String conversationId, + String storageRef, + String fileName, + String mimeType, + long sizeBytes) { + } /** * Lightweight conversation summary for export. diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java index a458b7ea8f..d30de7c170 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java @@ -78,12 +78,14 @@ public class AttachmentForwarder { private final SafeHttpClient httpClient; private final long maxForwardBytes; private final long maxAggregateBytes; + private final io.micrometer.core.instrument.MeterRegistry meterRegistry; @Inject public AttachmentForwarder(IAttachmentStore attachmentStore, ModelCapabilityService capabilityService, AttachmentTextExtractor textExtractor, SafeHttpClient httpClient, + io.micrometer.core.instrument.MeterRegistry meterRegistry, @ConfigProperty(name = "eddi.attachments.max-forward-bytes", defaultValue = "10485760") long maxForwardBytes, @ConfigProperty(name = "eddi.attachments.max-forward-aggregate-bytes", @@ -92,6 +94,7 @@ public AttachmentForwarder(IAttachmentStore attachmentStore, this.capabilityService = capabilityService; this.textExtractor = textExtractor; this.httpClient = httpClient; + this.meterRegistry = meterRegistry; this.maxForwardBytes = maxForwardBytes; this.maxAggregateBytes = maxAggregateBytes; } @@ -159,9 +162,22 @@ public void forward(List messages, IConversationMemory memory, Stri messages.set(lastUserIdx, UserMessage.from(contents)); LOGGER.debugf("Forwarded %d attachment content item(s) to the LLM", added); } + recordMetrics(added, errors.size()); persist(memory.getCurrentStep(), extracts, errors); } + private void recordMetrics(int forwarded, int errored) { + if (meterRegistry == null) { + return; + } + if (forwarded > 0) { + meterRegistry.counter("eddi.attachment.forwarded").increment(forwarded); + } + if (errored > 0) { + meterRegistry.counter("eddi.attachment.errors").increment(errored); + } + } + private Content process(Attachment att, String conversationId, String provider, String model, long[] aggregate, List extracts, List errors, ModelCapabilityService.Support visionOverride, diff --git a/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java b/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java index 7705ea0363..7a71715885 100644 --- a/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java +++ b/src/test/java/ai/labs/eddi/engine/gdpr/GdprComplianceServiceTest.java @@ -43,8 +43,11 @@ class GdprComplianceServiceTest { private IAuditStore auditStore; private AuditLedgerService auditLedgerService; private GdprComplianceService service; + private Instance attachmentStorageInstance; + private IAttachmentStore attachmentStore; @BeforeEach + @SuppressWarnings("unchecked") void setUp() { userMemoryStore = mock(IUserMemoryStore.class); conversationMemoryStore = mock(IConversationMemoryStore.class); @@ -53,8 +56,8 @@ void setUp() { auditStore = mock(IAuditStore.class); auditLedgerService = mock(AuditLedgerService.class); - @SuppressWarnings("unchecked") - Instance attachmentStorageInstance = mock(Instance.class); + attachmentStorageInstance = mock(Instance.class); + attachmentStore = mock(IAttachmentStore.class); when(attachmentStorageInstance.isResolvable()).thenReturn(false); service = new GdprComplianceService( @@ -251,6 +254,30 @@ void exportUserData_aggregatesAllStores() throws Exception { assertEquals(ConversationState.ENDED, convExport.state()); } + @Test + void exportUserData_includesAttachmentMetadata() throws Exception { + when(userMemoryStore.getAllEntries(USER_ID)).thenReturn(List.of()); + when(conversationMemoryStore.getConversationIdsByUserId(USER_ID)).thenReturn(List.of("conv-1")); + when(conversationMemoryStore.loadConversationMemorySnapshot("conv-1")).thenReturn(null); + when(userConversationStore.getAllForUser(USER_ID)).thenReturn(List.of()); + when(auditStore.getEntriesByUserId(eq(USER_ID), anyInt(), anyInt())).thenReturn(List.of()); + + when(attachmentStorageInstance.isResolvable()).thenReturn(true); + when(attachmentStorageInstance.get()).thenReturn(attachmentStore); + when(attachmentStore.listByConversation("conv-1")).thenReturn(List.of( + new IAttachmentStore.Attachment("ref-1", "report.pdf", "application/pdf", 2048, "conv-1"))); + + UserDataExport export = service.exportUserData(USER_ID); + + assertEquals(1, export.attachments().size()); + var a = export.attachments().getFirst(); + assertEquals("conv-1", a.conversationId()); + assertEquals("ref-1", a.storageRef()); + assertEquals("report.pdf", a.fileName()); + assertEquals("application/pdf", a.mimeType()); + assertEquals(2048, a.sizeBytes()); + } + @Test void exportUserData_handlesEmptyData() throws Exception { // Given — user has no data diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java index 2563e9e29e..be66fe728e 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java @@ -61,7 +61,8 @@ void setUp() { private AttachmentForwarder newForwarder(long perFile, long aggregate) { var capability = new ModelCapabilityService(k -> Optional.empty()); var extractor = new AttachmentTextExtractor(10_000); - return new AttachmentForwarder(store, capability, extractor, httpClient, perFile, aggregate); + return new AttachmentForwarder(store, capability, extractor, httpClient, + new io.micrometer.core.instrument.simple.SimpleMeterRegistry(), perFile, aggregate); } // ==================== No-op cases ==================== @@ -421,6 +422,25 @@ void invalidBase64_addsNote() { assertTrue(((TextContent) c).text().contains("invalid base64")); } + @Test + void metrics_recordForwardedAndErrors() { + var registry = new io.micrometer.core.instrument.simple.SimpleMeterRegistry(); + var f = new AttachmentForwarder(store, new ModelCapabilityService(k -> Optional.empty()), + new AttachmentTextExtractor(10_000), httpClient, registry, 10L * 1024 * 1024, 20L * 1024 * 1024); + Attachment ok = new Attachment(); + ok.setMimeType("image/png"); + ok.setBase64Data(Base64.getEncoder().encodeToString("png".getBytes())); + Attachment bad = new Attachment(); + bad.setMimeType("image/png"); // no source → error, no content + mockAttachments(ok, bad); + List messages = messages(UserMessage.from("look")); + + f.forward(messages, memory, "openai", "gpt-4o"); + + assertEquals(1.0, registry.counter("eddi.attachment.forwarded").count()); + assertEquals(1.0, registry.counter("eddi.attachment.errors").count()); + } + @Test void noContentSource_skipped() { Attachment att = new Attachment(); // NONE From 0d8c08328e3e5b9d2932c1372b809f7d4d521279 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 09:40:56 +0200 Subject: [PATCH 15/23] test(attachments): UserDataExport record coverage to 100% Cover the backward-compatible (no-attachments) constructor and all nested export-entry accessors. Every new/changed attachment class now clears the >90% instruction / >80% branch gate. --- .../eddi/engine/gdpr/UserDataExportTest.java | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 src/test/java/ai/labs/eddi/engine/gdpr/UserDataExportTest.java diff --git a/src/test/java/ai/labs/eddi/engine/gdpr/UserDataExportTest.java b/src/test/java/ai/labs/eddi/engine/gdpr/UserDataExportTest.java new file mode 100644 index 0000000000..34eeb8b9a0 --- /dev/null +++ b/src/test/java/ai/labs/eddi/engine/gdpr/UserDataExportTest.java @@ -0,0 +1,63 @@ +/* + * Copyright EDDI contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package ai.labs.eddi.engine.gdpr; + +import ai.labs.eddi.engine.memory.model.ConversationState; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +/** + * Unit tests for the {@link UserDataExport} export record and its nested + * entries, including the backward-compatible (no-attachments) constructor. + */ +class UserDataExportTest { + + @Test + void backwardCompatibleConstructor_defaultsAttachmentsEmpty() { + var now = Instant.now(); + var export = new UserDataExport("user-1", now, List.of(), List.of(), List.of(), List.of()); + + assertEquals("user-1", export.userId()); + assertEquals(now, export.exportedAt()); + assertTrue(export.memories().isEmpty()); + assertTrue(export.conversations().isEmpty()); + assertTrue(export.managedConversations().isEmpty()); + assertTrue(export.auditEntries().isEmpty()); + assertNotNull(export.attachments()); + assertTrue(export.attachments().isEmpty()); + } + + @Test + void fullConstructor_exposesAllComponents() { + var att = new UserDataExport.AttachmentExportEntry("conv-1", "ref-1", "a.pdf", "application/pdf", 42); + var conv = new UserDataExport.ConversationExportEntry("conv-1", "agent-1", 3, ConversationState.ENDED, List.of()); + var audit = new UserDataExport.AuditExportEntry("conv-1", "agent-1", "llm", 12L, Map.of("k", "v"), Instant.now()); + + var export = new UserDataExport("user-1", Instant.now(), List.of(), List.of(conv), List.of(), + List.of(audit), List.of(att)); + + assertEquals(1, export.attachments().size()); + assertEquals("conv-1", att.conversationId()); + assertEquals("ref-1", att.storageRef()); + assertEquals("a.pdf", att.fileName()); + assertEquals("application/pdf", att.mimeType()); + assertEquals(42, att.sizeBytes()); + + assertEquals("agent-1", conv.agentId()); + assertEquals(3, conv.agentVersion()); + assertEquals(ConversationState.ENDED, conv.state()); + assertTrue(conv.outputs().isEmpty()); + + assertEquals("llm", audit.taskType()); + assertEquals(12L, audit.durationMs()); + assertEquals("v", audit.llmDetail().get("k")); + assertNotNull(audit.timestamp()); + } +} From b7cfffb656ccbf4a16567fae31064387e782b15f Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 09:58:29 +0200 Subject: [PATCH 16/23] fix(attachments): correct two high-severity bugs found in adversarial review 1. Prefix-collision silent data loss (AttachmentForwarder / AgentOrchestrator): getLatestData is a PREFIX scan, and the ATTACHMENTS key "attachments" is a prefix of the attachments:extracts / attachments:errors keys the forwarder persist()s. A second forwarder (or readAttachment auto-add) invocation in the same conversation step reverse-scanned and returned a List extract/ error entry, so readAttachments() found no Attachment and forwarded ZERO attachments with no error note. Reachable with two langchain tasks sharing an action or two langchain workflow steps. Fixed by reading ATTACHMENTS via the exact-match getData(MemoryKey) instead of the prefix getLatestData. 2. Mirror-inverted history stitching (ConversationLogGenerator): withAttachmentExtracts passed the FORWARD conversation-output index into IConversationStepStack.get(), which is REVERSE-ordered (get(0)=newest). In a 3-turn conversation, turn 1's extract surfaced on turn 3's message and turn 1 lost it; only the middle turn aligned. Fixed by converting the forward index to the reverse accessor index (size-1-index). Both escaped the unit tests because they stubbed getLatestData directly and used single-turn (size==1) memories where the reversal is a no-op. Added regression tests: a real ConversationMemory with persisted extracts/errors proving the forwarder still forwards, and a 3-turn stitching test proving extracts land on the correct turn. --- .../memory/ConversationLogGenerator.java | 8 +++- .../modules/llm/impl/AgentOrchestrator.java | 5 ++- .../modules/llm/impl/AttachmentForwarder.java | 8 +++- .../memory/ConversationLogGeneratorTest.java | 37 +++++++++++++++++++ .../llm/impl/AgentOrchestratorBranchTest.java | 4 +- .../llm/impl/AttachmentForwarderTest.java | 29 ++++++++++++++- 6 files changed, 84 insertions(+), 7 deletions(-) diff --git a/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java b/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java index 0b5a65d58a..61307fd10b 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java +++ b/src/main/java/ai/labs/eddi/engine/memory/ConversationLogGenerator.java @@ -164,7 +164,13 @@ public static String withAttachmentExtracts(IConversationMemory.IConversationSte if (allSteps == null || input == null || stepIndex < 0 || stepIndex >= allSteps.size()) { return input; } - IData> data = allSteps.get(stepIndex).getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS); + // conversationOutputs is forward-ordered (0 = oldest) but + // IConversationStepStack.get() + // is reverse-ordered (get(0) = newest), so convert the forward output index to + // the + // reverse step index to land on the SAME turn (not its mirror). + IConversationMemory.IConversationStep step = allSteps.get(allSteps.size() - 1 - stepIndex); + IData> data = step.getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS); if (data == null || data.getResult() == null || data.getResult().isEmpty()) { return input; } diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java index d13e2a4190..b3217a044b 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AgentOrchestrator.java @@ -706,7 +706,10 @@ private void addReadAttachmentToolIfEnabled(List tools, IConversationMem if (attachmentStore == null || attachmentTextExtractor == null) { return; } - IData> attachmentData = memory.getCurrentStep().getLatestData(MemoryKeys.ATTACHMENTS); + // Exact-match read (getData, not the prefix-scanning getLatestData): + // "attachments" + // is a prefix of the attachments:extracts/errors keys the forwarder persists. + IData> attachmentData = memory.getCurrentStep().getData(MemoryKeys.ATTACHMENTS); if (attachmentData == null || attachmentData.getResult() == null || attachmentData.getResult().isEmpty()) { return; } diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java index d30de7c170..49c474632c 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java @@ -324,7 +324,13 @@ private Content note(List errors, String name, String mime, long sizeByt } private List readAttachments(IConversationMemory memory) { - IData> data = memory.getCurrentStep().getLatestData(ATTACHMENTS); + // Exact-match read: getLatestData is a prefix scan, and ATTACHMENTS + // ("attachments") + // is a prefix of the attachments:extracts / attachments:errors keys this + // forwarder + // persists — a prefix read would return the wrong entry on a second forwarder + // invocation in the same step. + IData> data = memory.getCurrentStep().getData(ATTACHMENTS); if (data == null || data.getResult() == null) { return List.of(); } diff --git a/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java b/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java index f17d5089b3..a1b92d29f4 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/ConversationLogGeneratorTest.java @@ -202,6 +202,43 @@ void noExtractsUnchanged() { var log = new ConversationLogGenerator(memory).generate(-1, true, true); assertEquals("Hi", log.getMessages().getFirst().getContent().getLast().getValue()); } + + @Test + @DisplayName("multi-turn: extract lands on its own turn, not the mirror turn") + void stitchesOntoCorrectTurnAcrossTurns() { + // Three turns; the extract lives on the OLDEST turn (output index 0). + var out0 = new ConversationOutput(); + out0.put("input", "turn0"); + var out1 = new ConversationOutput(); + out1.put("input", "turn1"); + var out2 = new ConversationOutput(); + out2.put("input", "turn2"); + var memory = mock(IConversationMemory.class); + when(memory.getConversationOutputs()).thenReturn(new ArrayList<>(List.of(out0, out1, out2))); + + var oldestStep = mock(IConversationMemory.IConversationStep.class); + @SuppressWarnings("unchecked") + IData> data = mock(IData.class); + when(data.getResult()).thenReturn(List.of("PDF EXTRACT")); + when(oldestStep.getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS)).thenReturn(data); + var otherStep = mock(IConversationMemory.IConversationStep.class); + when(otherStep.getLatestData(MemoryKeys.ATTACHMENT_EXTRACTS)).thenReturn(null); + + // Stack.get() is reverse-ordered: get(0)=newest(turn2) … get(2)=oldest(turn0). + var stack = mock(IConversationMemory.IConversationStepStack.class); + when(stack.size()).thenReturn(3); + when(stack.get(0)).thenReturn(otherStep); + when(stack.get(1)).thenReturn(otherStep); + when(stack.get(2)).thenReturn(oldestStep); + when(memory.getAllSteps()).thenReturn(stack); + + var log = new ConversationLogGenerator(memory).generate(-1, true, true); + + String turn0 = log.getMessages().get(0).getContent().getLast().getValue(); + String turn2 = log.getMessages().get(2).getContent().getLast().getValue(); + assertTrue(turn0.contains("PDF EXTRACT"), "extract must land on turn 0: " + turn0); + assertFalse(turn2.contains("PDF EXTRACT"), "extract must NOT leak onto turn 2: " + turn2); + } } // ─── Basic generation ─────────────────────────────────────── diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java index d141c3f4af..b96182b81a 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AgentOrchestratorBranchTest.java @@ -163,9 +163,9 @@ private void withAttachments(boolean present) { if (present) { IData data = mock(IData.class); doReturn(List.of(new Object())).when(data).getResult(); - doReturn(data).when(step).getLatestData(MemoryKeys.ATTACHMENTS); + doReturn(data).when(step).getData(MemoryKeys.ATTACHMENTS); } else { - doReturn(null).when(step).getLatestData(MemoryKeys.ATTACHMENTS); + doReturn(null).when(step).getData(MemoryKeys.ATTACHMENTS); } } diff --git a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java index be66fe728e..4a3199a9ec 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarderTest.java @@ -84,7 +84,7 @@ void emptyMessages() { @Test void noAttachments() { - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(null); + when(currentStep.getData(ATTACHMENTS)).thenReturn(null); List messages = messages(UserMessage.from("Hi")); forwarder.forward(messages, memory, "openai", "gpt-4o"); assertEquals(1, messages.size()); @@ -441,6 +441,31 @@ void metrics_recordForwardedAndErrors() { assertEquals(1.0, registry.counter("eddi.attachment.errors").count()); } + @Test + void secondInvocation_withPersistedExtractsErrors_stillForwards() { + // Regression: a prior forwarder pass in the same step persisted the + // attachments:extracts / attachments:errors keys. A prefix read of + // "attachments" would return one of those List entries and forward + // nothing; the exact-match read must still find the List. + var realMemory = new ai.labs.eddi.engine.memory.ConversationMemory("agent-1", 1, "user-1"); + var step = realMemory.getCurrentStep(); + Attachment att = new Attachment(); + att.setMimeType("image/png"); + att.setBase64Data(Base64.getEncoder().encodeToString("png".getBytes())); + step.storeData(new ai.labs.eddi.engine.memory.model.Data<>(ATTACHMENTS.key(), List.of(att))); + // Inserted AFTER attachments — this is what a prefix reverse-scan would return + // first. + step.storeData(new ai.labs.eddi.engine.memory.model.Data<>( + ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENT_ERRORS.key(), List.of("earlier error"))); + step.storeData(new ai.labs.eddi.engine.memory.model.Data<>( + ai.labs.eddi.engine.memory.MemoryKeys.ATTACHMENT_EXTRACTS.key(), List.of("doc: earlier extract"))); + + List messages = messages(UserMessage.from("look")); + forwarder.forward(messages, realMemory, "openai", "gpt-4o"); + + assertInstanceOf(ImageContent.class, ((UserMessage) messages.get(0)).contents().get(1)); + } + @Test void noContentSource_skipped() { Attachment att = new Attachment(); // NONE @@ -504,7 +529,7 @@ private static List messages(ChatMessage... m) { private void mockAttachments(Attachment... attachments) { IData data = mock(IData.class); when(data.getResult()).thenReturn(List.of(attachments)); - when(currentStep.getLatestData(ATTACHMENTS)).thenReturn(data); + when(currentStep.getData(ATTACHMENTS)).thenReturn(data); } private static byte[] tinyPdf(String text) throws Exception { From 1a40952a40db5b9c2a44a0719251fa95f71aeeb2 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 10:07:34 +0200 Subject: [PATCH 17/23] fix(attachments): harden ContentTypeMatcher against the attachments prefix collision ContentTypeMatcher (a behavior-rule condition) also reads the short "attachments" key via the prefix-scanning getLatestData and is vulnerable to the same collision with the forwarder-persisted attachments:extracts/errors keys. Switch it to the exact getData read, matching the AttachmentForwarder/AgentOrchestrator fixes. Record the adversarial review outcome in the changelog. --- docs/changelog.md | 13 +++++++++++++ .../rules/impl/conditions/ContentTypeMatcher.java | 5 ++++- .../impl/conditions/ContentTypeMatcherTest.java | 8 ++++---- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index b69b9fbf65..fedee4fe8c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,19 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Adversarial review + fixes (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) + +A multi-agent adversarial review of the whole implementation surfaced **two real high-severity defects** (both verified by an independent refutation pass, both missed by the unit tests because they stubbed `getLatestData` directly and used single-turn memories): + +1. **Prefix-collision silent data loss.** `IConversationStep.getLatestData` is a *prefix* scan, and the `ATTACHMENTS` key `"attachments"` is a prefix of the `attachments:extracts` / `attachments:errors` keys the forwarder `persist()`s. A second forwarder (or `readAttachment` auto-add, or `ContentTypeMatcher`) read in the same step reverse-scanned and returned a `List` instead of the `List` → **zero attachments forwarded, no error note**. Reachable with two langchain tasks sharing an action or two langchain workflow steps. Fixed by reading the exact key via `getData(MemoryKey)` in `AttachmentForwarder`, `AgentOrchestrator`, and `ContentTypeMatcher`. +2. **Mirror-inverted history stitching.** `ConversationLogGenerator.withAttachmentExtracts` passed the *forward* conversation-output index into `IConversationStepStack.get()`, which is *reverse*-ordered (`get(0)` = newest). In a 3-turn conversation, turn 1's extract surfaced on turn 3 and turn 1 lost it; only the middle turn aligned. Fixed by converting the forward index to the reverse accessor index (`size-1-index`). + +Regression tests added for both (a real `ConversationMemory` with persisted extract/error keys proving the forwarder still forwards; a 3-turn stitching test proving extracts land on the correct turn). All new/changed classes remain above the >90% instruction / >80% branch gate; 654 tests green across the touched surface. + --- ## 📎 Multimodal Attachments Completion — Phase 6 (partial): Metrics + GDPR portability (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcher.java b/src/main/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcher.java index 4a05625afe..7e1bcd1a64 100644 --- a/src/main/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcher.java +++ b/src/main/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcher.java @@ -90,7 +90,10 @@ public ExecutionState execute(IConversationMemory memory, List trace) { return FAIL; } - IData> data = memory.getCurrentStep().getLatestData(ATTACHMENTS); + // Exact-match read: "attachments" is a prefix of the attachments:extracts / + // attachments:errors keys the LLM forwarder persists, and getLatestData is a + // prefix scan — getData avoids returning the wrong entry. + IData> data = memory.getCurrentStep().getData(ATTACHMENTS); if (data == null || data.getResult() == null) { return FAIL; } diff --git a/src/test/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcherTest.java b/src/test/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcherTest.java index e3b839be73..18dfd91fc9 100644 --- a/src/test/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcherTest.java +++ b/src/test/java/ai/labs/eddi/modules/rules/impl/conditions/ContentTypeMatcherTest.java @@ -47,7 +47,7 @@ void setUp() { * arise with {@code when().thenReturn()} on generic methods. */ private void stubAttachments(IData> data) { - doReturn(data).when(currentStep).getLatestData(ArgumentMatchers.>any()); + doReturn(data).when(currentStep).getData(ArgumentMatchers.>any()); } private void stubAttachmentList(Attachment... attachments) { @@ -190,7 +190,7 @@ class NoAttachments { @DisplayName("should FAIL when no attachment data in memory") void failsWhenNoDataInMemory() { matcher.setConfigs(Map.of("mimeType", "image/png")); - doReturn(null).when(currentStep).getLatestData(ArgumentMatchers.>any()); + doReturn(null).when(currentStep).getData(ArgumentMatchers.>any()); assertEquals(FAIL, matcher.execute(memory, List.of())); } @@ -452,7 +452,7 @@ void ignoresNonAttachmentObjects() { att.setMimeType("image/png"); List mixed = List.of(att, "not-an-attachment", 42); doReturn(new Data<>("attachments", mixed)).when(currentStep) - .getLatestData(ArgumentMatchers.>any()); + .getData(ArgumentMatchers.>any()); assertEquals(SUCCESS, matcher.execute(memory, List.of())); } @@ -463,7 +463,7 @@ void failsWhenAllNonAttachment() { matcher.setConfigs(Map.of("mimeType", "image/*")); List nonAttachments = List.of("string", 123, Map.of("key", "val")); doReturn(new Data<>("attachments", nonAttachments)).when(currentStep) - .getLatestData(ArgumentMatchers.>any()); + .getData(ArgumentMatchers.>any()); assertEquals(FAIL, matcher.execute(memory, List.of())); } From 1359273df5f4b44b9ac9ca695d1fc4b34e10e2a0 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Fri, 3 Jul 2026 10:54:15 +0200 Subject: [PATCH 18/23] fix(attachments): address PR #588 automated review findings - (High) readAttachment couldn't see group-shared blobs: listByConversation is owner-only, but group attachments are owned by the group conversation and granted to members. Add IAttachmentStore.listAccessible (owned OR granted) to both backends (GridFS grants-array match / Postgres = ANY(grants)); the readAttachment tool lists/resolves through it. - (Medium) materializeAttachments dropped url attachments when the store is null; restructure so only the inline-base64 path requires a store. - (Medium) replace brittle message.contains("denied") with a typed AttachmentAccessDeniedException (thrown by both backends' authz/delete paths); REST maps it to 403 and other store errors to 404/500. - (Note) remove an unused local variable in a GridFS test. Tests updated/added; 277 green across affected classes. --- docs/changelog.md | 15 +++++++++++ .../mongo/GridFsAttachmentStore.java | 21 +++++++++++++--- .../postgres/PostgresAttachmentStore.java | 21 +++++++++++++--- .../engine/attachments/IAttachmentStore.java | 25 ++++++++++++++++++- .../internal/GroupConversationService.java | 15 +++++------ .../memory/rest/RestAttachmentUpload.java | 19 ++++++++------ .../llm/tools/impl/ReadAttachmentTool.java | 4 +-- .../mongo/GridFsAttachmentStoreTest.java | 23 +++++++++++++++-- .../GroupConversationServiceTest.java | 12 ++++++--- .../memory/rest/RestAttachmentUploadTest.java | 4 +-- .../tools/impl/ReadAttachmentToolTest.java | 22 ++++++++-------- 11 files changed, 137 insertions(+), 44 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index fedee4fe8c..81e791fad5 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,21 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Automated review fixes (2026-07-03) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) + +Addressed the GitHub code-quality / Copilot review of PR #588: + +1. **(High) `readAttachment` couldn't see group-shared blobs.** `listByConversation` returns only *owned* blobs, so a group member — whose shared attachments are owned by the group conversation and merely *granted* to it — got an empty list and couldn't recall them via the tool. Added `IAttachmentStore.listAccessible(conversationId)` (owned **OR** granted) in both backends (GridFS `metadata.grants` array match / Postgres `? = ANY(grants)`), and `ReadAttachmentTool` now lists/resolves through it. +2. **(Medium) URL attachments dropped when no store configured.** `GroupConversationService.materializeAttachments` returned early on a null store, discarding hosted-`url` attachments that don't need a store. Restructured to skip only the inline-base64 (store-requiring) path. +3. **(Medium) Brittle access-denied detection.** The download endpoint keyed 403-vs-404 off `message.contains("denied")`. Added a typed `IAttachmentStore.AttachmentAccessDeniedException` (thrown by both backends' authz/delete paths); the REST layer catches it for 403 and treats other store exceptions as 404/500. +4. **(Note) Unused local variable** removed from a GridFS test. + +Tests updated + added (grant-aware listing, url-without-store materialize, typed-exception 403 paths); 277 green across the affected classes, coverage gate still met. + --- ## 📎 Multimodal Attachments Completion — Adversarial review + fixes (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java b/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java index 290ecf1185..a293ab742f 100644 --- a/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java @@ -168,7 +168,7 @@ public boolean delete(String storageRef, String requestingConversationId) throws Document metadata = file.getMetadata(); String owner = metadata != null ? metadata.getString(META_CONVERSATION_ID) : null; if (owner != null && !owner.equals(requestingConversationId)) { - throw new AttachmentStoreException( + throw new AttachmentAccessDeniedException( "Delete denied: attachment belongs to '%s', requested from '%s'" .formatted(owner, requestingConversationId)); } @@ -191,8 +191,21 @@ public long deleteByConversation(String conversationId) { @Override public List listByConversation(String conversationId) { + return listMatching(Filters.eq("metadata." + META_CONVERSATION_ID, conversationId)); + } + + @Override + public List listAccessible(String conversationId) { + // Owned by the conversation OR granted to it. Filters.eq on the array field + // matches documents whose grants array contains the value (Mongo semantics). + return listMatching(Filters.or( + Filters.eq("metadata." + META_CONVERSATION_ID, conversationId), + Filters.eq("metadata." + META_GRANTS, conversationId))); + } + + private List listMatching(Bson filter) { List results = new ArrayList<>(); - for (GridFSFile file : gridFSBucket.find(Filters.eq("metadata." + META_CONVERSATION_ID, conversationId))) { + for (GridFSFile file : gridFSBucket.find(filter)) { Document metadata = file.getMetadata(); String ref = metadata != null && metadata.getString(META_STORAGE_REF) != null ? metadata.getString(META_STORAGE_REF) @@ -204,7 +217,7 @@ public List listByConversation(String conversationId) { ? metadata.getString(META_MIME_TYPE) : "application/octet-stream", file.getLength(), - conversationId)); + metadata != null ? metadata.getString(META_CONVERSATION_ID) : null)); } return results; } @@ -258,7 +271,7 @@ private void authorize(GridFSFile file, String requester) throws AttachmentStore if (grants != null && grants.contains(requester)) { return; } - throw new AttachmentStoreException( + throw new AttachmentAccessDeniedException( "Cross-conversation access denied: attachment belongs to '%s', requested from '%s'" .formatted(owner, requester)); } diff --git a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java index 27ee961ce6..40ff6204bd 100644 --- a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java @@ -213,7 +213,7 @@ public boolean delete(String storageRef, String requestingConversationId) throws } } if (owner != null && !owner.equals(requestingConversationId)) { - throw new AttachmentStoreException( + throw new AttachmentAccessDeniedException( "Delete denied: attachment belongs to '%s', requested from '%s'" .formatted(owner, requestingConversationId)); } @@ -245,12 +245,25 @@ public long deleteByConversation(String conversationId) { @Override public List listByConversation(String conversationId) { + return listWhere("conversation_id = ?", conversationId, null); + } + + @Override + public List listAccessible(String conversationId) { + // Owned by the conversation OR granted to it. + return listWhere("conversation_id = ? OR ? = ANY(COALESCE(grants, '{}'))", conversationId, conversationId); + } + + private List listWhere(String whereClause, String param1, String param2) { ensureSchema(); String sql = "SELECT storage_ref, filename, mime_type, size_bytes, conversation_id " - + "FROM attachments WHERE conversation_id = ? ORDER BY created_at"; + + "FROM attachments WHERE " + whereClause + " ORDER BY created_at"; try (Connection conn = dataSourceInstance.get().getConnection(); PreparedStatement ps = conn.prepareStatement(sql)) { - ps.setString(1, conversationId); + ps.setString(1, param1); + if (param2 != null) { + ps.setString(2, param2); + } try (ResultSet rs = ps.executeQuery()) { List results = new ArrayList<>(); while (rs.next()) { @@ -302,7 +315,7 @@ private void authorize(String owner, ResultSet rs, String requester) throws SQLE if (grantsContain(rs, requester)) { return; } - throw new AttachmentStoreException( + throw new AttachmentAccessDeniedException( "Cross-conversation access denied: attachment belongs to '%s', requested from '%s'" .formatted(owner, requester)); } diff --git a/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java b/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java index f2cd79a6be..94d9434430 100644 --- a/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java @@ -124,7 +124,7 @@ Attachment store(byte[] bytes, String declaredMime, String filename, long deleteByConversation(String conversationId); /** - * List all attachments for a conversation. + * List attachments owned by a conversation. * * @param conversationId * the conversation ID @@ -132,6 +132,18 @@ Attachment store(byte[] bytes, String declaredMime, String filename, */ List listByConversation(String conversationId); + /** + * List all attachments a conversation can read — those it owns plus + * those it has been {@linkplain #grantAccess granted}. Used by the + * {@code readAttachment} tool so group members can enumerate blobs shared with + * them (owned by the group conversation, granted to the member). + * + * @param conversationId + * the requesting conversation ID + * @return owned + granted attachment metadata + */ + List listAccessible(String conversationId); + /** * Attachment metadata record. * @@ -159,4 +171,15 @@ public AttachmentStoreException(String message, Throwable cause) { super(message, cause); } } + + /** + * Thrown specifically when a conversation is not permitted to access a blob + * (not the owner and no grant). Lets callers distinguish an authorization + * failure (403) from a missing blob (404) without matching on message text. + */ + class AttachmentAccessDeniedException extends AttachmentStoreException { + public AttachmentAccessDeniedException(String message) { + super(message); + } + } } diff --git a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java index b53f8d2274..4701cd64d8 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java @@ -251,19 +251,20 @@ void materializeAttachments(GroupConversation gc, List incoming) { if (incoming == null || incoming.isEmpty()) { return; } - if (attachmentStore == null) { - LOGGER.warn("Group attachments were provided but no attachment store is configured; ignoring them."); - return; - } List materialized = new ArrayList<>(); for (Attachment a : incoming) { try { - if (a.getBase64Data() != null && !a.getBase64Data().isBlank()) { + if (a.getUrl() != null && !a.getUrl().isBlank()) { + // Hosted URL — forwarded as-is; no blob store required. + materialized.add(a); + } else if (a.getBase64Data() != null && !a.getBase64Data().isBlank()) { + if (attachmentStore == null) { + LOGGER.warn("Inline group attachment provided but no attachment store is configured; skipping it."); + continue; + } byte[] bytes = Base64.getDecoder().decode(a.getBase64Data()); var stored = attachmentStore.store(bytes, a.getMimeType(), a.getFileName(), gc.getId(), defaultTenantId); materialized.add(new Attachment(stored.mimeType(), stored.filename(), stored.sizeBytes(), stored.storageRef())); - } else if (a.getUrl() != null && !a.getUrl().isBlank()) { - materialized.add(a); } else if (a.getStorageRef() != null && !a.getStorageRef().isBlank()) { materialized.add(a); } diff --git a/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java b/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java index a0d31c3fb2..9d39d66868 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java +++ b/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java @@ -226,14 +226,17 @@ public void downloadAttachment( .header("Content-Type", meta.mimeType() != null ? meta.mimeType() : "application/octet-stream") .header("Content-Disposition", "attachment; filename=\"" + downloadName + "\"") .build()); + } catch (IAttachmentStore.AttachmentAccessDeniedException e) { + LOGGER.debugf("Attachment download denied for conversation '%s': %s", + sanitize(conversationId), e.getMessage()); + asyncResponse.resume(Response.status(Response.Status.FORBIDDEN) + .entity(Map.of("error", e.getMessage(), "code", "ATTACHMENT_ACCESS_DENIED")) + .build()); } catch (IAttachmentStore.AttachmentStoreException e) { - boolean denied = e.getMessage() != null && e.getMessage().contains("denied"); - var status = denied ? Response.Status.FORBIDDEN : Response.Status.NOT_FOUND; - LOGGER.debugf("Attachment download %s for conversation '%s': %s", - denied ? "denied" : "not found", sanitize(conversationId), e.getMessage()); - asyncResponse.resume(Response.status(status) - .entity(Map.of("error", e.getMessage(), "code", - denied ? "ATTACHMENT_ACCESS_DENIED" : "ATTACHMENT_NOT_FOUND")) + LOGGER.debugf("Attachment download not found for conversation '%s': %s", + sanitize(conversationId), e.getMessage()); + asyncResponse.resume(Response.status(Response.Status.NOT_FOUND) + .entity(Map.of("error", e.getMessage(), "code", "ATTACHMENT_NOT_FOUND")) .build()); } catch (Exception e) { LOGGER.errorf(e, "Failed to download attachment for conversation '%s'", sanitize(conversationId)); @@ -270,7 +273,7 @@ public void deleteAttachment( .entity(Map.of("error", "Attachment not found", "code", "ATTACHMENT_NOT_FOUND")) .build()); } - } catch (IAttachmentStore.AttachmentStoreException e) { + } catch (IAttachmentStore.AttachmentAccessDeniedException e) { LOGGER.debugf("Attachment delete denied for conversation '%s': %s", sanitize(conversationId), e.getMessage()); asyncResponse.resume(Response.status(Response.Status.FORBIDDEN) diff --git a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java index 4ad485e929..4b82d0e438 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java +++ b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentTool.java @@ -48,7 +48,7 @@ public ReadAttachmentTool(IAttachmentStore attachmentStore, AttachmentTextExtrac @Tool("Lists the files attached to this conversation. Returns each attachment's name, type and size. " + "Use this to discover what is available before calling readAttachment.") public String listAttachments() { - List attachments = attachmentStore.listByConversation(conversationId); + List attachments = attachmentStore.listAccessible(conversationId); if (attachments.isEmpty()) { return "No attachments are available in this conversation."; } @@ -103,7 +103,7 @@ private Attachment resolve(String nameOrRef) { if (nameOrRef == null || nameOrRef.isBlank()) { return null; } - List attachments = attachmentStore.listByConversation(conversationId); + List attachments = attachmentStore.listAccessible(conversationId); // Prefer an exact storageRef match, then an exact file-name match, then // case-insensitive name. for (Attachment a : attachments) { diff --git a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java index 30acbc1ac1..80affed0c6 100644 --- a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java +++ b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java @@ -316,7 +316,6 @@ void getMetadata_grantedConversation_returnsMetadata() throws Exception { @Test void getMetadata_nullMetadataDefaults() throws Exception { - ObjectId id = new ObjectId(); GridFSFile f = mock(GridFSFile.class); when(f.getFilename()).thenReturn("x.bin"); when(f.getLength()).thenReturn(9L); @@ -406,7 +405,8 @@ void listByConversation_returnsAttachmentsWithUuidRef() { when(f1.getFilename()).thenReturn("image.png"); when(f1.getLength()).thenReturn(1024L); when(f1.getMetadata()).thenReturn(new Document() - .append("mimeType", "image/png").append("storageRef", "uuid-1")); + .append("mimeType", "image/png").append("storageRef", "uuid-1") + .append("conversationId", "conv-1")); whenFindIterate(f1); List results = sut.listByConversation("conv-1"); @@ -439,4 +439,23 @@ void listByConversation_emptyResults() { whenFindIterate(); assertTrue(sut.listByConversation("conv-empty").isEmpty()); } + + @Test + void listAccessible_returnsOwnerFromMetadata() { + // A blob owned by another conversation but granted to the requester: the + // OR filter returns it, and its owner (not the requester) is reported. + GridFSFile f = mock(GridFSFile.class); + when(f.getObjectId()).thenReturn(new ObjectId()); + when(f.getFilename()).thenReturn("shared.pdf"); + when(f.getLength()).thenReturn(10L); + when(f.getMetadata()).thenReturn(new Document() + .append("mimeType", "application/pdf").append("storageRef", "uuid-g") + .append("conversationId", "owner-conv")); + whenFindIterate(f); + + List results = sut.listAccessible("member-conv"); + assertEquals(1, results.size()); + assertEquals("uuid-g", results.getFirst().storageRef()); + assertEquals("owner-conv", results.getFirst().conversationId()); + } } diff --git a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java index 3dc3f91af8..6b0d93dcdd 100644 --- a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java +++ b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java @@ -135,14 +135,20 @@ void materialize_url_passesThrough() { } @Test - void materialize_noStore_ignored() { + void materialize_noStore_dropsInlineButKeepsUrl() { service.attachmentStore = null; var inline = new Attachment(); inline.setBase64Data("x"); + var url = new Attachment(); + url.setMimeType("image/png"); + url.setUrl("https://example.com/y.png"); var gc = gc("gc-1"); - service.materializeAttachments(gc, List.of(inline)); - assertNull(gc.getAttachments()); + service.materializeAttachments(gc, List.of(inline, url)); + + // inline base64 dropped (no store), url kept (no store needed) + assertEquals(1, gc.getAttachments().size()); + assertEquals("https://example.com/y.png", gc.getAttachments().get(0).getUrl()); } @Test diff --git a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java index a3b394ac2b..857c6b50ae 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java @@ -427,7 +427,7 @@ void shouldReturn404WhenNotFound() throws Exception { @Test void shouldReturn403WhenDenied() throws Exception { when(attachmentStore.getMetadata("ref-1", "conv-other")) - .thenThrow(new AttachmentStoreException( + .thenThrow(new IAttachmentStore.AttachmentAccessDeniedException( "Cross-conversation access denied: attachment belongs to 'conv-1', requested from 'conv-other'")); Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-other", "ref-1", ar)); @@ -491,7 +491,7 @@ void shouldReturn404WhenNotFound() throws Exception { @Test void shouldReturn403WhenNotOwner() throws Exception { when(attachmentStore.delete("ref-1", "conv-other")) - .thenThrow(new AttachmentStoreException( + .thenThrow(new IAttachmentStore.AttachmentAccessDeniedException( "Delete denied: attachment belongs to 'conv-1', requested from 'conv-other'")); Response response = captureAsync(ar -> endpoint.deleteAttachment("conv-other", "ref-1", ar)); diff --git a/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java index 1a32ddba88..20a278cb6e 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/tools/impl/ReadAttachmentToolTest.java @@ -46,13 +46,13 @@ private static Attachment att(String ref, String name, String mime, long size) { @Test void list_empty() { - when(store.listByConversation(CONV)).thenReturn(List.of()); + when(store.listAccessible(CONV)).thenReturn(List.of()); assertTrue(tool.listAttachments().contains("No attachments")); } @Test void list_formatsEntries() { - when(store.listByConversation(CONV)).thenReturn(List.of( + when(store.listAccessible(CONV)).thenReturn(List.of( att("r1", "report.pdf", "application/pdf", 2048), att("r2", "notes.txt", "text/plain", 12))); String out = tool.listAttachments(); @@ -66,7 +66,7 @@ void list_formatsEntries() { @Test void read_byFileName_extractsText() throws Exception { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "notes.txt", "text/plain", 5))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "notes.txt", "text/plain", 5))); when(store.load("r1", CONV)).thenReturn("hello world".getBytes(StandardCharsets.UTF_8)); String out = tool.readAttachment("notes.txt", 0); @@ -75,7 +75,7 @@ void read_byFileName_extractsText() throws Exception { @Test void read_byStorageRef_extractsText() throws Exception { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "notes.txt", "text/plain", 5))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "notes.txt", "text/plain", 5))); when(store.load("r1", CONV)).thenReturn("body".getBytes(StandardCharsets.UTF_8)); assertTrue(tool.readAttachment("r1", 0).contains("body")); @@ -83,7 +83,7 @@ void read_byStorageRef_extractsText() throws Exception { @Test void read_caseInsensitiveFileName() throws Exception { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "Notes.TXT", "text/plain", 5))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "Notes.TXT", "text/plain", 5))); when(store.load("r1", CONV)).thenReturn("x".getBytes(StandardCharsets.UTF_8)); assertTrue(tool.readAttachment("notes.txt", 0).contains("x")); } @@ -91,7 +91,7 @@ void read_caseInsensitiveFileName() throws Exception { @Test void read_pdfPage() throws Exception { byte[] pdf = multiPagePdf("Alpha page", "Beta page"); - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "doc.pdf", "application/pdf", pdf.length))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "doc.pdf", "application/pdf", pdf.length))); when(store.load("r1", CONV)).thenReturn(pdf); String out = tool.readAttachment("doc.pdf", 2); @@ -101,34 +101,34 @@ void read_pdfPage() throws Exception { @Test void read_notFound() { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); assertTrue(tool.readAttachment("missing.txt", 0).contains("No attachment named")); } @Test void read_nonExtractableType_note() throws Exception { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "pic.png", "image/png", 100))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "pic.png", "image/png", 100))); when(store.load("r1", CONV)).thenReturn(new byte[]{1, 2, 3}); assertTrue(tool.readAttachment("pic.png", 0).contains("no extractable text")); } @Test void read_loadDenied_error() throws Exception { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); when(store.load("r1", CONV)).thenThrow(new AttachmentStoreException("access denied")); assertTrue(tool.readAttachment("a.txt", 0).contains("Could not read attachment")); } @Test void read_emptyText_note() throws Exception { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "empty.txt", "text/plain", 0))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "empty.txt", "text/plain", 0))); when(store.load("r1", CONV)).thenReturn(new byte[0]); assertTrue(tool.readAttachment("empty.txt", 0).contains("no extractable text")); } @Test void read_blankRef_notFound() { - when(store.listByConversation(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); + when(store.listAccessible(CONV)).thenReturn(List.of(att("r1", "a.txt", "text/plain", 1))); assertTrue(tool.readAttachment(" ", 0).contains("No attachment named")); } From c767607a721f74ac15512c899237f8f403687589 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Mon, 13 Jul 2026 19:53:32 +0200 Subject: [PATCH 19/23] refactor(attachments): replace fully-qualified names with imports Address @niedch PR #588 review: the field-injected IAttachmentStore in ConversationService and GroupConversationService used fully-qualified type references (and, in GroupConversationService, a fully-qualified @jakarta.inject.Inject even though jakarta.inject.Inject was already imported). Add IAttachmentStore imports and use @Inject / simple type names. No behavior change. --- docs/changelog.md | 15 +++++++++++++++ .../eddi/engine/internal/ConversationService.java | 5 +++-- .../engine/internal/GroupConversationService.java | 5 +++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 81e791fad5..5ee1df9a1c 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,21 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 📎 Multimodal Attachments Completion — Human review fixes: FQN → imports (2026-07-13) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) + +Addressed @niedch's human review comments on PR #588: + +1. **`GroupConversationService`** — the field-injected attachment store used a fully-qualified `@jakarta.inject.Inject` and `ai.labs.eddi.engine.attachments.IAttachmentStore` type. `jakarta.inject.Inject` was already imported, so the annotation is now `@Inject`; added an `IAttachmentStore` import and the field reads `IAttachmentStore attachmentStore;`. +2. **`ConversationService`** — same FQN smell on the injected field **and** the anonymous `getAttachmentStore()` override (reviewer flagged the override; the field had it too). Added the `IAttachmentStore` import and simplified both usages. + +Compile clean (`mvnw compile` → exit 0). No behavior change — pure import hygiene. + +**Deferred (tracked separately):** @niedch also suggested a "general solution for the authorization to avoid duplicating it in multiple places" on `PostgresAttachmentStore.authorize`. Verified as a real duplication — the access policy is copy-pasted across **4 sites** (read owner-or-grant + delete owner-only, in both the Postgres and GridFS stores) with an identical denial message, and the **read** path has already drifted for the null-owner edge case (Postgres denies, Mongo allows; the delete path stays consistent). Because the reviewer framed it as future work and unifying the read path is a security-behavior change that deserves its own tested PR, it was **not** folded into this PR — spun off as a dedicated follow-up (extract a shared `AttachmentAccessPolicy`, standardize null-owner reads to deny-by-default, add a two-backend regression test). + --- ## 📎 Multimodal Attachments Completion — Automated review fixes (2026-07-03) diff --git a/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java b/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java index 0bcd338aaa..6d5a96f4c4 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/internal/ConversationService.java @@ -9,6 +9,7 @@ import ai.labs.eddi.datastore.IResourceStore.ResourceNotFoundException; import ai.labs.eddi.datastore.IResourceStore.ResourceStoreException; import ai.labs.eddi.engine.api.IConversationService; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.audit.AuditLedgerService; import ai.labs.eddi.engine.gdpr.GdprComplianceService; import ai.labs.eddi.engine.gdpr.ProcessingRestrictedException; @@ -88,7 +89,7 @@ public class ConversationService implements IConversationService { // Field-injected so the numerous direct-construction unit tests need no change; // used only to resolve stored-attachment metadata at conversation init. @Inject - ai.labs.eddi.engine.attachments.IAttachmentStore attachmentStore; + IAttachmentStore attachmentStore; @ConfigProperty(name = "eddi.attachments.max-per-turn", defaultValue = "5") int maxAttachmentsPerTurn; @@ -682,7 +683,7 @@ public String getUserId() { } @Override - public ai.labs.eddi.engine.attachments.IAttachmentStore getAttachmentStore() { + public IAttachmentStore getAttachmentStore() { return attachmentStore; } diff --git a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java index 4701cd64d8..46b29ad07e 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java @@ -36,6 +36,7 @@ import ai.labs.eddi.datastore.serialization.IJsonSerialization; import ai.labs.eddi.engine.api.IConversationService; import ai.labs.eddi.engine.api.IGroupConversationService; +import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.memory.ConversationOutputExtractor; import ai.labs.eddi.engine.memory.model.ConversationState; import ai.labs.eddi.engine.memory.model.Attachment; @@ -92,8 +93,8 @@ public class GroupConversationService implements IGroupConversationService { // Field-injected so the direct-construction unit tests stay unchanged; used to // materialize and share discussion attachments with member conversations. - @jakarta.inject.Inject - ai.labs.eddi.engine.attachments.IAttachmentStore attachmentStore; + @Inject + IAttachmentStore attachmentStore; // Incremental peer verification: tracks the last verified transcript index // per group conversation ID, so we only verify new entries each turn (O(N) From bdcad292467448f3436df75b7d5ab766c1f22490 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Mon, 13 Jul 2026 20:29:05 +0200 Subject: [PATCH 20/23] docs(agents): require simple-name imports over inline FQNs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an Imports subsection to AGENTS.md §4.7 (Best Practices & Common Pitfalls): always import types/annotations and reference them by simple name; the only acceptable inline fully-qualified name is disambiguating two same-named classes used in one file. Codifies the recurring PR review comment that prompted the FQN->import cleanup in ConversationService and GroupConversationService. --- AGENTS.md | 6 ++++++ docs/changelog.md | 2 ++ 2 files changed, 8 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 0affe00976..17e145a4b0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -590,6 +590,12 @@ When implementing a new feature, provide: - Include conversation context in logs - Use appropriate levels: DEBUG (verbose), INFO (important events), ERROR (failures) +#### Imports + +- **Always reference types and annotations by their simple name with a top-level `import`** — never inline a fully-qualified name (e.g. write `@Inject IAttachmentStore store;` with the imports, not `@jakarta.inject.Inject ai.labs.eddi.engine.attachments.IAttachmentStore store;`). FQNs in field declarations, method signatures, annotations, and generics hurt readability and are a common review comment. +- The **only** acceptable inline FQN is disambiguating two classes that share a simple name and are both used in the same file — and even then, prefer restructuring so only one is imported. +- Don't leave unused imports behind after a refactor; run `./mvnw formatter:format` and `./mvnw validate` (Checkstyle) before committing. + #### Production-Scale Thinking When designing any new feature, always consider these before finalizing the design: diff --git a/docs/changelog.md b/docs/changelog.md index d1586a22c9..c7f1e8307a 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -16,6 +16,8 @@ Addressed @niedch's human review comments on PR #588: Compile clean (`mvnw compile` → exit 0). No behavior change — pure import hygiene. +Also codified the convention in `AGENTS.md` §4.7 (new **Imports** subsection): always import types/annotations and reference them by simple name; the only acceptable inline FQN is disambiguating two same-named classes used in one file. Prevents this review comment from recurring. + **Deferred (tracked separately):** @niedch also suggested a "general solution for the authorization to avoid duplicating it in multiple places" on `PostgresAttachmentStore.authorize`. Verified as a real duplication — the access policy is copy-pasted across **4 sites** (read owner-or-grant + delete owner-only, in both the Postgres and GridFS stores) with an identical denial message, and the **read** path has already drifted for the null-owner edge case (Postgres denies, Mongo allows; the delete path stays consistent). Because the reviewer framed it as future work and unifying the read path is a security-behavior change that deserves its own tested PR, it was **not** folded into this PR — spun off as a dedicated follow-up (extract a shared `AttachmentAccessPolicy`, standardize null-owner reads to deny-by-default, add a two-backend regression test). --- From 0160f6edfa1a071c0203312fb615bb54556c8f6b Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Mon, 13 Jul 2026 21:02:31 +0200 Subject: [PATCH 21/23] fix(attachments): re-hydrate group attachments on HITL resume A critical adversarial re-review of the origin/main merge surfaced a merge-emergent bug: group-shared attachments were silently lost across a HITL pause/resume. GroupConversation.attachments is @JsonIgnore transient (the durable copy lives in the blob store). resumeDiscussion() reloads a fresh GC from the store, so getAttachments() is null; executeDiscussion() re-seeded the sibling transient field dynamicAgentConfig but not attachments. A member speaking for the first time after a resume therefore got neither the blob-store grant nor the attachment_* context. Add rehydrateAttachmentsFromStore(gc), called in executeDiscussion right after the dynamicAgentConfig re-seed, rebuilding the metadata list from IAttachmentStore.listByConversation(gc.getId()) when the in-memory list is empty. Keeps the blob store as the single source of truth (no dangling refs after erasure), no persistence-schema change. Guarded by null/empty rather than startPhaseIndex, because a task-level pause in phase 0 resumes at index 0. URL-only attachments are not blob-backed and are not recovered on resume (documented as a known limitation). Neither merge parent could exhibit this: our branch had group attachments but no resumeDiscussion; origin/main had resumeDiscussion but no group attachments. Adds 4 unit tests (rehydrate_*). --- docs/changelog.md | 14 +++++ .../internal/GroupConversationService.java | 39 +++++++++++++ .../GroupConversationServiceTest.java | 56 +++++++++++++++++++ 3 files changed, 109 insertions(+) diff --git a/docs/changelog.md b/docs/changelog.md index c7f1e8307a..a46a207068 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,20 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 🐛 Multimodal Attachments Completion — Fix: group attachments lost on HITL resume (2026-07-13) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) + +Found by a critical adversarial re-review of the `origin/main` merge (10-dimension workflow + per-finding refutation). A **merge-emergent** bug — neither parent could exhibit it alone: our branch added group-shared attachments; `origin/main` added group HITL pause/resume; combined, they interact badly. + +**Bug:** `GroupConversation.attachments` is `@JsonIgnore` transient (the durable copy is the blob store). `resumeDiscussion()` reloads a fresh GC from the store, so `getAttachments()` is null; `executeDiscussion()` re-seeded the sibling transient field `dynamicAgentConfig` but **not** `attachments`. Result: a member speaking for the first time *after* a HITL resume got neither the blob-store grant nor the `attachment_*` context — blind to the shared files. Compiles cleanly; runtime-only. + +**Fix:** new package-private `rehydrateAttachmentsFromStore(gc)`, called in `executeDiscussion` right after the `dynamicAgentConfig` re-seed (so the two transient fields are handled symmetrically in one place). It rebuilds the metadata list from `IAttachmentStore.listByConversation(gc.getId())` when the in-memory list is empty — keeping the blob store as the single source of truth (no dangling refs after erasure) with **no persistence-schema change**. Guarded by null/empty (not `startPhaseIndex`, since a task-level pause in phase 0 resumes at index 0). **Known limitation:** URL-only attachments are not blob-backed and are not recovered on resume (documented in code; a follow-up can persist those if it becomes a real need). + +4 unit tests added (`rehydrate_*`); `GroupConversationServiceTest` + `RestGroupConversationTest` green. The rest of the merge review came back clean — 9/10 dimensions no findings, and the integration sweep confirmed the conflict resolutions themselves are correct (clean unions, no mis-picked sides, consistent call sites). + --- ## 📎 Multimodal Attachments Completion — Human review fixes: FQN → imports (2026-07-13) diff --git a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java index 595f676f8a..f38fcfe216 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java +++ b/src/main/java/ai/labs/eddi/engine/internal/GroupConversationService.java @@ -310,6 +310,40 @@ void materializeAttachments(GroupConversation gc, List incoming) { } } + /** + * Re-hydrate a group conversation's shared attachments from the durable blob + * store. {@link GroupConversation#getAttachments()} is {@code @JsonIgnore} + * transient, so a GC reloaded on a HITL resume has lost them; without this, a + * member whose first turn lands after the resume gets neither the blob grant + * nor the {@code attachment_*} context from {@link #grantAndInjectAttachments}. + *

+ * No-op when attachments are already present (fresh discussion — set by + * {@link #materializeAttachments}) or when the store holds none. URL-only + * attachments are not blob-backed and are intentionally not recovered here. + */ + void rehydrateAttachmentsFromStore(GroupConversation gc) { + if (attachmentStore == null || gc.getAttachments() != null && !gc.getAttachments().isEmpty()) { + return; + } + try { + var storedAttachments = attachmentStore.listByConversation(gc.getId()); + if (storedAttachments.isEmpty()) { + return; + } + List rehydrated = new ArrayList<>(); + for (var stored : storedAttachments) { + rehydrated.add(new Attachment(stored.mimeType(), stored.filename(), + stored.sizeBytes(), stored.storageRef())); + } + gc.setAttachments(rehydrated); + LOGGER.infof("Re-hydrated %d shared attachment(s) for group conversation '%s' from the blob store", + rehydrated.size(), gc.getId()); + } catch (Exception e) { + LOGGER.warnf("Failed to re-hydrate shared attachments for group conversation '%s': %s", + gc.getId(), e.getMessage()); + } + } + /** * Grant a member conversation access to the group's stored attachments and * inject them as {@code attachment_*} context on the member's first turn. URL @@ -375,6 +409,11 @@ private GroupConversation executeDiscussion(GroupConversation gc, AgentGroupConf // AgentOrchestrator to enforce group-level guardrails on dynamic tools. gc.setDynamicAgentConfig(config.getDynamicAgents()); + // Re-hydrate shared attachments (transient like dynamicAgentConfig above) from + // the durable blob store so a HITL resume doesn't silently drop them for a + // member whose first turn lands after the resume. See the method comment. + rehydrateAttachmentsFromStore(gc); + // AtomicInteger: shared across the phase loop; parallel phases increment // from virtual threads. Seed from pausedTurnCount to preserve budget across // resumes (M3). diff --git a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java index ce7f4bc41b..09d3fa8f0e 100644 --- a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java +++ b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java @@ -216,6 +216,62 @@ void grantAndInject_noAttachments_noop() { service.grantAndInjectAttachments(gc("gc-1"), "m", context); assertTrue(context.isEmpty()); } + + // rehydrateAttachmentsFromStore — recovers the transient attachments list + // after a HITL resume reloads the GroupConversation (attachments are + // @JsonIgnore transient, so a reloaded GC has none). + + @Test + void rehydrate_nullAttachments_rebuildsFromStore() { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + when(store.listByConversation("gc-1")).thenReturn( + List.of(new IAttachmentStore.Attachment("ref-1", "doc.pdf", "application/pdf", 10, "gc-1"))); + var gc = gc("gc-1"); + + service.rehydrateAttachmentsFromStore(gc); + + assertEquals(1, gc.getAttachments().size()); + assertEquals("ref-1", gc.getAttachments().get(0).getStorageRef()); + assertEquals("doc.pdf", gc.getAttachments().get(0).getFileName()); + assertEquals("application/pdf", gc.getAttachments().get(0).getMimeType()); + assertEquals(10L, gc.getAttachments().get(0).getSizeBytes()); + } + + @Test + void rehydrate_attachmentsAlreadyPresent_noop() { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + var gc = gc("gc-1"); + gc.setAttachments(List.of(new Attachment("image/png", "keep.png", 5, "ref-keep"))); + + service.rehydrateAttachmentsFromStore(gc); + + // fresh discussion already has its list — the store must not be consulted + verifyNoInteractions(store); + assertEquals(1, gc.getAttachments().size()); + assertEquals("ref-keep", gc.getAttachments().get(0).getStorageRef()); + } + + @Test + void rehydrate_emptyStore_leavesAttachmentsNull() { + var store = mock(IAttachmentStore.class); + service.attachmentStore = store; + when(store.listByConversation("gc-1")).thenReturn(List.of()); + + var gc = gc("gc-1"); + service.rehydrateAttachmentsFromStore(gc); + + assertNull(gc.getAttachments()); + } + + @Test + void rehydrate_noStore_noop() { + service.attachmentStore = null; + var gc = gc("gc-1"); + service.rehydrateAttachmentsFromStore(gc); + assertNull(gc.getAttachments()); + } } // ================================================================= From 2d2e1ee6cefd3fb20cc51e4184e422b3259f421b Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Mon, 13 Jul 2026 21:50:58 +0200 Subject: [PATCH 22/23] fix(attachments): address PR #588 review comments Correctness: - Download 404-vs-500 (High): load/getMetadata threw a bare AttachmentStoreException for both a missing blob and an internal store failure, so downloadAttachment mapped SQL/backend errors to 404 at DEBUG. Add a typed AttachmentNotFoundException (symmetric with AttachmentAccessDeniedException); both stores throw it for missing blobs; the endpoint returns 404 for it and 500 (ERROR, ATTACHMENT_STORE_ERROR) for any other store exception. +regression test. - GDPR export isolation (Major): the attachment-metadata export wrapped the whole conversation loop in one try/catch, so one failing listByConversation truncated the export for every remaining conversation. Isolate per conversation, mirroring the conversation-snapshot block. - URL group attachment without mimeType (Medium): toAttachments kept URL refs with null/blank mimeType that AttachmentContextExtractor drops later; skip them up front so the loss is explicit. Observability: - AttachmentForwarder: init reusable Counters once (constructor) instead of resolving per forward(); import MeterRegistry/Counter. - AttachmentTextExtractor: per-extraction PDF logs INFO -> DEBUG. - Conversation: include conversation id in the attachment-issue warning. Style (AGENTS.md import guideline): - LlmTask @jakarta.inject.Inject -> @Inject; GroupConversation imports Attachment; GroupConversationServiceTest imports Context. - GridFsAttachmentStoreTest.whenFindIterate generalized to any file count. Declined: MultimodalOverride stays a mutable Jackson POJO for consistency with its sibling nested config types. --- docs/changelog.md | 28 +++++++++++++++++++ .../groups/model/GroupConversation.java | 7 +++-- .../mongo/GridFsAttachmentStore.java | 6 ++-- .../postgres/PostgresAttachmentStore.java | 4 +-- .../engine/attachments/IAttachmentStore.java | 13 +++++++++ .../engine/gdpr/GdprComplianceService.java | 13 +++++++-- .../internal/RestGroupConversation.java | 6 ++++ .../memory/rest/RestAttachmentUpload.java | 12 ++++++-- .../engine/runtime/internal/Conversation.java | 3 +- .../modules/llm/impl/AttachmentForwarder.java | 21 +++++++------- .../labs/eddi/modules/llm/impl/LlmTask.java | 4 +-- .../tools/impl/AttachmentTextExtractor.java | 4 +-- .../mongo/GridFsAttachmentStoreTest.java | 22 +++++++++------ .../GroupConversationServiceTest.java | 9 +++--- .../memory/rest/RestAttachmentUploadTest.java | 17 ++++++++++- 15 files changed, 127 insertions(+), 42 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index a46a207068..76e9d6c7f6 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,34 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 🔍 Multimodal Attachments Completion — PR #588 review-comment fixes (2026-07-13) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) + +Addressed CodeRabbit + Copilot review comments. + +**Correctness** +- **Download 404-vs-500 (High):** `IAttachmentStore.load`/`getMetadata` threw a bare `AttachmentStoreException` for *both* a missing blob and an internal store failure, so `RestAttachmentUpload.downloadAttachment` mapped SQL/backend errors to 404 (at DEBUG) — hiding outages. Added a typed `AttachmentNotFoundException` (symmetric with `AttachmentAccessDeniedException`); both stores throw it for genuinely-missing blobs; the endpoint returns 404 for it and 500 (ERROR log, `ATTACHMENT_STORE_ERROR`) for any other store exception. +regression test. +- **GDPR export isolation (Major):** the attachment-metadata export wrapped the whole conversation loop in one try/catch, so one failing `listByConversation` truncated the export for every remaining conversation. Each conversation is now isolated (mirrors the conversation-snapshot block above it). +- **URL group attachment without mimeType (Medium):** `RestGroupConversation.toAttachments` kept URL refs with null/blank mimeType that `AttachmentContextExtractor` silently drops later; now skipped up front so the loss is explicit. + +**Observability** +- `AttachmentForwarder`: reusable `Counter`s initialized once (in the constructor — the registry is constructor-injected, so `@PostConstruct` wouldn't fire in the direct-construction unit tests) instead of resolved per `forward()`; `MeterRegistry`/`Counter` imported. +- `AttachmentTextExtractor`: per-extraction PDF logs lowered INFO → DEBUG (they run on every user turn / tool call). +- `Conversation`: the attachment-issue warning now includes the conversation id. + +**Style** (the import guideline just added to AGENTS.md §4.7) +- `LlmTask` (`@jakarta.inject.Inject` → `@Inject`), `GroupConversation` (`Attachment` imported), `GroupConversationServiceTest` (`Context` imported), and the FQN `MeterRegistry` in `AttachmentForwarder`. +- `GridFsAttachmentStoreTest.whenFindIterate` generalized to any file count (was hardcoded to the 0/1/2-file cases). + +**Declined / documented** +- `LlmConfiguration.MultimodalOverride` kept as a mutable Jackson POJO (not a record) for consistency with every sibling nested config type in the file — converting only one would be inconsistent and need `@JsonCreator` wiring. +- URL-only group attachments still aren't recovered after a HITL resume — a deliberate, documented limitation (the blob store is the durable source; URLs aren't blob-backed). The PR description should note this. + +All affected unit tests green. + --- ## 🐛 Multimodal Attachments Completion — Fix: group attachments lost on HITL resume (2026-07-13) diff --git a/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java b/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java index 35f6c2e39a..d8479d35ff 100644 --- a/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java +++ b/src/main/java/ai/labs/eddi/configs/groups/model/GroupConversation.java @@ -5,6 +5,7 @@ package ai.labs.eddi.configs.groups.model; import ai.labs.eddi.configs.hitl.HitlTimeoutPolicy; +import ai.labs.eddi.engine.memory.model.Attachment; import com.fasterxml.jackson.annotation.JsonIgnore; import java.time.Instant; @@ -82,7 +83,7 @@ public class GroupConversation { * {@code IAttachmentStore} bound to this conversation's id. */ @JsonIgnore - private transient List attachments; + private transient List attachments; /** * A single entry in the discussion transcript. Each entry records one agent's @@ -337,11 +338,11 @@ public void setDynamicAgentConfig(AgentGroupConfiguration.DynamicAgentConfig dyn } @JsonIgnore - public List getAttachments() { + public List getAttachments() { return attachments; } - public void setAttachments(List attachments) { + public void setAttachments(List attachments) { this.attachments = attachments; } diff --git a/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java b/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java index a293ab742f..a993bb33e0 100644 --- a/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStore.java @@ -121,7 +121,7 @@ public Attachment store(byte[] bytes, String declaredMime, String filename, public byte[] load(String storageRef, String requestingConversationId) throws AttachmentStoreException { GridFSFile file = findFileByRef(storageRef); if (file == null) { - throw new AttachmentStoreException("Attachment not found: " + storageRef); + throw new AttachmentNotFoundException("Attachment not found: " + storageRef); } authorize(file, requestingConversationId); @@ -134,7 +134,7 @@ public byte[] load(String storageRef, String requestingConversationId) throws At public Attachment getMetadata(String storageRef, String requestingConversationId) throws AttachmentStoreException { GridFSFile file = findFileByRef(storageRef); if (file == null) { - throw new AttachmentStoreException("Attachment not found: " + storageRef); + throw new AttachmentNotFoundException("Attachment not found: " + storageRef); } authorize(file, requestingConversationId); @@ -153,7 +153,7 @@ public void grantAccess(String storageRef, String conversationId) throws Attachm var result = filesCollection.updateOne(refFilter(storageRef), Updates.addToSet("metadata." + META_GRANTS, conversationId)); if (result.getMatchedCount() == 0) { - throw new AttachmentStoreException("Attachment not found: " + storageRef); + throw new AttachmentNotFoundException("Attachment not found: " + storageRef); } LOGGER.debugf("Granted conversation '%s' access to attachment %s", sanitize(conversationId), storageRef); diff --git a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java index 40ff6204bd..c7fe11e93f 100644 --- a/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/datastore/postgres/PostgresAttachmentStore.java @@ -141,7 +141,7 @@ public byte[] load(String storageRef, String requestingConversationId) throws At ps.setString(1, storageRef); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) { - throw new AttachmentStoreException("Attachment not found: " + storageRef); + throw new AttachmentNotFoundException("Attachment not found: " + storageRef); } authorize(rs.getString("conversation_id"), rs, requestingConversationId); return rs.getBytes("data"); @@ -160,7 +160,7 @@ public Attachment getMetadata(String storageRef, String requestingConversationId ps.setString(1, storageRef); try (ResultSet rs = ps.executeQuery()) { if (!rs.next()) { - throw new AttachmentStoreException("Attachment not found: " + storageRef); + throw new AttachmentNotFoundException("Attachment not found: " + storageRef); } authorize(rs.getString("conversation_id"), rs, requestingConversationId); return new Attachment( diff --git a/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java b/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java index 94d9434430..f12ebbd7f8 100644 --- a/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java +++ b/src/main/java/ai/labs/eddi/engine/attachments/IAttachmentStore.java @@ -182,4 +182,17 @@ public AttachmentAccessDeniedException(String message) { super(message); } } + + /** + * Thrown when a storage reference does not resolve to any blob — a genuinely + * missing attachment, as opposed to a backend/store failure (which stays a + * plain {@link AttachmentStoreException}). Lets the REST layer return 404 for a + * missing blob while a store outage surfaces as 500 instead of being + * misreported as "not found". + */ + class AttachmentNotFoundException extends AttachmentStoreException { + public AttachmentNotFoundException(String message) { + super(message); + } + } } diff --git a/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java b/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java index 0f98ff8c79..bb966231d6 100644 --- a/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java +++ b/src/main/java/ai/labs/eddi/engine/gdpr/GdprComplianceService.java @@ -295,9 +295,16 @@ public UserDataExport exportUserData(String userId) { if (attachmentStorageInstance.isResolvable()) { var store = attachmentStorageInstance.get(); for (var convId : conversationMemoryStore.getConversationIdsByUserId(userId)) { - for (var a : store.listByConversation(convId)) { - attachmentEntries.add(new UserDataExport.AttachmentExportEntry( - convId, a.storageRef(), a.filename(), a.mimeType(), a.sizeBytes())); + try { + for (var a : store.listByConversation(convId)) { + attachmentEntries.add(new UserDataExport.AttachmentExportEntry( + convId, a.storageRef(), a.filename(), a.mimeType(), a.sizeBytes())); + } + } catch (Exception e) { + // Isolate per conversation so one bad lookup doesn't truncate + // the whole export (mirrors the conversation-snapshot block above). + LOGGER.warnf("[GDPR] Skipping attachments for conversation %s during export: %s", + convId, e.getMessage()); } } } diff --git a/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java b/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java index 3d6d3f70e6..2c7e87485c 100644 --- a/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java +++ b/src/main/java/ai/labs/eddi/engine/internal/RestGroupConversation.java @@ -131,6 +131,12 @@ private static List toAttachments(List refs) { if (r.data() != null && !r.data().isBlank()) { a.setBase64Data(r.data()); } else if (r.url() != null && !r.url().isBlank()) { + // A URL attachment with no mimeType is dropped later by + // AttachmentContextExtractor; skip it here so the loss is explicit + // rather than silent. + if (r.mimeType() == null || r.mimeType().isBlank()) { + continue; + } a.setUrl(r.url()); } else { continue; diff --git a/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java b/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java index 9d39d66868..164a3c3d86 100644 --- a/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java +++ b/src/main/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUpload.java @@ -204,7 +204,7 @@ public void listAttachments( @Operation( operationId = "downloadAttachment", summary = "Download a single attachment", - description = "Streams the raw bytes of one attachment. Access is checked against " + description = "Returns the raw bytes of one attachment (buffered, not chunked). Access is checked against " + "the owning conversation (owner or explicit grant); references are " + "unguessable.") @APIResponse(responseCode = "200", description = "Attachment bytes with Content-Type and Content-Disposition.") @@ -232,12 +232,20 @@ public void downloadAttachment( asyncResponse.resume(Response.status(Response.Status.FORBIDDEN) .entity(Map.of("error", e.getMessage(), "code", "ATTACHMENT_ACCESS_DENIED")) .build()); - } catch (IAttachmentStore.AttachmentStoreException e) { + } catch (IAttachmentStore.AttachmentNotFoundException e) { LOGGER.debugf("Attachment download not found for conversation '%s': %s", sanitize(conversationId), e.getMessage()); asyncResponse.resume(Response.status(Response.Status.NOT_FOUND) .entity(Map.of("error", e.getMessage(), "code", "ATTACHMENT_NOT_FOUND")) .build()); + } catch (IAttachmentStore.AttachmentStoreException e) { + // A store/backend failure (not a missing blob) — surface as 500 and + // log at ERROR so outages are not silently misreported as 404. + LOGGER.errorf(e, "Attachment store error downloading attachment for conversation '%s'", + sanitize(conversationId)); + asyncResponse.resume(Response.status(Response.Status.INTERNAL_SERVER_ERROR) + .entity(Map.of("error", "Failed to download attachment", "code", "ATTACHMENT_STORE_ERROR")) + .build()); } catch (Exception e) { LOGGER.errorf(e, "Failed to download attachment for conversation '%s'", sanitize(conversationId)); asyncResponse.resume(Response.status(Response.Status.INTERNAL_SERVER_ERROR) diff --git a/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java b/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java index 50bd3124ed..20ca6f0e28 100644 --- a/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java +++ b/src/main/java/ai/labs/eddi/engine/runtime/internal/Conversation.java @@ -271,7 +271,8 @@ private List> prepareLifecycleData(String message, Map conversationMemory.getConversationId(), propertiesHandler.getMaxAttachmentsPerTurn()); if (!extraction.errors().isEmpty()) { - extraction.errors().forEach(err -> LOGGER.warnv("Attachment issue: {0}", err)); + extraction.errors().forEach(err -> LOGGER.warnv("Attachment issue in conversation {0}: {1}", + conversationMemory.getConversationId(), err)); var errorData = new Data<>(MemoryKeys.ATTACHMENT_ERRORS.key(), extraction.errors()); errorData.setPublic(false); currentStep.storeData(errorData); diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java index 49c474632c..cafc3ce25f 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/AttachmentForwarder.java @@ -21,6 +21,8 @@ import dev.langchain4j.data.message.PdfFileContent; import dev.langchain4j.data.message.TextContent; import dev.langchain4j.data.message.UserMessage; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import org.eclipse.microprofile.config.inject.ConfigProperty; @@ -78,14 +80,15 @@ public class AttachmentForwarder { private final SafeHttpClient httpClient; private final long maxForwardBytes; private final long maxAggregateBytes; - private final io.micrometer.core.instrument.MeterRegistry meterRegistry; + private final Counter forwardedCounter; + private final Counter errorsCounter; @Inject public AttachmentForwarder(IAttachmentStore attachmentStore, ModelCapabilityService capabilityService, AttachmentTextExtractor textExtractor, SafeHttpClient httpClient, - io.micrometer.core.instrument.MeterRegistry meterRegistry, + MeterRegistry meterRegistry, @ConfigProperty(name = "eddi.attachments.max-forward-bytes", defaultValue = "10485760") long maxForwardBytes, @ConfigProperty(name = "eddi.attachments.max-forward-aggregate-bytes", @@ -94,7 +97,8 @@ public AttachmentForwarder(IAttachmentStore attachmentStore, this.capabilityService = capabilityService; this.textExtractor = textExtractor; this.httpClient = httpClient; - this.meterRegistry = meterRegistry; + this.forwardedCounter = meterRegistry != null ? meterRegistry.counter("eddi.attachment.forwarded") : null; + this.errorsCounter = meterRegistry != null ? meterRegistry.counter("eddi.attachment.errors") : null; this.maxForwardBytes = maxForwardBytes; this.maxAggregateBytes = maxAggregateBytes; } @@ -167,14 +171,11 @@ public void forward(List messages, IConversationMemory memory, Stri } private void recordMetrics(int forwarded, int errored) { - if (meterRegistry == null) { - return; - } - if (forwarded > 0) { - meterRegistry.counter("eddi.attachment.forwarded").increment(forwarded); + if (forwardedCounter != null && forwarded > 0) { + forwardedCounter.increment(forwarded); } - if (errored > 0) { - meterRegistry.counter("eddi.attachment.errors").increment(errored); + if (errorsCounter != null && errored > 0) { + errorsCounter.increment(errored); } } diff --git a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java index d417b46ca1..b15dc1338d 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java +++ b/src/main/java/ai/labs/eddi/modules/llm/impl/LlmTask.java @@ -115,10 +115,10 @@ public class LlmTask implements ILifecycleTask { // Field-injected so the many direct-construction unit tests are unaffected; // null-guarded at the call site. - @jakarta.inject.Inject + @Inject AttachmentForwarder attachmentForwarder; - @jakarta.inject.Inject + @Inject AttachmentTextExtractor attachmentTextExtractor; /** diff --git a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java index 62bfa80866..f18f4ee891 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java +++ b/src/main/java/ai/labs/eddi/modules/llm/tools/impl/AttachmentTextExtractor.java @@ -119,7 +119,7 @@ public String extractPdfText(byte[] pdfBytes, int maxChars) throws AttachmentExt try (PDDocument document = Loader.loadPDF(pdfBytes)) { PDFTextStripper stripper = new PDFTextStripper(); String text = stripper.getText(document); - LOGGER.infof("Extracted %d characters from PDF with %d pages", + LOGGER.debugf("Extracted %d characters from PDF with %d pages", text.length(), document.getNumberOfPages()); return cap(text, cap); } catch (Exception e) { @@ -147,7 +147,7 @@ public String extractPdfText(byte[] pdfBytes, int startPage, int endPage, int ma stripper.setStartPage(startPage); stripper.setEndPage(effectiveEnd); String text = stripper.getText(document); - LOGGER.infof("Extracted text from PDF pages %d-%d (%d characters)", + LOGGER.debugf("Extracted text from PDF pages %d-%d (%d characters)", startPage, effectiveEnd, text.length()); return cap(text, cap); } catch (AttachmentExtractionException e) { diff --git a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java index 80affed0c6..b13b15b379 100644 --- a/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java +++ b/src/test/java/ai/labs/eddi/datastore/mongo/GridFsAttachmentStoreTest.java @@ -107,18 +107,22 @@ private void whenFindIterate(GridFSFile... files) { @SuppressWarnings("unchecked") MongoCursor cursor = mock(MongoCursor.class); doReturn(cursor).when(it).iterator(); + // hasNext() returns true once per file then false; next() returns each file + // in order. Generalizes to any file count (not just 0/1/2). Boolean[] hasNext = new Boolean[files.length + 1]; - for (int i = 0; i < files.length; i++) + for (int i = 0; i < files.length; i++) { hasNext[i] = true; + } hasNext[files.length] = false; - if (files.length == 0) { - when(cursor.hasNext()).thenReturn(false); - } else if (files.length == 1) { - when(cursor.hasNext()).thenReturn(true, false); - when(cursor.next()).thenReturn(files[0]); - } else { - when(cursor.hasNext()).thenReturn(true, true, false); - when(cursor.next()).thenReturn(files[0], files[1]); + var hasNextStub = when(cursor.hasNext()); + for (Boolean b : hasNext) { + hasNextStub = hasNextStub.thenReturn(b); + } + if (files.length > 0) { + var nextStub = when(cursor.next()); + for (GridFSFile f : files) { + nextStub = nextStub.thenReturn(f); + } } } diff --git a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java index 09d3fa8f0e..ddb2e11723 100644 --- a/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java +++ b/src/test/java/ai/labs/eddi/engine/internal/GroupConversationServiceTest.java @@ -25,6 +25,7 @@ import ai.labs.eddi.engine.api.IGroupConversationService.GroupDepthExceededException; import ai.labs.eddi.engine.api.IGroupConversationService.GroupDiscussionException; import ai.labs.eddi.engine.attachments.IAttachmentStore; +import ai.labs.eddi.engine.model.Context; import ai.labs.eddi.engine.memory.model.Attachment; import ai.labs.eddi.engine.memory.model.ConversationOutput; import ai.labs.eddi.engine.memory.model.SimpleConversationMemorySnapshot; @@ -167,7 +168,7 @@ void grantAndInject_storedRef_grantsAndInjects() throws Exception { service.attachmentStore = store; var gc = gc("gc-1"); gc.setAttachments(List.of(new Attachment("application/pdf", "doc.pdf", 10, "ref-1"))); - Map context = new LinkedHashMap<>(); + Map context = new LinkedHashMap<>(); service.grantAndInjectAttachments(gc, "member-conv", context); @@ -187,7 +188,7 @@ void grantAndInject_url_injectsWithoutGrant() { url.setMimeType("image/png"); url.setUrl("https://example.com/y.png"); gc.setAttachments(List.of(url)); - Map context = new LinkedHashMap<>(); + Map context = new LinkedHashMap<>(); service.grantAndInjectAttachments(gc, "member-conv", context); @@ -203,7 +204,7 @@ void grantAndInject_grantFailure_skipsEntry() throws Exception { doThrow(new IAttachmentStore.AttachmentStoreException("nope")).when(store).grantAccess(any(), any()); var gc = gc("gc-1"); gc.setAttachments(List.of(new Attachment("application/pdf", "d.pdf", 1, "ref-1"))); - Map context = new LinkedHashMap<>(); + Map context = new LinkedHashMap<>(); service.grantAndInjectAttachments(gc, "m", context); assertFalse(context.containsKey("attachment_0")); @@ -212,7 +213,7 @@ void grantAndInject_grantFailure_skipsEntry() throws Exception { @Test void grantAndInject_noAttachments_noop() { service.attachmentStore = mock(IAttachmentStore.class); - Map context = new LinkedHashMap<>(); + Map context = new LinkedHashMap<>(); service.grantAndInjectAttachments(gc("gc-1"), "m", context); assertTrue(context.isEmpty()); } diff --git a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java index 857c6b50ae..b31f4b61ae 100644 --- a/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java +++ b/src/test/java/ai/labs/eddi/engine/memory/rest/RestAttachmentUploadTest.java @@ -6,6 +6,7 @@ import ai.labs.eddi.engine.attachments.IAttachmentStore; import ai.labs.eddi.engine.attachments.IAttachmentStore.Attachment; +import ai.labs.eddi.engine.attachments.IAttachmentStore.AttachmentNotFoundException; import ai.labs.eddi.engine.attachments.IAttachmentStore.AttachmentStoreException; import jakarta.ws.rs.container.AsyncResponse; import jakarta.ws.rs.core.Response; @@ -414,7 +415,7 @@ void shouldStreamBytesWithHeaders() throws Exception { @Test void shouldReturn404WhenNotFound() throws Exception { when(attachmentStore.getMetadata("missing", "conv-1")) - .thenThrow(new AttachmentStoreException("Attachment not found: missing")); + .thenThrow(new AttachmentNotFoundException("Attachment not found: missing")); Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-1", "missing", ar)); @@ -448,6 +449,20 @@ void shouldReturn500OnUnexpectedError() throws Exception { assertEquals(500, response.getStatus()); } + @Test + void shouldReturn500OnStoreError() throws Exception { + // A backend/store failure (not a missing blob) must be a 500, not a 404. + when(attachmentStore.getMetadata("ref-1", "conv-1")) + .thenThrow(new AttachmentStoreException("Failed to load attachment")); + + Response response = captureAsync(ar -> endpoint.downloadAttachment("conv-1", "ref-1", ar)); + + assertEquals(500, response.getStatus()); + @SuppressWarnings("unchecked") + var body = (Map) response.getEntity(); + assertEquals("ATTACHMENT_STORE_ERROR", body.get("code")); + } + @Test void shouldSanitizeContentDispositionFilename() throws Exception { var meta = new Attachment("ref-1", "bad\"name\r\n.png", "image/png", 2, "conv-1"); From c387b9d7016ee0e8ba17e4d82ba2d9c93e22b0d7 Mon Sep 17 00:00:00 2001 From: Gregor Jarisch Date: Mon, 13 Jul 2026 23:56:10 +0200 Subject: [PATCH 23/23] chore(config): remove no-op reattachTurns knob LlmConfiguration.Task.reattachTurns (@since 6.1.0, added on this branch) was dead config: getReattachTurns() is called nowhere in src/main, so setting it did nothing at runtime. Past-turn attachments already reach the model via text-extract stitching (attachments:extracts), not native re-attachment. Remove the field, getter/setter, and its round-trip test. Found by a codebase-wide dead-config audit; the other candidate no-op knobs it surfaced are triaged and tracked as follow-ups rather than mass-deleted (see changelog). --- docs/changelog.md | 15 +++++++++++ .../modules/llm/model/LlmConfiguration.java | 17 ------------ .../llm/model/LlmConfigurationTaskTest.java | 27 ------------------- 3 files changed, 15 insertions(+), 44 deletions(-) diff --git a/docs/changelog.md b/docs/changelog.md index 76e9d6c7f6..2282af54a1 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -3,6 +3,21 @@ > **Purpose:** Living document tracking all changes, decisions, and reasoning during implementation. Updated as work progresses for easy reference and review. +--- + +## 🧹 Multimodal Attachments Completion — Remove dead config knob `reattachTurns` (2026-07-13) + +**Repo:** EDDI (`feat/multimodal-attachments-completion`) + +`LlmConfiguration.Task.reattachTurns` (`@since 6.1.0`, added on this branch) was a no-op: `getReattachTurns()` is called nowhere in `src/main`, so setting it changed nothing at runtime. Past-turn PDFs/docs already reach the model via text-extract stitching (`attachments:extracts`), never native re-attachment. Removed the field, getter/setter, and its round-trip test. + +Surfaced by a codebase-wide dead-config audit (adversarial multi-agent sweep). The audit flagged ~26 other candidate no-op knobs; rather than mass-delete, they were **triaged** and tracked as follow-ups: +- **Genuinely dead** — `ModelCascadeConfig.strategy` ("parallel = future", never built), `dream.batchSize`. +- **Feature exists but knob unwired** — `enableParallelExecution` + `parallelExecutionTimeoutMs` (orphaned `ToolExecutionService` parallel machinery), RAG `injectionStrategy`/`contextTemplate`, `McpServerConfig.transport`, `autoRecallCategories`, `dream.schedule`/`maxUsersPerRun`. +- **⚠️ Unenforced guardrails** — `DynamicAgentConfig.allowRecruitment`/`allowDelegation`/`maxRecruitedAgentsPerDiscussion`/`maxDelegationsPerTask`/`inheritParentModel` are read nowhere; the guardrails silently don't apply (tracked as its own security/cost fix). +- **Roadmap scaffolding — keep** — `sessionManagement`/`autoSnapshot`/`maxCheckpointsPerConversation` (Session Forking is in-progress per roadmap). +- **Audit blind spot** — operator knobs selected via Quarkus `@IfBuildProfile`/`@LookupIfProperty` (e.g. `eddi.messaging.type`) are *not* dead; a getter-grep can't see build-time bean selection. Those need per-item verification, not deletion. + --- ## 🔍 Multimodal Attachments Completion — PR #588 review-comment fixes (2026-07-13) diff --git a/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java b/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java index f72a7ef4be..735029ecdb 100644 --- a/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java +++ b/src/main/java/ai/labs/eddi/modules/llm/model/LlmConfiguration.java @@ -308,15 +308,6 @@ public static class Task { */ private MultimodalOverride multimodal; - /** - * Number of past turns whose attachments are natively re-attached to the LLM on - * later turns (in addition to the always-on text-extract stitching). {@code 0} - * (default) means attachments attach only on their own turn. - * - * @since 6.1.0 - */ - private Integer reattachTurns = 0; - /** * Per-task tool-approval gating override (tool-level HITL). When present, it * FULLY REPLACES the agent-level {@code hitlConfig.toolApprovals} for this task @@ -672,14 +663,6 @@ public void setMultimodal(MultimodalOverride multimodal) { this.multimodal = multimodal; } - public Integer getReattachTurns() { - return reattachTurns != null ? reattachTurns : 0; - } - - public void setReattachTurns(Integer reattachTurns) { - this.reattachTurns = reattachTurns; - } - } /** diff --git a/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java b/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java index 606b73e007..1d097a3339 100644 --- a/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java +++ b/src/test/java/ai/labs/eddi/modules/llm/model/LlmConfigurationTaskTest.java @@ -53,33 +53,6 @@ void setAndGet() { } } - @Nested - @DisplayName("reattachTurns") - class ReattachTurns { - - @Test - @DisplayName("defaults to 0") - void defaultsZero() { - assertEquals(0, new LlmConfiguration.Task().getReattachTurns()); - } - - @Test - @DisplayName("null coalesces to 0") - void nullCoalescesZero() { - var task = new LlmConfiguration.Task(); - task.setReattachTurns(null); - assertEquals(0, task.getReattachTurns()); - } - - @Test - @DisplayName("getter/setter round-trip") - void setAndGet() { - var task = new LlmConfiguration.Task(); - task.setReattachTurns(3); - assertEquals(3, task.getReattachTurns()); - } - } - @Nested @DisplayName("isAgentMode") class IsAgentMode {