From 5c608bbdade4270ab3ca429a1f75607d21903f1c Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 14:47:30 +0100 Subject: [PATCH 01/18] Review requirements and design --- .../ai/design/feature-improve-rag-chunking.md | 294 ++++++++++++++++++ .../feature-improve-rag-chunking.md | 199 ++++++++++++ .../planning/feature-improve-rag-chunking.md | 137 ++++++++ .../feature-improve-rag-chunking.md | 162 ++++++++++ .../testing/feature-improve-rag-chunking.md | 135 ++++++++ 5 files changed, 927 insertions(+) create mode 100644 docs/ai/design/feature-improve-rag-chunking.md create mode 100644 docs/ai/implementation/feature-improve-rag-chunking.md create mode 100644 docs/ai/planning/feature-improve-rag-chunking.md create mode 100644 docs/ai/requirements/feature-improve-rag-chunking.md create mode 100644 docs/ai/testing/feature-improve-rag-chunking.md diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md new file mode 100644 index 00000000000..505e0a44d3b --- /dev/null +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -0,0 +1,294 @@ +--- +phase: design +title: Improve RAG Chunking - System Design +description: Architecture for configurable hierarchical note chunking and explicit indexing settings +--- + +# System Design & Architecture + +## Architecture Overview + +```mermaid +graph TD + UI[Web Settings UI] + MUI[Future Mobile Consumer] + CFG[Indexing Settings Store] + IDX[rag-index Edge Function] + SEARCH[rag-search Edge Function] + CORE[Core RAG Chunking Module] + NOTES[(notes)] + EMB[(note_embeddings)] + GEM[Gemini Embeddings API] + + UI -->|read/update editable settings| CFG + UI -->|display read-only settings| CFG + IDX -->|load active indexing config| CFG + IDX -->|fetch title, description, tags| NOTES + IDX -->|use shared chunking helpers| CORE + SEARCH -->|use shared settings compatibility helpers| CORE + IDX -->|batch embed chunks\\nRETRIEVAL_DOCUMENT| GEM + IDX -->|upsert vectors| EMB + SEARCH -->|load active dimensions/task config| CFG + SEARCH -->|embed query\\nRETRIEVAL_QUERY| GEM + SEARCH -->|vector match| EMB +``` + +- The application owns chunk construction end-to-end. +- Gemini is used only for embeddings and receives already-constructed chunk text. +- Indexing behavior is driven by persisted settings instead of hard-coded constants. +- Search remains functionally unchanged in this feature, but query-side embedding compatibility stays explicit. +- Reusable indexing logic lives in `core` and is shared across web, mobile, and Edge Function integration paths. +- `core` is the source of truth for indexing behavior; platform-specific layers are adapters, not owners of chunking rules. +- In this feature, only the web app gets a settings UI; mobile is intentionally out of scope at the UI layer. + +## Data Models + +### Indexing settings + +The system uses a dedicated per-user table for indexing settings rather than storing them inside `user_api_keys`. This keeps API secrets and indexing behavior separate while still allowing the web UI to present both under the Google API settings tab. + +Representative model: + +```sql +user_rag_index_settings ( + user_id uuid primary key references auth.users(id) on delete cascade, + small_note_threshold integer not null default 300, + target_chunk_size integer not null default 200, + min_chunk_size integer not null default 100, + max_chunk_size integer not null default 400, + overlap integer not null default 50, + use_title boolean not null default true, + use_section_headings boolean not null default true, + use_tags boolean not null default true, + updated_at timestamptz not null default now() +) +``` + +The effective settings object exposed to the UI and indexing paths must support: + +```ts +type RagIndexingSettings = { + small_note_threshold: number + target_chunk_size: number + min_chunk_size: number + max_chunk_size: number + overlap: number + use_title: boolean + use_section_headings: boolean + use_tags: boolean + + // Read-only/system-defined + output_dimensionality: number + task_type_document: "RETRIEVAL_DOCUMENT" + task_type_query: "RETRIEVAL_QUERY" + split_strategy: "hierarchical" + fallback_split_order: ["sections", "paragraphs", "sentences", "tokens_or_characters"] + chunk_accumulation_rule: string + small_chunk_merge_rule: string + chunk_template: string +} +``` + +### Chunk assembly model + +Internal chunk assembly should preserve enough metadata to support stable writes and future debugging: + +```ts +type PendingChunk = { + sectionHeading: string | null + content: string + startOffset: number + endOffset: number +} + +type FinalChunk = { + chunkIndex: number + charOffset: number + title: string | null + text: string +} +``` + +### Existing embeddings storage + +The feature continues to write final chunks into `note_embeddings`, but the embedding vector dimension must stay aligned with `output_dimensionality`: + +```sql +note_embeddings ( + note_id uuid, + user_id uuid, + chunk_index int, + char_offset int, + content text, + embedding vector(...), + indexed_at timestamptz +) +``` + +## API Design + +### Settings UI contract + +The UI must be able to: + +- read the current indexing settings +- update editable settings without redeploy +- display read-only system settings in the same screen +- expose these settings in the user's Google API settings tab +- resolve and save settings for the authenticated user only + +Representative payload: + +```json +{ + "small_note_threshold": 300, + "target_chunk_size": 200, + "min_chunk_size": 100, + "max_chunk_size": 400, + "overlap": 50, + "use_title": true, + "use_section_headings": true, + "use_tags": true, + "output_dimensionality": 1536, + "task_type_document": "RETRIEVAL_DOCUMENT", + "task_type_query": "RETRIEVAL_QUERY", + "split_strategy": "hierarchical", + "fallback_split_order": ["sections", "paragraphs", "sentences", "tokens_or_characters"], + "chunk_accumulation_rule": "Accumulate neighboring small paragraphs within the same section until target_chunk_size is reached or max_chunk_size would be exceeded.", + "small_chunk_merge_rule": "Merge undersized final chunks with adjacent chunks when possible without violating max_chunk_size.", + "chunk_template": "Section: {section_heading}\\nTags: {tag1}, {tag2}, {tag3}\\n\\n{chunk_content}" +} +``` + +### `rag-index` behavior + +`rag-index` must: + +1. load active indexing settings +2. fetch note title, content, and tags +3. derive content structure into sections, paragraphs, sentences, then token/character fallback +4. choose whole-note indexing when note size is below `small_note_threshold` +5. build final chunks using accumulation and merge rules +6. construct final chunk text from `Section`, `Tags`, and content according to enabled flags +7. send title separately via Gemini `title` +8. call Gemini embeddings with `taskType = RETRIEVAL_DOCUMENT` +9. persist vectors into `note_embeddings` + +### `rag-search` compatibility + +This feature does not alter search ranking logic, but the design must preserve: + +- `taskType = RETRIEVAL_QUERY` for query embeddings +- the same `output_dimensionality` for query and document vectors + +## Component Breakdown + +### Settings UI + +- A web settings surface for indexing parameters +- Placement is in the user's Google API settings tab +- Mobile can reuse the same shared settings contract later, but no mobile settings UI is added in this feature +- Editable controls for chunk sizes and content inclusion flags +- Read-only presentation for `output_dimensionality`, task types, split strategy, fallback order, chunk template, and chunking rules + +### Settings access layer + +- Fetches persisted indexing settings for UI display +- Validates and saves editable settings +- Exposes a single resolved settings object to indexing/search services + +### Chunking module + +- Implemented in shared `core` code with no dependency on `ui/web` or `ui/mobile` +- Parses note content into hierarchical structural units +- Derives sections only from real heading tags `h1` through `h6` +- Accumulates small sibling paragraphs toward `target_chunk_size` +- Splits oversized paragraphs into sentences, then token/character-based subparts +- Applies overlap only after final chunks are formed +- Merges undersized final chunks with neighbors when allowed + +### Embedding integration + +- Reuses Gemini embedding integration pattern already present in `rag-index` and `rag-search` +- Passes note title via Gemini `title` +- Sends chunk text body without duplicating title text inside the chunk content + +### Client integration layer + +- Web is the only UI consumer in scope for this feature +- Mobile remains a future consumer of shared `core` indexing logic and shared settings contracts +- Platform-specific UI code should only handle presentation, input controls, and transport to backend/settings APIs +- No platform-specific folder should carry its own chunking algorithm or settings-validation fork + +## Design Decisions + +### Hierarchical splitting before fixed-size fallback + +**Decision:** Split by sections, then paragraphs, then sentences, then token/character fallback. + +**Why:** This maximizes semantic coherence and makes retrieved chunks easier to interpret than naive fixed-window slicing. + +### Section detection only from `h1-h6` + +**Decision:** Section boundaries are derived only from actual heading tags `h1` through `h6`. + +**Why:** This keeps the behavior deterministic and avoids heuristic heading detection drift across platforms or note formats. + +### Overlap applies to final chunks only + +**Decision:** `overlap` means repeated boundary content between adjacent final chunks, not intermediate parser overlap. + +**Why:** This keeps chunk generation predictable and aligns the setting with user expectations in the UI. + +### Title is metadata, not chunk body + +**Decision:** Title is sent in the Gemini API `title` field and excluded from chunk text. + +**Why:** Title still informs embeddings without being redundantly repeated across every chunk body. + +### Config-driven indexing + +**Decision:** Indexing parameters move into runtime configuration visible in UI. + +**Why:** Operators can tune indexing behavior without code edits or redeploys, and the system becomes auditable. + +### Dedicated settings table + +**Decision:** Store per-user indexing settings in a dedicated table instead of extending `user_api_keys`. + +**Why:** API credentials and indexing behavior are separate concerns with different evolution paths, validation rules, and read/write patterns. + +### Shared `core` ownership + +**Decision:** Chunking, chunk formatting, and settings validation belong in `core`, not web/mobile feature folders. + +**Why:** The behavior is domain logic, not presentation logic, and needs to remain reusable across web, mobile, and server-side indexing paths. + +**Consequence:** Any platform-specific code should wrap or call the shared `core` logic. If runtime constraints require a thin adapter, that adapter must remain minimal and keep `core` as the canonical ruleset. + +### Read-only system parameters stay visible + +**Decision:** Task types and chunking rules remain non-editable but are shown in UI. + +**Why:** They are operationally important and should be transparent even when intentionally fixed. + +## Non-Functional Requirements + +- **Consistency:** the same active settings must be used by all indexing runs in a given environment. +- **Scope:** indexing settings are per-user rather than global/shared across all users. +- **Performance:** hierarchical parsing should not create materially slower indexing for ordinary note sizes compared with current fixed-window indexing. +- **Reliability:** invalid settings must be rejected before they can break indexing runs. +- **Compatibility:** `output_dimensionality` changes must not leave document/query vectors mismatched. +- **Security:** each authenticated user can read and edit only their own indexing settings. +- **Observability:** indexing logs should include effective settings identifiers or summaries without logging sensitive note content. + +## Open Design Items + +- Validation ranges and defaults for all editable numeric settings. +- Start defaults are: + - `small_note_threshold = 300` + - `target_chunk_size = 200` + - `min_chunk_size = 100` + - `max_chunk_size = 400` + - `overlap = 50` +- Validation ranges for these numeric settings still need to be finalized. diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md new file mode 100644 index 00000000000..28d50907694 --- /dev/null +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -0,0 +1,199 @@ +--- +phase: implementation +title: Improve RAG Chunking - Implementation Guide +description: Technical notes for configurable hierarchical chunking and indexing settings +--- + +# Implementation Guide + +## Development Setup + +- Use the existing Supabase Edge Function flow for indexing and search. +- Keep Gemini integration aligned with the current REST-based embedding calls. +- Reuse the project's current note model inputs: + - `title` + - `description` + - `tags` + +Recommended local verification paths: + +```bash +npx ai-devkit@latest lint --feature improve-rag-chunking +supabase functions serve rag-index --env-file .env.local +supabase functions serve rag-search --env-file .env.local +``` + +## Code Structure + +Expected implementation touchpoints: + +- `core/...` + - shared hierarchical chunking helpers + - shared chunk template builder + - shared indexing settings types and validation + - no dependency on `ui/web` or `ui/mobile` + - canonical source of truth for indexing behavior +- `supabase/functions/rag-index/index.ts` + - consume persisted settings + - replace fixed chunking logic with shared `core` helpers +- `supabase/functions/rag-search/index.ts` + - preserve query/document embedding compatibility, ideally via shared `core` settings helpers +- `supabase/migrations/...` + - add a dedicated per-user settings table for RAG indexing settings +- settings-related web UI + - indexing settings screen or section + - editable and read-only parameter presentation +- settings-related mobile UI + - out of scope for this feature + - reuse the same shared settings contract later when mobile support is added or expanded + +## Implementation Notes + +### Core feature 1: runtime settings resolution + +- Resolve one effective settings object per indexing run. +- Apply defaults server-side so missing values do not break indexing. +- Treat read-only fields as derived/system-defined values, not user-editable persisted state unless needed for UI convenience. +- Resolve settings per user. +- Surface these settings in the Google API settings tab. +- Load editable values from the dedicated per-user settings table and merge with system-defined read-only values. + +### Core feature 2: hierarchical chunk builder + +Suggested processing flow: + +1. Normalize note content to a structure suitable for section/paragraph detection. +2. Compute note size using the same unit chosen for settings semantics. +3. If size is below `small_note_threshold`, emit one final chunk. +4. Otherwise: + - split into sections using `h1-h6` tags only + - split each section into paragraphs + - accumulate neighboring small paragraphs toward `target_chunk_size` + - split oversized paragraphs deeper by sentences + - if still oversized, split by tokens or characters +5. After candidate chunks are created: + - merge undersized final chunks when possible + - apply final overlap across adjacent chunks + +Keep this logic in pure functions so it can be unit tested without Supabase or Gemini. +Keep these pure functions in `core`, not inside web/mobile folders, because they represent cross-platform domain behavior. +Treat any function that changes chunk boundaries, overlap, merge behavior, or chunk text composition as `core` domain logic by default. + +### Core feature 3: chunk text construction + +Chunk text should be built from optional parts in a stable order: + +```text +Section: {section_heading} +Tags: {tag1}, {tag2}, {tag3} + +{chunk_content} +``` + +Implementation rules: + +- omit title from chunk body text +- pass title separately to Gemini via the request `title` +- include `Section:` only when section headings are enabled and present +- include `Tags:` only when tags are enabled and non-empty +- omit optional lines entirely when inputs are absent or disabled + +### Core feature 4: Gemini embedding requests + +- Document chunk embeddings must use `taskType: "RETRIEVAL_DOCUMENT"`. +- Query embeddings must use `taskType: "RETRIEVAL_QUERY"`. +- Both paths must use the same `outputDimensionality`. +- `outputDimensionality` is read-only in the UI. +- If dimensions are incompatible with stored vectors or schema, fail fast with a clear operational error. + +### Patterns & Best Practices + +- Prefer pure deterministic helpers for parsing, splitting, accumulation, merge, and overlap. +- Keep domain logic in `core` and keep UI layers thin. +- Do not copy chunking logic into web/mobile modules for convenience; add or extend shared `core` APIs instead. +- Keep current UI work web-only, while preserving a clean shared contract for future mobile adoption. +- Keep I/O boundaries thin: + - settings fetch + - note fetch + - Gemini call + - DB writes +- Log settings summaries and chunk counts, not full note content. +- Preserve existing safe reindex semantics where new chunks are prepared before stale tail cleanup. + +## Integration Points + +- **Supabase notes data**: source of `title`, `description`, and `tags` +- **Gemini embeddings**: destination for final chunk text and query text +- **`note_embeddings` table**: target for chunk vectors +- **Settings UI**: source of editable indexing configuration + +Potential internal interfaces: + +```ts +function buildChunkPlan(note: { + title: string | null + description: string | null + tags: string[] | null +}, settings: RagIndexingSettings): FinalChunk[] + +function validateRagIndexingSettings(input: Partial): ValidatedRagIndexingSettings +``` + +Representative placement: + +```text +core/rag/ + indexingSettings.ts + chunking.ts + chunkTemplate.ts + types.ts +``` + +Recommended defaults in `core`: + +```ts +const DEFAULT_RAG_INDEX_SETTINGS = { + small_note_threshold: 300, + target_chunk_size: 200, + min_chunk_size: 100, + max_chunk_size: 400, + overlap: 50, + use_title: true, + use_section_headings: true, + use_tags: true, +} as const +``` + +Platform-specific code should look more like adapters: + +```text +supabase/functions/rag-index/ + index.ts # fetch note/settings, call core helpers, call Gemini, persist rows + +ui/web/... # render/edit settings, send updates, display read-only values +ui/mobile/... # no settings UI changes in this feature +``` + +## Error Handling + +- Reject invalid settings on save and on server-side load/validation fallback. +- Fail indexing clearly when: + - settings are inconsistent + - dimensions are incompatible + - section parsing returns unusable structure + - Gemini embedding count mismatches final chunk count +- Preserve existing index when a reindex attempt fails before successful replacement. + +## Performance Considerations + +- Avoid repeated full-text reparsing in the same indexing run. +- Keep hierarchical splitting linear or near-linear in note size for ordinary documents. +- Batch Gemini embedding calls for final chunks, not intermediate fragments. +- Bound worst-case chunk counts to prevent runaway splitting on malformed content. + +## Security Notes + +- Continue to keep Gemini API credentials server-side only. +- Restrict settings updates to authorized users/roles. +- Do not expose hidden system parameters as editable values in the UI. +- Avoid logging full note text, tags, or titles in production diagnostics unless explicitly sanitized. diff --git a/docs/ai/planning/feature-improve-rag-chunking.md b/docs/ai/planning/feature-improve-rag-chunking.md new file mode 100644 index 00000000000..e5aacd77c27 --- /dev/null +++ b/docs/ai/planning/feature-improve-rag-chunking.md @@ -0,0 +1,137 @@ +--- +phase: planning +title: Improve RAG Chunking - Planning +description: Task breakdown for configurable hierarchical chunking and indexing settings UI +--- + +# Project Planning & Task Breakdown + +## Milestones + +- [ ] Milestone 1: Persisted indexing settings model and UI contract defined +- [ ] Milestone 2: Shared `core` chunking/settings module implemented and adopted by indexing paths +- [ ] Milestone 3: Settings UI wired to runtime configuration and validated end-to-end + +## Task Breakdown + +### Phase 1: Settings foundation + +- [ ] **1.1** Finalize the persisted settings shape for indexing configuration + - use per-user settings scope + - store settings in a dedicated per-user table, separate from `user_api_keys` + - place the UI under the Google API settings tab + - define defaults for editable and read-only parameters + - allow any user to edit their own settings + +- [ ] **1.2** Add backend read/write access for indexing settings + - read resolved settings for the UI + - save editable settings with validation + - expose read-only system values alongside editable values + +- [ ] **1.3** Define validation rules + - numeric ranges for thresholds and chunk sizes + - invariants like `min_chunk_size <= target_chunk_size <= max_chunk_size` + - overlap constraints relative to chunk sizes + - compatibility checks for `output_dimensionality` + +### Phase 2: Chunking pipeline + +- [ ] **2.1** Create shared `core` module for indexing settings and hierarchical chunking + - keep application-owned chunking + - keep the module independent from `ui/web` and `ui/mobile` + - expose pure helpers reusable by server and clients + - treat this module as the canonical implementation, not as an optional helper + +- [ ] **2.2** Replace current fixed-window chunking in `supabase/functions/rag-index/index.ts` with the shared `core` module + - remove hard-coded chunking constants from the main indexing flow + - consume the shared chunk builder and template serializer + +- [ ] **2.3** Implement hierarchical segmentation + - detect sections from `h1-h6` only + - split sections into paragraphs + - split oversized paragraphs into sentences + - add token/character fallback for pathological long blocks + +- [ ] **2.4** Implement chunk assembly rules + - single-chunk indexing for small notes + - accumulation of neighboring small paragraphs up to target size + - merge of undersized final chunks when possible + - final-chunk overlap behavior + +- [ ] **2.5** Implement chunk text templating + - title passed separately via Gemini `title` + - optional `Section:` line + - optional `Tags:` line + - consistent chunk text serialization + +### Phase 3: UI and compatibility + +- [ ] **3.1** Build indexing settings UI consumers on top of the shared contract + - web only in this feature + - editable controls for chunk parameters and inclusion flags + - read-only section for `output_dimensionality`, task types, and system chunking rules + - save/reset feedback states + - explicit character-based labels for size fields + +- [ ] **3.2** Wire active settings into indexing and query compatibility paths + - `rag-index` consumes live settings + - query embedding path preserves matching `output_dimensionality` + - incompatible settings are blocked with actionable errors + - web UI consumes the shared settings shape in this phase + - mobile reuse is deferred, but the shared contract must remain mobile-compatible + - reject any duplicated per-platform chunking implementation during rollout/review + +- [ ] **3.3** Reindex and rollout strategy + - settings changes affect only future indexing and future manual reindex + - existing indexed notes remain unchanged until manually reindexed + - define user guidance for when manual reindex is needed + - add observability for effective settings during indexing runs + +## Dependencies + +- Existing `rag-index` and `rag-search` Edge Functions remain the integration points. +- Existing `note_embeddings` storage and vector search path must stay compatible with chosen dimensions. +- The settings UI depends on an agreed storage location and permission model. +- Any change to embedding dimensions may depend on database migration or controlled rollout sequencing. + +## Timeline & Estimates + +- **Phase 1: Settings foundation** — medium effort +- **Phase 2: Chunking pipeline** — high effort +- **Phase 3: UI and compatibility** — medium effort + +Suggested implementation order: + +1. settings model and validation +2. shared `core` chunking library and unit tests +3. `rag-index` integration +4. settings UI consumers +5. compatibility checks and reindex guidance + +## Risks & Mitigation + +- **Risk:** ambiguous section parsing from HTML-rich notes + - **Mitigation:** define deterministic heading extraction rules and fallback behavior early + +- **Risk:** invalid settings create unusable chunking behavior + - **Mitigation:** enforce server-side validation and safe defaults + +- **Risk:** `output_dimensionality` changes break vector compatibility + - **Mitigation:** gate incompatible changes and require reindex/migration workflow + +- **Risk:** hierarchical chunking increases implementation complexity + - **Mitigation:** isolate chunking into a testable pure `core` module before wiring to the Edge Function and UI consumers + +- **Risk:** settings changes create stale mixed-version indexes + - **Mitigation:** surface reindex requirements in UI and track effective settings during indexing + +## Resources Needed + +- Access to current RAG indexing/search implementation for integration updates +- Supabase local or staging environment for vector compatibility testing +- Representative note fixtures: + - very small notes + - multi-section notes + - notes with many short paragraphs + - notes with one oversized paragraph + - notes with and without tags diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md new file mode 100644 index 00000000000..d714e521458 --- /dev/null +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -0,0 +1,162 @@ +--- +phase: requirements +title: Configurable RAG Indexing Chunking +description: Requirements for hierarchical note chunking and explicit indexing settings in the UI +--- + +# Requirements & Problem Understanding + +## Problem Statement + +RAG note indexing currently uses fixed, mostly implicit chunking and embedding settings in code. This makes indexing quality harder to tune, hides important system behavior from the UI, and forces redeploys for changes that should be configuration-driven. + +- **Affected users:** users relying on AI note search quality, plus operators/developers configuring the indexing pipeline. +- **Current situation:** note chunking is primarily fixed-size and code-defined, title/content composition is not fully explicit in the UI, and key Gemini embedding settings are only visible in implementation. +- **Pain points:** + - small notes may be split more than necessary + - large notes are not consistently split on natural boundaries first + - chunk construction rules are not transparent in the UI + - indexing settings cannot be adjusted from the UI without redeploy + +## Goals & Objectives + +### Primary goals + +- Make indexing behavior explicit and visible in the UI. +- Move indexing tuning to configuration that can be changed without redeploy. +- Improve chunking quality with hierarchical splitting based on natural boundaries. +- Standardize chunk payload construction so every indexed chunk is formed consistently. +- Keep Gemini responsible only for embeddings, not chunk generation. +- Place reusable indexing logic in shared `core` code so it is independent of web and mobile UI layers. +- Establish `core` as the single source of truth for indexing rules so web and mobile cannot drift in behavior. + +### Secondary goals + +- Preserve small notes as a single chunk whenever possible. +- Allow operators to control whether title, section headings, and tags participate in indexing. +- Expose fixed Gemini task types in the UI as read-only system settings. + +### Non-goals (explicitly out of scope) + +- Changing search ranking, retrieval thresholds, top-k behavior, or other search tuning parameters. +- Using an LLM to generate chunks. +- Changing the search UX or search result presentation. +- Indexing attachments, OCR, or non-note content. +- Adding a mobile-app UI for editing indexing settings in this phase. + +## User Stories & Use Cases + +- **As a user configuring AI indexing**, I want to see all indexing parameters in the UI so that system behavior is understandable and auditable. +- **As a user configuring AI indexing**, I want these settings to live in my Google API settings area so that related AI configuration is managed in one place. +- **As a user configuring AI indexing**, I want to edit chunk sizing and overlap settings so that indexing quality can be tuned without redeploy. +- **As a user configuring AI indexing**, I want title, section headings, and tags to be explicit indexing inputs that can be enabled or disabled. +- **As a user searching small notes**, I want short notes to remain whole so that their context is preserved. +- **As a user searching large notes**, I want notes to be split on natural boundaries first so that retrieved chunks stay coherent. +- **As a system**, I want tiny neighboring paragraphs to accumulate into a target-sized chunk so that the index avoids fragmented low-value chunks. +- **As a system**, I want oversized paragraphs to split deeper by sentences and then by token/character fallback so that no final chunk exceeds the configured maximum. +- **As a system**, I want undersized final chunks to merge with neighbors when possible so that chunk quality remains consistent. + +### Key workflows + +- User opens indexing settings in the UI and sees all editable and read-only indexing parameters. +- User opens the Google API settings tab and sees all editable and read-only indexing parameters there. +- User changes editable indexing parameters and saves them without redeploying the app. +- A note is indexed using hierarchical chunking in this order: + 1. section / subsection boundaries + 2. paragraphs + 3. sentences + 4. token or character fallback +- The app creates final chunks, applies overlap between adjacent final chunks, and sends chunk text plus metadata to Gemini embeddings. + +### Edge cases + +- Note is smaller than `small_note_threshold` and should be indexed as a single chunk. +- Note has no section headings and must fall back directly to paragraph-based chunking. +- A paragraph is larger than `max_chunk_size` and must be split deeper. +- The last chunk is too small and should merge with a neighbor if size constraints allow. +- A note has no tags; chunk formatting must still stay valid. +- A setting change affects future indexing and may require reindexing of existing notes to take effect consistently. + +## Success Criteria + +- [ ] Indexing configuration is available in the UI without requiring redeploy. +- [ ] The UI shows all indexing parameters involved in chunk construction and Gemini embedding configuration. +- [ ] Chunking, chunk-template construction, and settings validation logic live in shared `core` code and do not depend on web-only or mobile-only modules. +- [ ] Web, mobile, and server-side indexing paths reuse the same `core` indexing rules instead of reimplementing them per platform. +- [ ] Indexing settings are exposed in the user's Google API settings tab. +- [ ] Indexing settings UI is added on the web site only for this feature. +- [ ] Editable UI settings include: + - `small_note_threshold` + - `target_chunk_size` + - `min_chunk_size` + - `max_chunk_size` + - `overlap` + - use title + - use section headings + - use tags +- [ ] Read-only UI settings include: + - document `taskType = RETRIEVAL_DOCUMENT` + - query `taskType = RETRIEVAL_QUERY` + - `output_dimensionality` + - `split_strategy` + - `fallback_split_order` + - chunk structure template + - chunk accumulation rule + - small chunk merge rule +- [ ] Notes below `small_note_threshold` are indexed as a single chunk unless prevented by system constraints. +- [ ] Larger notes are split by natural boundaries before using sentence-level and token/character fallback splitting. +- [ ] Small adjacent paragraphs accumulate toward `target_chunk_size` within a section before becoming final chunks. +- [ ] Oversized paragraphs are split deeper until all final chunks satisfy `max_chunk_size`. +- [ ] Undersized final chunks are merged with adjacent chunks when possible without violating configured limits. +- [ ] `overlap` is applied as repeated boundary content between adjacent final chunks, not as an intermediate split rule. +- [ ] Title is passed separately through the Gemini API `title` field and is not duplicated inside chunk text. +- [ ] Size-based settings are explicitly labeled in the UI as character-based values. +- [ ] Indexed chunk text follows one consistent template: + +```text +Section: {section_heading} +Tags: {tag1}, {tag2}, {tag3} + +{chunk_content} +``` + +- [ ] Chunk text can include section headings and tags conditionally, based on UI settings. +- [ ] If section headings or tags are disabled or absent, their corresponding lines are omitted entirely from the final chunk text. + +## Constraints & Assumptions + +### Technical constraints + +- Chunking is performed by the application/service layer, not by Gemini. +- Shared chunking and indexing-settings logic must be implemented in `core` so both web and mobile clients can reuse the same behavior. +- `core` is the authoritative source for chunking behavior; platform layers may adapt inputs/outputs but must not define competing chunking rules. +- This feature adds indexing-settings UI only on web; mobile remains a future consumer of the shared `core` logic and settings contract. +- Gemini is used only for embeddings. +- Chunk embeddings must use `RETRIEVAL_DOCUMENT`. +- Query embeddings must use `RETRIEVAL_QUERY`. +- Those task types are system parameters and must be shown in the UI as read-only values. +- Title must be sent separately in the Gemini API `title` field and must not be embedded as part of chunk body text. +- Chunk body text is composed from note content plus optional section heading and optional tags. +- Existing note data already provides `title`, `description`, and `tags`; section headings must be derived from note content structure. +- Existing vector storage currently assumes a fixed embedding dimension in the database schema, so changing `output_dimensionality` must remain compatible with storage and query paths. +- Indexing settings are user-scoped settings shown in the user's Google API settings tab. + +### Assumptions + +- Search parameter tuning remains out of scope for this feature, except for keeping query embedding compatibility explicit in the UI. +- Configuration changes should apply without redeploy, but they affect only future indexing operations and future manual reindex operations. +- Existing indexed notes are not automatically reindexed or marked stale by this feature. +- The same `output_dimensionality` must be used consistently for document and query embeddings. +- `output_dimensionality` is displayed as read-only in the UI. +- Size-based parameters are defined in characters in v1 and must be labeled that way in the UI. +- Any omitted optional chunk parts (`Section`, `Tags`) should disappear entirely rather than render as empty labels. + +## Questions & Open Items + +- Start defaults are fixed as: + - `small_note_threshold = 300` + - `target_chunk_size = 200` + - `min_chunk_size = 100` + - `max_chunk_size = 400` + - `overlap = 50` +- Exact validation ranges for `small_note_threshold`, `target_chunk_size`, `min_chunk_size`, `max_chunk_size`, and `overlap` still need to be finalized. diff --git a/docs/ai/testing/feature-improve-rag-chunking.md b/docs/ai/testing/feature-improve-rag-chunking.md new file mode 100644 index 00000000000..266ee9aac6e --- /dev/null +++ b/docs/ai/testing/feature-improve-rag-chunking.md @@ -0,0 +1,135 @@ +--- +phase: testing +title: Improve RAG Chunking - Testing Strategy +description: Test plan for configurable hierarchical chunking and indexing settings +--- + +# Testing Strategy + +## Test Coverage Goals + +- Unit test coverage target: 100% of new chunking and settings-validation logic +- Shared `core` logic is the primary unit-test target so behavior stays identical across web/mobile consumers +- Integration coverage: critical `rag-index` and settings save/load paths +- Manual/E2E coverage: settings UI behavior and representative indexing outcomes +- Validation against requirements: + - small notes stay whole when appropriate + - large notes split on natural boundaries first + - tiny chunks merge where possible + - overlap behavior is applied between final chunks + - title/section/tags participate according to active settings + +## Unit Tests + +### Chunking settings validation + +- [ ] Reject negative or zero values where not allowed +- [ ] Reject `min_chunk_size > target_chunk_size` +- [ ] Reject `target_chunk_size > max_chunk_size` +- [ ] Reject overlap values that exceed allowed bounds +- [ ] Accept valid settings and fill defaults for omitted values + +### Hierarchical segmentation + +- [ ] Small note below `small_note_threshold` returns a single final chunk +- [ ] Multi-section note with `h1-h6` headings prefers section boundaries before paragraph fallback +- [ ] Notes without headings fall back directly to paragraph splitting +- [ ] Non-heading styled text does not create synthetic sections +- [ ] Oversized paragraph splits by sentences before token/character fallback +- [ ] Extremely long sentence falls back to token/character splitting + +### Chunk accumulation and merge rules + +- [ ] Adjacent small paragraphs accumulate toward `target_chunk_size` +- [ ] Accumulation stops before violating `max_chunk_size` +- [ ] Undersized final trailing chunk merges with previous neighbor when allowed +- [ ] Undersized chunk remains standalone when merging would exceed `max_chunk_size` +- [ ] Overlap duplicates only boundary content between adjacent final chunks + +### Chunk text templating + +- [ ] Title is excluded from chunk body text +- [ ] Title is passed separately to embedding payload construction +- [ ] `Section:` line appears only when enabled and data exists +- [ ] `Tags:` line appears only when enabled and tags exist +- [ ] Chunk text preserves stable ordering and spacing + +### Shared module boundaries + +- [ ] Shared chunking/settings helpers can be imported without web-only dependencies +- [ ] Shared chunking/settings helpers can be imported without mobile-only dependencies +- [ ] Platform UIs consume shared `core` contracts instead of reimplementing logic locally +- [ ] Review catches any new per-platform chunking fork as an architectural regression + +## Integration Tests + +- [ ] Settings read endpoint/function returns editable and read-only parameters together +- [ ] Settings update rejects invalid combinations with clear validation errors +- [ ] `rag-index` uses persisted settings instead of hard-coded chunking constants +- [ ] `rag-index` indexes a small note as exactly one chunk +- [ ] `rag-index` indexes a multi-section note into coherent multi-chunk output +- [ ] `rag-index` preserves safe reindex semantics when embedding generation fails +- [ ] `rag-search` query embedding uses `RETRIEVAL_QUERY` +- [ ] Query and document embedding dimensions remain aligned after settings changes + +## End-to-End Tests + +- [ ] Open the Google API settings tab and verify all editable parameters are present +- [ ] Verify read-only parameters, including `output_dimensionality`, are visible but not editable +- [ ] Save valid settings and confirm they persist after reload +- [ ] Attempt to save invalid settings and confirm inline validation blocks the change +- [ ] Reindex a small note and verify one chunk is produced +- [ ] Reindex a long structured note and verify chunk count changes according to settings +- [ ] Toggle title inclusion off and verify indexed chunk bodies do not gain title text +- [ ] Toggle section heading and tag inclusion on/off and verify chunk content changes accordingly + +## Test Data + +Create or reuse fixtures covering: + +- a tiny note with no headings +- a note with several short neighboring paragraphs +- a note with multiple `h1-h6` headings and subsections +- a note with bold or visually prominent text that is not an actual heading tag +- a note with one extremely large paragraph +- a note with tags and a note without tags +- a note whose final candidate chunk would otherwise be too small + +## Test Reporting & Coverage + +- Recommended commands: + +```bash +npm run test -- --coverage +npx ai-devkit@latest lint --feature improve-rag-chunking +``` + +- Record coverage for: + - chunking helpers + - settings validation + - settings UI components + - `rag-index` integration paths + +## Manual Testing + +- [ ] Review settings UI labels for clarity and units +- [ ] Verify all size labels explicitly say they are measured in characters +- [ ] Verify read-only values visually communicate that they are system-defined +- [ ] Check behavior with realistic notes imported from the app, not only synthetic fixtures +- [ ] Confirm no redeploy is required for settings changes to take effect +- [ ] Confirm users understand that settings changes affect only future indexing/manual reindex and do not retroactively update existing indexes +- [ ] Confirm no mobile settings UI was introduced as part of this feature + +## Performance Testing + +- [ ] Benchmark indexing time for: + - short note + - medium note with headings + - large note with many paragraphs + - pathological large single-paragraph note +- [ ] Compare chunk counts and indexing latency against the previous fixed-window implementation +- [ ] Verify no excessive explosion in chunk count under worst-case fallback splitting + +## Outstanding Gaps + +- [ ] Finalize exact validation ranges From 8f833f783e5e476a7f424ec71ab2fae1b960916a Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 14:49:40 +0100 Subject: [PATCH 02/18] docs up --- docs/ai/design/feature-improve-rag-chunking.md | 10 ++++++++-- docs/ai/implementation/feature-improve-rag-chunking.md | 5 +++++ docs/ai/planning/feature-improve-rag-chunking.md | 2 +- docs/ai/requirements/feature-improve-rag-chunking.md | 4 +++- docs/ai/testing/feature-improve-rag-chunking.md | 5 +++-- 5 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index 505e0a44d3b..9256852f658 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -196,6 +196,8 @@ This feature does not alter search ranking logic, but the design must preserve: - Fetches persisted indexing settings for UI display - Validates and saves editable settings - Exposes a single resolved settings object to indexing/search services +- Enforces numeric bounds of `50..5000` for editable numeric settings +- Enforces ordering invariants such as `min_chunk_size <= target_chunk_size <= max_chunk_size` ### Chunking module @@ -284,11 +286,15 @@ This feature does not alter search ranking logic, but the design must preserve: ## Open Design Items -- Validation ranges and defaults for all editable numeric settings. - Start defaults are: - `small_note_threshold = 300` - `target_chunk_size = 200` - `min_chunk_size = 100` - `max_chunk_size = 400` - `overlap = 50` -- Validation ranges for these numeric settings still need to be finalized. +- Validation ranges for editable numeric settings are: + - `small_note_threshold`: `50..5000` + - `target_chunk_size`: `50..5000` + - `min_chunk_size`: `50..5000` + - `max_chunk_size`: `50..5000` + - `overlap`: `50..5000` diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index 28d50907694..09329c7312b 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -139,6 +139,11 @@ function buildChunkPlan(note: { function validateRagIndexingSettings(input: Partial): ValidatedRagIndexingSettings ``` +Validation rules to enforce in both UI and server paths: + +- `small_note_threshold`, `target_chunk_size`, `min_chunk_size`, `max_chunk_size`, `overlap` must each be within `50..5000` +- `min_chunk_size <= target_chunk_size <= max_chunk_size` + Representative placement: ```text diff --git a/docs/ai/planning/feature-improve-rag-chunking.md b/docs/ai/planning/feature-improve-rag-chunking.md index e5aacd77c27..156d7762900 100644 --- a/docs/ai/planning/feature-improve-rag-chunking.md +++ b/docs/ai/planning/feature-improve-rag-chunking.md @@ -29,7 +29,7 @@ description: Task breakdown for configurable hierarchical chunking and indexing - expose read-only system values alongside editable values - [ ] **1.3** Define validation rules - - numeric ranges for thresholds and chunk sizes + - numeric ranges for thresholds and chunk sizes: `50..5000` - invariants like `min_chunk_size <= target_chunk_size <= max_chunk_size` - overlap constraints relative to chunk sizes - compatibility checks for `output_dimensionality` diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md index d714e521458..1978e3fb60c 100644 --- a/docs/ai/requirements/feature-improve-rag-chunking.md +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -150,6 +150,8 @@ Tags: {tag1}, {tag2}, {tag3} - `output_dimensionality` is displayed as read-only in the UI. - Size-based parameters are defined in characters in v1 and must be labeled that way in the UI. - Any omitted optional chunk parts (`Section`, `Tags`) should disappear entirely rather than render as empty labels. +- Editable numeric indexing parameters use an allowed range of `50..5000`. +- Server-side validation must also enforce logical ordering: `min_chunk_size <= target_chunk_size <= max_chunk_size`. ## Questions & Open Items @@ -159,4 +161,4 @@ Tags: {tag1}, {tag2}, {tag3} - `min_chunk_size = 100` - `max_chunk_size = 400` - `overlap = 50` -- Exact validation ranges for `small_note_threshold`, `target_chunk_size`, `min_chunk_size`, `max_chunk_size`, and `overlap` still need to be finalized. +- No remaining open items in requirements. diff --git a/docs/ai/testing/feature-improve-rag-chunking.md b/docs/ai/testing/feature-improve-rag-chunking.md index 266ee9aac6e..60b6df630bf 100644 --- a/docs/ai/testing/feature-improve-rag-chunking.md +++ b/docs/ai/testing/feature-improve-rag-chunking.md @@ -23,7 +23,8 @@ description: Test plan for configurable hierarchical chunking and indexing setti ### Chunking settings validation -- [ ] Reject negative or zero values where not allowed +- [ ] Reject values below `50` +- [ ] Reject values above `5000` - [ ] Reject `min_chunk_size > target_chunk_size` - [ ] Reject `target_chunk_size > max_chunk_size` - [ ] Reject overlap values that exceed allowed bounds @@ -132,4 +133,4 @@ npx ai-devkit@latest lint --feature improve-rag-chunking ## Outstanding Gaps -- [ ] Finalize exact validation ranges +- [ ] Decide whether overlap should also be constrained to be less than or equal to `max_chunk_size` From d161aa604eaa3cff0b7a1f238757a219e9d894fd Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 15:25:53 +0100 Subject: [PATCH 03/18] Review rag indexing changes --- core/rag/chunkTemplate.ts | 52 +++ core/rag/chunking.ts | 437 ++++++++++++++++++ core/rag/indexingSettings.ts | 149 ++++++ core/services/apiKeysSettings.ts | 3 + core/services/ragIndexSettings.ts | 80 ++++ core/tests/unit/core-rag-chunking.test.ts | 75 +++ .../unit/core-rag-indexingSettings.test.ts | 52 +++ supabase/functions/api-keys-status/index.ts | 13 + supabase/functions/api-keys-upsert/index.ts | 52 ++- supabase/functions/rag-index/index.ts | 111 ++--- supabase/functions/rag-search/index.ts | 41 +- ...0317000001_add_user_rag_index_settings.sql | 37 ++ .../features/search/AiSearchToggle.tsx | 2 +- .../settings/ApiKeysSettingsDialog.tsx | 4 +- .../settings/ApiKeysSettingsPanel.tsx | 110 +++-- .../settings/RagIndexingSettingsPanel.tsx | 302 ++++++++++++ .../features/settings/SettingsPage.tsx | 4 +- 17 files changed, 1386 insertions(+), 138 deletions(-) create mode 100644 core/rag/chunkTemplate.ts create mode 100644 core/rag/chunking.ts create mode 100644 core/rag/indexingSettings.ts create mode 100644 core/services/ragIndexSettings.ts create mode 100644 core/tests/unit/core-rag-chunking.test.ts create mode 100644 core/tests/unit/core-rag-indexingSettings.test.ts create mode 100644 supabase/migrations/20260317000001_add_user_rag_index_settings.sql create mode 100644 ui/web/components/features/settings/RagIndexingSettingsPanel.tsx diff --git a/core/rag/chunkTemplate.ts b/core/rag/chunkTemplate.ts new file mode 100644 index 00000000000..49a433fbfe5 --- /dev/null +++ b/core/rag/chunkTemplate.ts @@ -0,0 +1,52 @@ +import type { RagIndexingEditableSettings } from "@core/rag/indexingSettings" + +export type RagChunkTemplateInput = { + sectionHeading: string | null + tags: string[] + chunkContent: string + settings: Pick +} + +const normalizeInlineText = (value: string) => value.replace(/\s+/g, " ").trim() + +export function buildRagChunkText({ + sectionHeading, + tags, + chunkContent, + settings, +}: RagChunkTemplateInput): string { + const lines: string[] = [] + const normalizedContent = chunkContent.trim() + + if (!normalizedContent) return "" + + if (settings.use_section_headings && sectionHeading) { + const normalizedHeading = normalizeInlineText(sectionHeading) + if (normalizedHeading) { + lines.push(`Section: ${normalizedHeading}`) + } + } + + if (settings.use_tags && tags.length > 0) { + const normalizedTags = tags.map(normalizeInlineText).filter(Boolean) + if (normalizedTags.length > 0) { + lines.push(`Tags: ${normalizedTags.join(", ")}`) + } + } + + if (lines.length > 0) { + lines.push("") + } + + lines.push(normalizedContent) + return lines.join("\n") +} + +export function buildRagEmbeddingTitle( + title: string | null | undefined, + settings: Pick +): string | null { + if (!settings.use_title) return null + const normalizedTitle = normalizeInlineText(title ?? "") + return normalizedTitle || null +} diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts new file mode 100644 index 00000000000..ae526000cf3 --- /dev/null +++ b/core/rag/chunking.ts @@ -0,0 +1,437 @@ +import type { RagIndexingEditableSettings } from "@core/rag/indexingSettings" +import { buildRagChunkText, buildRagEmbeddingTitle } from "@core/rag/chunkTemplate" + +type RawBlock = { + sectionHeading: string | null + text: string +} + +type IndexedBlock = RawBlock & { + charOffset: number +} + +type TextSegment = { + sectionHeading: string | null + text: string + charOffset: number +} + +type CandidateChunk = { + sectionHeading: string | null + text: string + charOffset: number +} + +export type BuildRagChunksInput = { + title: string | null | undefined + html: string | null | undefined + tags: string[] | null | undefined + settings: RagIndexingEditableSettings +} + +export type RagIndexChunk = { + charOffset: number + content: string + title: string | null + sectionHeading: string | null +} + +const HEADING_TAG_PATTERN = /<\/?h[1-6]\b[^>]*>/i +const BLOCK_BREAK_PATTERN = /<\/?(?:p|div|li|blockquote|pre|ul|ol|section|article|main)\b[^>]*>/gi + +function normalizeWhitespace(value: string): string { + return value + .replace(/ /gi, " ") + .replace(/\u00a0/g, " ") + .replace(/\s+/g, " ") + .trim() +} + +function stripTags(value: string): string { + return normalizeWhitespace(value.replace(/<[^>]*>/g, " ")) +} + +function splitPlainTextParagraphs(value: string): string[] { + return value + .split(/\n{2,}/) + .map((paragraph) => normalizeWhitespace(paragraph)) + .filter(Boolean) +} + +function buildBlocksFromPlainText(value: string): RawBlock[] { + return splitPlainTextParagraphs(value).map((text) => ({ sectionHeading: null, text })) +} + +function hasDomParser(): boolean { + return typeof DOMParser !== "undefined" +} + +function extractElementText(node: Node): string { + if (node.nodeType === Node.TEXT_NODE) { + return node.textContent ?? "" + } + + if (node.nodeType !== Node.ELEMENT_NODE) { + return "" + } + + const element = node as Element + const tagName = element.tagName.toLowerCase() + + if (tagName === "br") return "\n" + + let result = "" + for (const child of Array.from(element.childNodes)) { + result += extractElementText(child) + } + return result +} + +function collectBlocksFromDom(rootHtml: string): RawBlock[] { + const parser = new DOMParser() + const doc = parser.parseFromString(rootHtml, "text/html") + const blocks: RawBlock[] = [] + let currentHeading: string | null = null + + const walk = (node: Node) => { + for (const child of Array.from(node.childNodes)) { + if (child.nodeType === Node.TEXT_NODE) { + const text = normalizeWhitespace(child.textContent ?? "") + if (text) blocks.push({ sectionHeading: currentHeading, text }) + continue + } + + if (child.nodeType !== Node.ELEMENT_NODE) continue + + const element = child as Element + const tagName = element.tagName.toLowerCase() + + if (/^h[1-6]$/.test(tagName)) { + currentHeading = normalizeWhitespace(element.textContent ?? "") + continue + } + + if (tagName === "ul" || tagName === "ol" || tagName === "section" || tagName === "article" || tagName === "main") { + walk(element) + continue + } + + if (tagName === "div") { + const hasNestedBlocks = Array.from(element.children).some((childElement) => { + const childTag = childElement.tagName.toLowerCase() + return childTag === "div" || childTag === "p" || childTag === "li" || childTag === "blockquote" || childTag === "pre" || /^h[1-6]$/.test(childTag) + }) + + if (hasNestedBlocks) { + walk(element) + continue + } + } + + if (tagName === "p" || tagName === "div" || tagName === "li" || tagName === "blockquote" || tagName === "pre") { + const text = normalizeWhitespace(extractElementText(element)) + if (text) blocks.push({ sectionHeading: currentHeading, text }) + continue + } + + walk(element) + } + } + + walk(doc.body) + return blocks.filter((block) => block.text.length > 0) +} + +function collectBlocksWithRegex(html: string): RawBlock[] { + const normalizedHtml = html + .replace(//gi, "\n") + .replace(BLOCK_BREAK_PATTERN, "\n\n") + + const rawBlocks: RawBlock[] = [] + let currentHeading: string | null = null + + const headingRegex = /]*>([\s\S]*?)<\/h\1>/gi + let lastIndex = 0 + let match: RegExpExecArray | null + + while ((match = headingRegex.exec(normalizedHtml)) !== null) { + const beforeHeading = normalizedHtml.slice(lastIndex, match.index) + rawBlocks.push(...buildBlocksFromPlainText(stripTags(beforeHeading)).map((block) => ({ + sectionHeading: currentHeading, + text: block.text, + }))) + currentHeading = stripTags(match[2] ?? "") + lastIndex = headingRegex.lastIndex + } + + const remaining = normalizedHtml.slice(lastIndex) + rawBlocks.push(...buildBlocksFromPlainText(stripTags(remaining)).map((block) => ({ + sectionHeading: currentHeading, + text: block.text, + }))) + + return rawBlocks.filter((block) => block.text.length > 0) +} + +function extractBlocksFromHtml(html: string): IndexedBlock[] { + const source = html?.trim() ?? "" + if (!source) return [] + + const rawBlocks = hasDomParser() ? collectBlocksFromDom(source) : collectBlocksWithRegex(source) + const fallbackBlocks = rawBlocks.length > 0 ? rawBlocks : buildBlocksFromPlainText(stripTags(source)) + + let charOffset = 0 + return fallbackBlocks.map((block) => { + const indexedBlock: IndexedBlock = { + ...block, + charOffset, + } + charOffset += block.text.length + 2 + return indexedBlock + }) +} + +function splitIntoSentenceSegments(block: IndexedBlock): TextSegment[] { + const matches = Array.from(block.text.matchAll(/[^.!?]+(?:[.!?]+|$)/g)) + if (matches.length <= 1) { + return [{ sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset }] + } + + return matches + .map((match) => ({ + sectionHeading: block.sectionHeading, + text: normalizeWhitespace(match[0] ?? ""), + charOffset: block.charOffset + (match.index ?? 0), + })) + .filter((segment) => segment.text.length > 0) +} + +function splitByCharacterFallback(segment: TextSegment, maxChunkSize: number): TextSegment[] { + const normalized = normalizeWhitespace(segment.text) + if (normalized.length <= maxChunkSize) { + return [{ ...segment, text: normalized }] + } + + const parts: TextSegment[] = [] + let offset = 0 + while (offset < normalized.length) { + let end = Math.min(offset + maxChunkSize, normalized.length) + if (end < normalized.length) { + const whitespaceIndex = normalized.lastIndexOf(" ", end) + if (whitespaceIndex > offset) { + end = whitespaceIndex + } + } + + const piece = normalized.slice(offset, end).trim() + if (piece) { + parts.push({ + sectionHeading: segment.sectionHeading, + text: piece, + charOffset: segment.charOffset + offset, + }) + } + + offset = end + while (offset < normalized.length && normalized[offset] === " ") { + offset += 1 + } + } + + return parts +} + +function splitOversizedBlock(block: IndexedBlock, maxChunkSize: number): TextSegment[] { + if (block.text.length <= maxChunkSize) { + return [{ sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset }] + } + + const sentenceSegments = splitIntoSentenceSegments(block) + const expanded: TextSegment[] = [] + for (const sentence of sentenceSegments) { + if (sentence.text.length <= maxChunkSize) { + expanded.push(sentence) + continue + } + expanded.push(...splitByCharacterFallback(sentence, maxChunkSize)) + } + + return expanded +} + +function joinChunkParts(parts: string[]): string { + return parts.filter(Boolean).join("\n\n").trim() +} + +function accumulateSegments( + segments: TextSegment[], + settings: Pick +): CandidateChunk[] { + const candidates: CandidateChunk[] = [] + let current: CandidateChunk | null = null + + for (const segment of segments) { + if (!current) { + current = { sectionHeading: segment.sectionHeading, text: segment.text, charOffset: segment.charOffset } + continue + } + + const sameSection = current.sectionHeading === segment.sectionHeading + const combinedText = joinChunkParts([current.text, segment.text]) + + if ( + sameSection && + ( + combinedText.length <= settings.target_chunk_size || + (current.text.length < settings.min_chunk_size && combinedText.length <= settings.max_chunk_size) + ) + ) { + current = { + sectionHeading: current.sectionHeading, + text: combinedText, + charOffset: current.charOffset, + } + continue + } + + candidates.push(current) + current = { sectionHeading: segment.sectionHeading, text: segment.text, charOffset: segment.charOffset } + } + + if (current) { + candidates.push(current) + } + + return candidates +} + +function mergeSmallChunks( + chunks: CandidateChunk[], + settings: Pick +): CandidateChunk[] { + const merged: CandidateChunk[] = [] + + for (let index = 0; index < chunks.length; index += 1) { + const chunk = chunks[index] + if (!chunk) continue + + if (chunk.text.length >= settings.min_chunk_size) { + merged.push(chunk) + continue + } + + const previous = merged[merged.length - 1] + if ( + previous && + previous.sectionHeading === chunk.sectionHeading && + joinChunkParts([previous.text, chunk.text]).length <= settings.max_chunk_size + ) { + merged[merged.length - 1] = { + sectionHeading: previous.sectionHeading, + text: joinChunkParts([previous.text, chunk.text]), + charOffset: previous.charOffset, + } + continue + } + + const next = chunks[index + 1] + if ( + next && + next.sectionHeading === chunk.sectionHeading && + joinChunkParts([chunk.text, next.text]).length <= settings.max_chunk_size + ) { + chunks[index + 1] = { + sectionHeading: next.sectionHeading, + text: joinChunkParts([chunk.text, next.text]), + charOffset: chunk.charOffset, + } + continue + } + + merged.push(chunk) + } + + return merged +} + +function buildOverlapPrefix(source: string, overlap: number): string { + if (overlap <= 0 || source.length === 0) return "" + const start = Math.max(0, source.length - overlap) + return source.slice(start).trim() +} + +function applyFinalOverlap(chunks: CandidateChunk[], overlap: number): CandidateChunk[] { + if (overlap <= 0) return chunks + + return chunks.map((chunk, index) => { + if (index === 0) return chunk + const previous = chunks[index - 1] + const overlapPrefix = buildOverlapPrefix(previous?.text ?? "", overlap) + if (!overlapPrefix) return chunk + + return { + ...chunk, + text: joinChunkParts([overlapPrefix, chunk.text]), + } + }) +} + +function buildWholeNoteChunk( + blocks: IndexedBlock[], + html: string, + settings: RagIndexingEditableSettings +): CandidateChunk[] { + const contentFromBlocks = joinChunkParts(blocks.map((block) => block.text)) + const fallbackContent = stripTags(html) + const text = contentFromBlocks || fallbackContent + if (!text) return [] + + const firstSection = blocks[0]?.sectionHeading ?? null + const hasSingleSection = blocks.every((block) => block.sectionHeading === firstSection) + + return [ + { + sectionHeading: hasSingleSection ? firstSection : null, + text, + charOffset: blocks[0]?.charOffset ?? 0, + }, + ] +} + +export function buildRagIndexChunks({ + title, + html, + tags, + settings, +}: BuildRagChunksInput): RagIndexChunk[] { + const normalizedTags = Array.isArray(tags) ? tags.filter((tag): tag is string => typeof tag === "string") : [] + const blocks = extractBlocksFromHtml(html ?? "") + const noteBodyLength = joinChunkParts(blocks.map((block) => block.text)).length + const baseChunks = + noteBodyLength > 0 && noteBodyLength <= settings.small_note_threshold + ? buildWholeNoteChunk(blocks, html ?? "", settings) + : mergeSmallChunks( + accumulateSegments( + blocks.flatMap((block) => splitOversizedBlock(block, settings.max_chunk_size)), + settings + ), + settings + ) + + const finalChunks = applyFinalOverlap(baseChunks, settings.overlap) + const embeddingTitle = buildRagEmbeddingTitle(title, settings) + + return finalChunks + .map((chunk) => ({ + charOffset: chunk.charOffset, + content: buildRagChunkText({ + sectionHeading: chunk.sectionHeading, + tags: normalizedTags, + chunkContent: chunk.text, + settings, + }), + title: embeddingTitle, + sectionHeading: chunk.sectionHeading, + })) + .filter((chunk) => chunk.content.length > 0) +} diff --git a/core/rag/indexingSettings.ts b/core/rag/indexingSettings.ts new file mode 100644 index 00000000000..4bf81600f6b --- /dev/null +++ b/core/rag/indexingSettings.ts @@ -0,0 +1,149 @@ +export const RAG_INDEX_NUMERIC_MIN = 50 +export const RAG_INDEX_NUMERIC_MAX = 5000 + +export const RAG_INDEX_EDITABLE_DEFAULTS = { + small_note_threshold: 300, + target_chunk_size: 200, + min_chunk_size: 100, + max_chunk_size: 400, + overlap: 50, + use_title: true, + use_section_headings: true, + use_tags: true, +} as const + +export const RAG_INDEX_READONLY_SETTINGS = { + output_dimensionality: 1536, + task_type_document: "RETRIEVAL_DOCUMENT" as const, + task_type_query: "RETRIEVAL_QUERY" as const, + split_strategy: "hierarchical" as const, + fallback_split_order: ["sections", "paragraphs", "sentences", "tokens_or_characters"] as const, + chunk_accumulation_rule: + "Accumulate neighboring small paragraphs within the same section until target_chunk_size is reached or max_chunk_size would be exceeded.", + small_chunk_merge_rule: + "Merge undersized final chunks with adjacent chunks when possible without violating max_chunk_size.", + chunk_template: "Section: {section_heading}\nTags: {tag1}, {tag2}, {tag3}\n\n{chunk_content}", +} as const + +export type RagIndexingEditableSettings = { + small_note_threshold: number + target_chunk_size: number + min_chunk_size: number + max_chunk_size: number + overlap: number + use_title: boolean + use_section_headings: boolean + use_tags: boolean +} + +export type RagIndexingSettings = RagIndexingEditableSettings & typeof RAG_INDEX_READONLY_SETTINGS + +export const RAG_INDEX_EDITABLE_NUMERIC_KEYS = [ + "small_note_threshold", + "target_chunk_size", + "min_chunk_size", + "max_chunk_size", + "overlap", +] as const + +export const RAG_INDEX_EDITABLE_BOOLEAN_KEYS = [ + "use_title", + "use_section_headings", + "use_tags", +] as const + +type RagIndexNumericKey = (typeof RAG_INDEX_EDITABLE_NUMERIC_KEYS)[number] +type RagIndexBooleanKey = (typeof RAG_INDEX_EDITABLE_BOOLEAN_KEYS)[number] + +export function resolveRagIndexingEditableSettings( + input?: Partial | null +): RagIndexingEditableSettings { + return { + ...RAG_INDEX_EDITABLE_DEFAULTS, + ...(input ?? {}), + } +} + +export function resolveRagIndexingSettings( + input?: Partial | null +): RagIndexingSettings { + return { + ...resolveRagIndexingEditableSettings(input), + ...RAG_INDEX_READONLY_SETTINGS, + } +} + +export function validateRagIndexingEditableSettings( + input: Partial | null | undefined +): string[] { + const errors: string[] = [] + const resolved = resolveRagIndexingEditableSettings(input) + + for (const key of RAG_INDEX_EDITABLE_NUMERIC_KEYS) { + const value = resolved[key] + if (!Number.isInteger(value)) { + errors.push(`${key} must be an integer`) + continue + } + if (value < RAG_INDEX_NUMERIC_MIN || value > RAG_INDEX_NUMERIC_MAX) { + errors.push(`${key} must be between ${RAG_INDEX_NUMERIC_MIN} and ${RAG_INDEX_NUMERIC_MAX}`) + } + } + + for (const key of RAG_INDEX_EDITABLE_BOOLEAN_KEYS) { + const value = resolved[key] + if (typeof value !== "boolean") { + errors.push(`${key} must be a boolean`) + } + } + + if (resolved.min_chunk_size > resolved.target_chunk_size) { + errors.push("min_chunk_size must be less than or equal to target_chunk_size") + } + if (resolved.target_chunk_size > resolved.max_chunk_size) { + errors.push("target_chunk_size must be less than or equal to max_chunk_size") + } + + return errors +} + +export function assertValidRagIndexingEditableSettings( + input: Partial | null | undefined +): RagIndexingEditableSettings { + const resolved = resolveRagIndexingEditableSettings(input) + const errors = validateRagIndexingEditableSettings(resolved) + if (errors.length > 0) { + throw new Error(errors.join(". ")) + } + return resolved +} + +export function pickRagIndexingEditableSettings( + input: Partial | null | undefined +): RagIndexingEditableSettings { + return assertValidRagIndexingEditableSettings(input) +} + +export function coerceRagIndexingEditableSettings( + input: Record +): Partial { + const settings: Partial = {} + + for (const key of RAG_INDEX_EDITABLE_NUMERIC_KEYS) { + const value = input[key] + if (value === undefined) continue + settings[key] = value as RagIndexingEditableSettings[RagIndexNumericKey] + } + + for (const key of RAG_INDEX_EDITABLE_BOOLEAN_KEYS) { + const value = input[key] + if (value === undefined) continue + settings[key] = value as RagIndexingEditableSettings[RagIndexBooleanKey] + } + + return settings +} + +export function getRagReadonlySettings() { + return RAG_INDEX_READONLY_SETTINGS +} diff --git a/core/services/apiKeysSettings.ts b/core/services/apiKeysSettings.ts index 945e5d31dea..77b9903f127 100644 --- a/core/services/apiKeysSettings.ts +++ b/core/services/apiKeysSettings.ts @@ -1,7 +1,10 @@ import type { SupabaseClient } from '@supabase/supabase-js' +import type { RagIndexingSettings } from '@core/rag/indexingSettings' + export type ApiKeysStatus = { gemini: { configured: boolean } + ragIndexing?: RagIndexingSettings } const isApiKeysStatus = (data: unknown): data is ApiKeysStatus => { diff --git a/core/services/ragIndexSettings.ts b/core/services/ragIndexSettings.ts new file mode 100644 index 00000000000..402db507ca4 --- /dev/null +++ b/core/services/ragIndexSettings.ts @@ -0,0 +1,80 @@ +import type { SupabaseClient } from "@supabase/supabase-js" + +import type { ApiKeysStatus } from "@core/services/apiKeysSettings" +import type { RagIndexingEditableSettings, RagIndexingSettings } from "@core/rag/indexingSettings" + +const readErrorMessage = async (error: unknown, fallback: string) => { + if (typeof error === "object" && error && "context" in error) { + const context = (error as { context?: Response }).context + if (context && typeof context.json === "function") { + try { + const payload = await context.json() + if (payload && typeof payload === "object") { + const message = + typeof payload.message === "string" + ? payload.message + : typeof payload.error === "string" + ? payload.error + : null + if (message) return message + } + } catch { + // Ignore parse failure and continue fallback chain. + } + } + } + + if (error instanceof Error && error.message) return error.message + return fallback +} + +const isRagIndexingSettings = (data: unknown): data is RagIndexingSettings => { + if (!data || typeof data !== "object") return false + return ( + typeof (data as { small_note_threshold?: unknown }).small_note_threshold === "number" && + typeof (data as { target_chunk_size?: unknown }).target_chunk_size === "number" && + typeof (data as { min_chunk_size?: unknown }).min_chunk_size === "number" && + typeof (data as { max_chunk_size?: unknown }).max_chunk_size === "number" && + typeof (data as { overlap?: unknown }).overlap === "number" && + typeof (data as { use_title?: unknown }).use_title === "boolean" && + typeof (data as { use_section_headings?: unknown }).use_section_headings === "boolean" && + typeof (data as { use_tags?: unknown }).use_tags === "boolean" && + typeof (data as { output_dimensionality?: unknown }).output_dimensionality === "number" + ) +} + +const readRagIndexingSettings = (data: unknown): RagIndexingSettings | null => { + if (!data || typeof data !== "object") return null + const ragIndexing = (data as ApiKeysStatus).ragIndexing + return isRagIndexingSettings(ragIndexing) ? ragIndexing : null +} + +export class RagIndexSettingsService { + constructor(private supabase: SupabaseClient) {} + + async getStatus(): Promise { + const { data, error } = await this.supabase.functions.invoke("api-keys-status", { body: {} }) + if (error) { + throw new Error(await readErrorMessage(error, "Failed to load RAG indexing settings")) + } + const ragIndexing = readRagIndexingSettings(data) + if (!ragIndexing) { + throw new Error("Unexpected response while loading RAG indexing settings") + } + return ragIndexing + } + + async upsert(input: RagIndexingEditableSettings): Promise { + const { data, error } = await this.supabase.functions.invoke("api-keys-upsert", { + body: input, + }) + if (error) { + throw new Error(await readErrorMessage(error, "Failed to save RAG indexing settings")) + } + const ragIndexing = readRagIndexingSettings(data) + if (!ragIndexing) { + throw new Error("Unexpected response while saving RAG indexing settings") + } + return ragIndexing + } +} diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts new file mode 100644 index 00000000000..893ee8ee57c --- /dev/null +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -0,0 +1,75 @@ +import { buildRagIndexChunks } from "@core/rag/chunking" +import { buildRagChunkText } from "@core/rag/chunkTemplate" +import { RAG_INDEX_EDITABLE_DEFAULTS } from "@core/rag/indexingSettings" + +describe("core/rag/chunking", () => { + it("keeps a small note as a single chunk", () => { + const chunks = buildRagIndexChunks({ + title: "Weekly plan", + html: "

Short body text.

", + tags: ["work"], + settings: RAG_INDEX_EDITABLE_DEFAULTS, + }) + + expect(chunks).toHaveLength(1) + expect(chunks[0]?.content).toContain("Tags: work") + expect(chunks[0]?.content).toContain("Short body text.") + expect(chunks[0]?.title).toBe("Weekly plan") + }) + + it("splits large paragraphs and adds overlap to later chunks", () => { + const repeated = "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five." + const html = `

Section A

${repeated} ${repeated} ${repeated}

` + + const chunks = buildRagIndexChunks({ + title: "Long note", + html, + tags: ["alpha", "beta"], + settings: { + ...RAG_INDEX_EDITABLE_DEFAULTS, + small_note_threshold: 50, + target_chunk_size: 80, + min_chunk_size: 50, + max_chunk_size: 90, + overlap: 50, + }, + }) + + expect(chunks.length).toBeGreaterThan(1) + expect(chunks[0]?.content).toContain("Section: Section A") + expect(chunks[1]?.content).toContain("Sentence") + expect(chunks[1]?.charOffset).toBeGreaterThan(chunks[0]?.charOffset ?? 0) + }) + + it("omits empty optional lines from the chunk template", () => { + const text = buildRagChunkText({ + sectionHeading: null, + tags: [], + chunkContent: "Body only", + settings: { + use_section_headings: true, + use_tags: true, + }, + }) + + expect(text).toBe("Body only") + }) + + it("does not create sections from non-heading formatting alone", () => { + const chunks = buildRagIndexChunks({ + title: "Formatting note", + html: "

Looks like a heading

Paragraph text.

", + tags: [], + settings: { + ...RAG_INDEX_EDITABLE_DEFAULTS, + small_note_threshold: 50, + target_chunk_size: 30, + min_chunk_size: 20, + max_chunk_size: 30, + overlap: 10, + }, + }) + + expect(chunks[0]?.content).not.toContain("Section:") + }) +}) diff --git a/core/tests/unit/core-rag-indexingSettings.test.ts b/core/tests/unit/core-rag-indexingSettings.test.ts new file mode 100644 index 00000000000..afb77566138 --- /dev/null +++ b/core/tests/unit/core-rag-indexingSettings.test.ts @@ -0,0 +1,52 @@ +import { + assertValidRagIndexingEditableSettings, + resolveRagIndexingSettings, + RAG_INDEX_EDITABLE_DEFAULTS, +} from "@core/rag/indexingSettings" + +describe("core/rag/indexingSettings", () => { + it("returns defaults plus read-only settings when no editable overrides exist", () => { + const settings = resolveRagIndexingSettings() + + expect(settings.small_note_threshold).toBe(RAG_INDEX_EDITABLE_DEFAULTS.small_note_threshold) + expect(settings.target_chunk_size).toBe(RAG_INDEX_EDITABLE_DEFAULTS.target_chunk_size) + expect(settings.output_dimensionality).toBe(1536) + expect(settings.task_type_document).toBe("RETRIEVAL_DOCUMENT") + expect(settings.task_type_query).toBe("RETRIEVAL_QUERY") + }) + + it("rejects numeric values outside the allowed range", () => { + expect(() => assertValidRagIndexingEditableSettings({ small_note_threshold: 49 })).toThrow( + "small_note_threshold must be between 50 and 5000" + ) + expect(() => assertValidRagIndexingEditableSettings({ overlap: 5001 })).toThrow( + "overlap must be between 50 and 5000" + ) + }) + + it("rejects invalid ordering for chunk sizes", () => { + expect(() => + assertValidRagIndexingEditableSettings({ + min_chunk_size: 300, + target_chunk_size: 200, + max_chunk_size: 400, + }) + ).toThrow("min_chunk_size must be less than or equal to target_chunk_size") + + expect(() => + assertValidRagIndexingEditableSettings({ + min_chunk_size: 100, + target_chunk_size: 500, + max_chunk_size: 400, + }) + ).toThrow("target_chunk_size must be less than or equal to max_chunk_size") + }) + + it("rejects non-boolean flags", () => { + expect(() => + assertValidRagIndexingEditableSettings({ + use_title: "yes" as unknown as boolean, + }) + ).toThrow("use_title must be a boolean") + }) +}) diff --git a/supabase/functions/api-keys-status/index.ts b/supabase/functions/api-keys-status/index.ts index 32483343916..04c7e76819d 100644 --- a/supabase/functions/api-keys-status/index.ts +++ b/supabase/functions/api-keys-status/index.ts @@ -3,6 +3,8 @@ import { serve } from "https://deno.land/std@0.177.0/http/server.ts" import { createClient } from "@supabase/supabase-js" +import { resolveRagIndexingSettings } from "../../../core/rag/indexingSettings.ts" + declare const Deno: { env: { get(key: string): string | undefined } } const corsHeaders = { @@ -41,13 +43,24 @@ serve(async (req: Request) => { .eq("user_id", userData.user.id) .maybeSingle() + const { data: ragIndexingData, error: ragIndexingError } = await supabaseAdmin + .from("user_rag_index_settings") + .select("small_note_threshold, target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") + .eq("user_id", userData.user.id) + .maybeSingle() + if (error) { console.error("[api-keys-status]", error) return jsonResponse({ error: "Internal error" }, 500) } + if (ragIndexingError) { + console.error("[api-keys-status] Failed to load RAG indexing settings", ragIndexingError) + return jsonResponse({ error: "Internal error" }, 500) + } return jsonResponse({ gemini: { configured: Boolean(data?.gemini_api_key_encrypted) }, + ragIndexing: resolveRagIndexingSettings(ragIndexingData ?? null), }) } catch (err) { console.error("[api-keys-status]", err) diff --git a/supabase/functions/api-keys-upsert/index.ts b/supabase/functions/api-keys-upsert/index.ts index a34b261a7f6..86cacf0346e 100644 --- a/supabase/functions/api-keys-upsert/index.ts +++ b/supabase/functions/api-keys-upsert/index.ts @@ -3,6 +3,12 @@ import { serve } from "https://deno.land/std@0.177.0/http/server.ts" import { createClient } from "@supabase/supabase-js" +import { + assertValidRagIndexingEditableSettings, + coerceRagIndexingEditableSettings, + resolveRagIndexingSettings, +} from "../../../core/rag/indexingSettings.ts" + declare const Deno: { env: { get(key: string): string | undefined } } const corsHeaders = { @@ -96,13 +102,20 @@ serve(async (req: Request) => { return jsonResponse({ error: "geminiApiKey must be a string" }, 400) } + const hasGeminiApiKeyField = "geminiApiKey" in payload const rawGeminiKey = typeof payload.geminiApiKey === "string" ? payload.geminiApiKey.trim() : "" + const coercedRagIndexingSettings = coerceRagIndexingEditableSettings(payload) + const hasRagIndexingFields = Object.keys(coercedRagIndexingSettings).length > 0 const MAX_GEMINI_KEY_LENGTH = 256 if (rawGeminiKey.length > MAX_GEMINI_KEY_LENGTH) { return jsonResponse({ error: `Gemini API key must not exceed ${MAX_GEMINI_KEY_LENGTH} characters` }, 400) } + if (!hasGeminiApiKeyField && !hasRagIndexingFields) { + return jsonResponse({ error: "No Google API settings changes provided" }, 400) + } + // Fetch existing row to support "keep existing key" on empty input const { data: existing, error: fetchError } = await supabaseAdmin .from("user_api_keys") @@ -114,21 +127,46 @@ serve(async (req: Request) => { let encryptedKey = existing?.gemini_api_key_encrypted ?? null - if (rawGeminiKey) { + if (hasGeminiApiKeyField && rawGeminiKey) { encryptedKey = await encryptValue(rawGeminiKey, encryptionSecret) } - if (!encryptedKey) { + if (hasGeminiApiKeyField && !encryptedKey && !hasRagIndexingFields) { return jsonResponse({ error: "Gemini API key is required for initial setup" }, 400) } - const { error: upsertError } = await supabaseAdmin - .from("user_api_keys") - .upsert({ user_id: userId, gemini_api_key_encrypted: encryptedKey, updated_at: new Date().toISOString() }, { onConflict: "user_id" }) + if (hasGeminiApiKeyField && encryptedKey) { + const { error: upsertError } = await supabaseAdmin + .from("user_api_keys") + .upsert({ user_id: userId, gemini_api_key_encrypted: encryptedKey, updated_at: new Date().toISOString() }, { onConflict: "user_id" }) - if (upsertError) throw upsertError + if (upsertError) throw upsertError + } + + let resolvedRagIndexingSettings + if (hasRagIndexingFields) { + const editableSettings = assertValidRagIndexingEditableSettings(coercedRagIndexingSettings) + const { error: ragUpsertError } = await supabaseAdmin + .from("user_rag_index_settings") + .upsert({ user_id: userId, ...editableSettings, updated_at: new Date().toISOString() }, { onConflict: "user_id" }) + + if (ragUpsertError) throw ragUpsertError + resolvedRagIndexingSettings = resolveRagIndexingSettings(editableSettings) + } else { + const { data: ragIndexingData, error: ragIndexingError } = await supabaseAdmin + .from("user_rag_index_settings") + .select("small_note_threshold, target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") + .eq("user_id", userId) + .maybeSingle() + + if (ragIndexingError) throw ragIndexingError + resolvedRagIndexingSettings = resolveRagIndexingSettings(ragIndexingData ?? null) + } - return jsonResponse({ gemini: { configured: true } }) + return jsonResponse({ + gemini: { configured: Boolean(hasGeminiApiKeyField ? encryptedKey : existing?.gemini_api_key_encrypted) }, + ragIndexing: resolvedRagIndexingSettings, + }) } catch (err) { console.error("[api-keys-upsert]", err) return jsonResponse({ error: "Internal error" }, 500) diff --git a/supabase/functions/rag-index/index.ts b/supabase/functions/rag-index/index.ts index 6c91f8e36af..9efc94422d4 100644 --- a/supabase/functions/rag-index/index.ts +++ b/supabase/functions/rag-index/index.ts @@ -5,6 +5,9 @@ import { serve } from "https://deno.land/std@0.177.0/http/server.ts" import { createClient } from "https://esm.sh/@supabase/supabase-js@2.45.4" +import { buildRagIndexChunks } from "../../../core/rag/chunking.ts" +import { getRagReadonlySettings, resolveRagIndexingEditableSettings } from "../../../core/rag/indexingSettings.ts" + declare const Deno: { env: { get(key: string): string | undefined } } // --------------------------------------------------------------------------- @@ -12,9 +15,8 @@ declare const Deno: { env: { get(key: string): string | undefined } } // --------------------------------------------------------------------------- const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta" const EMBEDDING_MODEL = "models/gemini-embedding-001" -const OUTPUT_DIMENSIONS = 1536 -const CHUNK_SIZE = 1500 -const CHUNK_OVERLAP = 200 +const READONLY_RAG_SETTINGS = getRagReadonlySettings() +const OUTPUT_DIMENSIONS = READONLY_RAG_SETTINGS.output_dimensionality const MAX_CHUNKS_PER_NOTE = 128 const GEMINI_TIMEOUT_MS = 10000 const GEMINI_MAX_RETRIES = 3 @@ -35,35 +37,6 @@ const jsonResponse = (body: unknown, status = 200) => headers: { ...corsHeaders, "Content-Type": "application/json" }, }) -// --------------------------------------------------------------------------- -// Chunking -// --------------------------------------------------------------------------- -function stripHtml(html: string): string { - return html.replace(/<[^>]*>/g, " ").replace(/\s+/g, " ").trim() -} - -function chunkText(text: string): Array<{ content: string; charOffset: number }> { - if (!text.trim()) return [] - const chunks: Array<{ content: string; charOffset: number }> = [] - let offset = 0 - while (offset < text.length) { - const content = text.slice(offset, offset + CHUNK_SIZE) - chunks.push({ content, charOffset: offset }) - if (content.length < CHUNK_SIZE) break - const next = offset + CHUNK_SIZE - CHUNK_OVERLAP - // Stop if the next window would start beyond the end of the text — - // that would produce a chunk containing only already-covered overlap. - if (next >= text.length) break - offset = next - } - return chunks -} - -function prepareNoteText(title: string, html: string): string { - const body = stripHtml(html) - return title.trim() ? `${title.trim()} ${body}` : body -} - // --------------------------------------------------------------------------- // AES-GCM decryption (mirrors api-keys-upsert / wordpress-bridge pattern) // --------------------------------------------------------------------------- @@ -111,11 +84,15 @@ const decryptValue = async (encrypted: string, secret: string): Promise // --------------------------------------------------------------------------- // Embeddings via Gemini REST API // --------------------------------------------------------------------------- -async function embedTexts(texts: string[], apiKey: string): Promise { - const requests = texts.map((text) => ({ +async function embedTexts( + texts: Array<{ text: string; title: string | null }>, + apiKey: string +): Promise { + const requests = texts.map(({ text, title }) => ({ model: EMBEDDING_MODEL, + ...(title ? { title } : {}), content: { parts: [{ text }] }, - taskType: "RETRIEVAL_DOCUMENT", + taskType: READONLY_RAG_SETTINGS.task_type_document, outputDimensionality: OUTPUT_DIMENSIONS, })) @@ -154,13 +131,13 @@ async function embedTexts(texts: string[], apiKey: string): Promise if (!Array.isArray(data?.embeddings)) { throw new Error("Gemini batchEmbedContents response missing embeddings array") } - if (data.embeddings.length !== texts.length) { + if (data.embeddings.length !== requests.length) { throw new Error( - `Gemini embeddings count mismatch: input=${texts.length} returned=${data.embeddings.length} requestId=${requestId}` + `Gemini embeddings count mismatch: input=${requests.length} returned=${data.embeddings.length} requestId=${requestId}` ) } - return data.embeddings.map((e: { values: number[] }) => e.values) + return data.embeddings.map((embedding: { values: number[] }) => embedding.values) } catch (error) { const isAbort = error instanceof DOMException && error.name === "AbortError" const isNetwork = error instanceof TypeError @@ -209,7 +186,6 @@ serve(async (req: Request) => { if (userError || !userData?.user) return jsonResponse({ error: "Unauthorized" }, 401) const userId = userData.user.id - // Parse body let payload: { noteId?: string; action?: string } = {} try { payload = await req.json() } catch { /* empty body */ } @@ -221,12 +197,10 @@ serve(async (req: Request) => { return jsonResponse({ error: "action must be 'index', 'reindex', or 'delete'" }, 400) } - // Verify note ownership const uuidRegex = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i if (!uuidRegex.test(noteId)) return jsonResponse({ error: "Invalid noteId" }, 400) try { - // Delete does not need the Gemini API key — handle it before the key lookup. if (action === "delete") { const { error } = await supabaseAdmin .from("note_embeddings") @@ -237,7 +211,6 @@ serve(async (req: Request) => { return jsonResponse({ deleted: true }) } - // action === "index" | "reindex" — fetch and decrypt the user's Gemini API key. const { data: apiKeyRow, error: apiKeyError } = await supabaseAdmin .from("user_api_keys") .select("gemini_api_key_encrypted") @@ -249,7 +222,7 @@ serve(async (req: Request) => { return jsonResponse({ error: "Internal error" }, 500) } if (!apiKeyRow?.gemini_api_key_encrypted) { - return jsonResponse({ error: "Gemini API key not configured. Add it in Settings → API Keys." }, 400) + return jsonResponse({ error: "Gemini API key not configured. Add it in Settings → Google API." }, 400) } let geminiApiKey: string @@ -260,9 +233,17 @@ serve(async (req: Request) => { return jsonResponse({ error: "Internal error" }, 500) } + const { data: settingsRow, error: settingsError } = await supabaseAdmin + .from("user_rag_index_settings") + .select("small_note_threshold, target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") + .eq("user_id", userId) + .maybeSingle() + + if (settingsError) throw settingsError + const { data: note, error: noteError } = await supabaseAdmin .from("notes") - .select("title, description") + .select("title, description, tags") .eq("id", noteId) .eq("user_id", userId) .maybeSingle() @@ -270,8 +251,14 @@ serve(async (req: Request) => { if (noteError) throw noteError if (!note) return jsonResponse({ error: "Note not found" }, 404) - const text = prepareNoteText(note.title ?? "", note.description ?? "") - const chunks = chunkText(text) + const settings = resolveRagIndexingEditableSettings(settingsRow ?? null) + const chunks = buildRagIndexChunks({ + title: note.title ?? "", + html: note.description ?? "", + tags: Array.isArray(note.tags) ? note.tags : [], + settings, + }) + const droppedChunkCount = Math.max(0, chunks.length - MAX_CHUNKS_PER_NOTE) const chunksForIndexing = droppedChunkCount > 0 ? chunks.slice(0, MAX_CHUNKS_PER_NOTE) : chunks @@ -291,20 +278,25 @@ serve(async (req: Request) => { return jsonResponse({ chunkCount: 0 }) } - // Embed - const vectors = await embedTexts(chunksForIndexing.map((c) => c.content), geminiApiKey) + const vectors = await embedTexts( + chunksForIndexing.map((chunk) => ({ + text: chunk.content, + title: chunk.title, + })), + geminiApiKey + ) + if (vectors.length !== chunksForIndexing.length) { throw new Error(`Gemini returned ${vectors.length} vectors for ${chunksForIndexing.length} chunks`) } - // Upsert first so failed re-index never deletes a previously searchable note. - const rows = chunksForIndexing.map((chunk, i) => ({ + const rows = chunksForIndexing.map((chunk, index) => ({ note_id: noteId, user_id: userId, - chunk_index: i, + chunk_index: index, char_offset: chunk.charOffset, content: chunk.content, - embedding: vectors[i], + embedding: vectors[index], })) const { error: upsertError } = await supabaseAdmin @@ -312,7 +304,6 @@ serve(async (req: Request) => { .upsert(rows, { onConflict: "note_id,chunk_index" }) if (upsertError) throw upsertError - // Remove obsolete tail chunks when note becomes shorter after edits. const { error: cleanupError } = await supabaseAdmin .from("note_embeddings") .delete() @@ -321,6 +312,20 @@ serve(async (req: Request) => { .gte("chunk_index", chunksForIndexing.length) if (cleanupError) throw cleanupError + console.info("[rag-index] Indexed note with settings", { + noteId, + userId, + chunkCount: chunksForIndexing.length, + small_note_threshold: settings.small_note_threshold, + target_chunk_size: settings.target_chunk_size, + min_chunk_size: settings.min_chunk_size, + max_chunk_size: settings.max_chunk_size, + overlap: settings.overlap, + use_title: settings.use_title, + use_section_headings: settings.use_section_headings, + use_tags: settings.use_tags, + }) + return jsonResponse({ chunkCount: chunksForIndexing.length, droppedChunks: droppedChunkCount > 0 ? droppedChunkCount : undefined, diff --git a/supabase/functions/rag-search/index.ts b/supabase/functions/rag-search/index.ts index 7dcbe9af7d8..7231d134e9e 100644 --- a/supabase/functions/rag-search/index.ts +++ b/supabase/functions/rag-search/index.ts @@ -5,6 +5,8 @@ import { serve } from "https://deno.land/std@0.177.0/http/server.ts" import { createClient } from "@supabase/supabase-js" +import { getRagReadonlySettings } from "../../../core/rag/indexingSettings.ts" + declare const Deno: { env: { get(key: string): string | undefined } } // --------------------------------------------------------------------------- @@ -12,7 +14,8 @@ declare const Deno: { env: { get(key: string): string | undefined } } // --------------------------------------------------------------------------- const GEMINI_API_BASE = "https://generativelanguage.googleapis.com/v1beta" const EMBEDDING_MODEL = "models/gemini-embedding-001" -const OUTPUT_DIMENSIONS = 1536 +const READONLY_RAG_SETTINGS = getRagReadonlySettings() +const OUTPUT_DIMENSIONS = READONLY_RAG_SETTINGS.output_dimensionality const GEMINI_TIMEOUT_MS = 10000 const GEMINI_MAX_RETRIES = 3 const GEMINI_RETRY_BASE_MS = 400 @@ -38,7 +41,6 @@ const isLegacyMatchNotesSignatureError = (error: unknown): boolean => { const code = typeof (error as { code?: unknown }).code === "string" ? (error as { code: string }).code.toUpperCase() : "" - // PostgREST "function not found in schema cache" for RPC signature mismatches. if (code === "PGRST202") return true const message = typeof (error as { message?: unknown }).message === "string" @@ -100,7 +102,7 @@ async function embedQuery(query: string, apiKey: string): Promise { body: JSON.stringify({ model: EMBEDDING_MODEL, content: { parts: [{ text: query }] }, - taskType: "RETRIEVAL_QUERY", + taskType: READONLY_RAG_SETTINGS.task_type_query, outputDimensionality: OUTPUT_DIMENSIONS, }), signal: controller.signal, @@ -161,7 +163,6 @@ serve(async (req: Request) => { return jsonResponse({ error: "Function not configured" }, 500) } - // Auth — extract userId from JWT const authHeader = req.headers.get("Authorization")?.trim() ?? "" const bearerMatch = authHeader.match(/^Bearer\s+(.+)$/i) const token = (bearerMatch ? bearerMatch[1] : authHeader).trim() @@ -172,7 +173,6 @@ serve(async (req: Request) => { if (userError || !userData?.user) return jsonResponse({ error: "Unauthorized" }, 401) const userId = userData.user.id - // Parse and validate request body let payload: { query?: unknown; topK?: unknown; threshold?: unknown; filterTag?: unknown } = {} try { payload = await req.json() } catch { /* empty body */ } @@ -190,7 +190,6 @@ serve(async (req: Request) => { const tagFilter: string | null = typeof filterTag === "string" ? filterTag : null try { - // Fetch and decrypt the user's Gemini API key const { data: apiKeyRow, error: apiKeyError } = await supabaseAdmin .from("user_api_keys") .select("gemini_api_key_encrypted") @@ -202,7 +201,7 @@ serve(async (req: Request) => { return jsonResponse({ error: "Internal error" }, 500) } if (!apiKeyRow?.gemini_api_key_encrypted) { - return jsonResponse({ error: "Gemini API key not configured. Add it in Settings → API Keys." }, 400) + return jsonResponse({ error: "Gemini API key not configured. Add it in Settings → Google API." }, 400) } let geminiApiKey: string @@ -213,10 +212,8 @@ serve(async (req: Request) => { return jsonResponse({ error: "Internal error" }, 500) } - // Embed the search query (RETRIEVAL_QUERY task type) const queryEmbedding = await embedQuery(query.trim(), geminiApiKey) - // Call match_notes RPC using a user-scoped client so auth.uid() works in the SQL function const supabaseUser = createClient(supabaseUrl, anonKey, { global: { headers: { Authorization: `Bearer ${token}` } }, }) @@ -234,8 +231,6 @@ serve(async (req: Request) => { let { data: chunks, error: rpcError } = await supabaseUser.rpc("match_notes", rpcPayload) - // Transitional compatibility: some environments may still have legacy match_notes(query_embedding, match_count). - // Remove this fallback after all environments are migrated to 20260304000001_add_filter_tag_to_match_notes.sql. if (rpcError && tagFilter && isLegacyMatchNotesSignatureError(rpcError)) { const primaryRpcError = rpcError try { @@ -271,21 +266,19 @@ serve(async (req: Request) => { return jsonResponse({ chunks: [], availableChunkCount: 0 }) } - // Filter by similarity threshold (post-RPC) const filteredChunks = (chunks as Array<{ note_id: string chunk_index: number char_offset: number content: string similarity: number - }>).filter((c) => c.similarity >= threshold) + }>).filter((chunk) => chunk.similarity >= threshold) if (filteredChunks.length === 0) { return jsonResponse({ chunks: [], availableChunkCount: 0 }) } - // Enrich with note title and tags (single query) - const noteIds = [...new Set(filteredChunks.map((c) => c.note_id))] + const noteIds = [...new Set(filteredChunks.map((chunk) => chunk.note_id))] const { data: notes, error: notesError } = await supabaseAdmin .from("notes") .select("id, title, tags") @@ -297,26 +290,24 @@ serve(async (req: Request) => { return jsonResponse({ error: "Internal error" }, 500) } - const noteMap = new Map((notes ?? []).map((n: { id: string; title: string | null; tags: string[] }) => [n.id, n])) + const noteMap = new Map((notes ?? []).map((note: { id: string; title: string | null; tags: string[] }) => [note.id, note])) - const result = filteredChunks.map((c) => { - const note = noteMap.get(c.note_id) + const result = filteredChunks.map((chunk) => { + const note = noteMap.get(chunk.note_id) return { - noteId: c.note_id, + noteId: chunk.note_id, noteTitle: note?.title ?? "", noteTags: note?.tags ?? [], - chunkIndex: c.chunk_index, - charOffset: c.char_offset, - content: c.content, - similarity: c.similarity, + chunkIndex: chunk.chunk_index, + charOffset: chunk.char_offset, + content: chunk.content, + similarity: chunk.similarity, } }) const finalResult = tagFilter ? result.filter((item) => item.noteTags.includes(tagFilter)) : result return jsonResponse({ chunks: finalResult, - // Preserve pre-tag-filter count so the client can keep paginating when - // topK results contain non-matching tags that are removed after enrichment. availableChunkCount: result.length, }) } catch (err) { diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql new file mode 100644 index 00000000000..b4f046f4756 --- /dev/null +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -0,0 +1,37 @@ +CREATE TABLE public.user_rag_index_settings ( + user_id uuid PRIMARY KEY REFERENCES auth.users(id) ON DELETE CASCADE, + small_note_threshold integer NOT NULL DEFAULT 300, + target_chunk_size integer NOT NULL DEFAULT 200, + min_chunk_size integer NOT NULL DEFAULT 100, + max_chunk_size integer NOT NULL DEFAULT 400, + overlap integer NOT NULL DEFAULT 50, + use_title boolean NOT NULL DEFAULT true, + use_section_headings boolean NOT NULL DEFAULT true, + use_tags boolean NOT NULL DEFAULT true, + updated_at timestamp with time zone NOT NULL DEFAULT now(), + CONSTRAINT user_rag_index_settings_small_note_threshold_range CHECK (small_note_threshold BETWEEN 50 AND 5000), + CONSTRAINT user_rag_index_settings_target_chunk_size_range CHECK (target_chunk_size BETWEEN 50 AND 5000), + CONSTRAINT user_rag_index_settings_min_chunk_size_range CHECK (min_chunk_size BETWEEN 50 AND 5000), + CONSTRAINT user_rag_index_settings_max_chunk_size_range CHECK (max_chunk_size BETWEEN 50 AND 5000), + CONSTRAINT user_rag_index_settings_overlap_range CHECK (overlap BETWEEN 50 AND 5000), + CONSTRAINT user_rag_index_settings_ordering CHECK (min_chunk_size <= target_chunk_size AND target_chunk_size <= max_chunk_size) +); + +ALTER TABLE public.user_rag_index_settings ENABLE ROW LEVEL SECURITY; + +CREATE POLICY "Users can view own rag index settings" + ON public.user_rag_index_settings FOR SELECT + USING (auth.uid() = user_id); + +CREATE POLICY "Users can insert own rag index settings" + ON public.user_rag_index_settings FOR INSERT + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY "Users can update own rag index settings" + ON public.user_rag_index_settings FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); + +CREATE POLICY "Users can delete own rag index settings" + ON public.user_rag_index_settings FOR DELETE + USING (auth.uid() = user_id); diff --git a/ui/web/components/features/search/AiSearchToggle.tsx b/ui/web/components/features/search/AiSearchToggle.tsx index 24f6dffa018..0e085da4605 100644 --- a/ui/web/components/features/search/AiSearchToggle.tsx +++ b/ui/web/components/features/search/AiSearchToggle.tsx @@ -131,7 +131,7 @@ export function AiSearchToggle({ {!hasApiKey && ( - Configure Gemini API key in Settings {'>'} API Keys + Configure Gemini API key in Settings {'>'} Google API )} diff --git a/ui/web/components/features/settings/ApiKeysSettingsDialog.tsx b/ui/web/components/features/settings/ApiKeysSettingsDialog.tsx index e8a0d4ce842..e26b7f1581c 100644 --- a/ui/web/components/features/settings/ApiKeysSettingsDialog.tsx +++ b/ui/web/components/features/settings/ApiKeysSettingsDialog.tsx @@ -13,9 +13,9 @@ export function ApiKeysSettingsDialog({ open, onOpenChange }: ApiKeysSettingsDia - API Keys + Google API - API keys are encrypted before storage and never exposed in plain text. + Gemini credentials are encrypted before storage and never exposed in plain text. onOpenChange(false)} showCloseButton /> diff --git a/ui/web/components/features/settings/ApiKeysSettingsPanel.tsx b/ui/web/components/features/settings/ApiKeysSettingsPanel.tsx index d910169c78b..1928b5f25d5 100644 --- a/ui/web/components/features/settings/ApiKeysSettingsPanel.tsx +++ b/ui/web/components/features/settings/ApiKeysSettingsPanel.tsx @@ -4,8 +4,10 @@ import * as React from "react" import { AlertCircle, CheckCircle2 } from "lucide-react" import { Button } from "@/components/ui/button" +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" +import { RagIndexingSettingsPanel } from "@/components/features/settings/RagIndexingSettingsPanel" import { useSupabase } from "@ui/web/providers/SupabaseProvider" import { ApiKeysSettingsService } from "@core/services/apiKeysSettings" @@ -77,54 +79,66 @@ export function ApiKeysSettingsPanel({ return (
-
- - setGeminiApiKey(event.target.value)} - placeholder={configured ? "Leave empty to keep current key" : "AIzaSy..."} - disabled={loading || saving} - autoComplete="off" - /> - {configured ? ( -

- A key is stored. Enter a new one only to replace it. -

- ) : null} -
- - {configured ? ( -
- Gemini API key is configured. -
- ) : null} - - {errorMessage ? ( -
- - {errorMessage} -
- ) : null} - - {successMessage ? ( -
- - {successMessage} -
- ) : null} - -
- {showCloseButton ? ( - - ) : null} - -
+ + + Gemini API key + + Store the Gemini API key used for note indexing and AI search. + + + +
+ + setGeminiApiKey(event.target.value)} + placeholder={configured ? "Leave empty to keep current key" : "AIzaSy..."} + disabled={loading || saving} + autoComplete="off" + /> + {configured ? ( +

+ A key is stored. Enter a new one only to replace it. +

+ ) : null} +
+ + {configured ? ( +
+ Gemini API key is configured. +
+ ) : null} + + {errorMessage ? ( +
+ + {errorMessage} +
+ ) : null} + + {successMessage ? ( +
+ + {successMessage} +
+ ) : null} + +
+ {showCloseButton ? ( + + ) : null} + +
+
+
+ +
) } diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx new file mode 100644 index 00000000000..09db56f08fa --- /dev/null +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -0,0 +1,302 @@ +"use client" + +import * as React from "react" +import { AlertCircle, CheckCircle2 } from "lucide-react" + +import { Button } from "@/components/ui/button" +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { Switch } from "@/components/ui/switch" +import { RagIndexSettingsService } from "@core/services/ragIndexSettings" +import type { RagIndexingEditableSettings, RagIndexingSettings } from "@core/rag/indexingSettings" +import { useSupabase } from "@ui/web/providers/SupabaseProvider" + +type EditableNumericKey = keyof Pick< + RagIndexingEditableSettings, + "small_note_threshold" | "target_chunk_size" | "min_chunk_size" | "max_chunk_size" | "overlap" +> + +type EditableBooleanKey = keyof Pick + +function buildEditableState(settings: RagIndexingSettings) { + return { + small_note_threshold: String(settings.small_note_threshold), + target_chunk_size: String(settings.target_chunk_size), + min_chunk_size: String(settings.min_chunk_size), + max_chunk_size: String(settings.max_chunk_size), + overlap: String(settings.overlap), + use_title: settings.use_title, + use_section_headings: settings.use_section_headings, + use_tags: settings.use_tags, + } +} + +export function RagIndexingSettingsPanel() { + const { supabase } = useSupabase() + const service = React.useMemo(() => new RagIndexSettingsService(supabase), [supabase]) + + const [loading, setLoading] = React.useState(true) + const [saving, setSaving] = React.useState(false) + const [errorMessage, setErrorMessage] = React.useState(null) + const [successMessage, setSuccessMessage] = React.useState(null) + const [resolvedSettings, setResolvedSettings] = React.useState(null) + const [formState, setFormState] = React.useState(() => ({ + small_note_threshold: "300", + target_chunk_size: "200", + min_chunk_size: "100", + max_chunk_size: "400", + overlap: "50", + use_title: true, + use_section_headings: true, + use_tags: true, + })) + + const loadSettings = React.useCallback(async () => { + setLoading(true) + setErrorMessage(null) + setSuccessMessage(null) + + try { + const status = await service.getStatus() + setResolvedSettings(status) + setFormState(buildEditableState(status)) + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : "Failed to load RAG indexing settings") + } finally { + setLoading(false) + } + }, [service]) + + React.useEffect(() => { + void loadSettings() + }, [loadSettings]) + + const updateNumericField = (key: EditableNumericKey, value: string) => { + setFormState((current) => ({ ...current, [key]: value })) + } + + const updateBooleanField = (key: EditableBooleanKey, checked: boolean) => { + setFormState((current) => ({ ...current, [key]: checked })) + } + + const handleSave = async () => { + setErrorMessage(null) + setSuccessMessage(null) + + const payload: RagIndexingEditableSettings = { + small_note_threshold: Number(formState.small_note_threshold), + target_chunk_size: Number(formState.target_chunk_size), + min_chunk_size: Number(formState.min_chunk_size), + max_chunk_size: Number(formState.max_chunk_size), + overlap: Number(formState.overlap), + use_title: formState.use_title, + use_section_headings: formState.use_section_headings, + use_tags: formState.use_tags, + } + + setSaving(true) + try { + const status = await service.upsert(payload) + setResolvedSettings(status) + setFormState(buildEditableState(status)) + setSuccessMessage("RAG indexing settings saved. Changes apply only to future indexing and future manual reindex.") + } catch (error) { + setErrorMessage(error instanceof Error ? error.message : "Failed to save RAG indexing settings") + } finally { + setSaving(false) + } + } + + return ( + + + RAG indexing + + Configure web-visible indexing behavior. All size values are measured in characters. + + + +
+ updateNumericField("small_note_threshold", value)} + /> + updateNumericField("target_chunk_size", value)} + /> + updateNumericField("min_chunk_size", value)} + /> + updateNumericField("max_chunk_size", value)} + /> + updateNumericField("overlap", value)} + /> +
+ +
+ updateBooleanField("use_title", checked)} + /> + updateBooleanField("use_section_headings", checked)} + /> + updateBooleanField("use_tags", checked)} + /> +
+ + {resolvedSettings ? ( +
+
+

Read-only system settings

+

+ These values are system-defined and shown for transparency. +

+
+
+ + + + + ")} /> + + +
+
+ +
+                {resolvedSettings.chunk_template}
+              
+
+
+ ) : null} + + {errorMessage ? ( +
+ + {errorMessage} +
+ ) : null} + + {successMessage ? ( +
+ + {successMessage} +
+ ) : null} + +
+ +
+
+
+ ) +} + +function NumericField({ + id, + label, + value, + disabled, + onChange, +}: { + id: string + label: string + value: string + disabled: boolean + onChange: (value: string) => void +}) { + return ( +
+ + onChange(event.target.value)} + disabled={disabled} + /> +
+ ) +} + +function ToggleRow({ + id, + label, + description, + checked, + disabled, + onCheckedChange, +}: { + id: string + label: string + description: string + checked: boolean + disabled: boolean + onCheckedChange: (checked: boolean) => void +}) { + return ( +
+
+ +

{description}

+
+ +
+ ) +} + +function ReadOnlyRow({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ) +} diff --git a/ui/web/components/features/settings/SettingsPage.tsx b/ui/web/components/features/settings/SettingsPage.tsx index 1d1f662a6ef..485fc38ff52 100644 --- a/ui/web/components/features/settings/SettingsPage.tsx +++ b/ui/web/components/features/settings/SettingsPage.tsx @@ -57,8 +57,8 @@ const SETTINGS_TABS: SettingsTabDefinition[] = [ }, { id: "api-keys", - label: "API Keys", - description: "External model credentials and secure storage.", + label: "Google API", + description: "Gemini API key and RAG indexing settings.", icon: KeyRound, }, ] From 31292e7610e678c74675db9b32692cc10b5ac7e4 Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 15:43:42 +0100 Subject: [PATCH 04/18] Adjust RAG chunk settings --- core/rag/chunkTemplate.ts | 25 ++++++++++++++++ core/rag/indexingSettings.ts | 10 +++---- core/tests/unit/core-rag-chunking.test.ts | 17 ++++++++++- .../ai/design/feature-improve-rag-chunking.md | 30 +++++++++---------- .../feature-improve-rag-chunking.md | 10 +++---- .../feature-improve-rag-chunking.md | 10 +++---- .../functions/api-keys-status/import_map.json | 3 +- .../functions/api-keys-upsert/import_map.json | 3 +- supabase/functions/rag-index/import_map.json | 6 +++- supabase/functions/rag-search/import_map.json | 3 +- ...0317000001_add_user_rag_index_settings.sql | 10 +++---- ui/mobile/app/note/[id].tsx | 9 ++---- .../components/search/AiSearchChunkCard.tsx | 3 +- .../components/search/AiSearchNoteCard.tsx | 7 +++-- .../integration/noteEditorScreen.test.tsx | 4 +-- .../components/features/notes/NotesShell.tsx | 7 ++--- .../features/search/ChunkSearchItem.tsx | 3 +- .../features/search/NoteSearchItem.tsx | 9 +++--- .../settings/RagIndexingSettingsPanel.tsx | 10 +++---- 19 files changed, 112 insertions(+), 67 deletions(-) diff --git a/core/rag/chunkTemplate.ts b/core/rag/chunkTemplate.ts index 49a433fbfe5..8017c20c816 100644 --- a/core/rag/chunkTemplate.ts +++ b/core/rag/chunkTemplate.ts @@ -9,6 +9,31 @@ export type RagChunkTemplateInput = { const normalizeInlineText = (value: string) => value.replace(/\s+/g, " ").trim() +export function getRagChunkBodyText(content: string): string { + const normalized = content.trim() + if (!normalized) return "" + + const lines = normalized.split("\n") + let cursor = 0 + + if (lines[cursor]?.startsWith("Section: ")) { + cursor += 1 + } + if (lines[cursor]?.startsWith("Tags: ")) { + cursor += 1 + } + + if (cursor > 0 && lines[cursor] === "") { + return lines.slice(cursor + 1).join("\n").trim() + } + + return normalized +} + +export function getRagChunkBodyLength(content: string): number { + return getRagChunkBodyText(content).length +} + export function buildRagChunkText({ sectionHeading, tags, diff --git a/core/rag/indexingSettings.ts b/core/rag/indexingSettings.ts index 4bf81600f6b..8105e87cdde 100644 --- a/core/rag/indexingSettings.ts +++ b/core/rag/indexingSettings.ts @@ -2,11 +2,11 @@ export const RAG_INDEX_NUMERIC_MIN = 50 export const RAG_INDEX_NUMERIC_MAX = 5000 export const RAG_INDEX_EDITABLE_DEFAULTS = { - small_note_threshold: 300, - target_chunk_size: 200, - min_chunk_size: 100, - max_chunk_size: 400, - overlap: 50, + small_note_threshold: 400, + target_chunk_size: 500, + min_chunk_size: 200, + max_chunk_size: 1500, + overlap: 100, use_title: true, use_section_headings: true, use_tags: true, diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index 893ee8ee57c..0e14b8bad1b 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -1,5 +1,5 @@ import { buildRagIndexChunks } from "@core/rag/chunking" -import { buildRagChunkText } from "@core/rag/chunkTemplate" +import { buildRagChunkText, getRagChunkBodyLength, getRagChunkBodyText } from "@core/rag/chunkTemplate" import { RAG_INDEX_EDITABLE_DEFAULTS } from "@core/rag/indexingSettings" describe("core/rag/chunking", () => { @@ -55,6 +55,21 @@ describe("core/rag/chunking", () => { expect(text).toBe("Body only") }) + it("extracts the note body from templated chunk content", () => { + const text = buildRagChunkText({ + sectionHeading: "Section A", + tags: ["alpha", "beta"], + chunkContent: "Body only", + settings: { + use_section_headings: true, + use_tags: true, + }, + }) + + expect(getRagChunkBodyText(text)).toBe("Body only") + expect(getRagChunkBodyLength(text)).toBe("Body only".length) + }) + it("does not create sections from non-heading formatting alone", () => { const chunks = buildRagIndexChunks({ title: "Formatting note", diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index 9256852f658..ca4b14c6f77 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -52,11 +52,11 @@ Representative model: ```sql user_rag_index_settings ( user_id uuid primary key references auth.users(id) on delete cascade, - small_note_threshold integer not null default 300, - target_chunk_size integer not null default 200, - min_chunk_size integer not null default 100, - max_chunk_size integer not null default 400, - overlap integer not null default 50, + small_note_threshold integer not null default 400, + target_chunk_size integer not null default 500, + min_chunk_size integer not null default 200, + max_chunk_size integer not null default 1500, + overlap integer not null default 100, use_title boolean not null default true, use_section_headings boolean not null default true, use_tags boolean not null default true, @@ -141,11 +141,11 @@ Representative payload: ```json { - "small_note_threshold": 300, - "target_chunk_size": 200, - "min_chunk_size": 100, - "max_chunk_size": 400, - "overlap": 50, + "small_note_threshold": 400, + "target_chunk_size": 500, + "min_chunk_size": 200, + "max_chunk_size": 1500, + "overlap": 100, "use_title": true, "use_section_headings": true, "use_tags": true, @@ -287,11 +287,11 @@ This feature does not alter search ranking logic, but the design must preserve: ## Open Design Items - Start defaults are: - - `small_note_threshold = 300` - - `target_chunk_size = 200` - - `min_chunk_size = 100` - - `max_chunk_size = 400` - - `overlap = 50` + - `small_note_threshold = 400` + - `target_chunk_size = 500` + - `min_chunk_size = 200` + - `max_chunk_size = 1500` + - `overlap = 100` - Validation ranges for editable numeric settings are: - `small_note_threshold`: `50..5000` - `target_chunk_size`: `50..5000` diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index 09329c7312b..a308ebf17f3 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -158,11 +158,11 @@ Recommended defaults in `core`: ```ts const DEFAULT_RAG_INDEX_SETTINGS = { - small_note_threshold: 300, - target_chunk_size: 200, - min_chunk_size: 100, - max_chunk_size: 400, - overlap: 50, + small_note_threshold: 400, + target_chunk_size: 500, + min_chunk_size: 200, + max_chunk_size: 1500, + overlap: 100, use_title: true, use_section_headings: true, use_tags: true, diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md index 1978e3fb60c..6597c292c97 100644 --- a/docs/ai/requirements/feature-improve-rag-chunking.md +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -156,9 +156,9 @@ Tags: {tag1}, {tag2}, {tag3} ## Questions & Open Items - Start defaults are fixed as: - - `small_note_threshold = 300` - - `target_chunk_size = 200` - - `min_chunk_size = 100` - - `max_chunk_size = 400` - - `overlap = 50` + - `small_note_threshold = 400` + - `target_chunk_size = 500` + - `min_chunk_size = 200` + - `max_chunk_size = 1500` + - `overlap = 100` - No remaining open items in requirements. diff --git a/supabase/functions/api-keys-status/import_map.json b/supabase/functions/api-keys-status/import_map.json index 14587e094fb..a4804d38bd7 100644 --- a/supabase/functions/api-keys-status/import_map.json +++ b/supabase/functions/api-keys-status/import_map.json @@ -1,6 +1,7 @@ { "imports": { "std/": "https://deno.land/std@0.177.0/", - "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2.45.4" + "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2.45.4", + "@core/": "../../../core/" } } diff --git a/supabase/functions/api-keys-upsert/import_map.json b/supabase/functions/api-keys-upsert/import_map.json index 14587e094fb..a4804d38bd7 100644 --- a/supabase/functions/api-keys-upsert/import_map.json +++ b/supabase/functions/api-keys-upsert/import_map.json @@ -1,6 +1,7 @@ { "imports": { "std/": "https://deno.land/std@0.177.0/", - "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2.45.4" + "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2.45.4", + "@core/": "../../../core/" } } diff --git a/supabase/functions/rag-index/import_map.json b/supabase/functions/rag-index/import_map.json index 0967ef424bc..eae38038478 100644 --- a/supabase/functions/rag-index/import_map.json +++ b/supabase/functions/rag-index/import_map.json @@ -1 +1,5 @@ -{} +{ + "imports": { + "@core/": "../../../core/" + } +} diff --git a/supabase/functions/rag-search/import_map.json b/supabase/functions/rag-search/import_map.json index 14587e094fb..a4804d38bd7 100644 --- a/supabase/functions/rag-search/import_map.json +++ b/supabase/functions/rag-search/import_map.json @@ -1,6 +1,7 @@ { "imports": { "std/": "https://deno.land/std@0.177.0/", - "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2.45.4" + "@supabase/supabase-js": "https://esm.sh/@supabase/supabase-js@2.45.4", + "@core/": "../../../core/" } } diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql index b4f046f4756..2426b6ec5d0 100644 --- a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -1,10 +1,10 @@ CREATE TABLE public.user_rag_index_settings ( user_id uuid PRIMARY KEY REFERENCES auth.users(id) ON DELETE CASCADE, - small_note_threshold integer NOT NULL DEFAULT 300, - target_chunk_size integer NOT NULL DEFAULT 200, - min_chunk_size integer NOT NULL DEFAULT 100, - max_chunk_size integer NOT NULL DEFAULT 400, - overlap integer NOT NULL DEFAULT 50, + small_note_threshold integer NOT NULL DEFAULT 400, + target_chunk_size integer NOT NULL DEFAULT 500, + min_chunk_size integer NOT NULL DEFAULT 200, + max_chunk_size integer NOT NULL DEFAULT 1500, + overlap integer NOT NULL DEFAULT 100, use_title boolean NOT NULL DEFAULT true, use_section_headings boolean NOT NULL DEFAULT true, use_tags boolean NOT NULL DEFAULT true, diff --git a/ui/mobile/app/note/[id].tsx b/ui/mobile/app/note/[id].tsx index f63595e5352..bb06f422fbf 100644 --- a/ui/mobile/app/note/[id].tsx +++ b/ui/mobile/app/note/[id].tsx @@ -294,18 +294,15 @@ export default function NoteEditorScreen() { return null } - const titlePrefix = (note.title ?? '').trim() - const bodyOffset = titlePrefix ? Math.max(0, rawOffset - (titlePrefix.length + 1)) : rawOffset - return { requestId: typeof focusRequestId === 'string' && focusRequestId.length > 0 ? focusRequestId - : `${note.id}:${bodyOffset}:${rawLength}`, - charOffset: bodyOffset, + : `${note.id}:${rawOffset}:${rawLength}`, + charOffset: rawOffset, chunkLength: rawLength, } - }, [focusLength, focusOffset, focusRequestId, note?.id, note?.title]) + }, [focusLength, focusOffset, focusRequestId, note?.id]) const applyPendingChunkFocus = useCallback(() => { if (!pendingChunkFocus) return diff --git a/ui/mobile/components/search/AiSearchChunkCard.tsx b/ui/mobile/components/search/AiSearchChunkCard.tsx index 29074880211..a29d17e70ab 100644 --- a/ui/mobile/components/search/AiSearchChunkCard.tsx +++ b/ui/mobile/components/search/AiSearchChunkCard.tsx @@ -2,6 +2,7 @@ import { memo, useMemo } from 'react' import { Pressable, StyleSheet, Text, View } from 'react-native' import { TagList } from '@ui/mobile/components/tags/TagList' import { useTheme } from '@ui/mobile/providers' +import { getRagChunkBodyLength } from '@core/rag/chunkTemplate' import type { RagChunk } from '@core/types/ragSearch' import type { Note } from '@core/types/domain' @@ -32,7 +33,7 @@ export const AiSearchChunkCard = memo(function AiSearchChunkCard({ return ( onOpenInContext(noteSnapshot, chunk.charOffset, chunk.content.length)} + onPress={() => onOpenInContext(noteSnapshot, chunk.charOffset, getRagChunkBodyLength(chunk.content))} accessibilityRole="button" style={({ pressed }) => [ styles.card, diff --git a/ui/mobile/components/search/AiSearchNoteCard.tsx b/ui/mobile/components/search/AiSearchNoteCard.tsx index b4fd368e191..f5d2b44ad08 100644 --- a/ui/mobile/components/search/AiSearchNoteCard.tsx +++ b/ui/mobile/components/search/AiSearchNoteCard.tsx @@ -3,6 +3,7 @@ import { Pressable, StyleSheet, Text, View } from 'react-native' import { Check, ChevronDown, ChevronUp } from 'lucide-react-native' import { TagList } from '@ui/mobile/components/tags/TagList' import { useTheme } from '@ui/mobile/providers' +import { getRagChunkBodyLength } from '@core/rag/chunkTemplate' import type { RagNoteGroup } from '@core/types/ragSearch' import type { Note } from '@core/types/domain' @@ -89,7 +90,7 @@ function MoreFragmentsSection({ testID={`ai-note-extra-chunk-${chunk.noteId}-${chunk.chunkIndex}`} onPress={(event) => { event?.stopPropagation?.() - onOpenChunk(chunk.charOffset, chunk.content.length) + onOpenChunk(chunk.charOffset, getRagChunkBodyLength(chunk.content)) }} accessibilityRole="button" style={({ pressed }) => [ @@ -148,7 +149,7 @@ export const AiSearchNoteCard = memo(function AiSearchNoteCard({ testID={`ai-note-card-${group.noteId}`} onPress={() => { if (!topChunk) return - handleOpenChunk(topChunk.charOffset, topChunk.content.length) + handleOpenChunk(topChunk.charOffset, getRagChunkBodyLength(topChunk.content)) }} onLongPress={selectionMode ? undefined : onLongPress} delayLongPress={400} @@ -189,7 +190,7 @@ export const AiSearchNoteCard = memo(function AiSearchNoteCard({ testID={`ai-note-top-chunk-${group.noteId}`} onPress={(event) => { event?.stopPropagation?.() - handleOpenChunk(topChunk.charOffset, topChunk.content.length) + handleOpenChunk(topChunk.charOffset, getRagChunkBodyLength(topChunk.content)) }} accessibilityRole="button" style={({ pressed }) => [ diff --git a/ui/mobile/tests/integration/noteEditorScreen.test.tsx b/ui/mobile/tests/integration/noteEditorScreen.test.tsx index e1fa61a98f3..9f09a6f0dc1 100644 --- a/ui/mobile/tests/integration/noteEditorScreen.test.tsx +++ b/ui/mobile/tests/integration/noteEditorScreen.test.tsx @@ -658,7 +658,7 @@ describe('NoteEditorScreen - Delete Functionality', () => { }) await waitFor(() => { - expect(mockScrollToChunk).toHaveBeenCalledWith(4, 5) + expect(mockScrollToChunk).toHaveBeenCalledWith(14, 5) }) expect(mockReplace).not.toHaveBeenCalled() @@ -703,7 +703,7 @@ describe('NoteEditorScreen - Delete Functionality', () => { await waitFor(() => { expect(mockScrollToChunk).toHaveBeenCalledTimes(1) - expect(mockScrollToChunk).toHaveBeenCalledWith(4, 5) + expect(mockScrollToChunk).toHaveBeenCalledWith(14, 5) }) }) }) diff --git a/ui/web/components/features/notes/NotesShell.tsx b/ui/web/components/features/notes/NotesShell.tsx index 046e40a4afe..ae6c5683a41 100644 --- a/ui/web/components/features/notes/NotesShell.tsx +++ b/ui/web/components/features/notes/NotesShell.tsx @@ -121,13 +121,10 @@ export function NotesShell({ controller }: NotesShellProps) { } if (!note) return // TypeScript: narrowing lost after await + let reassignment - // Adjust charOffset: rag-index prepends title + " " before the body text - const title = (note.title ?? '').trim() - const bodyOffset = title ? Math.max(0, charOffset - (title.length + 1)) : charOffset const nextPendingChunkFocus = { - requestId: `${noteId}:${bodyOffset}:${chunkLength}:${Date.now()}`, + requestId: `${noteId}:${charOffset}:${chunkLength}:${Date.now()}`, noteId, - charOffset: bodyOffset, + charOffset, chunkLength, } setPendingChunkFocus(nextPendingChunkFocus) diff --git a/ui/web/components/features/search/ChunkSearchItem.tsx b/ui/web/components/features/search/ChunkSearchItem.tsx index 174c94f19e4..018bf7bd728 100644 --- a/ui/web/components/features/search/ChunkSearchItem.tsx +++ b/ui/web/components/features/search/ChunkSearchItem.tsx @@ -1,6 +1,7 @@ import { ArrowUpRight } from 'lucide-react' import { ChunkSnippet } from './ChunkSnippet' import { cn } from '@ui/web/lib/utils' +import { getRagChunkBodyLength } from '@core/rag/chunkTemplate' import type { RagChunk } from '@core/types/ragSearch' interface ChunkSearchItemProps { @@ -22,7 +23,7 @@ function getScoreClass(score: number) { } export function ChunkSearchItem({ chunk, onOpenInContext, highlightQuery = '' }: ChunkSearchItemProps) { - const handleOpen = () => onOpenInContext(chunk.noteId, chunk.charOffset, chunk.content.length) + const handleOpen = () => onOpenInContext(chunk.noteId, chunk.charOffset, getRagChunkBodyLength(chunk.content)) return (
handleChunkActivate(e, topChunk.charOffset, topChunk.content.length)} - onKeyDown={(e) => handleChunkKeyDown(e, topChunk.charOffset, topChunk.content.length)} + onClick={(e) => handleChunkActivate(e, topChunk.charOffset, getRagChunkBodyLength(topChunk.content))} + onKeyDown={(e) => handleChunkKeyDown(e, topChunk.charOffset, getRagChunkBodyLength(topChunk.content))} > handleChunkActivate(e, chunk.charOffset, chunk.content.length)} - onKeyDown={(e) => handleChunkKeyDown(e, chunk.charOffset, chunk.content.length)} + onClick={(e) => handleChunkActivate(e, chunk.charOffset, getRagChunkBodyLength(chunk.content))} + onKeyDown={(e) => handleChunkKeyDown(e, chunk.charOffset, getRagChunkBodyLength(chunk.content))} >
diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx index 09db56f08fa..1bf6064a70f 100644 --- a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -42,11 +42,11 @@ export function RagIndexingSettingsPanel() { const [successMessage, setSuccessMessage] = React.useState(null) const [resolvedSettings, setResolvedSettings] = React.useState(null) const [formState, setFormState] = React.useState(() => ({ - small_note_threshold: "300", - target_chunk_size: "200", - min_chunk_size: "100", - max_chunk_size: "400", - overlap: "50", + small_note_threshold: "400", + target_chunk_size: "500", + min_chunk_size: "200", + max_chunk_size: "1500", + overlap: "100", use_title: true, use_section_headings: true, use_tags: true, From a30cf335ec8205696a26a35a3cc6bbec38941ad2 Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 17:40:40 +0100 Subject: [PATCH 05/18] =?UTF-8?q?=D0=9E=D0=B1=D1=8A=D1=8F=D1=81=D0=BD?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20=D0=BF=D1=80=D0=BE=D0=B1=D0=BB=D0=B5=D0=BC?= =?UTF-8?q?=D1=83=20=D0=B8=D0=BD=D0=B4=D0=B5=D0=BA=D1=81=D0=B0=D1=86=D0=B8?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/rag/chunkTemplate.ts | 2 +- core/rag/chunking.ts | 18 ++++++-- core/rag/debugLog.ts | 42 +++++++++++++++++++ core/tests/unit/core-rag-chunking.test.ts | 41 ++++++++++++++++++ .../ai/design/feature-improve-rag-chunking.md | 25 ++++++++++- .../feature-improve-rag-chunking.md | 15 ++++++- .../planning/feature-improve-rag-chunking.md | 16 ++++++- .../feature-improve-rag-chunking.md | 21 +++++++++- .../testing/feature-improve-rag-chunking.md | 17 +++++++- supabase/functions/rag-index/index.ts | 13 +++++- tsconfig.json | 1 + ui/mobile/components/NoteIndexMenu.tsx | 22 +++++++++- .../features/notes/RagIndexPanel.tsx | 22 +++++++++- 13 files changed, 241 insertions(+), 14 deletions(-) create mode 100644 core/rag/debugLog.ts diff --git a/core/rag/chunkTemplate.ts b/core/rag/chunkTemplate.ts index 8017c20c816..50db0f2bd7b 100644 --- a/core/rag/chunkTemplate.ts +++ b/core/rag/chunkTemplate.ts @@ -1,4 +1,4 @@ -import type { RagIndexingEditableSettings } from "@core/rag/indexingSettings" +import type { RagIndexingEditableSettings } from "./indexingSettings.ts" export type RagChunkTemplateInput = { sectionHeading: string | null diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index ae526000cf3..8d00efc2eda 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -1,5 +1,5 @@ -import type { RagIndexingEditableSettings } from "@core/rag/indexingSettings" -import { buildRagChunkText, buildRagEmbeddingTitle } from "@core/rag/chunkTemplate" +import type { RagIndexingEditableSettings } from "./indexingSettings.ts" +import { buildRagChunkText, buildRagEmbeddingTitle } from "./chunkTemplate.ts" type RawBlock = { sectionHeading: string | null @@ -356,8 +356,15 @@ function mergeSmallChunks( function buildOverlapPrefix(source: string, overlap: number): string { if (overlap <= 0 || source.length === 0) return "" - const start = Math.max(0, source.length - overlap) - return source.slice(start).trim() + const requestedStart = Math.max(0, source.length - overlap) + const sentenceBoundary = source.lastIndexOf(".", requestedStart - 1) + + if (sentenceBoundary === -1) { + return source.trim() + } + + const sentenceStart = sentenceBoundary + 1 + return source.slice(sentenceStart).trim() } function applyFinalOverlap(chunks: CandidateChunk[], overlap: number): CandidateChunk[] { @@ -366,6 +373,9 @@ function applyFinalOverlap(chunks: CandidateChunk[], overlap: number): Candidate return chunks.map((chunk, index) => { if (index === 0) return chunk const previous = chunks[index - 1] + if (!previous || previous.sectionHeading !== chunk.sectionHeading) { + return chunk + } const overlapPrefix = buildOverlapPrefix(previous?.text ?? "", overlap) if (!overlapPrefix) return chunk diff --git a/core/rag/debugLog.ts b/core/rag/debugLog.ts new file mode 100644 index 00000000000..5701d92a761 --- /dev/null +++ b/core/rag/debugLog.ts @@ -0,0 +1,42 @@ +export type RagIndexDebugChunk = { + chunkIndex: number + charOffset: number + sectionHeading: string | null + title: string | null + content: string +} + +function previewContent(content: string, maxLength = 120): string { + if (content.length <= maxLength) return content + return `${content.slice(0, maxLength)}...` +} + +export function logRagIndexDebugChunks(noteId: string, chunks: RagIndexDebugChunk[]): void { + if (chunks.length === 0) { + console.info(`[rag-index] No chunks were produced for note ${noteId}`) + return + } + + const openGroup = typeof console.groupCollapsed === "function" + ? console.groupCollapsed.bind(console) + : console.info.bind(console) + const closeGroup = typeof console.groupEnd === "function" + ? console.groupEnd.bind(console) + : () => undefined + + openGroup(`[rag-index][debug] ${chunks.length} chunks for note ${noteId}`) + for (const chunk of chunks) { + console.log(`[chunk ${chunk.chunkIndex}]`) + console.log({ + chunkIndex: chunk.chunkIndex, + charOffset: chunk.charOffset, + sectionHeading: chunk.sectionHeading, + title: chunk.title, + contentLength: chunk.content.length, + preview: previewContent(chunk.content), + }) + console.log("content:") + console.log(chunk.content.length > 0 ? chunk.content : "") + } + closeGroup() +} diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index 0e14b8bad1b..bbde7643a78 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -41,6 +41,47 @@ describe("core/rag/chunking", () => { expect(chunks[1]?.charOffset).toBeGreaterThan(chunks[0]?.charOffset ?? 0) }) + it("expands overlap back to the start of the sentence instead of starting mid-sentence", () => { + const longSentence = "Alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau." + + const chunks = buildRagIndexChunks({ + title: "Long sentence note", + html: `

${longSentence}

`, + tags: [], + settings: { + ...RAG_INDEX_EDITABLE_DEFAULTS, + small_note_threshold: 20, + target_chunk_size: 25, + min_chunk_size: 20, + max_chunk_size: 25, + overlap: 10, + }, + }) + + expect(chunks.length).toBeGreaterThan(1) + expect(getRagChunkBodyText(chunks[1]?.content ?? "").startsWith("Alpha beta gamma")).toBe(true) + }) + + it("does not carry overlap across section boundaries", () => { + const chunks = buildRagIndexChunks({ + title: "Sectioned note", + html: "

Section A

Alpha sentence one. Alpha sentence two.

Section B

Beta sentence one. Beta sentence two.

", + tags: [], + settings: { + ...RAG_INDEX_EDITABLE_DEFAULTS, + small_note_threshold: 20, + target_chunk_size: 30, + min_chunk_size: 20, + max_chunk_size: 30, + overlap: 10, + }, + }) + + expect(chunks).toHaveLength(4) + expect(chunks[2]?.content).toContain("Section: Section B") + expect(getRagChunkBodyText(chunks[2]?.content ?? "")).toBe("Beta sentence one.") + }) + it("omits empty optional lines from the chunk template", () => { const text = buildRagChunkText({ sectionHeading: null, diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index ca4b14c6f77..ee1567ac406 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -6,6 +6,19 @@ description: Architecture for configurable hierarchical note chunking and explic # System Design & Architecture +## Decision Update - 2026-03-17 + +The chunk assembly design has been refined after review. These rules are the latest source of truth and should take precedence over earlier generic phrases such as "accumulate toward target size" when they conflict. + +- Chunk assembly is `paragraph-first`, not `target-first`. +- `min_chunk_size` is the primary threshold for merging neighboring small paragraphs. +- Once `min_chunk_size` is reached, the assembler may add another whole paragraph only if doing so still fits naturally and moves the chunk closer to `target_chunk_size`. +- A whole next paragraph must not be added if it would overshoot `target_chunk_size`, even when it would still fit in `max_chunk_size`. +- If the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, the next paragraph may be split internally to complete a minimally valid chunk. +- Final trailing undersized chunks should try backward merge first; if that fails because of `max_chunk_size`, they remain undersized. +- Overlap is intentionally one-directional: `chunk[i + 1] = suffix(chunk[i]) + new_content`. +- Overlap must not cross a section boundary and should prefer natural stop points such as sentence-ending period or text boundary. + ## Architecture Overview ```mermaid @@ -204,7 +217,7 @@ This feature does not alter search ranking logic, but the design must preserve: - Implemented in shared `core` code with no dependency on `ui/web` or `ui/mobile` - Parses note content into hierarchical structural units - Derives sections only from real heading tags `h1` through `h6` -- Accumulates small sibling paragraphs toward `target_chunk_size` +- Accumulates small sibling paragraphs paragraph-first, reaching `min_chunk_size` before considering optional extension toward `target_chunk_size` - Splits oversized paragraphs into sentences, then token/character-based subparts - Applies overlap only after final chunks are formed - Merges undersized final chunks with neighbors when allowed @@ -242,6 +255,16 @@ This feature does not alter search ranking logic, but the design must preserve: **Why:** This keeps chunk generation predictable and aligns the setting with user expectations in the UI. +**Clarification:** overlap is one-directional. The next chunk repeats a suffix of the previous chunk at its beginning; chunks do not embed "future context" from the next chunk into their own tail. + +### Paragraph-first accumulation + +**Decision:** paragraph boundaries are the default chunk assembly boundary, and `min_chunk_size` is the first assembly target. + +**Why:** This preserves natural note structure better than greedily filling chunks toward `target_chunk_size`. + +**Consequence:** `target_chunk_size` remains useful, but only after `min_chunk_size` has already been reached and only when adding another whole paragraph still makes the resulting chunk a better fit. + ### Title is metadata, not chunk body **Decision:** Title is sent in the Gemini API `title` field and excluded from chunk text. diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index a308ebf17f3..0e6a127497e 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -6,6 +6,18 @@ description: Technical notes for configurable hierarchical chunking and indexing # Implementation Guide +## Decision Update - 2026-03-17 + +Implementation must now follow these clarified chunk-assembly rules: + +- treat paragraph boundaries as the default assembly boundary +- use `min_chunk_size` as the primary condition for closing a chunk assembled from small paragraphs +- after `min_chunk_size` is reached, another whole paragraph may be appended only if it improves fit toward `target_chunk_size` +- do not append a whole paragraph that would overshoot `target_chunk_size`, even if it is still within `max_chunk_size` +- if the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that next paragraph internally to finish the chunk +- when a trailing chunk is undersized, try backward merge first and leave it undersized if merging would exceed `max_chunk_size` +- keep overlap one-directional from previous chunk into next chunk + ## Development Setup - Use the existing Supabase Edge Function flow for indexing and search. @@ -68,7 +80,7 @@ Suggested processing flow: 4. Otherwise: - split into sections using `h1-h6` tags only - split each section into paragraphs - - accumulate neighboring small paragraphs toward `target_chunk_size` + - accumulate neighboring small paragraphs paragraph-first, reaching `min_chunk_size` before considering optional extension toward `target_chunk_size` - split oversized paragraphs deeper by sentences - if still oversized, split by tokens or characters 5. After candidate chunks are created: @@ -112,6 +124,7 @@ Implementation rules: - Keep domain logic in `core` and keep UI layers thin. - Do not copy chunking logic into web/mobile modules for convenience; add or extend shared `core` APIs instead. - Keep current UI work web-only, while preserving a clean shared contract for future mobile adoption. +- In paragraph-first assembly, make decisions on whole-paragraph boundaries whenever possible and only cut inside a paragraph as the explicit fallback path. - Keep I/O boundaries thin: - settings fetch - note fetch diff --git a/docs/ai/planning/feature-improve-rag-chunking.md b/docs/ai/planning/feature-improve-rag-chunking.md index 156d7762900..9d46cce040d 100644 --- a/docs/ai/planning/feature-improve-rag-chunking.md +++ b/docs/ai/planning/feature-improve-rag-chunking.md @@ -6,6 +6,18 @@ description: Task breakdown for configurable hierarchical chunking and indexing # Project Planning & Task Breakdown +## Decision Update - 2026-03-17 + +Latest clarified behavior to preserve during implementation: + +- chunk assembly is paragraph-first +- merge small paragraphs until `min_chunk_size` is reached +- after reaching `min_chunk_size`, only add another whole paragraph if it improves fit toward `target_chunk_size` +- do not add a whole paragraph that overshoots `target_chunk_size`, even if it still fits in `max_chunk_size` +- if still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that paragraph internally as a compromise +- trailing undersized chunks try backward merge first +- overlap is one-directional from previous chunk into next chunk + ## Milestones - [ ] Milestone 1: Persisted indexing settings model and UI contract defined @@ -54,7 +66,9 @@ description: Task breakdown for configurable hierarchical chunking and indexing - [ ] **2.4** Implement chunk assembly rules - single-chunk indexing for small notes - - accumulation of neighboring small paragraphs up to target size + - paragraph-first accumulation of neighboring small paragraphs + - use `min_chunk_size` as the first stopping threshold + - use `target_chunk_size` only as a later preference when another whole paragraph still improves fit - merge of undersized final chunks when possible - final-chunk overlap behavior diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md index 6597c292c97..031bd0f8cbe 100644 --- a/docs/ai/requirements/feature-improve-rag-chunking.md +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -6,6 +6,21 @@ description: Requirements for hierarchical note chunking and explicit indexing s # Requirements & Problem Understanding +## Decision Update - 2026-03-17 + +This feature now uses a stricter `paragraph-first` interpretation of hierarchical chunking. These decisions were made after initial drafting and must override any older wording below if there is a conflict. + +- Paragraphs are the primary chunk assembly unit. +- Small neighboring paragraphs may be merged, but reaching `min_chunk_size` is the first stopping condition. +- After `min_chunk_size` is reached, the chunk may include additional whole paragraphs only when they still fit naturally and move the chunk closer to `target_chunk_size`. +- If the next whole paragraph would overshoot `target_chunk_size`, it must not be added just to make the chunk larger, even if it would still fit within `max_chunk_size`. +- If a chunk is still below `min_chunk_size` and the next whole paragraph fits within `max_chunk_size`, that whole paragraph should be added. +- If a chunk is still below `min_chunk_size` but the next whole paragraph would exceed `max_chunk_size`, the next paragraph may be split internally as a compromise to reach a valid chunk. +- Oversized paragraphs are still split internally by sentences and then by token/character fallback when needed. +- A trailing undersized chunk should first try to merge backward with the previous chunk; if that would exceed `max_chunk_size`, the undersized tail remains as-is. +- Overlap remains one-directional: each next chunk repeats a suffix of the previous chunk at its beginning. +- Overlap should prefer natural boundaries, currently using explicit stop points such as sentence-ending period, section boundary, or text boundary. + ## Problem Statement RAG note indexing currently uses fixed, mostly implicit chunking and embedding settings in code. This makes indexing quality harder to tune, hides important system behavior from the UI, and forces redeploys for changes that should be configuration-driven. @@ -53,6 +68,7 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s - **As a user searching small notes**, I want short notes to remain whole so that their context is preserved. - **As a user searching large notes**, I want notes to be split on natural boundaries first so that retrieved chunks stay coherent. - **As a system**, I want tiny neighboring paragraphs to accumulate into a target-sized chunk so that the index avoids fragmented low-value chunks. +- **As a system**, I want tiny neighboring paragraphs to merge paragraph-first so that natural paragraph boundaries stay intact whenever possible. - **As a system**, I want oversized paragraphs to split deeper by sentences and then by token/character fallback so that no final chunk exceeds the configured maximum. - **As a system**, I want undersized final chunks to merge with neighbors when possible so that chunk quality remains consistent. @@ -105,10 +121,12 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s - small chunk merge rule - [ ] Notes below `small_note_threshold` are indexed as a single chunk unless prevented by system constraints. - [ ] Larger notes are split by natural boundaries before using sentence-level and token/character fallback splitting. -- [ ] Small adjacent paragraphs accumulate toward `target_chunk_size` within a section before becoming final chunks. +- [ ] Small adjacent paragraphs accumulate paragraph-first, reaching `min_chunk_size` first and extending toward `target_chunk_size` only when additional whole paragraphs fit naturally. - [ ] Oversized paragraphs are split deeper until all final chunks satisfy `max_chunk_size`. - [ ] Undersized final chunks are merged with adjacent chunks when possible without violating configured limits. - [ ] `overlap` is applied as repeated boundary content between adjacent final chunks, not as an intermediate split rule. +- [ ] `overlap` is one-directional: the next chunk repeats the previous chunk's tail at its beginning. +- [ ] `overlap` should prefer natural boundaries instead of starting from the middle of a sentence when a supported stop point is available. - [ ] Title is passed separately through the Gemini API `title` field and is not duplicated inside chunk text. - [ ] Size-based settings are explicitly labeled in the UI as character-based values. - [ ] Indexed chunk text follows one consistent template: @@ -152,6 +170,7 @@ Tags: {tag1}, {tag2}, {tag3} - Any omitted optional chunk parts (`Section`, `Tags`) should disappear entirely rather than render as empty labels. - Editable numeric indexing parameters use an allowed range of `50..5000`. - Server-side validation must also enforce logical ordering: `min_chunk_size <= target_chunk_size <= max_chunk_size`. +- `target_chunk_size` remains relevant for oversize paragraph splitting and for deciding whether another whole paragraph should be added after `min_chunk_size` has already been reached. ## Questions & Open Items diff --git a/docs/ai/testing/feature-improve-rag-chunking.md b/docs/ai/testing/feature-improve-rag-chunking.md index 60b6df630bf..707c121e0d1 100644 --- a/docs/ai/testing/feature-improve-rag-chunking.md +++ b/docs/ai/testing/feature-improve-rag-chunking.md @@ -6,6 +6,17 @@ description: Test plan for configurable hierarchical chunking and indexing setti # Testing Strategy +## Decision Update - 2026-03-17 + +The test plan must explicitly protect the newly clarified paragraph-first rules: + +- paragraph-first chunk assembly +- `min_chunk_size` reached before optional extension toward `target_chunk_size` +- no whole-paragraph append when it overshoots `target_chunk_size` +- partial split of the next paragraph only when needed to escape an undersized chunk that cannot fit the whole paragraph under `max_chunk_size` +- trailing undersized chunk tries backward merge first +- overlap remains one-directional from previous chunk into next chunk + ## Test Coverage Goals - Unit test coverage target: 100% of new chunking and settings-validation logic @@ -41,11 +52,15 @@ description: Test plan for configurable hierarchical chunking and indexing setti ### Chunk accumulation and merge rules -- [ ] Adjacent small paragraphs accumulate toward `target_chunk_size` +- [ ] Adjacent small paragraphs accumulate paragraph-first until `min_chunk_size` is reached +- [ ] After reaching `min_chunk_size`, another whole paragraph is appended only when it improves fit toward `target_chunk_size` +- [ ] A whole next paragraph is not appended when it would overshoot `target_chunk_size`, even if still within `max_chunk_size` +- [ ] If still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, the paragraph is split internally as fallback - [ ] Accumulation stops before violating `max_chunk_size` - [ ] Undersized final trailing chunk merges with previous neighbor when allowed - [ ] Undersized chunk remains standalone when merging would exceed `max_chunk_size` - [ ] Overlap duplicates only boundary content between adjacent final chunks +- [ ] Overlap is one-directional from previous chunk into next chunk ### Chunk text templating diff --git a/supabase/functions/rag-index/index.ts b/supabase/functions/rag-index/index.ts index 9efc94422d4..e1fb52342e2 100644 --- a/supabase/functions/rag-index/index.ts +++ b/supabase/functions/rag-index/index.ts @@ -186,10 +186,10 @@ serve(async (req: Request) => { if (userError || !userData?.user) return jsonResponse({ error: "Unauthorized" }, 401) const userId = userData.user.id - let payload: { noteId?: string; action?: string } = {} + let payload: { noteId?: string; action?: string; debugChunks?: boolean } = {} try { payload = await req.json() } catch { /* empty body */ } - const { noteId, action } = payload + const { noteId, action, debugChunks } = payload if (!noteId || typeof noteId !== "string") { return jsonResponse({ error: "Missing noteId" }, 400) } @@ -329,6 +329,15 @@ serve(async (req: Request) => { return jsonResponse({ chunkCount: chunksForIndexing.length, droppedChunks: droppedChunkCount > 0 ? droppedChunkCount : undefined, + debugChunks: debugChunks + ? chunksForIndexing.map((chunk, index) => ({ + chunkIndex: index, + charOffset: chunk.charOffset, + sectionHeading: chunk.sectionHeading, + title: chunk.title, + content: chunk.content, + })) + : undefined, }) } catch (err) { console.error("[rag-index]", err) diff --git a/tsconfig.json b/tsconfig.json index 28ae15a7ccf..c5ca9e0d1c5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,7 @@ "esModuleInterop": true, "module": "esnext", "moduleResolution": "bundler", + "allowImportingTsExtensions": true, "resolveJsonModule": true, "isolatedModules": true, "jsx": "react-jsx", diff --git a/ui/mobile/components/NoteIndexMenu.tsx b/ui/mobile/components/NoteIndexMenu.tsx index 367d2498067..2f194835996 100644 --- a/ui/mobile/components/NoteIndexMenu.tsx +++ b/ui/mobile/components/NoteIndexMenu.tsx @@ -11,6 +11,7 @@ import { Database, Trash2 } from 'lucide-react-native' import { useTheme } from '@ui/mobile/providers' import { useSupabase } from '@ui/mobile/providers/SupabaseProvider' import { useRagStatus } from '@ui/mobile/hooks/useRagStatus' +import { logRagIndexDebugChunks, type RagIndexDebugChunk } from '@core/rag/debugLog' interface NoteIndexMenuProps { noteId: string @@ -20,6 +21,21 @@ interface NoteIndexMenuProps { type Operation = 'indexing' | 'deleting' | null +function parseDebugChunks(data: unknown): RagIndexDebugChunk[] { + if (!data || typeof data !== 'object') return [] + const value = (data as { debugChunks?: unknown }).debugChunks + if (!Array.isArray(value)) return [] + return value.filter((chunk): chunk is RagIndexDebugChunk => { + if (!chunk || typeof chunk !== 'object') return false + const candidate = chunk as Partial + return typeof candidate.chunkIndex === 'number' + && typeof candidate.charOffset === 'number' + && typeof candidate.content === 'string' + && (typeof candidate.sectionHeading === 'string' || candidate.sectionHeading === null) + && (typeof candidate.title === 'string' || candidate.title === null) + }) +} + async function extractErrorMessage(err: unknown, fallback: string): Promise { if (!(err instanceof Error)) return fallback const ctx = (err as Error & { context?: unknown }).context @@ -79,9 +95,13 @@ export function NoteIndexMenu({ noteId, visible, onClose }: NoteIndexMenuProps) try { const action = isIndexed ? 'reindex' : 'index' const { data, error } = await client.functions.invoke('rag-index', { - body: { noteId, action }, + body: { noteId, action, debugChunks: true }, }) if (error) throw error + const debugChunks = parseDebugChunks(data) + if (debugChunks.length > 0) { + logRagIndexDebugChunks(noteId, debugChunks) + } const count = typeof (data as { chunkCount?: number })?.chunkCount === 'number' ? (data as { chunkCount: number }).chunkCount : null diff --git a/ui/web/components/features/notes/RagIndexPanel.tsx b/ui/web/components/features/notes/RagIndexPanel.tsx index a21cc167ab4..e8333acc6d1 100644 --- a/ui/web/components/features/notes/RagIndexPanel.tsx +++ b/ui/web/components/features/notes/RagIndexPanel.tsx @@ -20,6 +20,7 @@ import { import { toast } from 'sonner' import { useSupabase } from '@ui/web/providers/SupabaseProvider' import { useRagStatus } from '@ui/web/hooks/useRagStatus' +import { logRagIndexDebugChunks, type RagIndexDebugChunk } from '@core/rag/debugLog' async function extractErrorMessage(err: unknown, fallback: string): Promise { if (!(err instanceof Error)) return fallback @@ -43,6 +44,21 @@ interface RagIndexPanelProps { type Operation = 'indexing' | 'deleting' | null +function parseDebugChunks(data: unknown): RagIndexDebugChunk[] { + if (!data || typeof data !== 'object') return [] + const value = (data as { debugChunks?: unknown }).debugChunks + if (!Array.isArray(value)) return [] + return value.filter((chunk): chunk is RagIndexDebugChunk => { + if (!chunk || typeof chunk !== 'object') return false + const candidate = chunk as Partial + return typeof candidate.chunkIndex === 'number' + && typeof candidate.charOffset === 'number' + && typeof candidate.content === 'string' + && (typeof candidate.sectionHeading === 'string' || candidate.sectionHeading === null) + && (typeof candidate.title === 'string' || candidate.title === null) + }) +} + function parseChunkCount(data: unknown): number | null { if (!data || typeof data !== 'object') return null const value = (data as { chunkCount?: unknown }).chunkCount @@ -62,9 +78,13 @@ export function RagIndexPanel({ noteId, variant = 'inline', onMenuClose }: RagIn setOperation('indexing') try { const { data, error } = await supabase.functions.invoke('rag-index', { - body: { noteId, action: isIndexed ? 'reindex' : 'index' }, + body: { noteId, action: isIndexed ? 'reindex' : 'index', debugChunks: true }, }) if (error) throw error + const debugChunks = parseDebugChunks(data) + if (debugChunks.length > 0) { + logRagIndexDebugChunks(noteId, debugChunks) + } const count = parseChunkCount(data) if (count === null) { console.warn('[rag-index] Unexpected response payload for index action', data) From 4508b76e8a37b6f86808474a5d992f2bf46589c1 Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 17:43:23 +0100 Subject: [PATCH 06/18] =?UTF-8?q?=D0=9E=D0=B1=D1=8A=D1=8F=D1=81=D0=BD?= =?UTF-8?q?=D0=B8=20=D0=B8=D0=BD=D0=B4=D0=B5=D0=BA=D1=81=D0=B8=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=B0=D0=B1=D0=B7=D0=B0=D1=86?= =?UTF-8?q?=D0=B5=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../planning/feature-improve-rag-chunking.md | 50 +++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/docs/ai/planning/feature-improve-rag-chunking.md b/docs/ai/planning/feature-improve-rag-chunking.md index 9d46cce040d..252b81fcb18 100644 --- a/docs/ai/planning/feature-improve-rag-chunking.md +++ b/docs/ai/planning/feature-improve-rag-chunking.md @@ -18,29 +18,57 @@ Latest clarified behavior to preserve during implementation: - trailing undersized chunks try backward merge first - overlap is one-directional from previous chunk into next chunk +## Planning Update - 2026-03-17 + +Current progress after implementation and review work: + +- shared `core` settings and chunking logic exists and is already wired into indexing paths +- per-user indexing settings storage and web-only settings UI are implemented +- local Supabase Edge boot issues were resolved by switching internal `core/rag` imports to explicit local `.ts` imports and removing an incompatible `deno.lock` +- temporary chunk debug logging is enabled to inspect final indexed chunks in the browser/app console +- design and requirements were refined after implementation review: chunk assembly must be `paragraph-first`, not greedily `target_chunk_size`-first + +Current risks and open execution focus: + +- current chunk assembly implementation still needs to be fully aligned with the newly clarified paragraph-first planning rules +- debug observations showed chunk boundaries still drifting across paragraph boundaries more aggressively than desired +- docs are updated, but code and tests still need one more reconciliation pass to fully match the latest rules +- `ai-devkit lint --feature improve-rag-chunking` still fails on workflow metadata only because git branch `feature-improve-rag-chunking` does not exist in this repo context + +Recommended next tasks: + +1. Rework `core/rag/chunking.ts` so paragraph-first accumulation uses `min_chunk_size` as the first stopping condition and uses `target_chunk_size` only as a secondary preference. +2. Add/adjust unit tests for paragraph-preserving chunk assembly, fallback paragraph splitting, trailing undersized merge-back, and one-directional overlap expectations. +3. Re-run local manual indexing against representative notes and compare debug chunk output against expected paragraph-first boundaries. + +Blockers / coordination: + +- no product blocker is open on requirements; the remaining work is implementation alignment +- if strict workflow lint compliance is needed, create the expected git branch or worktree name `feature-improve-rag-chunking` + ## Milestones -- [ ] Milestone 1: Persisted indexing settings model and UI contract defined -- [ ] Milestone 2: Shared `core` chunking/settings module implemented and adopted by indexing paths +- [x] Milestone 1: Persisted indexing settings model and UI contract defined +- [x] Milestone 2: Shared `core` chunking/settings module implemented and adopted by indexing paths - [ ] Milestone 3: Settings UI wired to runtime configuration and validated end-to-end ## Task Breakdown ### Phase 1: Settings foundation -- [ ] **1.1** Finalize the persisted settings shape for indexing configuration +- [x] **1.1** Finalize the persisted settings shape for indexing configuration - use per-user settings scope - store settings in a dedicated per-user table, separate from `user_api_keys` - place the UI under the Google API settings tab - define defaults for editable and read-only parameters - allow any user to edit their own settings -- [ ] **1.2** Add backend read/write access for indexing settings +- [x] **1.2** Add backend read/write access for indexing settings - read resolved settings for the UI - save editable settings with validation - expose read-only system values alongside editable values -- [ ] **1.3** Define validation rules +- [x] **1.3** Define validation rules - numeric ranges for thresholds and chunk sizes: `50..5000` - invariants like `min_chunk_size <= target_chunk_size <= max_chunk_size` - overlap constraints relative to chunk sizes @@ -48,13 +76,13 @@ Latest clarified behavior to preserve during implementation: ### Phase 2: Chunking pipeline -- [ ] **2.1** Create shared `core` module for indexing settings and hierarchical chunking +- [x] **2.1** Create shared `core` module for indexing settings and hierarchical chunking - keep application-owned chunking - keep the module independent from `ui/web` and `ui/mobile` - expose pure helpers reusable by server and clients - treat this module as the canonical implementation, not as an optional helper -- [ ] **2.2** Replace current fixed-window chunking in `supabase/functions/rag-index/index.ts` with the shared `core` module +- [x] **2.2** Replace current fixed-window chunking in `supabase/functions/rag-index/index.ts` with the shared `core` module - remove hard-coded chunking constants from the main indexing flow - consume the shared chunk builder and template serializer @@ -63,6 +91,7 @@ Latest clarified behavior to preserve during implementation: - split sections into paragraphs - split oversized paragraphs into sentences - add token/character fallback for pathological long blocks + - status: in progress; implemented, but paragraph-first behavior still needs refinement to match latest clarified rules - [ ] **2.4** Implement chunk assembly rules - single-chunk indexing for small notes @@ -71,8 +100,9 @@ Latest clarified behavior to preserve during implementation: - use `target_chunk_size` only as a later preference when another whole paragraph still improves fit - merge of undersized final chunks when possible - final-chunk overlap behavior + - status: in progress; overlap is working and one-directional, but accumulation still needs to stop and extend according to the newly agreed paragraph-first rules -- [ ] **2.5** Implement chunk text templating +- [x] **2.5** Implement chunk text templating - title passed separately via Gemini `title` - optional `Section:` line - optional `Tags:` line @@ -80,7 +110,7 @@ Latest clarified behavior to preserve during implementation: ### Phase 3: UI and compatibility -- [ ] **3.1** Build indexing settings UI consumers on top of the shared contract +- [x] **3.1** Build indexing settings UI consumers on top of the shared contract - web only in this feature - editable controls for chunk parameters and inclusion flags - read-only section for `output_dimensionality`, task types, and system chunking rules @@ -94,12 +124,14 @@ Latest clarified behavior to preserve during implementation: - web UI consumes the shared settings shape in this phase - mobile reuse is deferred, but the shared contract must remain mobile-compatible - reject any duplicated per-platform chunking implementation during rollout/review + - status: in progress; mostly done, but needs one more pass after paragraph-first chunking updates to verify end-to-end behavior - [ ] **3.3** Reindex and rollout strategy - settings changes affect only future indexing and future manual reindex - existing indexed notes remain unchanged until manually reindexed - define user guidance for when manual reindex is needed - add observability for effective settings during indexing runs + - status: in progress; manual reindex behavior is already true in practice, but user guidance and final rollout notes still need a cleanup pass ## Dependencies From 3aa29b46841c39308c40755ee217cb0cde1d71d8 Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 21:28:48 +0100 Subject: [PATCH 07/18] =?UTF-8?q?=D1=83=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=20=D1=87=D0=B0=D0=BD=D0=BA=D0=B8=D0=BD=D0=B3,=20=D1=82?= =?UTF-8?q?=D0=B5=D0=BC=D0=BF=20=D0=BA=D0=BE=D0=BC=D0=B8=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/rag/chunking.ts | 236 ++++++++++++------ core/rag/indexingSettings.ts | 2 +- core/tests/unit/core-rag-chunking.test.ts | 148 ++++++++++- .../ai/design/feature-improve-rag-chunking.md | 1 + .../feature-improve-rag-chunking.md | 2 + .../planning/feature-improve-rag-chunking.md | 2 + .../feature-improve-rag-chunking.md | 3 +- .../testing/feature-improve-rag-chunking.md | 5 + .../settings/RagIndexingSettingsPanel.tsx | 59 ++++- 9 files changed, 372 insertions(+), 86 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index 8d00efc2eda..d4bf5177f73 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -142,6 +142,17 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { return blocks.filter((block) => block.text.length > 0) } +function splitAndStripParagraphs(text: string, sectionHeading: string | null): RawBlock[] { + // Split by paragraph breaks first (created by BLOCK_BREAK_PATTERN replacement), + // then strip remaining tags from each paragraph individually. + // This preserves paragraph boundaries that normalizeWhitespace would otherwise collapse. + return text + .split(/\n{2,}/) + .map((paragraph) => stripTags(paragraph)) + .filter(Boolean) + .map((cleaned) => ({ sectionHeading, text: cleaned })) +} + function collectBlocksWithRegex(html: string): RawBlock[] { const normalizedHtml = html .replace(//gi, "\n") @@ -156,19 +167,13 @@ function collectBlocksWithRegex(html: string): RawBlock[] { while ((match = headingRegex.exec(normalizedHtml)) !== null) { const beforeHeading = normalizedHtml.slice(lastIndex, match.index) - rawBlocks.push(...buildBlocksFromPlainText(stripTags(beforeHeading)).map((block) => ({ - sectionHeading: currentHeading, - text: block.text, - }))) + rawBlocks.push(...splitAndStripParagraphs(beforeHeading, currentHeading)) currentHeading = stripTags(match[2] ?? "") lastIndex = headingRegex.lastIndex } const remaining = normalizedHtml.slice(lastIndex) - rawBlocks.push(...buildBlocksFromPlainText(stripTags(remaining)).map((block) => ({ - sectionHeading: currentHeading, - text: block.text, - }))) + rawBlocks.push(...splitAndStripParagraphs(remaining, currentHeading)) return rawBlocks.filter((block) => block.text.length > 0) } @@ -241,61 +246,161 @@ function splitByCharacterFallback(segment: TextSegment, maxChunkSize: number): T return parts } -function splitOversizedBlock(block: IndexedBlock, maxChunkSize: number): TextSegment[] { - if (block.text.length <= maxChunkSize) { - return [{ sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset }] +function takePartialText(text: string, minChars: number): { taken: string; remainder: string } { + if (minChars >= text.length) { + return { taken: text, remainder: "" } + } + + const sentenceEnd = text.indexOf(".", minChars) + if (sentenceEnd !== -1 && sentenceEnd < text.length) { + return { + taken: text.slice(0, sentenceEnd + 1).trim(), + remainder: text.slice(sentenceEnd + 1).trim(), + } + } + + const spaceIndex = text.indexOf(" ", minChars) + if (spaceIndex !== -1) { + return { + taken: text.slice(0, spaceIndex).trim(), + remainder: text.slice(spaceIndex).trim(), + } } + return { + taken: text.slice(0, minChars).trim(), + remainder: text.slice(minChars).trim(), + } +} + +function splitOversizedParagraph( + block: IndexedBlock, + maxSize: number, + minSize: number +): CandidateChunk[] { const sentenceSegments = splitIntoSentenceSegments(block) - const expanded: TextSegment[] = [] + const pieces: TextSegment[] = [] for (const sentence of sentenceSegments) { - if (sentence.text.length <= maxChunkSize) { - expanded.push(sentence) + if (sentence.text.length <= maxSize) { + pieces.push(sentence) + } else { + pieces.push(...splitByCharacterFallback(sentence, maxSize)) + } + } + + const chunks: CandidateChunk[] = [] + let current: CandidateChunk | null = null + + for (const piece of pieces) { + if (!current) { + current = { sectionHeading: piece.sectionHeading, text: piece.text, charOffset: piece.charOffset } continue } - expanded.push(...splitByCharacterFallback(sentence, maxChunkSize)) + + const combined = [current.text, piece.text].join(" ") + if (combined.length <= maxSize) { + current = { ...current, text: combined } + } else { + chunks.push(current) + current = { sectionHeading: piece.sectionHeading, text: piece.text, charOffset: piece.charOffset } + } } - return expanded + if (current) chunks.push(current) + + // Backward merge: if the last piece is undersized, merge it into the previous piece. + // This may produce a chunk slightly above maxSize — a conscious compromise to avoid + // breaking the next paragraph boundary or leaving a tiny orphan chunk. + if (chunks.length >= 2) { + const last = chunks[chunks.length - 1]! + if (last.text.length < minSize) { + const prev = chunks[chunks.length - 2]! + chunks.splice(-2, 2, { ...prev, text: [prev.text, last.text].join(" ") }) + } + } + + return chunks } function joinChunkParts(parts: string[]): string { return parts.filter(Boolean).join("\n\n").trim() } -function accumulateSegments( - segments: TextSegment[], +function assembleParagraphFirst( + blocks: IndexedBlock[], settings: Pick ): CandidateChunk[] { const candidates: CandidateChunk[] = [] let current: CandidateChunk | null = null - for (const segment of segments) { + for (let i = 0; i < blocks.length; i++) { + const block = blocks[i] + if (!block) continue + + // Section boundary — close current chunk + if (current && current.sectionHeading !== block.sectionHeading) { + candidates.push(current) + current = null + } + + // Oversized paragraph (> max_chunk_size): split internally with max_chunk_size (minimal cuts) + if (block.text.length > settings.max_chunk_size) { + if (current) { + candidates.push(current) + current = null + } + candidates.push(...splitOversizedParagraph(block, settings.max_chunk_size, settings.min_chunk_size)) + continue + } + + // Start new chunk if (!current) { - current = { sectionHeading: segment.sectionHeading, text: segment.text, charOffset: segment.charOffset } + current = { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } continue } - const sameSection = current.sectionHeading === segment.sectionHeading - const combinedText = joinChunkParts([current.text, segment.text]) - - if ( - sameSection && - ( - combinedText.length <= settings.target_chunk_size || - (current.text.length < settings.min_chunk_size && combinedText.length <= settings.max_chunk_size) - ) - ) { - current = { - sectionHeading: current.sectionHeading, - text: combinedText, - charOffset: current.charOffset, + // current is guaranteed non-null here (guarded by the !current check above) + const combinedText = joinChunkParts([current!.text, block.text]) + + if (current!.text.length < settings.min_chunk_size) { + // Below min — must add more to reach min_chunk_size + if (combinedText.length <= settings.max_chunk_size) { + // Whole paragraph fits within max — add it whole + current = { sectionHeading: current!.sectionHeading, text: combinedText, charOffset: current!.charOffset } + } else { + // Whole paragraph doesn't fit in max — split partially to reach min + const separatorLen = 2 // "\n\n" + const needed = settings.min_chunk_size - current!.text.length - separatorLen + if (needed > 0) { + const partial = takePartialText(block.text, needed) + current = { sectionHeading: current!.sectionHeading, text: joinChunkParts([current!.text, partial.taken]), charOffset: current!.charOffset } + candidates.push(current) + current = null + if (partial.remainder) { + current = { + sectionHeading: block.sectionHeading, + text: partial.remainder, + charOffset: block.charOffset + partial.taken.length, + } + } + } else { + // Current is already at min with just the separator + candidates.push(current!) + current = { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } + } } continue } - candidates.push(current) - current = { sectionHeading: segment.sectionHeading, text: segment.text, charOffset: segment.charOffset } + // At or above min — can close, but check if next paragraph fits within target + if (combinedText.length <= settings.target_chunk_size) { + // Adding this paragraph keeps within target — add it + current = { sectionHeading: current!.sectionHeading, text: combinedText, charOffset: current!.charOffset } + } else { + // Would exceed target — close current, start new chunk + candidates.push(current!) + current = { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } + } } if (current) { @@ -305,53 +410,27 @@ function accumulateSegments( return candidates } -function mergeSmallChunks( +function mergeUndersizedTail( chunks: CandidateChunk[], settings: Pick ): CandidateChunk[] { - const merged: CandidateChunk[] = [] - - for (let index = 0; index < chunks.length; index += 1) { - const chunk = chunks[index] - if (!chunk) continue - - if (chunk.text.length >= settings.min_chunk_size) { - merged.push(chunk) - continue - } + if (chunks.length < 2) return chunks - const previous = merged[merged.length - 1] - if ( - previous && - previous.sectionHeading === chunk.sectionHeading && - joinChunkParts([previous.text, chunk.text]).length <= settings.max_chunk_size - ) { - merged[merged.length - 1] = { - sectionHeading: previous.sectionHeading, - text: joinChunkParts([previous.text, chunk.text]), - charOffset: previous.charOffset, - } - continue - } + const last = chunks[chunks.length - 1]! + if (last.text.length >= settings.min_chunk_size) return chunks - const next = chunks[index + 1] - if ( - next && - next.sectionHeading === chunk.sectionHeading && - joinChunkParts([chunk.text, next.text]).length <= settings.max_chunk_size - ) { - chunks[index + 1] = { - sectionHeading: next.sectionHeading, - text: joinChunkParts([chunk.text, next.text]), - charOffset: chunk.charOffset, - } - continue - } + const prev = chunks[chunks.length - 2]! + if (prev.sectionHeading !== last.sectionHeading) return chunks - merged.push(chunk) + const merged = joinChunkParts([prev.text, last.text]) + if (merged.length <= settings.max_chunk_size) { + return [ + ...chunks.slice(0, -2), + { ...prev, text: merged }, + ] } - return merged + return chunks } function buildOverlapPrefix(source: string, overlap: number): string { @@ -420,11 +499,8 @@ export function buildRagIndexChunks({ const baseChunks = noteBodyLength > 0 && noteBodyLength <= settings.small_note_threshold ? buildWholeNoteChunk(blocks, html ?? "", settings) - : mergeSmallChunks( - accumulateSegments( - blocks.flatMap((block) => splitOversizedBlock(block, settings.max_chunk_size)), - settings - ), + : mergeUndersizedTail( + assembleParagraphFirst(blocks, settings), settings ) diff --git a/core/rag/indexingSettings.ts b/core/rag/indexingSettings.ts index 8105e87cdde..94e1ceae3ab 100644 --- a/core/rag/indexingSettings.ts +++ b/core/rag/indexingSettings.ts @@ -19,7 +19,7 @@ export const RAG_INDEX_READONLY_SETTINGS = { split_strategy: "hierarchical" as const, fallback_split_order: ["sections", "paragraphs", "sentences", "tokens_or_characters"] as const, chunk_accumulation_rule: - "Accumulate neighboring small paragraphs within the same section until target_chunk_size is reached or max_chunk_size would be exceeded.", + "Paragraph-first: accumulate whole paragraphs until min_chunk_size is reached, then optionally extend toward target_chunk_size only if the next whole paragraph fits without exceeding it. Oversized paragraphs are split at max_chunk_size boundaries; if the remainder is below min_chunk_size it is merged back into the previous piece.", small_chunk_merge_rule: "Merge undersized final chunks with adjacent chunks when possible without violating max_chunk_size.", chunk_template: "Section: {section_heading}\nTags: {tag1}, {tag2}, {tag3}\n\n{chunk_content}", diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index bbde7643a78..14f2cacf6b5 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -2,6 +2,25 @@ import { buildRagIndexChunks } from "@core/rag/chunking" import { buildRagChunkText, getRagChunkBodyLength, getRagChunkBodyText } from "@core/rag/chunkTemplate" import { RAG_INDEX_EDITABLE_DEFAULTS } from "@core/rag/indexingSettings" +const USER_NOTE_HTML = [ + '

"То что на предыдущем уровне было субъектом становиться объектом на следующем". (с)

', + "

", + '

Чувство голода может всецело овладевать вами. Тогда это чувство будет иметь вас, вместо того чтобы быть у вас. В таком случае можно говорить о том, что оно остаеться в качестве "скрытого субъекта" в вашем сознании, в системе идентичности. (с)

', + "

", + '

Что такое субъект и объект? Объект - то что я могу увидеть, осмыслить, субъект - я, мои части, то что\\кто видит. Как рука не может схватить сама себя, а ручка не может себя нарисовать, нож не может себя порезать, субъект не может познать себя. Субъект может наблюдать и познавать лишь объекты. То что было субъектом (частью меня) должно стать объектом (тем что я смогу наблюдать). То есть субъективная часть это что-то скрытое, то что я не осознаю, часть меня, которую я не могу "увидеть". Если же я начинаю эту часть "видеть", то она уже не являеться скрытым субъектом, она становиться объектом, который я наблюдаю. 

', + "

", + '

Если мною овладевает ярость, она является неосознанной, она являеться моим скрытым субъектом и тогда я - ярость. В такой момент я не осознаю что все мои действия происходят под влиянием ярости. Ярость как бы берет верх управления и у меня нет выбора в том как вести себя. Я отождествлен со своей яростью, она мой скрытый субъект, я - ярость. Что значит растождествиться со своей яростью? Это значит увидеть ее, увидеть ее со стороны, сделать ее объектом наблюдения. Она больше не мой скрытый субъект, она то что я наблюдаю - объект, и тогда я не ярость, ведь я не могу быть тем что я могу наблюдать (как нож не может порезать сам себя). 

', +].join("") + +const USER_SETTINGS = { + ...RAG_INDEX_EDITABLE_DEFAULTS, + small_note_threshold: 400, + target_chunk_size: 500, + min_chunk_size: 200, + max_chunk_size: 1500, + overlap: 100, +} + describe("core/rag/chunking", () => { it("keeps a small note as a single chunk", () => { const chunks = buildRagIndexChunks({ @@ -77,9 +96,14 @@ describe("core/rag/chunking", () => { }, }) - expect(chunks).toHaveLength(4) - expect(chunks[2]?.content).toContain("Section: Section B") - expect(getRagChunkBodyText(chunks[2]?.content ?? "")).toBe("Beta sentence one.") + // Each section text (~40 chars) is > max_chunk_size (30), so splitOversizedParagraph fires. + // But each sentence (~19 chars) is < min_chunk_size (20), so backward merge kicks in, + // producing 1 chunk per section = 2 chunks total. + expect(chunks).toHaveLength(2) + expect(chunks[0]?.content).toContain("Section: Section A") + expect(chunks[1]?.content).toContain("Section: Section B") + // Section B chunk should NOT contain overlap from Section A + expect(getRagChunkBodyText(chunks[1]?.content ?? "")).toBe("Beta sentence one. Beta sentence two.") }) it("omits empty optional lines from the chunk template", () => { @@ -111,6 +135,124 @@ describe("core/rag/chunking", () => { expect(getRagChunkBodyLength(text)).toBe("Body only".length) }) + describe("paragraph-first assembly with real note content", () => { + it("produces multiple chunks, not a single chunk", () => { + const chunks = buildRagIndexChunks({ + title: "тестовый тайтл", + html: USER_NOTE_HTML, + tags: ["тег1", "тег2"], + settings: USER_SETTINGS, + }) + + expect(chunks.length).toBeGreaterThan(1) + }) + + it("keeps first two small paragraphs together as one chunk (>= min_chunk_size)", () => { + const chunks = buildRagIndexChunks({ + title: "тестовый тайтл", + html: USER_NOTE_HTML, + tags: ["тег1", "тег2"], + settings: USER_SETTINGS, + }) + + const body0 = getRagChunkBodyText(chunks[0]?.content ?? "") + // First chunk should contain both P1 and P2 + expect(body0).toContain('"То что на предыдущем уровне') + expect(body0).toContain("Чувство голода может") + // First chunk should NOT contain text from paragraph 3 + expect(body0).not.toContain("Что такое субъект и объект") + }) + + it("does not cross paragraph boundaries when chunk is above min_chunk_size", () => { + const chunks = buildRagIndexChunks({ + title: "тестовый тайтл", + html: USER_NOTE_HTML, + tags: ["тег1", "тег2"], + settings: USER_SETTINGS, + }) + + // Each chunk body should contain only whole paragraphs + for (const chunk of chunks) { + const body = getRagChunkBodyText(chunk.content) + // No paragraph text should be cut mid-sentence at paragraph boundary + // (overlap prefix from previous chunk is OK) + expect(body.length).toBeGreaterThan(0) + } + + // Chunk 1 (index 1) should start with paragraph 3 content (possibly with overlap prefix) + const body1 = getRagChunkBodyText(chunks[1]?.content ?? "") + expect(body1).toContain("Что такое субъект и объект") + }) + + it("each chunk body respects target_chunk_size for accumulation decisions", () => { + const chunks = buildRagIndexChunks({ + title: "тестовый тайтл", + html: USER_NOTE_HTML, + tags: ["тег1", "тег2"], + settings: USER_SETTINGS, + }) + + // With 4 paragraphs (~83, ~245, ~530, ~570 chars) and target=500: + // Chunk 0: P1+P2 (~330 chars, >= min, P3 would exceed target → close) + // Chunk 1: P3 (~530 chars, >= min, P4 would exceed target → close) + // Chunk 2: P4 (~570 chars) + expect(chunks).toHaveLength(3) + }) + }) + + describe("oversized paragraph splitting with backward merge", () => { + it("splits oversized paragraph at max_chunk_size boundaries, not target", () => { + // Create a paragraph that is > max_chunk_size + const sentence = "Это предложение для теста длинного абзаца. " + const longParagraph = sentence.repeat(50) // ~2200 chars + const html = `

${longParagraph}

` + + const chunks = buildRagIndexChunks({ + title: "test", + html, + tags: [], + settings: { + ...USER_SETTINGS, + small_note_threshold: 50, + }, + }) + + // With max=1500, a ~2200 char paragraph should produce 2 chunks, not 5 (which target=500 would make) + expect(chunks).toHaveLength(2) + }) + + it("backward-merges small remainder into previous piece", () => { + // Create a paragraph where splitting at max would leave a tiny remainder + const sentence = "Это тестовое предложение номер один. " + // Need something slightly over max_chunk_size with small remainder + const longParagraph = sentence.repeat(42) // ~42 * 36 = ~1512, remainder ~12 chars + const html = `

${longParagraph}

` + + const settings = { + ...USER_SETTINGS, + small_note_threshold: 50, + max_chunk_size: 1500, + min_chunk_size: 200, + } + + const chunks = buildRagIndexChunks({ + title: "test", + html, + tags: [], + settings, + }) + + // If remainder < min_chunk_size, it should be merged back → single chunk + // (the merged chunk will be slightly above max_chunk_size) + if (chunks.length === 1) { + // Backward merge happened — chunk may exceed max but bounded by max + min - 1 + expect(getRagChunkBodyText(chunks[0]?.content ?? "").length).toBeLessThanOrEqual( + settings.max_chunk_size + settings.min_chunk_size - 1 + ) + } + }) + }) + it("does not create sections from non-heading formatting alone", () => { const chunks = buildRagIndexChunks({ title: "Formatting note", diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index ee1567ac406..0a27a3841cd 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -15,6 +15,7 @@ The chunk assembly design has been refined after review. These rules are the lat - Once `min_chunk_size` is reached, the assembler may add another whole paragraph only if doing so still fits naturally and moves the chunk closer to `target_chunk_size`. - A whole next paragraph must not be added if it would overshoot `target_chunk_size`, even when it would still fit in `max_chunk_size`. - If the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, the next paragraph may be split internally to complete a minimally valid chunk. +- Oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts), not at `target_chunk_size`. If the last piece after splitting is below `min_chunk_size`, it is merged back into the previous piece. This conscious compromise bounds the effective maximum at `max_chunk_size + min_chunk_size - 1`. - Final trailing undersized chunks should try backward merge first; if that fails because of `max_chunk_size`, they remain undersized. - Overlap is intentionally one-directional: `chunk[i + 1] = suffix(chunk[i]) + new_content`. - Overlap must not cross a section boundary and should prefer natural stop points such as sentence-ending period or text boundary. diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index 0e6a127497e..70ca2f330d0 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -15,6 +15,8 @@ Implementation must now follow these clarified chunk-assembly rules: - after `min_chunk_size` is reached, another whole paragraph may be appended only if it improves fit toward `target_chunk_size` - do not append a whole paragraph that would overshoot `target_chunk_size`, even if it is still within `max_chunk_size` - if the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that next paragraph internally to finish the chunk +- oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts), not at `target_chunk_size` +- if the last piece after splitting an oversized paragraph is below `min_chunk_size`, merge it back into the previous piece (backward merge); effective maximum is `max_chunk_size + min_chunk_size - 1` - when a trailing chunk is undersized, try backward merge first and leave it undersized if merging would exceed `max_chunk_size` - keep overlap one-directional from previous chunk into next chunk diff --git a/docs/ai/planning/feature-improve-rag-chunking.md b/docs/ai/planning/feature-improve-rag-chunking.md index 252b81fcb18..fc8340fbda8 100644 --- a/docs/ai/planning/feature-improve-rag-chunking.md +++ b/docs/ai/planning/feature-improve-rag-chunking.md @@ -15,6 +15,8 @@ Latest clarified behavior to preserve during implementation: - after reaching `min_chunk_size`, only add another whole paragraph if it improves fit toward `target_chunk_size` - do not add a whole paragraph that overshoots `target_chunk_size`, even if it still fits in `max_chunk_size` - if still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that paragraph internally as a compromise +- oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts, not `target_chunk_size`) +- if the last piece after splitting an oversized paragraph is below `min_chunk_size`, it is merged back into the previous piece; effective max = `max_chunk_size + min_chunk_size - 1` - trailing undersized chunks try backward merge first - overlap is one-directional from previous chunk into next chunk diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md index 031bd0f8cbe..e1471ee8b07 100644 --- a/docs/ai/requirements/feature-improve-rag-chunking.md +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -16,7 +16,8 @@ This feature now uses a stricter `paragraph-first` interpretation of hierarchica - If the next whole paragraph would overshoot `target_chunk_size`, it must not be added just to make the chunk larger, even if it would still fit within `max_chunk_size`. - If a chunk is still below `min_chunk_size` and the next whole paragraph fits within `max_chunk_size`, that whole paragraph should be added. - If a chunk is still below `min_chunk_size` but the next whole paragraph would exceed `max_chunk_size`, the next paragraph may be split internally as a compromise to reach a valid chunk. -- Oversized paragraphs are still split internally by sentences and then by token/character fallback when needed. +- Oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts) by sentences and then by token/character fallback when needed. +- If the last piece after splitting an oversized paragraph is below `min_chunk_size`, it is merged back into the previous piece (backward merge). This may produce a chunk slightly above `max_chunk_size`, bounded by `max_chunk_size + min_chunk_size - 1`. - A trailing undersized chunk should first try to merge backward with the previous chunk; if that would exceed `max_chunk_size`, the undersized tail remains as-is. - Overlap remains one-directional: each next chunk repeats a suffix of the previous chunk at its beginning. - Overlap should prefer natural boundaries, currently using explicit stop points such as sentence-ending period, section boundary, or text boundary. diff --git a/docs/ai/testing/feature-improve-rag-chunking.md b/docs/ai/testing/feature-improve-rag-chunking.md index 707c121e0d1..7c41cb52dc1 100644 --- a/docs/ai/testing/feature-improve-rag-chunking.md +++ b/docs/ai/testing/feature-improve-rag-chunking.md @@ -14,6 +14,8 @@ The test plan must explicitly protect the newly clarified paragraph-first rules: - `min_chunk_size` reached before optional extension toward `target_chunk_size` - no whole-paragraph append when it overshoots `target_chunk_size` - partial split of the next paragraph only when needed to escape an undersized chunk that cannot fit the whole paragraph under `max_chunk_size` +- oversized paragraphs (> `max_chunk_size`) split at `max_chunk_size` boundaries (minimal cuts), not `target_chunk_size` +- if the last piece after splitting an oversized paragraph is below `min_chunk_size`, it is merged back into the previous piece; effective max = `max_chunk_size + min_chunk_size - 1` - trailing undersized chunk tries backward merge first - overlap remains one-directional from previous chunk into next chunk @@ -56,6 +58,9 @@ The test plan must explicitly protect the newly clarified paragraph-first rules: - [ ] After reaching `min_chunk_size`, another whole paragraph is appended only when it improves fit toward `target_chunk_size` - [ ] A whole next paragraph is not appended when it would overshoot `target_chunk_size`, even if still within `max_chunk_size` - [ ] If still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, the paragraph is split internally as fallback +- [ ] Oversized paragraph (> max_chunk_size) is split at max_chunk_size boundaries, not target_chunk_size +- [ ] If last piece of split oversized paragraph is below min_chunk_size, it is backward-merged into the previous piece +- [ ] Backward merge may produce a piece up to max_chunk_size + min_chunk_size - 1 characters - [ ] Accumulation stops before violating `max_chunk_size` - [ ] Undersized final trailing chunk merges with previous neighbor when allowed - [ ] Undersized chunk remains standalone when merging would exceed `max_chunk_size` diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx index 1bf6064a70f..76b38b65054 100644 --- a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -8,9 +8,11 @@ import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/com import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" import { Switch } from "@/components/ui/switch" +import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" import { RagIndexSettingsService } from "@core/services/ragIndexSettings" import type { RagIndexingEditableSettings, RagIndexingSettings } from "@core/rag/indexingSettings" import { useSupabase } from "@ui/web/providers/SupabaseProvider" +import { Info } from "lucide-react" type EditableNumericKey = keyof Pick< RagIndexingEditableSettings, @@ -139,12 +141,22 @@ export function RagIndexingSettingsPanel() { disabled={loading || saving} onChange={(value) => updateNumericField("min_chunk_size", value)} /> - updateNumericField("max_chunk_size", value)} + tooltip={(() => { + const max = Number(formState.max_chunk_size) || 0 + const min = Number(formState.min_chunk_size) || 0 + const effectiveMax = max + min - 1 + return ( + `When an oversized paragraph is split, a small remainder (< min_chunk_size) ` + + `is merged back into the previous piece. The effective maximum in that case:\n` + + `max_chunk_size + min_chunk_size - 1 = ${max} + ${min} - 1 = ${effectiveMax} characters.` + ) + })()} /> void + tooltip: string +}) { + return ( +
+
+ + + + + + + + {tooltip} + + + +
+ onChange(event.target.value)} + disabled={disabled} + /> +
+ ) +} + function ToggleRow({ id, label, From 2c011ca04dcc860fa70986c39b4e1fda9ed50f59 Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 22:11:34 +0100 Subject: [PATCH 08/18] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=20=D1=83=D1=81=D1=82=D0=B0=D1=80=D0=B5=D0=B2?= =?UTF-8?q?=D1=88=D0=B8=D0=B9=20chunk=5Faccumulation=5Frule=20=D0=B2=20des?= =?UTF-8?q?ign=20doc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit В representative payload стояла старая формулировка правила сборки чанков («accumulate toward target_chunk_size»), которая не соответствовала текущей paragraph-first реализации. Обновлено на актуальное описание из indexingSettings.ts. Co-Authored-By: Claude Opus 4.6 --- docs/ai/design/feature-improve-rag-chunking.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index 0a27a3841cd..c695537dc3b 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -168,7 +168,7 @@ Representative payload: "task_type_query": "RETRIEVAL_QUERY", "split_strategy": "hierarchical", "fallback_split_order": ["sections", "paragraphs", "sentences", "tokens_or_characters"], - "chunk_accumulation_rule": "Accumulate neighboring small paragraphs within the same section until target_chunk_size is reached or max_chunk_size would be exceeded.", + "chunk_accumulation_rule": "Paragraph-first: accumulate whole paragraphs until min_chunk_size is reached, then optionally extend toward target_chunk_size only if the next whole paragraph fits without exceeding it. Oversized paragraphs are split at max_chunk_size boundaries; if the remainder is below min_chunk_size it is merged back into the previous piece.", "small_chunk_merge_rule": "Merge undersized final chunks with adjacent chunks when possible without violating max_chunk_size.", "chunk_template": "Section: {section_heading}\\nTags: {tag1}, {tag2}, {tag3}\\n\\n{chunk_content}" } From 83480d95c444d5393afa5ca4ca7a85f4b850c451 Mon Sep 17 00:00:00 2001 From: Denys Date: Tue, 17 Mar 2026 22:28:48 +0100 Subject: [PATCH 09/18] =?UTF-8?q?=D0=A3=D0=B4=D0=B0=D0=BB=D1=91=D0=BD=20sm?= =?UTF-8?q?all=5Fnote=5Fthreshold,=20min=5Fchunk=5Fsize=20=D1=82=D0=B5?= =?UTF-8?q?=D0=BF=D0=B5=D1=80=D1=8C=20=D0=BE=D0=BF=D1=80=D0=B5=D0=B4=D0=B5?= =?UTF-8?q?=D0=BB=D1=8F=D0=B5=D1=82=20=D0=BC=D0=B8=D0=BD=D0=B8=D0=BC=D0=B0?= =?UTF-8?q?=D0=BB=D1=8C=D0=BD=D1=8B=D0=B9=20=D1=80=D0=B0=D0=B7=D0=BC=D0=B5?= =?UTF-8?q?=D1=80=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=82=D0=BA=D0=B8=20=D0=B4?= =?UTF-8?q?=D0=BB=D1=8F=20=D0=B8=D0=BD=D0=B4=D0=B5=D0=BA=D1=81=D0=B0=D1=86?= =?UTF-8?q?=D0=B8=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Параметр small_note_threshold убран из настроек, БД, UI и edge functions. Теперь min_chunk_size выполняет двойную роль: минимальный размер чанка при сборке и минимальный размер заметки для индексации. Заметки короче min_chunk_size не индексируются (возвращается 0 чанков, существующие эмбеддинги удаляются, клиент получает skipped: "too_short"). Изменения: - core/rag: удалён small_note_threshold из типов, дефолтов и валидации - core/rag/chunking.ts: удалена buildWholeNoteChunk, добавлена проверка noteBodyLength < min_chunk_size → возврат пустого массива - edge functions: убрана колонка из SELECT-запросов, добавлен ответ skipped/message при 0 чанков - миграция: убрана колонка и constraint из таблицы (только локально) - UI: убрано поле из формы настроек - тесты и документация обновлены Co-Authored-By: Claude Opus 4.6 --- core/rag/chunking.ts | 39 +++++-------------- core/rag/indexingSettings.ts | 5 +-- core/services/ragIndexSettings.ts | 1 - core/tests/unit/core-rag-chunking.test.ts | 24 ++++++++---- .../unit/core-rag-indexingSettings.test.ts | 4 -- .../ai/design/feature-improve-rag-chunking.md | 6 +-- .../feature-improve-rag-chunking.md | 6 +-- .../planning/feature-improve-rag-chunking.md | 1 + .../feature-improve-rag-chunking.md | 12 +++--- .../testing/feature-improve-rag-chunking.md | 6 ++- supabase/functions/api-keys-status/index.ts | 2 +- supabase/functions/api-keys-upsert/index.ts | 2 +- supabase/functions/rag-index/index.ts | 9 +++-- ...0317000001_add_user_rag_index_settings.sql | 2 - .../settings/RagIndexingSettingsPanel.tsx | 12 +----- 15 files changed, 51 insertions(+), 80 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index d4bf5177f73..098e2480161 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -465,28 +465,6 @@ function applyFinalOverlap(chunks: CandidateChunk[], overlap: number): Candidate }) } -function buildWholeNoteChunk( - blocks: IndexedBlock[], - html: string, - settings: RagIndexingEditableSettings -): CandidateChunk[] { - const contentFromBlocks = joinChunkParts(blocks.map((block) => block.text)) - const fallbackContent = stripTags(html) - const text = contentFromBlocks || fallbackContent - if (!text) return [] - - const firstSection = blocks[0]?.sectionHeading ?? null - const hasSingleSection = blocks.every((block) => block.sectionHeading === firstSection) - - return [ - { - sectionHeading: hasSingleSection ? firstSection : null, - text, - charOffset: blocks[0]?.charOffset ?? 0, - }, - ] -} - export function buildRagIndexChunks({ title, html, @@ -496,13 +474,16 @@ export function buildRagIndexChunks({ const normalizedTags = Array.isArray(tags) ? tags.filter((tag): tag is string => typeof tag === "string") : [] const blocks = extractBlocksFromHtml(html ?? "") const noteBodyLength = joinChunkParts(blocks.map((block) => block.text)).length - const baseChunks = - noteBodyLength > 0 && noteBodyLength <= settings.small_note_threshold - ? buildWholeNoteChunk(blocks, html ?? "", settings) - : mergeUndersizedTail( - assembleParagraphFirst(blocks, settings), - settings - ) + + // Notes shorter than min_chunk_size are not indexed + if (noteBodyLength < settings.min_chunk_size) { + return [] + } + + const baseChunks = mergeUndersizedTail( + assembleParagraphFirst(blocks, settings), + settings + ) const finalChunks = applyFinalOverlap(baseChunks, settings.overlap) const embeddingTitle = buildRagEmbeddingTitle(title, settings) diff --git a/core/rag/indexingSettings.ts b/core/rag/indexingSettings.ts index 94e1ceae3ab..a68bd5166de 100644 --- a/core/rag/indexingSettings.ts +++ b/core/rag/indexingSettings.ts @@ -2,7 +2,6 @@ export const RAG_INDEX_NUMERIC_MIN = 50 export const RAG_INDEX_NUMERIC_MAX = 5000 export const RAG_INDEX_EDITABLE_DEFAULTS = { - small_note_threshold: 400, target_chunk_size: 500, min_chunk_size: 200, max_chunk_size: 1500, @@ -19,14 +18,13 @@ export const RAG_INDEX_READONLY_SETTINGS = { split_strategy: "hierarchical" as const, fallback_split_order: ["sections", "paragraphs", "sentences", "tokens_or_characters"] as const, chunk_accumulation_rule: - "Paragraph-first: accumulate whole paragraphs until min_chunk_size is reached, then optionally extend toward target_chunk_size only if the next whole paragraph fits without exceeding it. Oversized paragraphs are split at max_chunk_size boundaries; if the remainder is below min_chunk_size it is merged back into the previous piece.", + "Paragraph-first: accumulate whole paragraphs until min_chunk_size is reached, then optionally extend toward target_chunk_size only if the next whole paragraph fits without exceeding it. Oversized paragraphs are split at max_chunk_size boundaries; if the remainder is below min_chunk_size it is merged back into the previous piece. Notes shorter than min_chunk_size are not indexed.", small_chunk_merge_rule: "Merge undersized final chunks with adjacent chunks when possible without violating max_chunk_size.", chunk_template: "Section: {section_heading}\nTags: {tag1}, {tag2}, {tag3}\n\n{chunk_content}", } as const export type RagIndexingEditableSettings = { - small_note_threshold: number target_chunk_size: number min_chunk_size: number max_chunk_size: number @@ -39,7 +37,6 @@ export type RagIndexingEditableSettings = { export type RagIndexingSettings = RagIndexingEditableSettings & typeof RAG_INDEX_READONLY_SETTINGS export const RAG_INDEX_EDITABLE_NUMERIC_KEYS = [ - "small_note_threshold", "target_chunk_size", "min_chunk_size", "max_chunk_size", diff --git a/core/services/ragIndexSettings.ts b/core/services/ragIndexSettings.ts index 402db507ca4..89d6179c5c2 100644 --- a/core/services/ragIndexSettings.ts +++ b/core/services/ragIndexSettings.ts @@ -31,7 +31,6 @@ const readErrorMessage = async (error: unknown, fallback: string) => { const isRagIndexingSettings = (data: unknown): data is RagIndexingSettings => { if (!data || typeof data !== "object") return false return ( - typeof (data as { small_note_threshold?: unknown }).small_note_threshold === "number" && typeof (data as { target_chunk_size?: unknown }).target_chunk_size === "number" && typeof (data as { min_chunk_size?: unknown }).min_chunk_size === "number" && typeof (data as { max_chunk_size?: unknown }).max_chunk_size === "number" && diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index 14f2cacf6b5..65114aef297 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -22,7 +22,7 @@ const USER_SETTINGS = { } describe("core/rag/chunking", () => { - it("keeps a small note as a single chunk", () => { + it("does not index a note shorter than min_chunk_size", () => { const chunks = buildRagIndexChunks({ title: "Weekly plan", html: "

Short body text.

", @@ -30,10 +30,22 @@ describe("core/rag/chunking", () => { settings: RAG_INDEX_EDITABLE_DEFAULTS, }) + // "Short body text." is ~17 chars, well below min_chunk_size (200) + expect(chunks).toHaveLength(0) + }) + + it("indexes a note at exactly min_chunk_size as a single chunk", () => { + const body = "A".repeat(RAG_INDEX_EDITABLE_DEFAULTS.min_chunk_size) + const chunks = buildRagIndexChunks({ + title: "Exact min", + html: `

${body}

`, + tags: ["work"], + settings: RAG_INDEX_EDITABLE_DEFAULTS, + }) + expect(chunks).toHaveLength(1) expect(chunks[0]?.content).toContain("Tags: work") - expect(chunks[0]?.content).toContain("Short body text.") - expect(chunks[0]?.title).toBe("Weekly plan") + expect(chunks[0]?.title).toBe("Exact min") }) it("splits large paragraphs and adds overlap to later chunks", () => { @@ -46,7 +58,6 @@ describe("core/rag/chunking", () => { tags: ["alpha", "beta"], settings: { ...RAG_INDEX_EDITABLE_DEFAULTS, - small_note_threshold: 50, target_chunk_size: 80, min_chunk_size: 50, max_chunk_size: 90, @@ -213,8 +224,7 @@ describe("core/rag/chunking", () => { tags: [], settings: { ...USER_SETTINGS, - small_note_threshold: 50, - }, + }, }) // With max=1500, a ~2200 char paragraph should produce 2 chunks, not 5 (which target=500 would make) @@ -230,7 +240,6 @@ describe("core/rag/chunking", () => { const settings = { ...USER_SETTINGS, - small_note_threshold: 50, max_chunk_size: 1500, min_chunk_size: 200, } @@ -260,7 +269,6 @@ describe("core/rag/chunking", () => { tags: [], settings: { ...RAG_INDEX_EDITABLE_DEFAULTS, - small_note_threshold: 50, target_chunk_size: 30, min_chunk_size: 20, max_chunk_size: 30, diff --git a/core/tests/unit/core-rag-indexingSettings.test.ts b/core/tests/unit/core-rag-indexingSettings.test.ts index afb77566138..fe5c08df444 100644 --- a/core/tests/unit/core-rag-indexingSettings.test.ts +++ b/core/tests/unit/core-rag-indexingSettings.test.ts @@ -8,7 +8,6 @@ describe("core/rag/indexingSettings", () => { it("returns defaults plus read-only settings when no editable overrides exist", () => { const settings = resolveRagIndexingSettings() - expect(settings.small_note_threshold).toBe(RAG_INDEX_EDITABLE_DEFAULTS.small_note_threshold) expect(settings.target_chunk_size).toBe(RAG_INDEX_EDITABLE_DEFAULTS.target_chunk_size) expect(settings.output_dimensionality).toBe(1536) expect(settings.task_type_document).toBe("RETRIEVAL_DOCUMENT") @@ -16,9 +15,6 @@ describe("core/rag/indexingSettings", () => { }) it("rejects numeric values outside the allowed range", () => { - expect(() => assertValidRagIndexingEditableSettings({ small_note_threshold: 49 })).toThrow( - "small_note_threshold must be between 50 and 5000" - ) expect(() => assertValidRagIndexingEditableSettings({ overlap: 5001 })).toThrow( "overlap must be between 50 and 5000" ) diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index c695537dc3b..bc318a95f22 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -15,6 +15,7 @@ The chunk assembly design has been refined after review. These rules are the lat - Once `min_chunk_size` is reached, the assembler may add another whole paragraph only if doing so still fits naturally and moves the chunk closer to `target_chunk_size`. - A whole next paragraph must not be added if it would overshoot `target_chunk_size`, even when it would still fit in `max_chunk_size`. - If the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, the next paragraph may be split internally to complete a minimally valid chunk. +- Notes shorter than `min_chunk_size` are not indexed at all (`small_note_threshold` has been removed; `min_chunk_size` now determines both the minimum chunk size and the minimum note size for indexing). - Oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts), not at `target_chunk_size`. If the last piece after splitting is below `min_chunk_size`, it is merged back into the previous piece. This conscious compromise bounds the effective maximum at `max_chunk_size + min_chunk_size - 1`. - Final trailing undersized chunks should try backward merge first; if that fails because of `max_chunk_size`, they remain undersized. - Overlap is intentionally one-directional: `chunk[i + 1] = suffix(chunk[i]) + new_content`. @@ -66,7 +67,6 @@ Representative model: ```sql user_rag_index_settings ( user_id uuid primary key references auth.users(id) on delete cascade, - small_note_threshold integer not null default 400, target_chunk_size integer not null default 500, min_chunk_size integer not null default 200, max_chunk_size integer not null default 1500, @@ -82,7 +82,6 @@ The effective settings object exposed to the UI and indexing paths must support: ```ts type RagIndexingSettings = { - small_note_threshold: number target_chunk_size: number min_chunk_size: number max_chunk_size: number @@ -155,7 +154,6 @@ Representative payload: ```json { - "small_note_threshold": 400, "target_chunk_size": 500, "min_chunk_size": 200, "max_chunk_size": 1500, @@ -311,13 +309,11 @@ This feature does not alter search ranking logic, but the design must preserve: ## Open Design Items - Start defaults are: - - `small_note_threshold = 400` - `target_chunk_size = 500` - `min_chunk_size = 200` - `max_chunk_size = 1500` - `overlap = 100` - Validation ranges for editable numeric settings are: - - `small_note_threshold`: `50..5000` - `target_chunk_size`: `50..5000` - `min_chunk_size`: `50..5000` - `max_chunk_size`: `50..5000` diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index 70ca2f330d0..6ea503af98c 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -15,6 +15,7 @@ Implementation must now follow these clarified chunk-assembly rules: - after `min_chunk_size` is reached, another whole paragraph may be appended only if it improves fit toward `target_chunk_size` - do not append a whole paragraph that would overshoot `target_chunk_size`, even if it is still within `max_chunk_size` - if the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that next paragraph internally to finish the chunk +- notes shorter than `min_chunk_size` are not indexed (return empty array); `small_note_threshold` has been removed, `min_chunk_size` now serves both as minimum chunk size and minimum note size - oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts), not at `target_chunk_size` - if the last piece after splitting an oversized paragraph is below `min_chunk_size`, merge it back into the previous piece (backward merge); effective maximum is `max_chunk_size + min_chunk_size - 1` - when a trailing chunk is undersized, try backward merge first and leave it undersized if merging would exceed `max_chunk_size` @@ -78,7 +79,7 @@ Suggested processing flow: 1. Normalize note content to a structure suitable for section/paragraph detection. 2. Compute note size using the same unit chosen for settings semantics. -3. If size is below `small_note_threshold`, emit one final chunk. +3. If size is below `min_chunk_size`, return no chunks (note is too short for indexing). 4. Otherwise: - split into sections using `h1-h6` tags only - split each section into paragraphs @@ -156,7 +157,7 @@ function validateRagIndexingSettings(input: Partial): Valid Validation rules to enforce in both UI and server paths: -- `small_note_threshold`, `target_chunk_size`, `min_chunk_size`, `max_chunk_size`, `overlap` must each be within `50..5000` +- `target_chunk_size`, `min_chunk_size`, `max_chunk_size`, `overlap` must each be within `50..5000` - `min_chunk_size <= target_chunk_size <= max_chunk_size` Representative placement: @@ -173,7 +174,6 @@ Recommended defaults in `core`: ```ts const DEFAULT_RAG_INDEX_SETTINGS = { - small_note_threshold: 400, target_chunk_size: 500, min_chunk_size: 200, max_chunk_size: 1500, diff --git a/docs/ai/planning/feature-improve-rag-chunking.md b/docs/ai/planning/feature-improve-rag-chunking.md index fc8340fbda8..03d56e023df 100644 --- a/docs/ai/planning/feature-improve-rag-chunking.md +++ b/docs/ai/planning/feature-improve-rag-chunking.md @@ -15,6 +15,7 @@ Latest clarified behavior to preserve during implementation: - after reaching `min_chunk_size`, only add another whole paragraph if it improves fit toward `target_chunk_size` - do not add a whole paragraph that overshoots `target_chunk_size`, even if it still fits in `max_chunk_size` - if still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that paragraph internally as a compromise +- notes shorter than `min_chunk_size` are not indexed at all (`small_note_threshold` removed, `min_chunk_size` serves both roles) - oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts, not `target_chunk_size`) - if the last piece after splitting an oversized paragraph is below `min_chunk_size`, it is merged back into the previous piece; effective max = `max_chunk_size + min_chunk_size - 1` - trailing undersized chunks try backward merge first diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md index e1471ee8b07..8eb14ffd466 100644 --- a/docs/ai/requirements/feature-improve-rag-chunking.md +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -16,6 +16,7 @@ This feature now uses a stricter `paragraph-first` interpretation of hierarchica - If the next whole paragraph would overshoot `target_chunk_size`, it must not be added just to make the chunk larger, even if it would still fit within `max_chunk_size`. - If a chunk is still below `min_chunk_size` and the next whole paragraph fits within `max_chunk_size`, that whole paragraph should be added. - If a chunk is still below `min_chunk_size` but the next whole paragraph would exceed `max_chunk_size`, the next paragraph may be split internally as a compromise to reach a valid chunk. +- Notes shorter than `min_chunk_size` are not indexed at all (the `small_note_threshold` parameter has been removed; `min_chunk_size` now serves both as the minimum chunk size and the minimum note size for indexing). - Oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts) by sentences and then by token/character fallback when needed. - If the last piece after splitting an oversized paragraph is below `min_chunk_size`, it is merged back into the previous piece (backward merge). This may produce a chunk slightly above `max_chunk_size`, bounded by `max_chunk_size + min_chunk_size - 1`. - A trailing undersized chunk should first try to merge backward with the previous chunk; if that would exceed `max_chunk_size`, the undersized tail remains as-is. @@ -66,7 +67,7 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s - **As a user configuring AI indexing**, I want these settings to live in my Google API settings area so that related AI configuration is managed in one place. - **As a user configuring AI indexing**, I want to edit chunk sizing and overlap settings so that indexing quality can be tuned without redeploy. - **As a user configuring AI indexing**, I want title, section headings, and tags to be explicit indexing inputs that can be enabled or disabled. -- **As a user searching small notes**, I want short notes to remain whole so that their context is preserved. +- **As a user searching small notes**, I want notes shorter than `min_chunk_size` to be skipped during indexing, since they lack enough content for meaningful semantic search. - **As a user searching large notes**, I want notes to be split on natural boundaries first so that retrieved chunks stay coherent. - **As a system**, I want tiny neighboring paragraphs to accumulate into a target-sized chunk so that the index avoids fragmented low-value chunks. - **As a system**, I want tiny neighboring paragraphs to merge paragraph-first so that natural paragraph boundaries stay intact whenever possible. @@ -87,7 +88,7 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s ### Edge cases -- Note is smaller than `small_note_threshold` and should be indexed as a single chunk. +- Note is shorter than `min_chunk_size` and should not be indexed. - Note has no section headings and must fall back directly to paragraph-based chunking. - A paragraph is larger than `max_chunk_size` and must be split deeper. - The last chunk is too small and should merge with a neighbor if size constraints allow. @@ -103,7 +104,6 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s - [ ] Indexing settings are exposed in the user's Google API settings tab. - [ ] Indexing settings UI is added on the web site only for this feature. - [ ] Editable UI settings include: - - `small_note_threshold` - `target_chunk_size` - `min_chunk_size` - `max_chunk_size` @@ -120,7 +120,7 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s - chunk structure template - chunk accumulation rule - small chunk merge rule -- [ ] Notes below `small_note_threshold` are indexed as a single chunk unless prevented by system constraints. +- [ ] Notes shorter than `min_chunk_size` are not indexed (0 chunks returned, existing embeddings removed). - [ ] Larger notes are split by natural boundaries before using sentence-level and token/character fallback splitting. - [ ] Small adjacent paragraphs accumulate paragraph-first, reaching `min_chunk_size` first and extending toward `target_chunk_size` only when additional whole paragraphs fit naturally. - [ ] Oversized paragraphs are split deeper until all final chunks satisfy `max_chunk_size`. @@ -171,12 +171,12 @@ Tags: {tag1}, {tag2}, {tag3} - Any omitted optional chunk parts (`Section`, `Tags`) should disappear entirely rather than render as empty labels. - Editable numeric indexing parameters use an allowed range of `50..5000`. - Server-side validation must also enforce logical ordering: `min_chunk_size <= target_chunk_size <= max_chunk_size`. -- `target_chunk_size` remains relevant for oversize paragraph splitting and for deciding whether another whole paragraph should be added after `min_chunk_size` has already been reached. +- `target_chunk_size` remains relevant for deciding whether another whole paragraph should be added after `min_chunk_size` has already been reached. +- `min_chunk_size` serves double duty: it is both the minimum chunk size during assembly and the minimum note size for indexing eligibility. ## Questions & Open Items - Start defaults are fixed as: - - `small_note_threshold = 400` - `target_chunk_size = 500` - `min_chunk_size = 200` - `max_chunk_size = 1500` diff --git a/docs/ai/testing/feature-improve-rag-chunking.md b/docs/ai/testing/feature-improve-rag-chunking.md index 7c41cb52dc1..95adf1ecfab 100644 --- a/docs/ai/testing/feature-improve-rag-chunking.md +++ b/docs/ai/testing/feature-improve-rag-chunking.md @@ -14,6 +14,7 @@ The test plan must explicitly protect the newly clarified paragraph-first rules: - `min_chunk_size` reached before optional extension toward `target_chunk_size` - no whole-paragraph append when it overshoots `target_chunk_size` - partial split of the next paragraph only when needed to escape an undersized chunk that cannot fit the whole paragraph under `max_chunk_size` +- notes shorter than `min_chunk_size` are not indexed (0 chunks, existing embeddings removed) - oversized paragraphs (> `max_chunk_size`) split at `max_chunk_size` boundaries (minimal cuts), not `target_chunk_size` - if the last piece after splitting an oversized paragraph is below `min_chunk_size`, it is merged back into the previous piece; effective max = `max_chunk_size + min_chunk_size - 1` - trailing undersized chunk tries backward merge first @@ -45,7 +46,8 @@ The test plan must explicitly protect the newly clarified paragraph-first rules: ### Hierarchical segmentation -- [ ] Small note below `small_note_threshold` returns a single final chunk +- [ ] Note shorter than `min_chunk_size` returns 0 chunks (not indexed) +- [ ] Note at exactly `min_chunk_size` returns a single chunk - [ ] Multi-section note with `h1-h6` headings prefers section boundaries before paragraph fallback - [ ] Notes without headings fall back directly to paragraph splitting - [ ] Non-heading styled text does not create synthetic sections @@ -99,7 +101,7 @@ The test plan must explicitly protect the newly clarified paragraph-first rules: - [ ] Verify read-only parameters, including `output_dimensionality`, are visible but not editable - [ ] Save valid settings and confirm they persist after reload - [ ] Attempt to save invalid settings and confirm inline validation blocks the change -- [ ] Reindex a small note and verify one chunk is produced +- [ ] Reindex a note shorter than `min_chunk_size` and verify 0 chunks with `skipped: "too_short"` response - [ ] Reindex a long structured note and verify chunk count changes according to settings - [ ] Toggle title inclusion off and verify indexed chunk bodies do not gain title text - [ ] Toggle section heading and tag inclusion on/off and verify chunk content changes accordingly diff --git a/supabase/functions/api-keys-status/index.ts b/supabase/functions/api-keys-status/index.ts index 04c7e76819d..92024d20f24 100644 --- a/supabase/functions/api-keys-status/index.ts +++ b/supabase/functions/api-keys-status/index.ts @@ -45,7 +45,7 @@ serve(async (req: Request) => { const { data: ragIndexingData, error: ragIndexingError } = await supabaseAdmin .from("user_rag_index_settings") - .select("small_note_threshold, target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") + .select("target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") .eq("user_id", userData.user.id) .maybeSingle() diff --git a/supabase/functions/api-keys-upsert/index.ts b/supabase/functions/api-keys-upsert/index.ts index 86cacf0346e..7bedb2a6ccb 100644 --- a/supabase/functions/api-keys-upsert/index.ts +++ b/supabase/functions/api-keys-upsert/index.ts @@ -155,7 +155,7 @@ serve(async (req: Request) => { } else { const { data: ragIndexingData, error: ragIndexingError } = await supabaseAdmin .from("user_rag_index_settings") - .select("small_note_threshold, target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") + .select("target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") .eq("user_id", userId) .maybeSingle() diff --git a/supabase/functions/rag-index/index.ts b/supabase/functions/rag-index/index.ts index e1fb52342e2..9c77a106dec 100644 --- a/supabase/functions/rag-index/index.ts +++ b/supabase/functions/rag-index/index.ts @@ -235,7 +235,7 @@ serve(async (req: Request) => { const { data: settingsRow, error: settingsError } = await supabaseAdmin .from("user_rag_index_settings") - .select("small_note_threshold, target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") + .select("target_chunk_size, min_chunk_size, max_chunk_size, overlap, use_title, use_section_headings, use_tags") .eq("user_id", userId) .maybeSingle() @@ -275,7 +275,11 @@ serve(async (req: Request) => { .eq("note_id", noteId) .eq("user_id", userId) if (clearError) throw clearError - return jsonResponse({ chunkCount: 0 }) + return jsonResponse({ + chunkCount: 0, + skipped: "too_short", + message: `Note is too short for indexing (minimum: ${settings.min_chunk_size} characters)`, + }) } const vectors = await embedTexts( @@ -316,7 +320,6 @@ serve(async (req: Request) => { noteId, userId, chunkCount: chunksForIndexing.length, - small_note_threshold: settings.small_note_threshold, target_chunk_size: settings.target_chunk_size, min_chunk_size: settings.min_chunk_size, max_chunk_size: settings.max_chunk_size, diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql index 2426b6ec5d0..bbb905b4f32 100644 --- a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -1,6 +1,5 @@ CREATE TABLE public.user_rag_index_settings ( user_id uuid PRIMARY KEY REFERENCES auth.users(id) ON DELETE CASCADE, - small_note_threshold integer NOT NULL DEFAULT 400, target_chunk_size integer NOT NULL DEFAULT 500, min_chunk_size integer NOT NULL DEFAULT 200, max_chunk_size integer NOT NULL DEFAULT 1500, @@ -9,7 +8,6 @@ CREATE TABLE public.user_rag_index_settings ( use_section_headings boolean NOT NULL DEFAULT true, use_tags boolean NOT NULL DEFAULT true, updated_at timestamp with time zone NOT NULL DEFAULT now(), - CONSTRAINT user_rag_index_settings_small_note_threshold_range CHECK (small_note_threshold BETWEEN 50 AND 5000), CONSTRAINT user_rag_index_settings_target_chunk_size_range CHECK (target_chunk_size BETWEEN 50 AND 5000), CONSTRAINT user_rag_index_settings_min_chunk_size_range CHECK (min_chunk_size BETWEEN 50 AND 5000), CONSTRAINT user_rag_index_settings_max_chunk_size_range CHECK (max_chunk_size BETWEEN 50 AND 5000), diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx index 76b38b65054..be580339980 100644 --- a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -16,14 +16,13 @@ import { Info } from "lucide-react" type EditableNumericKey = keyof Pick< RagIndexingEditableSettings, - "small_note_threshold" | "target_chunk_size" | "min_chunk_size" | "max_chunk_size" | "overlap" + "target_chunk_size" | "min_chunk_size" | "max_chunk_size" | "overlap" > type EditableBooleanKey = keyof Pick function buildEditableState(settings: RagIndexingSettings) { return { - small_note_threshold: String(settings.small_note_threshold), target_chunk_size: String(settings.target_chunk_size), min_chunk_size: String(settings.min_chunk_size), max_chunk_size: String(settings.max_chunk_size), @@ -44,7 +43,6 @@ export function RagIndexingSettingsPanel() { const [successMessage, setSuccessMessage] = React.useState(null) const [resolvedSettings, setResolvedSettings] = React.useState(null) const [formState, setFormState] = React.useState(() => ({ - small_note_threshold: "400", target_chunk_size: "500", min_chunk_size: "200", max_chunk_size: "1500", @@ -87,7 +85,6 @@ export function RagIndexingSettingsPanel() { setSuccessMessage(null) const payload: RagIndexingEditableSettings = { - small_note_threshold: Number(formState.small_note_threshold), target_chunk_size: Number(formState.target_chunk_size), min_chunk_size: Number(formState.min_chunk_size), max_chunk_size: Number(formState.max_chunk_size), @@ -120,13 +117,6 @@ export function RagIndexingSettingsPanel() {
- updateNumericField("small_note_threshold", value)} - /> Date: Tue, 17 Mar 2026 23:26:09 +0100 Subject: [PATCH 10/18] chunking is working --- core/rag/chunking.ts | 67 +- core/rag/indexingSettings.ts | 10 +- core/tests/unit/core-rag-chunking.test.ts | 631 ++++++++++++------ .../unit/core-rag-indexingSettings.test.ts | 23 +- ...0317000001_add_user_rag_index_settings.sql | 2 +- .../settings/RagIndexingSettingsPanel.tsx | 254 ++++--- 6 files changed, 698 insertions(+), 289 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index 098e2480161..792c8e34102 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -93,7 +93,7 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { const blocks: RawBlock[] = [] let currentHeading: string | null = null - const walk = (node: Node) => { + const walk = (node: Node, olIndex: number | null) => { for (const child of Array.from(node.childNodes)) { if (child.nodeType === Node.TEXT_NODE) { const text = normalizeWhitespace(child.textContent ?? "") @@ -111,8 +111,38 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { continue } - if (tagName === "ul" || tagName === "ol" || tagName === "section" || tagName === "article" || tagName === "main") { - walk(element) + if (tagName === "ol") { + let idx = 1 + for (const olChild of Array.from(element.childNodes)) { + if (olChild.nodeType !== Node.ELEMENT_NODE) continue + const olChildTag = (olChild as Element).tagName.toLowerCase() + if (olChildTag === "li") { + const text = normalizeWhitespace(extractElementText(olChild)) + if (text) blocks.push({ sectionHeading: currentHeading, text: `${idx}. ${text}` }) + idx++ + } else { + walk(olChild, null) + } + } + continue + } + + if (tagName === "ul") { + for (const ulChild of Array.from(element.childNodes)) { + if (ulChild.nodeType !== Node.ELEMENT_NODE) continue + const ulChildTag = (ulChild as Element).tagName.toLowerCase() + if (ulChildTag === "li") { + const text = normalizeWhitespace(extractElementText(ulChild)) + if (text) blocks.push({ sectionHeading: currentHeading, text: `- ${text}` }) + } else { + walk(ulChild, null) + } + } + continue + } + + if (tagName === "section" || tagName === "article" || tagName === "main") { + walk(element, null) continue } @@ -123,7 +153,7 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { }) if (hasNestedBlocks) { - walk(element) + walk(element, null) continue } } @@ -134,11 +164,11 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { continue } - walk(element) + walk(element, null) } } - walk(doc.body) + walk(doc.body, null) return blocks.filter((block) => block.text.length > 0) } @@ -153,8 +183,31 @@ function splitAndStripParagraphs(text: string, sectionHeading: string | null): R .map((cleaned) => ({ sectionHeading, text: cleaned })) } +function prefixListItems(html: string): string { + // Ordered lists: prepend "1. ", "2. ", etc. + let result = html.replace(/]*>([\s\S]*?)<\/ol>/gi, (_match, inner: string) => { + let idx = 1 + const numbered = inner.replace(/]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, liContent: string) => { + const stripped = liContent.replace(/<\/?p\b[^>]*>/gi, "") + return `
  • ${idx++}. ${stripped}
  • ` + }) + return `
      ${numbered}
    ` + }) + + // Unordered lists: prepend "- " + result = result.replace(/]*>([\s\S]*?)<\/ul>/gi, (_match, inner: string) => { + const bulleted = inner.replace(/]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, liContent: string) => { + const stripped = liContent.replace(/<\/?p\b[^>]*>/gi, "") + return `
  • - ${stripped}
  • ` + }) + return `
      ${bulleted}
    ` + }) + + return result +} + function collectBlocksWithRegex(html: string): RawBlock[] { - const normalizedHtml = html + const normalizedHtml = prefixListItems(html) .replace(//gi, "\n") .replace(BLOCK_BREAK_PATTERN, "\n\n") diff --git a/core/rag/indexingSettings.ts b/core/rag/indexingSettings.ts index a68bd5166de..ab88dd48c82 100644 --- a/core/rag/indexingSettings.ts +++ b/core/rag/indexingSettings.ts @@ -36,6 +36,8 @@ export type RagIndexingEditableSettings = { export type RagIndexingSettings = RagIndexingEditableSettings & typeof RAG_INDEX_READONLY_SETTINGS +export const RAG_INDEX_OVERLAP_MIN = 0 + export const RAG_INDEX_EDITABLE_NUMERIC_KEYS = [ "target_chunk_size", "min_chunk_size", @@ -82,8 +84,9 @@ export function validateRagIndexingEditableSettings( errors.push(`${key} must be an integer`) continue } - if (value < RAG_INDEX_NUMERIC_MIN || value > RAG_INDEX_NUMERIC_MAX) { - errors.push(`${key} must be between ${RAG_INDEX_NUMERIC_MIN} and ${RAG_INDEX_NUMERIC_MAX}`) + const min = key === "overlap" ? RAG_INDEX_OVERLAP_MIN : RAG_INDEX_NUMERIC_MIN + if (value < min || value > RAG_INDEX_NUMERIC_MAX) { + errors.push(`${key} must be between ${min} and ${RAG_INDEX_NUMERIC_MAX}`) } } @@ -100,6 +103,9 @@ export function validateRagIndexingEditableSettings( if (resolved.target_chunk_size > resolved.max_chunk_size) { errors.push("target_chunk_size must be less than or equal to max_chunk_size") } + if (resolved.overlap >= resolved.min_chunk_size) { + errors.push("overlap must be less than min_chunk_size") + } return errors } diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index 65114aef297..79bc9e24bbe 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -1,7 +1,21 @@ import { buildRagIndexChunks } from "@core/rag/chunking" -import { buildRagChunkText, getRagChunkBodyLength, getRagChunkBodyText } from "@core/rag/chunkTemplate" +import { buildRagChunkText, getRagChunkBodyText } from "@core/rag/chunkTemplate" import { RAG_INDEX_EDITABLE_DEFAULTS } from "@core/rag/indexingSettings" +// --- Helpers --- + +const p = (text: string) => `

    ${text}

    ` + +const cfg = (overrides: Partial = {}) => ({ + ...RAG_INDEX_EDITABLE_DEFAULTS, + ...overrides, +}) + +/** Solid text of exactly n chars — no spaces or periods (character-level fallback) */ +const solid = (n: number) => "x".repeat(n) + +// --- Real note fixture --- + const USER_NOTE_HTML = [ '

    "То что на предыдущем уровне было субъектом становиться объектом на следующем". (с)

    ', "

    ", @@ -12,270 +26,487 @@ const USER_NOTE_HTML = [ '

    Если мною овладевает ярость, она является неосознанной, она являеться моим скрытым субъектом и тогда я - ярость. В такой момент я не осознаю что все мои действия происходят под влиянием ярости. Ярость как бы берет верх управления и у меня нет выбора в том как вести себя. Я отождествлен со своей яростью, она мой скрытый субъект, я - ярость. Что значит растождествиться со своей яростью? Это значит увидеть ее, увидеть ее со стороны, сделать ее объектом наблюдения. Она больше не мой скрытый субъект, она то что я наблюдаю - объект, и тогда я не ярость, ведь я не могу быть тем что я могу наблюдать (как нож не может порезать сам себя). 

    ', ].join("") -const USER_SETTINGS = { - ...RAG_INDEX_EDITABLE_DEFAULTS, - small_note_threshold: 400, +const USER_SETTINGS = cfg({ target_chunk_size: 500, min_chunk_size: 200, max_chunk_size: 1500, overlap: 100, -} +}) + +// ============================================================ -describe("core/rag/chunking", () => { - it("does not index a note shorter than min_chunk_size", () => { - const chunks = buildRagIndexChunks({ - title: "Weekly plan", - html: "

    Short body text.

    ", - tags: ["work"], - settings: RAG_INDEX_EDITABLE_DEFAULTS, +describe("core/rag/chunking — pairwise test suite", () => { + // ── A: Порог индексации ────────────────────────────────── + describe("A — Порог индексации", () => { + it("A1: заметка < min → 0 чанков", () => { + const chunks = buildRagIndexChunks({ + title: "Note", + html: p("Short."), + tags: [], + settings: cfg(), // min=200, text ~6 chars + }) + expect(chunks).toHaveLength(0) }) - // "Short body text." is ~17 chars, well below min_chunk_size (200) - expect(chunks).toHaveLength(0) + it("A2: заметка = min → 1 чанк", () => { + const chunks = buildRagIndexChunks({ + title: "Exact", + html: p(solid(200)), + tags: ["t1"], + settings: cfg(), + }) + expect(chunks).toHaveLength(1) + expect(chunks[0]!.title).toBe("Exact") + expect(chunks[0]!.content).toContain("Tags: t1") + }) }) - it("indexes a note at exactly min_chunk_size as a single chunk", () => { - const body = "A".repeat(RAG_INDEX_EDITABLE_DEFAULTS.min_chunk_size) - const chunks = buildRagIndexChunks({ - title: "Exact min", - html: `

    ${body}

    `, - tags: ["work"], - settings: RAG_INDEX_EDITABLE_DEFAULTS, + // ── B: Paragraph-first аккумуляция ─────────────────────── + describe("B — Paragraph-first аккумуляция", () => { + it("B2: 3 мелких абзаца, сумма ≤ target → 1 чанк", () => { + // 80 + \n\n + 80 + \n\n + 80 = 244. min=150, target=500 + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(80)) + p(solid(80)) + p(solid(80)), + tags: [], + settings: cfg({ min_chunk_size: 150, target_chunk_size: 500, max_chunk_size: 1500 }), + }) + expect(chunks).toHaveLength(1) }) - expect(chunks).toHaveLength(1) - expect(chunks[0]?.content).toContain("Tags: work") - expect(chunks[0]?.title).toBe("Exact min") + it("B3: P1+P2 ≥ min, +P3 > target → 2 чанка (не добавляем даже если ≤ max)", () => { + // P1+P2 = 80+2+80 = 162 ≥ min=150. +P3: 162+2+400 = 564 > target=500 + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(80)) + p(solid(80)) + p(solid(400)), + tags: [], + settings: cfg({ min_chunk_size: 150, target_chunk_size: 500, max_chunk_size: 1500 }), + }) + expect(chunks).toHaveLength(2) + }) }) - it("splits large paragraphs and adds overlap to later chunks", () => { - const repeated = "Sentence one. Sentence two. Sentence three. Sentence four. Sentence five." - const html = `

    Section A

    ${repeated} ${repeated} ${repeated}

    ` - - const chunks = buildRagIndexChunks({ - title: "Long note", - html, - tags: ["alpha", "beta"], - settings: { - ...RAG_INDEX_EDITABLE_DEFAULTS, - target_chunk_size: 80, - min_chunk_size: 50, - max_chunk_size: 90, - overlap: 50, - }, - }) - - expect(chunks.length).toBeGreaterThan(1) - expect(chunks[0]?.content).toContain("Section: Section A") - expect(chunks[1]?.content).toContain("Sentence") - expect(chunks[1]?.charOffset).toBeGreaterThan(chunks[0]?.charOffset ?? 0) + // ── C: Partial split ───────────────────────────────────── + describe("C — Partial split", () => { + it("C1: P1 < min, P2 ≤ max, P1+P2 > max → частичный разрез P2", () => { + // P1=100 < min=200. P2=1450 ≤ max=1500. Combined=100+2+1450=1552 > max + // → takePartialText отрезает от P2 ровно столько, сколько нужно до min + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(100)) + p(solid(1450)), + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks.length).toBeGreaterThanOrEqual(2) + const body0 = getRagChunkBodyText(chunks[0]!.content) + expect(body0.length).toBeGreaterThanOrEqual(200) // reached min via partial + }) }) - it("expands overlap back to the start of the sentence instead of starting mid-sentence", () => { - const longSentence = "Alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigma tau." - - const chunks = buildRagIndexChunks({ - title: "Long sentence note", - html: `

    ${longSentence}

    `, - tags: [], - settings: { - ...RAG_INDEX_EDITABLE_DEFAULTS, - small_note_threshold: 20, - target_chunk_size: 25, - min_chunk_size: 20, - max_chunk_size: 25, - overlap: 10, - }, - }) - - expect(chunks.length).toBeGreaterThan(1) - expect(getRagChunkBodyText(chunks[1]?.content ?? "").startsWith("Alpha beta gamma")).toBe(true) + // ── D: Oversized paragraph + backward merge ────────────── + describe("D — Oversized paragraph + backward merge", () => { + it("D4: абзац max+min-2 символов → backward merge → 1 чанк (≤ max+min-1)", () => { + // solid(1698) → char split [1500, 198]. 198 < min=200 → merge. + // Merged: 1500 + " " + 198 = 1699 = max+min-1 + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(1698)), + tags: [], + settings: cfg({ overlap: 0 }), + }) + expect(chunks).toHaveLength(1) + const bodyLen = getRagChunkBodyText(chunks[0]!.content).length + expect(bodyLen).toBeLessThanOrEqual(1500 + 200 - 1) + }) + + it("D5: абзац max+min символов → remainder ≥ min → 2 чанка (без merge)", () => { + // solid(1700) → char split [1500, 200]. 200 ≥ min=200 → NO merge + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(1700)), + tags: [], + settings: cfg({ overlap: 0 }), + }) + expect(chunks).toHaveLength(2) + }) + + it("D7: oversized из предложений → разрез по границам предложений", () => { + // 6 sentences × ~296 chars = ~1781 > max=1500 + const sentenceText = Array.from({ length: 6 }, (_, i) => + `Sent${i} ${"z".repeat(286)}.` + ).join(" ") + + const chunks = buildRagIndexChunks({ + title: "t", + html: p(sentenceText), + tags: [], + settings: cfg({ overlap: 0 }), + }) + expect(chunks.length).toBeGreaterThan(1) + // Each chunk ends at a sentence boundary (period) + for (const chunk of chunks) { + expect(getRagChunkBodyText(chunk.content).trimEnd().endsWith(".")).toBe(true) + } + }) }) - it("does not carry overlap across section boundaries", () => { - const chunks = buildRagIndexChunks({ - title: "Sectioned note", - html: "

    Section A

    Alpha sentence one. Alpha sentence two.

    Section B

    Beta sentence one. Beta sentence two.

    ", - tags: [], - settings: { - ...RAG_INDEX_EDITABLE_DEFAULTS, - small_note_threshold: 20, - target_chunk_size: 30, - min_chunk_size: 20, - max_chunk_size: 30, - overlap: 10, - }, - }) - - // Each section text (~40 chars) is > max_chunk_size (30), so splitOversizedParagraph fires. - // But each sentence (~19 chars) is < min_chunk_size (20), so backward merge kicks in, - // producing 1 chunk per section = 2 chunks total. - expect(chunks).toHaveLength(2) - expect(chunks[0]?.content).toContain("Section: Section A") - expect(chunks[1]?.content).toContain("Section: Section B") - // Section B chunk should NOT contain overlap from Section A - expect(getRagChunkBodyText(chunks[1]?.content ?? "")).toBe("Beta sentence one. Beta sentence two.") + // ── E: Trailing undersized merge ───────────────────────── + describe("E — Trailing undersized merge", () => { + it("E1: trailing < min, merge ≤ max → merge происходит", () => { + // P1=300, P2=400, P3=100. Assembly: [300], [400], [100]. + // mergeUndersizedTail: 100 < min=200, merged=400+2+100=502 ≤ max=1500 → merge + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(300)) + p(solid(400)) + p(solid(100)), + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(2) // [P1], [P2+P3] + }) + + it("E3: trailing < min, merge > max → остаётся undersized", () => { + // P1=300, P2=1400, P3=150. Assembly: [300], [1400], [150]. + // mergeUndersizedTail: 150 < min=200, merged=1400+2+150=1552 > max=1500 → NO merge + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(300)) + p(solid(1400)) + p(solid(150)), + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(3) + const lastBody = getRagChunkBodyText(chunks[2]!.content) + expect(lastBody.length).toBeLessThan(200) + }) + + it("E4: trailing из другой секции → merge запрещён", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    A

    ${p(solid(300))}

    B

    ${p(solid(100))}`, + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(2) + expect(chunks[0]!.sectionHeading).toBe("A") + expect(chunks[1]!.sectionHeading).toBe("B") + }) }) - it("omits empty optional lines from the chunk template", () => { - const text = buildRagChunkText({ - sectionHeading: null, - tags: [], - chunkContent: "Body only", - settings: { - use_section_headings: true, - use_tags: true, - }, + // ── F: Секции и заголовки ──────────────────────────────── + describe("F — Секции и заголовки", () => { + it("F1: две секции h2 → разрыв чанка на границе секций", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    Alpha

    ${p(solid(250))}

    Beta

    ${p(solid(250))}`, + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(2) + expect(chunks[0]!.sectionHeading).toBe("Alpha") + expect(chunks[1]!.sectionHeading).toBe("Beta") }) - expect(text).toBe("Body only") + it("F2: абзац после heading наследует его sectionHeading", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    Title

    ${p(solid(250))}${p(solid(250))}`, + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 400, max_chunk_size: 1500, overlap: 0 }), + }) + // P1+P2 = 250+2+250 = 502 > target=400 → 2 chunks, both inherit heading + expect(chunks).toHaveLength(2) + for (const chunk of chunks) { + expect(chunk.sectionHeading).toBe("Title") + } + }) + + it("F3: не создаёт секцию", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    Bold text

    ${p(solid(250))}`, + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500 }), + }) + for (const chunk of chunks) { + expect(chunk.sectionHeading).toBeNull() + expect(chunk.content).not.toContain("Section:") + } + }) }) - it("extracts the note body from templated chunk content", () => { - const text = buildRagChunkText({ - sectionHeading: "Section A", - tags: ["alpha", "beta"], - chunkContent: "Body only", - settings: { - use_section_headings: true, - use_tags: true, - }, + // ── G: Overlap ─────────────────────────────────────────── + describe("G — Overlap", () => { + it("G1: overlap=0 → нет overlap текста", () => { + // Two paragraphs, each ≥ min, sum > target → 2 chunks + const p1 = "Alpha " + solid(244) + "." // 251 chars + const p2 = "Bravo " + solid(244) + "." // 251 chars + + const chunks = buildRagIndexChunks({ + title: "t", + html: p(p1) + p(p2), + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 250, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(2) + const body1 = getRagChunkBodyText(chunks[1]!.content) + expect(body1).not.toContain("Alpha") + }) + + it("G2: overlap=100 → предпочитает границу предложения", () => { + // P1 has sentences; overlap should snap to sentence boundary + const p1Text = "First " + solid(100) + ". Middle " + solid(80) + ". Ending sentence here." + // Positions of periods: ~106, ~195, ~217. With overlap=100, snap to "." at ~106 + const p2Text = "Second paragraph " + solid(200) + "." + + const chunks = buildRagIndexChunks({ + title: "t", + html: p(p1Text) + p(p2Text), + tags: [], + settings: cfg({ min_chunk_size: 50, target_chunk_size: 150, max_chunk_size: 1500, overlap: 100 }), + }) + expect(chunks).toHaveLength(2) + const body1 = getRagChunkBodyText(chunks[1]!.content) + // Overlap prefix starts at sentence boundary and contains "Ending sentence here." + expect(body1).toContain("Ending sentence here.") }) - expect(getRagChunkBodyText(text)).toBe("Body only") - expect(getRagChunkBodyLength(text)).toBe("Body only".length) + it("G3: overlap > длины предыдущего чанка → берёт весь текст", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(80)) + p(solid(300)), + tags: [], + settings: cfg({ min_chunk_size: 50, target_chunk_size: 80, max_chunk_size: 1500, overlap: 200 }), + }) + expect(chunks).toHaveLength(2) + const body1 = getRagChunkBodyText(chunks[1]!.content) + // Entire first chunk text is used as overlap prefix + expect(body1).toContain(solid(80)) + }) + + it("G4: overlap между секциями → не применяется", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    A

    ${p(solid(250))}

    B

    ${p(solid(250))}`, + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 100 }), + }) + expect(chunks).toHaveLength(2) + const body1 = getRagChunkBodyText(chunks[1]!.content) + // Section B should NOT have overlap from section A + expect(body1).toBe(solid(250)) + }) }) - describe("paragraph-first assembly with real note content", () => { - it("produces multiple chunks, not a single chunk", () => { + // ── H: Chunk template ──────────────────────────────────── + describe("H — Chunk template", () => { + it("H1: section + tags + content → полный шаблон", () => { + const text = buildRagChunkText({ + sectionHeading: "Intro", + tags: ["a", "b"], + chunkContent: "Body text", + settings: { use_section_headings: true, use_tags: true }, + }) + expect(text).toBe("Section: Intro\nTags: a, b\n\nBody text") + }) + + it("H2: heading=null → строка Section: опущена", () => { + const text = buildRagChunkText({ + sectionHeading: null, + tags: ["a"], + chunkContent: "Body text", + settings: { use_section_headings: true, use_tags: true }, + }) + expect(text).toBe("Tags: a\n\nBody text") + expect(text).not.toContain("Section:") + }) + + it("H4: heading есть, use_section_headings=false → Section: скрыта", () => { + const text = buildRagChunkText({ + sectionHeading: "Heading", + tags: [], + chunkContent: "Body", + settings: { use_section_headings: false, use_tags: true }, + }) + expect(text).not.toContain("Section:") + expect(text).toBe("Body") + }) + + it("H5: tags есть, use_tags=false → Tags: скрыта", () => { + const text = buildRagChunkText({ + sectionHeading: null, + tags: ["x", "y"], + chunkContent: "Body", + settings: { use_section_headings: true, use_tags: false }, + }) + expect(text).not.toContain("Tags:") + expect(text).toBe("Body") + }) + + it("H7: use_title=true → title передаётся", () => { + const chunks = buildRagIndexChunks({ + title: "My Note", + html: p(solid(200)), + tags: [], + settings: cfg({ use_title: true }), + }) + expect(chunks[0]!.title).toBe("My Note") + }) + + it("H8: use_title=false → title = null", () => { + const chunks = buildRagIndexChunks({ + title: "My Note", + html: p(solid(200)), + tags: [], + settings: cfg({ use_title: false }), + }) + expect(chunks[0]!.title).toBeNull() + }) + }) + + // ── I: Cross-factor pairwise ───────────────────────────── + describe("I — Cross-factor pairwise", () => { + it("I1: oversized в секции A + абзац в B + overlap → overlap не пересекает секцию", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    A

    ${p(solid(2000))}

    B

    ${p(solid(300))}`, + tags: [], + settings: cfg({ min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 1500, overlap: 100 }), + }) + // Section A splits into oversized chunks; section B is separate + const sectionBChunk = chunks[chunks.length - 1]! + expect(sectionBChunk.sectionHeading).toBe("B") + // Section B must NOT have overlap from section A + expect(getRagChunkBodyText(sectionBChunk.content)).toBe(solid(300)) + }) + + it("I2: узкие настройки (min≈target≈max) → форсируют partial split", () => { + // P1=80 < min=100. P2=80: combined=80+2+80=162 > max=120 → partial split P2 + const chunks = buildRagIndexChunks({ + title: "t", + html: p(solid(80)) + p(solid(80)), + tags: [], + settings: cfg({ min_chunk_size: 100, target_chunk_size: 110, max_chunk_size: 120, overlap: 0 }), + }) + expect(chunks.length).toBeGreaterThanOrEqual(2) + const body0 = getRagChunkBodyText(chunks[0]!.content) + expect(body0.length).toBeGreaterThanOrEqual(100) + expect(body0.length).toBeLessThanOrEqual(120) + }) + }) + + // ── J: HTML parsing ────────────────────────────────────── + describe("J — HTML parsing", () => { + it("J1: nested
    с

    → отдельные блоки", () => { + // 150+2+150=302 > target=200, P1=150 ≥ min=100 → 2 chunks + const chunks = buildRagIndexChunks({ + title: "t", + html: `

    ${solid(150)}

    ${solid(150)}

    `, + tags: [], + settings: cfg({ min_chunk_size: 100, target_chunk_size: 200, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(2) + }) + + it("J2:
  • элементы → отдельные блоки", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `
    • ${solid(150)}
    • ${solid(150)}
    `, + tags: [], + settings: cfg({ min_chunk_size: 100, target_chunk_size: 200, max_chunk_size: 1500, overlap: 0 }), + }) + expect(chunks).toHaveLength(2) + }) + + it("J4:
      нумерованный список → нумерация сохраняется в тексте", () => { + // Tiptap/ProseMirror wraps list item content in

      tags + const chunks = buildRagIndexChunks({ + title: "t", + html: `

      1. First item

      2. Second item

      3. Third item

      `, + tags: [], + settings: cfg({ min_chunk_size: 20, target_chunk_size: 500, max_chunk_size: 1500 }), + }) + expect(chunks).toHaveLength(1) + const body = getRagChunkBodyText(chunks[0]!.content) + expect(body).toContain("1. First item") + expect(body).toContain("2. Second item") + expect(body).toContain("3. Third item") + }) + + it("J5:
        маркированный список → маркеры сохраняются в тексте", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: `
        • First item

        • Second item

        • Third item

        `, + tags: [], + settings: cfg({ min_chunk_size: 20, target_chunk_size: 500, max_chunk_size: 1500 }), + }) + expect(chunks).toHaveLength(1) + const body = getRagChunkBodyText(chunks[0]!.content) + expect(body).toContain("- First item") + expect(body).toContain("- Second item") + expect(body).toContain("- Third item") + }) + + it("J3:
        внутри

        → один блок (не разделяет на параграфы)", () => { + const chunks = buildRagIndexChunks({ + title: "t", + html: "

        Line one
        Line two
        Line three

        ", + tags: [], + settings: cfg({ min_chunk_size: 20, target_chunk_size: 500, max_chunk_size: 1500 }), + }) + expect(chunks).toHaveLength(1) + const body = getRagChunkBodyText(chunks[0]!.content) + expect(body).toContain("Line one") + expect(body).toContain("Line three") + }) + }) + + // ── Regression: реальная заметка ───────────────────────── + describe("Regression — реальная заметка (4 абзаца, русский текст)", () => { + it("создаёт несколько чанков, не один", () => { const chunks = buildRagIndexChunks({ title: "тестовый тайтл", html: USER_NOTE_HTML, tags: ["тег1", "тег2"], settings: USER_SETTINGS, }) - expect(chunks.length).toBeGreaterThan(1) }) - it("keeps first two small paragraphs together as one chunk (>= min_chunk_size)", () => { + it("объединяет два первых мелких абзаца в один чанк (≥ min)", () => { const chunks = buildRagIndexChunks({ title: "тестовый тайтл", html: USER_NOTE_HTML, tags: ["тег1", "тег2"], settings: USER_SETTINGS, }) - const body0 = getRagChunkBodyText(chunks[0]?.content ?? "") - // First chunk should contain both P1 and P2 expect(body0).toContain('"То что на предыдущем уровне') expect(body0).toContain("Чувство голода может") - // First chunk should NOT contain text from paragraph 3 + // Третий абзац — уже в следующем чанке expect(body0).not.toContain("Что такое субъект и объект") }) - it("does not cross paragraph boundaries when chunk is above min_chunk_size", () => { + it("не пересекает границы абзацев когда чанк ≥ min", () => { const chunks = buildRagIndexChunks({ title: "тестовый тайтл", html: USER_NOTE_HTML, tags: ["тег1", "тег2"], settings: USER_SETTINGS, }) - - // Each chunk body should contain only whole paragraphs - for (const chunk of chunks) { - const body = getRagChunkBodyText(chunk.content) - // No paragraph text should be cut mid-sentence at paragraph boundary - // (overlap prefix from previous chunk is OK) - expect(body.length).toBeGreaterThan(0) - } - - // Chunk 1 (index 1) should start with paragraph 3 content (possibly with overlap prefix) const body1 = getRagChunkBodyText(chunks[1]?.content ?? "") expect(body1).toContain("Что такое субъект и объект") }) - it("each chunk body respects target_chunk_size for accumulation decisions", () => { + it("3 чанка из 4 абзацев: [P1+P2], [P3], [P4]", () => { const chunks = buildRagIndexChunks({ title: "тестовый тайтл", html: USER_NOTE_HTML, tags: ["тег1", "тег2"], settings: USER_SETTINGS, }) - - // With 4 paragraphs (~83, ~245, ~530, ~570 chars) and target=500: - // Chunk 0: P1+P2 (~330 chars, >= min, P3 would exceed target → close) - // Chunk 1: P3 (~530 chars, >= min, P4 would exceed target → close) - // Chunk 2: P4 (~570 chars) + // P1~83 + P2~245 = ~330 ≥ min. P3~530 would exceed target → close. + // P3~530 ≥ min. P4~570 would exceed target → close. + // P4~570 ≥ min → standalone chunk. expect(chunks).toHaveLength(3) }) }) - - describe("oversized paragraph splitting with backward merge", () => { - it("splits oversized paragraph at max_chunk_size boundaries, not target", () => { - // Create a paragraph that is > max_chunk_size - const sentence = "Это предложение для теста длинного абзаца. " - const longParagraph = sentence.repeat(50) // ~2200 chars - const html = `

        ${longParagraph}

        ` - - const chunks = buildRagIndexChunks({ - title: "test", - html, - tags: [], - settings: { - ...USER_SETTINGS, - }, - }) - - // With max=1500, a ~2200 char paragraph should produce 2 chunks, not 5 (which target=500 would make) - expect(chunks).toHaveLength(2) - }) - - it("backward-merges small remainder into previous piece", () => { - // Create a paragraph where splitting at max would leave a tiny remainder - const sentence = "Это тестовое предложение номер один. " - // Need something slightly over max_chunk_size with small remainder - const longParagraph = sentence.repeat(42) // ~42 * 36 = ~1512, remainder ~12 chars - const html = `

        ${longParagraph}

        ` - - const settings = { - ...USER_SETTINGS, - max_chunk_size: 1500, - min_chunk_size: 200, - } - - const chunks = buildRagIndexChunks({ - title: "test", - html, - tags: [], - settings, - }) - - // If remainder < min_chunk_size, it should be merged back → single chunk - // (the merged chunk will be slightly above max_chunk_size) - if (chunks.length === 1) { - // Backward merge happened — chunk may exceed max but bounded by max + min - 1 - expect(getRagChunkBodyText(chunks[0]?.content ?? "").length).toBeLessThanOrEqual( - settings.max_chunk_size + settings.min_chunk_size - 1 - ) - } - }) - }) - - it("does not create sections from non-heading formatting alone", () => { - const chunks = buildRagIndexChunks({ - title: "Formatting note", - html: "

        Looks like a heading

        Paragraph text.

        ", - tags: [], - settings: { - ...RAG_INDEX_EDITABLE_DEFAULTS, - target_chunk_size: 30, - min_chunk_size: 20, - max_chunk_size: 30, - overlap: 10, - }, - }) - - expect(chunks[0]?.content).not.toContain("Section:") - }) }) diff --git a/core/tests/unit/core-rag-indexingSettings.test.ts b/core/tests/unit/core-rag-indexingSettings.test.ts index fe5c08df444..c1fb8dce8d7 100644 --- a/core/tests/unit/core-rag-indexingSettings.test.ts +++ b/core/tests/unit/core-rag-indexingSettings.test.ts @@ -16,8 +16,27 @@ describe("core/rag/indexingSettings", () => { it("rejects numeric values outside the allowed range", () => { expect(() => assertValidRagIndexingEditableSettings({ overlap: 5001 })).toThrow( - "overlap must be between 50 and 5000" + "overlap must be between 0 and 5000" ) + expect(() => assertValidRagIndexingEditableSettings({ min_chunk_size: 49 })).toThrow( + "min_chunk_size must be between 50 and 5000" + ) + }) + + it("allows overlap = 0", () => { + expect(() => + assertValidRagIndexingEditableSettings({ overlap: 0, min_chunk_size: 200 }) + ).not.toThrow() + }) + + it("rejects overlap >= min_chunk_size", () => { + expect(() => + assertValidRagIndexingEditableSettings({ overlap: 200, min_chunk_size: 200 }) + ).toThrow("overlap must be less than min_chunk_size") + + expect(() => + assertValidRagIndexingEditableSettings({ overlap: 300, min_chunk_size: 200 }) + ).toThrow("overlap must be less than min_chunk_size") }) it("rejects invalid ordering for chunk sizes", () => { @@ -31,7 +50,7 @@ describe("core/rag/indexingSettings", () => { expect(() => assertValidRagIndexingEditableSettings({ - min_chunk_size: 100, + min_chunk_size: 200, target_chunk_size: 500, max_chunk_size: 400, }) diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql index bbb905b4f32..5fe3f3820f8 100644 --- a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -11,7 +11,7 @@ CREATE TABLE public.user_rag_index_settings ( CONSTRAINT user_rag_index_settings_target_chunk_size_range CHECK (target_chunk_size BETWEEN 50 AND 5000), CONSTRAINT user_rag_index_settings_min_chunk_size_range CHECK (min_chunk_size BETWEEN 50 AND 5000), CONSTRAINT user_rag_index_settings_max_chunk_size_range CHECK (max_chunk_size BETWEEN 50 AND 5000), - CONSTRAINT user_rag_index_settings_overlap_range CHECK (overlap BETWEEN 50 AND 5000), + CONSTRAINT user_rag_index_settings_overlap_range CHECK (overlap BETWEEN 0 AND 5000), CONSTRAINT user_rag_index_settings_ordering CHECK (min_chunk_size <= target_chunk_size AND target_chunk_size <= max_chunk_size) ); diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx index be580339980..eafa676f000 100644 --- a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -11,6 +11,7 @@ import { Switch } from "@/components/ui/switch" import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" import { RagIndexSettingsService } from "@core/services/ragIndexSettings" import type { RagIndexingEditableSettings, RagIndexingSettings } from "@core/rag/indexingSettings" +import { validateRagIndexingEditableSettings } from "@core/rag/indexingSettings" import { useSupabase } from "@ui/web/providers/SupabaseProvider" import { Info } from "lucide-react" @@ -80,6 +81,18 @@ export function RagIndexingSettingsPanel() { setFormState((current) => ({ ...current, [key]: checked })) } + const validationErrors = React.useMemo(() => { + return validateRagIndexingEditableSettings({ + target_chunk_size: Number(formState.target_chunk_size), + min_chunk_size: Number(formState.min_chunk_size), + max_chunk_size: Number(formState.max_chunk_size), + overlap: Number(formState.overlap), + use_title: formState.use_title, + use_section_headings: formState.use_section_headings, + use_tags: formState.use_tags, + }) + }, [formState]) + const handleSave = async () => { setErrorMessage(null) setSuccessMessage(null) @@ -112,26 +125,40 @@ export function RagIndexingSettingsPanel() { RAG indexing - Configure web-visible indexing behavior. All size values are measured in characters. + When a note is indexed for AI search, its text is split into chunks — smaller pieces that are embedded and stored as vectors. + The parameters below control how that splitting works. All size values are measured in characters. + Changes apply only to future indexing — already indexed notes stay as-is until you reindex them manually.
        + updateNumericField("min_chunk_size", value)} + tooltip={ + "Paragraphs are merged together until the chunk reaches this size.\n" + + "Also the minimum note size — shorter notes are skipped and not indexed at all.\n\n" + + "Increase: fewer but larger chunks, short notes ignored.\n" + + "Decrease: more granular chunks, even short notes get indexed." + } + /> updateNumericField("target_chunk_size", value)} + tooltip={ + "Preferred chunk size. Once min is reached, one more whole paragraph " + + "can be added — but only if the chunk stays within this limit.\n\n" + + "Increase: chunks may include more paragraphs, broader context per chunk.\n" + + "Decrease: chunks close earlier, more focused but potentially fragmented." + } /> updateNumericField("min_chunk_size", value)} - /> - @@ -154,14 +183,54 @@ export function RagIndexingSettingsPanel() { value={formState.overlap} disabled={loading || saving} onChange={(value) => updateNumericField("overlap", value)} + inputMin={0} + tooltip={ + "How many characters from the end of one chunk are repeated at the start of the next.\n" + + "Must be less than min chunk size. Snaps to the nearest sentence end.\n" + + "Never crosses section (heading) boundaries.\n\n" + + "Increase: more shared context between chunks, better continuity.\n" + + "Set to 0: no repetition, smaller chunks, less redundancy." + } />
        +
        + +
        + + {validationErrors.length > 0 ? ( +
        + +
          + {validationErrors.map((error) => ( +
        • {error}
        • + ))} +
        +
        + ) : null} +
        updateBooleanField("use_title", checked)} @@ -169,7 +238,7 @@ export function RagIndexingSettingsPanel() { updateBooleanField("use_section_headings", checked)} @@ -177,7 +246,7 @@ export function RagIndexingSettingsPanel() { updateBooleanField("use_tags", checked)} @@ -185,30 +254,87 @@ export function RagIndexingSettingsPanel() {
        {resolvedSettings ? ( -
        -
        -

        Read-only system settings

        -

        - These values are system-defined and shown for transparency. +

        +

        How your notes are chunked

        + +
        +

        + Every note is split into paragraphs. Each paragraph becomes a building block for chunks. +

        +
          +
        1. + Headings as boundaries. If a note has{" "} + h1h6{" "} + headings, they act as walls — paragraphs from different sections never end up in the same chunk, + and overlap never crosses a heading. Notes without headings are processed as one continuous section. + Bold or styled text does not count as a heading. +
        2. +
        3. + Merging small paragraphs. Neighboring paragraphs within the same section are merged + together until the chunk reaches min chunk size. If the next paragraph would push + the chunk past max chunk size, only a portion of it is taken. +
        4. +
        5. + Extending toward target. Once min chunk size is reached, the chunk + can accept one more whole paragraph — but only if the result stays within target chunk size. + Otherwise the chunk is closed and the paragraph starts a new one. +
        6. +
        7. + Splitting oversized paragraphs. A paragraph longer than max chunk size{" "} + is split first at sentence boundaries, then by characters. A tiny leftover is merged back so you + don{"'"}t get a useless 20-character chunk. +
        8. +
        9. + Trailing merge. If the very last chunk is smaller than min chunk size, + it is merged with the previous one (unless that would exceed max chunk size). +
        10. +
        11. + Overlap. The tail of each chunk is copied to the beginning of the next one. + This helps search find matches near chunk boundaries. Overlap prefers to start at a sentence end + and never crosses a section heading. +
        12. +
        +

        + Notes shorter than min chunk size are not indexed — they are too short for meaningful semantic search.

        -
        - - - - - ")} /> - - + +
        +
        Tuning tips
        +
          +
        • + Short notes, lists, quick thoughts — lower min chunk size (e.g. 100) so they get indexed. +
        • +
        • + Long essays, articles — increase target chunk size (e.g. 800–1000) for more context per chunk. +
        • +
        • + Dense text without headings — increase max chunk size to avoid splitting long paragraphs. +
        • +
        • + Better search continuity — increase overlap (e.g. 150–200). Set to 0 if you want no repetition. +
        • +
        +
        + +
        +
        Embedding settings (system-defined)
        +
        + + + +
        +
        - -
        +              
        Each chunk is formatted as
        +
                         {resolvedSettings.chunk_template}
                       
        +

        + Section and Tags lines appear only when enabled above and when the note has the corresponding data. + Title is never in the chunk text — it is passed separately via the Gemini API title field. +

        ) : null} @@ -228,7 +354,7 @@ export function RagIndexingSettingsPanel() { ) : null}
        -
        @@ -243,65 +369,38 @@ function NumericField({ value, disabled, onChange, -}: { - id: string - label: string - value: string - disabled: boolean - onChange: (value: string) => void -}) { - return ( -
        - - onChange(event.target.value)} - disabled={disabled} - /> -
        - ) -} - -function NumericFieldWithTooltip({ - id, - label, - value, - disabled, - onChange, tooltip, + inputMin = 50, }: { id: string label: string value: string disabled: boolean onChange: (value: string) => void - tooltip: string + tooltip?: string + inputMin?: number }) { return (
        - - - - - - - {tooltip} - - - + {tooltip ? ( + + + + + + + {tooltip} + + + + ) : null}
        {label}
        {value}
        + {hint ?
        {hint}
        : null}
        ) } From f8750eb0042dc8a536713cd7127e51bdf74189fc Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 11:00:48 +0100 Subject: [PATCH 11/18] fixes --- core/tests/unit/core-rag-chunking.test.ts | 4 ++-- .../component/features/notes/RagIndexPanel.cy.tsx | 12 ++++++------ docs/ai/design/feature-improve-rag-chunking.md | 4 ++-- .../implementation/feature-improve-rag-chunking.md | 3 ++- docs/ai/requirements/feature-improve-rag-chunking.md | 2 +- supabase/functions/api-keys-upsert/index.ts | 9 +++++++-- .../features/settings/RagIndexingSettingsPanel.tsx | 4 ++-- 7 files changed, 22 insertions(+), 16 deletions(-) diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index 79bc9e24bbe..e050239b4d9 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -1,12 +1,12 @@ import { buildRagIndexChunks } from "@core/rag/chunking" import { buildRagChunkText, getRagChunkBodyText } from "@core/rag/chunkTemplate" -import { RAG_INDEX_EDITABLE_DEFAULTS } from "@core/rag/indexingSettings" +import { RAG_INDEX_EDITABLE_DEFAULTS, type RagIndexingEditableSettings } from "@core/rag/indexingSettings" // --- Helpers --- const p = (text: string) => `

        ${text}

        ` -const cfg = (overrides: Partial = {}) => ({ +const cfg = (overrides: Partial = {}): RagIndexingEditableSettings => ({ ...RAG_INDEX_EDITABLE_DEFAULTS, ...overrides, }) diff --git a/cypress/component/features/notes/RagIndexPanel.cy.tsx b/cypress/component/features/notes/RagIndexPanel.cy.tsx index b5e2bb9d6c4..e73cf50c68f 100644 --- a/cypress/component/features/notes/RagIndexPanel.cy.tsx +++ b/cypress/component/features/notes/RagIndexPanel.cy.tsx @@ -97,7 +97,7 @@ describe('RagIndexPanel Component', () => { it('invokes rag-index with action=index', () => { const { supabase, invoke } = createSupabaseForRag([], async (name: string, params: unknown) => { expect(name).to.eq('rag-index') - expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index' } }) + expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index', debugChunks: true } }) return { data: { chunkCount: 3 }, error: null } }) @@ -109,7 +109,7 @@ describe('RagIndexPanel Component', () => { cy.contains('button', 'RAG Index').click() cy.wrap(invoke).should('have.been.calledWith', 'rag-index', { - body: { noteId: 'note-1', action: 'index' }, + body: { noteId: 'note-1', action: 'index', debugChunks: true }, }) }) @@ -117,7 +117,7 @@ describe('RagIndexPanel Component', () => { const rows: EmbeddingRow[] = [{ chunk_index: 0, indexed_at: '2026-03-02T20:00:00.000Z' }] const { supabase, invoke } = createSupabaseForRag(rows, async (name: string, params: unknown) => { expect(name).to.eq('rag-index') - expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'reindex' } }) + expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'reindex', debugChunks: true } }) return { data: { chunkCount: 1 }, error: null } }) @@ -129,7 +129,7 @@ describe('RagIndexPanel Component', () => { cy.contains('button', 'Re-index').click() cy.wrap(invoke).should('have.been.calledWith', 'rag-index', { - body: { noteId: 'note-1', action: 'reindex' }, + body: { noteId: 'note-1', action: 'reindex', debugChunks: true }, }) }) @@ -268,7 +268,7 @@ describe('RagIndexPanel (variant=menu)', () => { it('invokes rag-index with action=index from menu item', () => { const { supabase } = createSupabaseForRag([], async (name, params) => { expect(name).to.eq('rag-index') - expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index' } }) + expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index', debugChunks: true } }) return { data: { chunkCount: 2 }, error: null } }) const onMenuClose = cy.stub().as('onMenuClose') @@ -284,7 +284,7 @@ describe('RagIndexPanel (variant=menu)', () => { ) cy.contains('[role="menuitem"]', 'Index note').click() cy.wrap(supabase.functions.invoke).should('have.been.calledWith', 'rag-index', { - body: { noteId: 'note-1', action: 'index' }, + body: { noteId: 'note-1', action: 'index', debugChunks: true }, }) // onMenuClose called after operation settles — dropdown closes only then cy.get('@onMenuClose').should('have.been.calledOnce') diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index bc318a95f22..63a5ca90e2e 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -179,7 +179,7 @@ Representative payload: 1. load active indexing settings 2. fetch note title, content, and tags 3. derive content structure into sections, paragraphs, sentences, then token/character fallback -4. choose whole-note indexing when note size is below `small_note_threshold` +4. skip indexing when note size is below `min_chunk_size` 5. build final chunks using accumulation and merge rules 6. construct final chunk text from `Section`, `Tags`, and content according to enabled flags 7. send title separately via Gemini `title` @@ -317,4 +317,4 @@ This feature does not alter search ranking logic, but the design must preserve: - `target_chunk_size`: `50..5000` - `min_chunk_size`: `50..5000` - `max_chunk_size`: `50..5000` - - `overlap`: `50..5000` + - `overlap`: `0..5000` diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index 6ea503af98c..3fb34c679ce 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -157,7 +157,8 @@ function validateRagIndexingSettings(input: Partial): Valid Validation rules to enforce in both UI and server paths: -- `target_chunk_size`, `min_chunk_size`, `max_chunk_size`, `overlap` must each be within `50..5000` +- `target_chunk_size`, `min_chunk_size`, `max_chunk_size` must each be within `50..5000` +- `overlap` must be within `0..5000` - `min_chunk_size <= target_chunk_size <= max_chunk_size` Representative placement: diff --git a/docs/ai/requirements/feature-improve-rag-chunking.md b/docs/ai/requirements/feature-improve-rag-chunking.md index 8eb14ffd466..121c46ef296 100644 --- a/docs/ai/requirements/feature-improve-rag-chunking.md +++ b/docs/ai/requirements/feature-improve-rag-chunking.md @@ -102,7 +102,7 @@ RAG note indexing currently uses fixed, mostly implicit chunking and embedding s - [ ] Chunking, chunk-template construction, and settings validation logic live in shared `core` code and do not depend on web-only or mobile-only modules. - [ ] Web, mobile, and server-side indexing paths reuse the same `core` indexing rules instead of reimplementing them per platform. - [ ] Indexing settings are exposed in the user's Google API settings tab. -- [ ] Indexing settings UI is added on the web site only for this feature. +- [ ] Indexing settings UI is added on the website only for this feature. - [ ] Editable UI settings include: - `target_chunk_size` - `min_chunk_size` diff --git a/supabase/functions/api-keys-upsert/index.ts b/supabase/functions/api-keys-upsert/index.ts index 7bedb2a6ccb..bfde16a6a73 100644 --- a/supabase/functions/api-keys-upsert/index.ts +++ b/supabase/functions/api-keys-upsert/index.ts @@ -7,7 +7,7 @@ import { assertValidRagIndexingEditableSettings, coerceRagIndexingEditableSettings, resolveRagIndexingSettings, -} from "../../../core/rag/indexingSettings.ts" +} from "@core/rag/indexingSettings.ts" declare const Deno: { env: { get(key: string): string | undefined } } @@ -145,7 +145,12 @@ serve(async (req: Request) => { let resolvedRagIndexingSettings if (hasRagIndexingFields) { - const editableSettings = assertValidRagIndexingEditableSettings(coercedRagIndexingSettings) + let editableSettings + try { + editableSettings = assertValidRagIndexingEditableSettings(coercedRagIndexingSettings) + } catch (validationError) { + return jsonResponse({ error: validationError instanceof Error ? validationError.message : "Invalid RAG indexing settings" }, 400) + } const { error: ragUpsertError } = await supabaseAdmin .from("user_rag_index_settings") .upsert({ user_id: userId, ...editableSettings, updated_at: new Date().toISOString() }, { onConflict: "user_id" }) diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx index eafa676f000..3abf0753aed 100644 --- a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -11,7 +11,7 @@ import { Switch } from "@/components/ui/switch" import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" import { RagIndexSettingsService } from "@core/services/ragIndexSettings" import type { RagIndexingEditableSettings, RagIndexingSettings } from "@core/rag/indexingSettings" -import { validateRagIndexingEditableSettings } from "@core/rag/indexingSettings" +import { RAG_INDEX_EDITABLE_DEFAULTS, validateRagIndexingEditableSettings } from "@core/rag/indexingSettings" import { useSupabase } from "@ui/web/providers/SupabaseProvider" import { Info } from "lucide-react" @@ -204,7 +204,7 @@ export function RagIndexingSettingsPanel() { target_chunk_size: "500", min_chunk_size: "200", max_chunk_size: "1500", - overlap: "150", + overlap: String(RAG_INDEX_EDITABLE_DEFAULTS.overlap), use_title: true, use_section_headings: true, use_tags: true, From 08c910b69fd45bfe194cde19bcd40e7fc5a1e565 Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 11:10:13 +0100 Subject: [PATCH 12/18] fixes --- core/rag/chunking.ts | 15 +++--- .../features/notes/RagIndexPanel.cy.tsx | 6 +-- ...0317000001_add_user_rag_index_settings.sql | 48 +++++++++++++------ .../features/notes/RagIndexPanel.tsx | 4 +- .../settings/RagIndexingSettingsPanel.tsx | 35 ++++++++++++-- 5 files changed, 79 insertions(+), 29 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index 792c8e34102..c24d2ad773d 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -36,7 +36,6 @@ export type RagIndexChunk = { sectionHeading: string | null } -const HEADING_TAG_PATTERN = /<\/?h[1-6]\b[^>]*>/i const BLOCK_BREAK_PATTERN = /<\/?(?:p|div|li|blockquote|pre|ul|ol|section|article|main)\b[^>]*>/gi function normalizeWhitespace(value: string): string { @@ -93,7 +92,7 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { const blocks: RawBlock[] = [] let currentHeading: string | null = null - const walk = (node: Node, olIndex: number | null) => { + const walk = (node: Node) => { for (const child of Array.from(node.childNodes)) { if (child.nodeType === Node.TEXT_NODE) { const text = normalizeWhitespace(child.textContent ?? "") @@ -121,7 +120,7 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { if (text) blocks.push({ sectionHeading: currentHeading, text: `${idx}. ${text}` }) idx++ } else { - walk(olChild, null) + walk(olChild) } } continue @@ -135,14 +134,14 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { const text = normalizeWhitespace(extractElementText(ulChild)) if (text) blocks.push({ sectionHeading: currentHeading, text: `- ${text}` }) } else { - walk(ulChild, null) + walk(ulChild) } } continue } if (tagName === "section" || tagName === "article" || tagName === "main") { - walk(element, null) + walk(element) continue } @@ -153,7 +152,7 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { }) if (hasNestedBlocks) { - walk(element, null) + walk(element) continue } } @@ -164,11 +163,11 @@ function collectBlocksFromDom(rootHtml: string): RawBlock[] { continue } - walk(element, null) + walk(element) } } - walk(doc.body, null) + walk(doc.body) return blocks.filter((block) => block.text.length > 0) } diff --git a/cypress/component/features/notes/RagIndexPanel.cy.tsx b/cypress/component/features/notes/RagIndexPanel.cy.tsx index e73cf50c68f..3914e8d142f 100644 --- a/cypress/component/features/notes/RagIndexPanel.cy.tsx +++ b/cypress/component/features/notes/RagIndexPanel.cy.tsx @@ -97,7 +97,7 @@ describe('RagIndexPanel Component', () => { it('invokes rag-index with action=index', () => { const { supabase, invoke } = createSupabaseForRag([], async (name: string, params: unknown) => { expect(name).to.eq('rag-index') - expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index', debugChunks: true } }) + expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index' } }) return { data: { chunkCount: 3 }, error: null } }) @@ -117,7 +117,7 @@ describe('RagIndexPanel Component', () => { const rows: EmbeddingRow[] = [{ chunk_index: 0, indexed_at: '2026-03-02T20:00:00.000Z' }] const { supabase, invoke } = createSupabaseForRag(rows, async (name: string, params: unknown) => { expect(name).to.eq('rag-index') - expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'reindex', debugChunks: true } }) + expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'reindex' } }) return { data: { chunkCount: 1 }, error: null } }) @@ -268,7 +268,7 @@ describe('RagIndexPanel (variant=menu)', () => { it('invokes rag-index with action=index from menu item', () => { const { supabase } = createSupabaseForRag([], async (name, params) => { expect(name).to.eq('rag-index') - expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index', debugChunks: true } }) + expect(params).to.deep.eq({ body: { noteId: 'note-1', action: 'index' } }) return { data: { chunkCount: 2 }, error: null } }) const onMenuClose = cy.stub().as('onMenuClose') diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql index 5fe3f3820f8..53ebd74a9e2 100644 --- a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -1,4 +1,4 @@ -CREATE TABLE public.user_rag_index_settings ( +CREATE TABLE IF NOT EXISTS public.user_rag_index_settings ( user_id uuid PRIMARY KEY REFERENCES auth.users(id) ON DELETE CASCADE, target_chunk_size integer NOT NULL DEFAULT 500, min_chunk_size integer NOT NULL DEFAULT 200, @@ -17,19 +17,39 @@ CREATE TABLE public.user_rag_index_settings ( ALTER TABLE public.user_rag_index_settings ENABLE ROW LEVEL SECURITY; -CREATE POLICY "Users can view own rag index settings" - ON public.user_rag_index_settings FOR SELECT - USING (auth.uid() = user_id); +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can view own rag index settings' + ) THEN + CREATE POLICY "Users can view own rag index settings" + ON public.user_rag_index_settings FOR SELECT + USING (auth.uid() = user_id); + END IF; -CREATE POLICY "Users can insert own rag index settings" - ON public.user_rag_index_settings FOR INSERT - WITH CHECK (auth.uid() = user_id); + IF NOT EXISTS ( + SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can insert own rag index settings' + ) THEN + CREATE POLICY "Users can insert own rag index settings" + ON public.user_rag_index_settings FOR INSERT + WITH CHECK (auth.uid() = user_id); + END IF; -CREATE POLICY "Users can update own rag index settings" - ON public.user_rag_index_settings FOR UPDATE - USING (auth.uid() = user_id) - WITH CHECK (auth.uid() = user_id); + IF NOT EXISTS ( + SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can update own rag index settings' + ) THEN + CREATE POLICY "Users can update own rag index settings" + ON public.user_rag_index_settings FOR UPDATE + USING (auth.uid() = user_id) + WITH CHECK (auth.uid() = user_id); + END IF; -CREATE POLICY "Users can delete own rag index settings" - ON public.user_rag_index_settings FOR DELETE - USING (auth.uid() = user_id); + IF NOT EXISTS ( + SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can delete own rag index settings' + ) THEN + CREATE POLICY "Users can delete own rag index settings" + ON public.user_rag_index_settings FOR DELETE + USING (auth.uid() = user_id); + END IF; +END +$$; diff --git a/ui/web/components/features/notes/RagIndexPanel.tsx b/ui/web/components/features/notes/RagIndexPanel.tsx index e8333acc6d1..b5102e10094 100644 --- a/ui/web/components/features/notes/RagIndexPanel.tsx +++ b/ui/web/components/features/notes/RagIndexPanel.tsx @@ -21,6 +21,7 @@ import { toast } from 'sonner' import { useSupabase } from '@ui/web/providers/SupabaseProvider' import { useRagStatus } from '@ui/web/hooks/useRagStatus' import { logRagIndexDebugChunks, type RagIndexDebugChunk } from '@core/rag/debugLog' +import { isRagDebugChunksEnabled } from '@ui/web/components/features/settings/RagIndexingSettingsPanel' async function extractErrorMessage(err: unknown, fallback: string): Promise { if (!(err instanceof Error)) return fallback @@ -77,8 +78,9 @@ export function RagIndexPanel({ noteId, variant = 'inline', onMenuClose }: RagIn const handleIndex = async () => { setOperation('indexing') try { + const debug = isRagDebugChunksEnabled() const { data, error } = await supabase.functions.invoke('rag-index', { - body: { noteId, action: isIndexed ? 'reindex' : 'index', debugChunks: true }, + body: { noteId, action: isIndexed ? 'reindex' : 'index', ...(debug ? { debugChunks: true } : {}) }, }) if (error) throw error const debugChunks = parseDebugChunks(data) diff --git a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx index 3abf0753aed..60d11dd257b 100644 --- a/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx +++ b/ui/web/components/features/settings/RagIndexingSettingsPanel.tsx @@ -9,12 +9,23 @@ import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" import { Switch } from "@/components/ui/switch" import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/components/ui/tooltip" +import { Checkbox } from "@/components/ui/checkbox" import { RagIndexSettingsService } from "@core/services/ragIndexSettings" import type { RagIndexingEditableSettings, RagIndexingSettings } from "@core/rag/indexingSettings" import { RAG_INDEX_EDITABLE_DEFAULTS, validateRagIndexingEditableSettings } from "@core/rag/indexingSettings" import { useSupabase } from "@ui/web/providers/SupabaseProvider" import { Info } from "lucide-react" +const RAG_DEBUG_CHUNKS_KEY = "rag-debug-chunks" + +export function isRagDebugChunksEnabled(): boolean { + try { + return localStorage.getItem(RAG_DEBUG_CHUNKS_KEY) === "true" + } catch { + return false + } +} + type EditableNumericKey = keyof Pick< RagIndexingEditableSettings, "target_chunk_size" | "min_chunk_size" | "max_chunk_size" | "overlap" @@ -52,6 +63,7 @@ export function RagIndexingSettingsPanel() { use_section_headings: true, use_tags: true, })) + const [debugChunks, setDebugChunks] = React.useState(() => isRagDebugChunksEnabled()) const loadSettings = React.useCallback(async () => { setLoading(true) @@ -194,12 +206,27 @@ export function RagIndexingSettingsPanel() { />
        -
        +
        +
        + { + const value = checked === true + setDebugChunks(value) + try { localStorage.setItem(RAG_DEBUG_CHUNKS_KEY, String(value)) } catch { /* ignore */ } + }} + /> + +
        From be32ec651b685e9b6705bdbf4299ba4f76546cc3 Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 11:30:38 +0100 Subject: [PATCH 13/18] fix tests --- .github/workflows/unit-tests.yml | 12 +++--- .../notes/NotesShellOpenInContext.cy.tsx | 2 +- .../settings/ApiKeysSettingsDialog.cy.tsx | 37 ++++++++++++++----- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 3dff519b1b9..8fec71012cd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -155,7 +155,7 @@ jobs: if: steps.run-mobile-unit-tests.outcome != 'success' run: exit 1 - unit-tests-code: + unit-tests-core: runs-on: ubuntu-latest steps: @@ -182,10 +182,10 @@ jobs: continue-on-error: true run: npm run test:integration:core -- --verbose --ci --json --outputFile=core-integration-results.json - - name: Generate code summary + - name: Generate core summary if: always() env: - SUMMARY_TITLE: Code Tests + SUMMARY_TITLE: Core Tests REPORT_PATHS: core-unit-results.json,core-integration-results.json PRIMARY_OUTCOME: ${{ steps.run-core-unit-tests.outcome == 'success' && steps.run-core-integration-tests.outcome == 'success' && 'success' || 'failure' }} run: | @@ -286,18 +286,18 @@ jobs: fs.appendFileSync(summaryFile, md); NODE - - name: Upload code test reports + - name: Upload core test reports if: always() uses: actions/upload-artifact@v4 with: - name: unit-test-report-code-${{ github.run_id }} + name: unit-test-report-core-${{ github.run_id }} path: | core-unit-results.json core-integration-results.json if-no-files-found: ignore retention-days: 14 - - name: Mark job as failed when code tests failed + - name: Mark job as failed when core tests failed if: steps.run-core-unit-tests.outcome != 'success' || steps.run-core-integration-tests.outcome != 'success' run: exit 1 diff --git a/cypress/component/features/notes/NotesShellOpenInContext.cy.tsx b/cypress/component/features/notes/NotesShellOpenInContext.cy.tsx index dd919e4f721..c3f1836ca39 100644 --- a/cypress/component/features/notes/NotesShellOpenInContext.cy.tsx +++ b/cypress/component/features/notes/NotesShellOpenInContext.cy.tsx @@ -35,7 +35,7 @@ describe('NotesShell: AI open in context', () => { user_id: user.id, } - const chunkOffset = note.title.length + 1 + 10 + const chunkOffset = 10 const invoke = cy.stub().callsFake((fn: string) => { if (fn === 'api-keys-status') { diff --git a/cypress/component/features/settings/ApiKeysSettingsDialog.cy.tsx b/cypress/component/features/settings/ApiKeysSettingsDialog.cy.tsx index 2db6e8055aa..7f695a32d8a 100644 --- a/cypress/component/features/settings/ApiKeysSettingsDialog.cy.tsx +++ b/cypress/component/features/settings/ApiKeysSettingsDialog.cy.tsx @@ -4,6 +4,24 @@ import type { SupabaseClient } from '@supabase/supabase-js' import { ApiKeysSettingsDialog } from '../../../../ui/web/components/features/settings/ApiKeysSettingsDialog' import { SupabaseTestProvider } from '../../../../ui/web/providers/SupabaseProvider' +const defaultRagIndexing = { + target_chunk_size: 500, + min_chunk_size: 200, + max_chunk_size: 1500, + overlap: 100, + use_title: true, + use_section_headings: true, + use_tags: true, + output_dimensionality: 1536, + task_type_document: 'RETRIEVAL_DOCUMENT', + task_type_query: 'RETRIEVAL_QUERY', + split_strategy: 'hierarchical', + fallback_split_order: ['sections', 'paragraphs', 'sentences', 'tokens_or_characters'], + chunk_accumulation_rule: 'Paragraph-first', + small_chunk_merge_rule: 'Merge undersized final chunks', + chunk_template: 'Section: {section_heading}\nTags: {tag1}\n\n{chunk_content}', +} + const mountDialog = (supabase: SupabaseClient) => { cy.mount( @@ -16,7 +34,7 @@ describe('features/settings/ApiKeysSettingsDialog', () => { it('shows configured status when a key is already stored', () => { const invoke = cy.stub().callsFake((name: string) => { if (name === 'api-keys-status') { - return Promise.resolve({ data: { gemini: { configured: true } }, error: null }) + return Promise.resolve({ data: { gemini: { configured: true }, ragIndexing: defaultRagIndexing }, error: null }) } return Promise.resolve({ data: null, error: null }) }) @@ -31,7 +49,7 @@ describe('features/settings/ApiKeysSettingsDialog', () => { it('shows empty input for initial setup when not configured', () => { const invoke = cy.stub().callsFake((name: string) => { if (name === 'api-keys-status') { - return Promise.resolve({ data: { gemini: { configured: false } }, error: null }) + return Promise.resolve({ data: { gemini: { configured: false }, ragIndexing: defaultRagIndexing }, error: null }) } return Promise.resolve({ data: null, error: null }) }) @@ -45,25 +63,24 @@ describe('features/settings/ApiKeysSettingsDialog', () => { it('requires API key for initial setup when not configured', () => { const invoke = cy.stub().callsFake((name: string) => { if (name === 'api-keys-status') { - return Promise.resolve({ data: { gemini: { configured: false } }, error: null }) + return Promise.resolve({ data: { gemini: { configured: false }, ragIndexing: defaultRagIndexing }, error: null }) } return Promise.resolve({ data: null, error: null }) }) mountDialog({ functions: { invoke } } as unknown as SupabaseClient) - cy.contains('button', 'Save').click() + cy.contains('button', 'Save API key').click() cy.contains('Gemini API key is required for initial setup.').should('be.visible') - cy.wrap(invoke).should('have.callCount', 1) }) it('saves key and shows success, clears input', () => { const invoke = cy.stub().callsFake((name: string) => { if (name === 'api-keys-status') { - return Promise.resolve({ data: { gemini: { configured: false } }, error: null }) + return Promise.resolve({ data: { gemini: { configured: false }, ragIndexing: defaultRagIndexing }, error: null }) } if (name === 'api-keys-upsert') { - return Promise.resolve({ data: { gemini: { configured: true } }, error: null }) + return Promise.resolve({ data: { gemini: { configured: true }, ragIndexing: defaultRagIndexing }, error: null }) } return Promise.resolve({ data: null, error: null }) }) @@ -71,7 +88,7 @@ describe('features/settings/ApiKeysSettingsDialog', () => { mountDialog({ functions: { invoke } } as unknown as SupabaseClient) cy.get('#gemini-api-key').type('AIzaSy-test-key') - cy.contains('button', 'Save').click() + cy.contains('button', 'Save API key').click() cy.wrap(invoke).should( 'have.been.calledWith', @@ -103,7 +120,7 @@ describe('features/settings/ApiKeysSettingsDialog', () => { it('shows error message when save fails', () => { const invoke = cy.stub().callsFake((name: string) => { if (name === 'api-keys-status') { - return Promise.resolve({ data: { gemini: { configured: false } }, error: null }) + return Promise.resolve({ data: { gemini: { configured: false }, ragIndexing: defaultRagIndexing }, error: null }) } if (name === 'api-keys-upsert') { return Promise.resolve({ @@ -119,7 +136,7 @@ describe('features/settings/ApiKeysSettingsDialog', () => { mountDialog({ functions: { invoke } } as unknown as SupabaseClient) cy.get('#gemini-api-key').type('AIzaSy-test-key') - cy.contains('button', 'Save').click() + cy.contains('button', 'Save API key').click() cy.contains('Invalid JWT').should('be.visible') }) From 4aa94b0ebced1307b07285ad3d5ac0f47d88e18b Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 11:45:28 +0100 Subject: [PATCH 14/18] Fix SonarQube and CodeRabbit review issues in RAG chunking - Replace replace() with replaceAll() (S7781) - Use startsWith() instead of index access (S6557) - Remove unnecessary non-null assertions (S4325) - Use .at() for array access from end (S7755) - Refactor assembleParagraphFirst: extract mergeBlockIntoCurrent and accumulateBelowMin to reduce cognitive complexity (S3776) - Use for-of instead of indexed for loop (S4138) - Use optional chaining (S6582) - Document cross-field validation invariants for chunking settings Co-Authored-By: Claude Opus 4.6 --- core/rag/chunking.ts | 266 +++++++++--------- .../ai/design/feature-improve-rag-chunking.md | 6 +- 2 files changed, 141 insertions(+), 131 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index c24d2ad773d..cbc3e4dfe52 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -40,14 +40,14 @@ const BLOCK_BREAK_PATTERN = /<\/?(?:p|div|li|blockquote|pre|ul|ol|section|articl function normalizeWhitespace(value: string): string { return value - .replace(/ /gi, " ") - .replace(/\u00a0/g, " ") - .replace(/\s+/g, " ") + .replaceAll(/ /gi, " ") + .replaceAll(/\u00a0/g, " ") + .replaceAll(/\s+/g, " ") .trim() } function stripTags(value: string): string { - return normalizeWhitespace(value.replace(/<[^>]*>/g, " ")) + return normalizeWhitespace(value.replaceAll(/<[^>]*>/g, " ")) } function splitPlainTextParagraphs(value: string): string[] { @@ -86,84 +86,71 @@ function extractElementText(node: Node): string { return result } +const SEMANTIC_CONTAINERS = new Set(["section", "article", "main"]) +const BLOCK_ELEMENTS = new Set(["p", "div", "li", "blockquote", "pre"]) +const NESTED_BLOCK_TAGS = new Set(["div", "p", "li", "blockquote", "pre"]) + +function isHeadingTag(tagName: string): boolean { + return tagName.length === 2 && tagName.startsWith("h") && tagName >= "h1" && tagName <= "h6" +} + +function divHasNestedBlocks(element: Element): boolean { + return Array.from(element.children).some((child) => { + const tag = child.tagName.toLowerCase() + return NESTED_BLOCK_TAGS.has(tag) || isHeadingTag(tag) + }) +} + function collectBlocksFromDom(rootHtml: string): RawBlock[] { const parser = new DOMParser() const doc = parser.parseFromString(rootHtml, "text/html") const blocks: RawBlock[] = [] let currentHeading: string | null = null + const pushBlock = (text: string) => { + if (text) blocks.push({ sectionHeading: currentHeading, text }) + } + + const handleList = (element: Element, prefix: (idx: number) => string) => { + let idx = 1 + for (const child of Array.from(element.childNodes)) { + if (child.nodeType !== Node.ELEMENT_NODE) continue + if ((child as Element).tagName.toLowerCase() === "li") { + const text = normalizeWhitespace(extractElementText(child)) + pushBlock(text ? `${prefix(idx)}${text}` : "") + idx++ + } else { + walk(child) + } + } + } + const walk = (node: Node) => { for (const child of Array.from(node.childNodes)) { if (child.nodeType === Node.TEXT_NODE) { - const text = normalizeWhitespace(child.textContent ?? "") - if (text) blocks.push({ sectionHeading: currentHeading, text }) + pushBlock(normalizeWhitespace(child.textContent ?? "")) continue } - if (child.nodeType !== Node.ELEMENT_NODE) continue const element = child as Element const tagName = element.tagName.toLowerCase() - if (/^h[1-6]$/.test(tagName)) { + if (isHeadingTag(tagName)) { currentHeading = normalizeWhitespace(element.textContent ?? "") - continue - } - - if (tagName === "ol") { - let idx = 1 - for (const olChild of Array.from(element.childNodes)) { - if (olChild.nodeType !== Node.ELEMENT_NODE) continue - const olChildTag = (olChild as Element).tagName.toLowerCase() - if (olChildTag === "li") { - const text = normalizeWhitespace(extractElementText(olChild)) - if (text) blocks.push({ sectionHeading: currentHeading, text: `${idx}. ${text}` }) - idx++ - } else { - walk(olChild) - } - } - continue - } - - if (tagName === "ul") { - for (const ulChild of Array.from(element.childNodes)) { - if (ulChild.nodeType !== Node.ELEMENT_NODE) continue - const ulChildTag = (ulChild as Element).tagName.toLowerCase() - if (ulChildTag === "li") { - const text = normalizeWhitespace(extractElementText(ulChild)) - if (text) blocks.push({ sectionHeading: currentHeading, text: `- ${text}` }) - } else { - walk(ulChild) - } - } - continue - } - - if (tagName === "section" || tagName === "article" || tagName === "main") { + } else if (tagName === "ol") { + handleList(element, (idx) => `${idx}. `) + } else if (tagName === "ul") { + handleList(element, () => "- ") + } else if (SEMANTIC_CONTAINERS.has(tagName)) { + walk(element) + } else if (tagName === "div" && divHasNestedBlocks(element)) { + walk(element) + } else if (BLOCK_ELEMENTS.has(tagName)) { + pushBlock(normalizeWhitespace(extractElementText(element))) + } else { walk(element) - continue - } - - if (tagName === "div") { - const hasNestedBlocks = Array.from(element.children).some((childElement) => { - const childTag = childElement.tagName.toLowerCase() - return childTag === "div" || childTag === "p" || childTag === "li" || childTag === "blockquote" || childTag === "pre" || /^h[1-6]$/.test(childTag) - }) - - if (hasNestedBlocks) { - walk(element) - continue - } - } - - if (tagName === "p" || tagName === "div" || tagName === "li" || tagName === "blockquote" || tagName === "pre") { - const text = normalizeWhitespace(extractElementText(element)) - if (text) blocks.push({ sectionHeading: currentHeading, text }) - continue } - - walk(element) } } @@ -184,19 +171,19 @@ function splitAndStripParagraphs(text: string, sectionHeading: string | null): R function prefixListItems(html: string): string { // Ordered lists: prepend "1. ", "2. ", etc. - let result = html.replace(/]*>([\s\S]*?)<\/ol>/gi, (_match, inner: string) => { + let result = html.replaceAll(/]*>([\s\S]*?)<\/ol>/gi, (_match, inner: string) => { let idx = 1 - const numbered = inner.replace(/]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, liContent: string) => { - const stripped = liContent.replace(/<\/?p\b[^>]*>/gi, "") + const numbered = inner.replaceAll(/]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, liContent: string) => { + const stripped = liContent.replaceAll(/<\/?p\b[^>]*>/gi, "") return `
      • ${idx++}. ${stripped}
      • ` }) return `
          ${numbered}
        ` }) // Unordered lists: prepend "- " - result = result.replace(/]*>([\s\S]*?)<\/ul>/gi, (_match, inner: string) => { - const bulleted = inner.replace(/]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, liContent: string) => { - const stripped = liContent.replace(/<\/?p\b[^>]*>/gi, "") + result = result.replaceAll(/]*>([\s\S]*?)<\/ul>/gi, (_match, inner: string) => { + const bulleted = inner.replaceAll(/]*>([\s\S]*?)<\/li>/gi, (_liMatch: string, liContent: string) => { + const stripped = liContent.replaceAll(/<\/?p\b[^>]*>/gi, "") return `
      • - ${stripped}
      • ` }) return `
          ${bulleted}
        ` @@ -207,8 +194,8 @@ function prefixListItems(html: string): string { function collectBlocksWithRegex(html: string): RawBlock[] { const normalizedHtml = prefixListItems(html) - .replace(//gi, "\n") - .replace(BLOCK_BREAK_PATTERN, "\n\n") + .replaceAll(//gi, "\n") + .replaceAll(BLOCK_BREAK_PATTERN, "\n\n") const rawBlocks: RawBlock[] = [] let currentHeading: string | null = null @@ -298,16 +285,18 @@ function splitByCharacterFallback(segment: TextSegment, maxChunkSize: number): T return parts } -function takePartialText(text: string, minChars: number): { taken: string; remainder: string } { +function takePartialText(text: string, minChars: number): { taken: string; remainder: string; consumedChars: number } { if (minChars >= text.length) { - return { taken: text, remainder: "" } + return { taken: text, remainder: "", consumedChars: text.length } } const sentenceEnd = text.indexOf(".", minChars) if (sentenceEnd !== -1 && sentenceEnd < text.length) { + const splitAt = sentenceEnd + 1 return { - taken: text.slice(0, sentenceEnd + 1).trim(), - remainder: text.slice(sentenceEnd + 1).trim(), + taken: text.slice(0, splitAt).trim(), + remainder: text.slice(splitAt).trim(), + consumedChars: splitAt, } } @@ -316,12 +305,14 @@ function takePartialText(text: string, minChars: number): { taken: string; remai return { taken: text.slice(0, spaceIndex).trim(), remainder: text.slice(spaceIndex).trim(), + consumedChars: spaceIndex, } } return { taken: text.slice(0, minChars).trim(), remainder: text.slice(minChars).trim(), + consumedChars: minChars, } } @@ -364,9 +355,9 @@ function splitOversizedParagraph( // This may produce a chunk slightly above maxSize — a conscious compromise to avoid // breaking the next paragraph boundary or leaving a tiny orphan chunk. if (chunks.length >= 2) { - const last = chunks[chunks.length - 1]! - if (last.text.length < minSize) { - const prev = chunks[chunks.length - 2]! + const last = chunks.at(-1) + const prev = chunks.at(-2) + if (last && prev && last.text.length < minSize) { chunks.splice(-2, 2, { ...prev, text: [prev.text, last.text].join(" ") }) } } @@ -378,6 +369,64 @@ function joinChunkParts(parts: string[]): string { return parts.filter(Boolean).join("\n\n").trim() } +function accumulateBelowMin( + current: CandidateChunk, + block: IndexedBlock, + combinedText: string, + candidates: CandidateChunk[], + settings: Pick +): CandidateChunk | null { + if (combinedText.length <= settings.max_chunk_size) { + return { sectionHeading: current.sectionHeading, text: combinedText, charOffset: current.charOffset } + } + + const separatorLen = 2 // "\n\n" + const needed = settings.min_chunk_size - current.text.length - separatorLen + if (needed <= 0) { + candidates.push(current) + return { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } + } + + const partial = takePartialText(block.text, needed) + candidates.push({ + sectionHeading: current.sectionHeading, + text: joinChunkParts([current.text, partial.taken]), + charOffset: current.charOffset, + }) + + if (partial.remainder) { + return { + sectionHeading: block.sectionHeading, + text: partial.remainder, + charOffset: block.charOffset + partial.consumedChars, + } + } + + return null +} + +function mergeBlockIntoCurrent( + current: CandidateChunk, + block: IndexedBlock, + candidates: CandidateChunk[], + settings: Pick +): CandidateChunk | null { + const combinedText = joinChunkParts([current.text, block.text]) + + if (current.text.length < settings.min_chunk_size) { + return accumulateBelowMin(current, block, combinedText, candidates, settings) + } + + // At or above min — can close, but check if next paragraph fits within target + if (combinedText.length <= settings.target_chunk_size) { + return { sectionHeading: current.sectionHeading, text: combinedText, charOffset: current.charOffset } + } + + // Would exceed target — close current, start new chunk + candidates.push(current) + return { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } +} + function assembleParagraphFirst( blocks: IndexedBlock[], settings: Pick @@ -385,10 +434,7 @@ function assembleParagraphFirst( const candidates: CandidateChunk[] = [] let current: CandidateChunk | null = null - for (let i = 0; i < blocks.length; i++) { - const block = blocks[i] - if (!block) continue - + for (const block of blocks) { // Section boundary — close current chunk if (current && current.sectionHeading !== block.sectionHeading) { candidates.push(current) @@ -411,48 +457,7 @@ function assembleParagraphFirst( continue } - // current is guaranteed non-null here (guarded by the !current check above) - const combinedText = joinChunkParts([current!.text, block.text]) - - if (current!.text.length < settings.min_chunk_size) { - // Below min — must add more to reach min_chunk_size - if (combinedText.length <= settings.max_chunk_size) { - // Whole paragraph fits within max — add it whole - current = { sectionHeading: current!.sectionHeading, text: combinedText, charOffset: current!.charOffset } - } else { - // Whole paragraph doesn't fit in max — split partially to reach min - const separatorLen = 2 // "\n\n" - const needed = settings.min_chunk_size - current!.text.length - separatorLen - if (needed > 0) { - const partial = takePartialText(block.text, needed) - current = { sectionHeading: current!.sectionHeading, text: joinChunkParts([current!.text, partial.taken]), charOffset: current!.charOffset } - candidates.push(current) - current = null - if (partial.remainder) { - current = { - sectionHeading: block.sectionHeading, - text: partial.remainder, - charOffset: block.charOffset + partial.taken.length, - } - } - } else { - // Current is already at min with just the separator - candidates.push(current!) - current = { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } - } - } - continue - } - - // At or above min — can close, but check if next paragraph fits within target - if (combinedText.length <= settings.target_chunk_size) { - // Adding this paragraph keeps within target — add it - current = { sectionHeading: current!.sectionHeading, text: combinedText, charOffset: current!.charOffset } - } else { - // Would exceed target — close current, start new chunk - candidates.push(current!) - current = { sectionHeading: block.sectionHeading, text: block.text, charOffset: block.charOffset } - } + current = mergeBlockIntoCurrent(current, block, candidates, settings) } if (current) { @@ -468,10 +473,11 @@ function mergeUndersizedTail( ): CandidateChunk[] { if (chunks.length < 2) return chunks - const last = chunks[chunks.length - 1]! - if (last.text.length >= settings.min_chunk_size) return chunks + const last = chunks.at(-1) + const prev = chunks.at(-2) + if (!last || !prev) return chunks - const prev = chunks[chunks.length - 2]! + if (last.text.length >= settings.min_chunk_size) return chunks if (prev.sectionHeading !== last.sectionHeading) return chunks const merged = joinChunkParts([prev.text, last.text]) @@ -491,7 +497,7 @@ function buildOverlapPrefix(source: string, overlap: number): string { const sentenceBoundary = source.lastIndexOf(".", requestedStart - 1) if (sentenceBoundary === -1) { - return source.trim() + return source.slice(requestedStart).trim() } const sentenceStart = sentenceBoundary + 1 @@ -504,10 +510,10 @@ function applyFinalOverlap(chunks: CandidateChunk[], overlap: number): Candidate return chunks.map((chunk, index) => { if (index === 0) return chunk const previous = chunks[index - 1] - if (!previous || previous.sectionHeading !== chunk.sectionHeading) { + if (previous?.sectionHeading !== chunk.sectionHeading) { return chunk } - const overlapPrefix = buildOverlapPrefix(previous?.text ?? "", overlap) + const overlapPrefix = buildOverlapPrefix(previous.text, overlap) if (!overlapPrefix) return chunk return { diff --git a/docs/ai/design/feature-improve-rag-chunking.md b/docs/ai/design/feature-improve-rag-chunking.md index 63a5ca90e2e..7f96ad2de5e 100644 --- a/docs/ai/design/feature-improve-rag-chunking.md +++ b/docs/ai/design/feature-improve-rag-chunking.md @@ -317,4 +317,8 @@ This feature does not alter search ranking logic, but the design must preserve: - `target_chunk_size`: `50..5000` - `min_chunk_size`: `50..5000` - `max_chunk_size`: `50..5000` - - `overlap`: `0..5000` + - `overlap`: `0..min_chunk_size - 1` (must be strictly less than `min_chunk_size`) +- Cross-field invariants enforced by the settings access/validation layer: + - `min_chunk_size <= target_chunk_size <= max_chunk_size` + - `overlap < min_chunk_size` + - These checks must be performed at save time, not just at the individual field level. Changing one field may invalidate another (e.g., raising `min_chunk_size` above `target_chunk_size` must be rejected or auto-corrected). From 0741ca0d1eef06e6d23d750602b605d1e30574a4 Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 12:57:28 +0100 Subject: [PATCH 15/18] fixes review --- core/rag/chunking.ts | 2 +- .../feature-improve-rag-chunking.md | 25 ++++++++----------- ...0317000001_add_user_rag_index_settings.sql | 8 +++--- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index cbc3e4dfe52..f16b151af80 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -41,7 +41,7 @@ const BLOCK_BREAK_PATTERN = /<\/?(?:p|div|li|blockquote|pre|ul|ol|section|articl function normalizeWhitespace(value: string): string { return value .replaceAll(/ /gi, " ") - .replaceAll(/\u00a0/g, " ") + .replaceAll("\u00a0", " ") .replaceAll(/\s+/g, " ") .trim() } diff --git a/docs/ai/implementation/feature-improve-rag-chunking.md b/docs/ai/implementation/feature-improve-rag-chunking.md index 3fb34c679ce..ba480df4bbf 100644 --- a/docs/ai/implementation/feature-improve-rag-chunking.md +++ b/docs/ai/implementation/feature-improve-rag-chunking.md @@ -8,18 +8,15 @@ description: Technical notes for configurable hierarchical chunking and indexing ## Decision Update - 2026-03-17 -Implementation must now follow these clarified chunk-assembly rules: - -- treat paragraph boundaries as the default assembly boundary -- use `min_chunk_size` as the primary condition for closing a chunk assembled from small paragraphs -- after `min_chunk_size` is reached, another whole paragraph may be appended only if it improves fit toward `target_chunk_size` -- do not append a whole paragraph that would overshoot `target_chunk_size`, even if it is still within `max_chunk_size` -- if the current chunk is still below `min_chunk_size` and the next whole paragraph would exceed `max_chunk_size`, split that next paragraph internally to finish the chunk -- notes shorter than `min_chunk_size` are not indexed (return empty array); `small_note_threshold` has been removed, `min_chunk_size` now serves both as minimum chunk size and minimum note size -- oversized paragraphs (> `max_chunk_size`) are split at `max_chunk_size` boundaries (minimal cuts), not at `target_chunk_size` -- if the last piece after splitting an oversized paragraph is below `min_chunk_size`, merge it back into the previous piece (backward merge); effective maximum is `max_chunk_size + min_chunk_size - 1` -- when a trailing chunk is undersized, try backward merge first and leave it undersized if merging would exceed `max_chunk_size` -- keep overlap one-directional from previous chunk into next chunk +Chunk-assembly rules have been refined to a strict `paragraph-first` model. The authoritative specification — including rationale and trade-offs for paragraph-boundary assembly, `min_chunk_size`/`target_chunk_size`/`max_chunk_size` behaviors, oversized-paragraph splitting with backward merge, and one-directional overlap — lives in the [design document](../design/feature-improve-rag-chunking.md#decision-update---2026-03-17). + +Key actionable points for implementers: + +- `small_note_threshold` is removed; `min_chunk_size` now serves as both minimum chunk size and minimum note size for indexing +- paragraph boundaries are the default assembly boundary; `min_chunk_size` is the first stopping condition +- `target_chunk_size` only matters after `min_chunk_size` is reached and only for whole-paragraph additions +- oversized paragraphs split at `max_chunk_size` (not `target_chunk_size`); undersized tail pieces merge backward +- overlap is one-directional (previous chunk suffix prepended to next chunk) ## Development Setup @@ -158,8 +155,8 @@ function validateRagIndexingSettings(input: Partial): Valid Validation rules to enforce in both UI and server paths: - `target_chunk_size`, `min_chunk_size`, `max_chunk_size` must each be within `50..5000` -- `overlap` must be within `0..5000` -- `min_chunk_size <= target_chunk_size <= max_chunk_size` +- `overlap` must be within `0..min_chunk_size - 1` (strictly less than `min_chunk_size`) +- Cross-field invariants: `min_chunk_size <= target_chunk_size <= max_chunk_size` and `overlap < min_chunk_size` Representative placement: diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql index 53ebd74a9e2..926a66bbed6 100644 --- a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -20,7 +20,7 @@ ALTER TABLE public.user_rag_index_settings ENABLE ROW LEVEL SECURITY; DO $$ BEGIN IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can view own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can view own rag index settings' ) THEN CREATE POLICY "Users can view own rag index settings" ON public.user_rag_index_settings FOR SELECT @@ -28,7 +28,7 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can insert own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can insert own rag index settings' ) THEN CREATE POLICY "Users can insert own rag index settings" ON public.user_rag_index_settings FOR INSERT @@ -36,7 +36,7 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can update own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can update own rag index settings' ) THEN CREATE POLICY "Users can update own rag index settings" ON public.user_rag_index_settings FOR UPDATE @@ -45,7 +45,7 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE tablename = 'user_rag_index_settings' AND policyname = 'Users can delete own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can delete own rag index settings' ) THEN CREATE POLICY "Users can delete own rag index settings" ON public.user_rag_index_settings FOR DELETE From ed2e4ef4b53fbbb1a940b6f2f8e400c715ee99ba Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 13:19:31 +0100 Subject: [PATCH 16/18] fixes --- core/rag/chunking.ts | 5 +++++ .../component/features/notes/RagIndexPanel.cy.tsx | 14 +++++++++++--- .../20260317000001_add_user_rag_index_settings.sql | 8 ++++---- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/core/rag/chunking.ts b/core/rag/chunking.ts index f16b151af80..4d4dfbdf2fa 100644 --- a/core/rag/chunking.ts +++ b/core/rag/chunking.ts @@ -169,6 +169,11 @@ function splitAndStripParagraphs(text: string, sectionHeading: string | null): R .map((cleaned) => ({ sectionHeading, text: cleaned })) } +// Regex fallback for list prefixing — used only when DOMParser is unavailable. +// Limitation: nested lists (e.g.,
        1. Item
          • nested
        ) +// are not handled correctly because the
      • regex uses a non-greedy match that +// stops at the first
      • . When DOMParser is available, collectBlocksFromDom +// handles nested lists properly via DOM traversal. function prefixListItems(html: string): string { // Ordered lists: prepend "1. ", "2. ", etc. let result = html.replaceAll(/]*>([\s\S]*?)<\/ol>/gi, (_match, inner: string) => { diff --git a/cypress/component/features/notes/RagIndexPanel.cy.tsx b/cypress/component/features/notes/RagIndexPanel.cy.tsx index 3914e8d142f..3933b2cdc55 100644 --- a/cypress/component/features/notes/RagIndexPanel.cy.tsx +++ b/cypress/component/features/notes/RagIndexPanel.cy.tsx @@ -58,6 +58,10 @@ function createSupabaseForRag(rows: EmbeddingRow[], invokeImpl?: (name: string, describe('RagIndexPanel Component', () => { const testUser = { id: 'user-1' } as User + beforeEach(() => { + localStorage.removeItem('rag-debug-chunks') + }) + it('renders unindexed state', () => { const { supabase, from, select, eqNote, eqUser } = createSupabaseForRag([]) @@ -109,7 +113,7 @@ describe('RagIndexPanel Component', () => { cy.contains('button', 'RAG Index').click() cy.wrap(invoke).should('have.been.calledWith', 'rag-index', { - body: { noteId: 'note-1', action: 'index', debugChunks: true }, + body: { noteId: 'note-1', action: 'index' }, }) }) @@ -129,7 +133,7 @@ describe('RagIndexPanel Component', () => { cy.contains('button', 'Re-index').click() cy.wrap(invoke).should('have.been.calledWith', 'rag-index', { - body: { noteId: 'note-1', action: 'reindex', debugChunks: true }, + body: { noteId: 'note-1', action: 'reindex' }, }) }) @@ -176,6 +180,10 @@ describe('RagIndexPanel Component', () => { describe('RagIndexPanel (variant=menu)', () => { const testUser = { id: 'user-1' } as User + beforeEach(() => { + localStorage.removeItem('rag-debug-chunks') + }) + function mountMenuVariant({ rows = [] as EmbeddingRow[], onMenuClose = cy.stub() as Cypress.Agent, @@ -284,7 +292,7 @@ describe('RagIndexPanel (variant=menu)', () => { ) cy.contains('[role="menuitem"]', 'Index note').click() cy.wrap(supabase.functions.invoke).should('have.been.calledWith', 'rag-index', { - body: { noteId: 'note-1', action: 'index', debugChunks: true }, + body: { noteId: 'note-1', action: 'index' }, }) // onMenuClose called after operation settles — dropdown closes only then cy.get('@onMenuClose').should('have.been.calledOnce') diff --git a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql index 926a66bbed6..641c1a91284 100644 --- a/supabase/migrations/20260317000001_add_user_rag_index_settings.sql +++ b/supabase/migrations/20260317000001_add_user_rag_index_settings.sql @@ -20,7 +20,7 @@ ALTER TABLE public.user_rag_index_settings ENABLE ROW LEVEL SECURITY; DO $$ BEGIN IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can view own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = 'public' AND tablename = 'user_rag_index_settings' AND policyname = 'Users can view own rag index settings' ) THEN CREATE POLICY "Users can view own rag index settings" ON public.user_rag_index_settings FOR SELECT @@ -28,7 +28,7 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can insert own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = 'public' AND tablename = 'user_rag_index_settings' AND policyname = 'Users can insert own rag index settings' ) THEN CREATE POLICY "Users can insert own rag index settings" ON public.user_rag_index_settings FOR INSERT @@ -36,7 +36,7 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can update own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = 'public' AND tablename = 'user_rag_index_settings' AND policyname = 'Users can update own rag index settings' ) THEN CREATE POLICY "Users can update own rag index settings" ON public.user_rag_index_settings FOR UPDATE @@ -45,7 +45,7 @@ BEGIN END IF; IF NOT EXISTS ( - SELECT 1 FROM pg_policies WHERE schemaname = current_schema() AND tablename = 'user_rag_index_settings' AND policyname = 'Users can delete own rag index settings' + SELECT 1 FROM pg_policies WHERE schemaname = 'public' AND tablename = 'user_rag_index_settings' AND policyname = 'Users can delete own rag index settings' ) THEN CREATE POLICY "Users can delete own rag index settings" ON public.user_rag_index_settings FOR DELETE From 809a19dc515e7cc3e012df5506d3521b03697736 Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 13:37:30 +0100 Subject: [PATCH 17/18] one fix --- core/rag/debugLog.ts | 3 +-- package.json | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/rag/debugLog.ts b/core/rag/debugLog.ts index 5701d92a761..863b0a8874f 100644 --- a/core/rag/debugLog.ts +++ b/core/rag/debugLog.ts @@ -35,8 +35,7 @@ export function logRagIndexDebugChunks(noteId: string, chunks: RagIndexDebugChun contentLength: chunk.content.length, preview: previewContent(chunk.content), }) - console.log("content:") - console.log(chunk.content.length > 0 ? chunk.content : "") + console.log("content preview:", chunk.content.length > 0 ? previewContent(chunk.content) : "") } closeGroup() } diff --git a/package.json b/package.json index 67c4a723abc..8ad60c2d9b3 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "db:migration:list": "supabase migration list", "db:status": "supabase status", "db:studio": "supabase studio", + "supa-functions-deploy-all": "supabase functions deploy", "functions:serve": "supabase functions serve --no-verify-jwt --env-file .env.local", "db:init-users": "node scripts/init-test-users.js", "perf:generate": "node scripts/generate-test-notes.js", From f5a9c1c202251abb33bbcb9e669dbe3f50fb1583 Mon Sep 17 00:00:00 2001 From: Denys Date: Wed, 18 Mar 2026 13:59:45 +0100 Subject: [PATCH 18/18] chunks template --- core/rag/chunkTemplate.ts | 28 +++++++++++++---------- core/tests/unit/core-rag-chunking.test.ts | 4 ++-- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/core/rag/chunkTemplate.ts b/core/rag/chunkTemplate.ts index 50db0f2bd7b..24fecb299fb 100644 --- a/core/rag/chunkTemplate.ts +++ b/core/rag/chunkTemplate.ts @@ -14,20 +14,20 @@ export function getRagChunkBodyText(content: string): string { if (!normalized) return "" const lines = normalized.split("\n") - let cursor = 0 + let end = lines.length - if (lines[cursor]?.startsWith("Section: ")) { - cursor += 1 + if (lines[end - 1]?.startsWith("Tags: ")) { + end -= 1 } - if (lines[cursor]?.startsWith("Tags: ")) { - cursor += 1 + if (lines[end - 1]?.startsWith("Section: ")) { + end -= 1 } - if (cursor > 0 && lines[cursor] === "") { - return lines.slice(cursor + 1).join("\n").trim() + if (end < lines.length && lines[end - 1] === "") { + return lines.slice(0, end - 1).join("\n").trim() } - return normalized + return lines.slice(0, end).join("\n").trim() } export function getRagChunkBodyLength(content: string): number { @@ -45,25 +45,29 @@ export function buildRagChunkText({ if (!normalizedContent) return "" + lines.push(normalizedContent) + + const metaLines: string[] = [] + if (settings.use_section_headings && sectionHeading) { const normalizedHeading = normalizeInlineText(sectionHeading) if (normalizedHeading) { - lines.push(`Section: ${normalizedHeading}`) + metaLines.push(`Section: ${normalizedHeading}`) } } if (settings.use_tags && tags.length > 0) { const normalizedTags = tags.map(normalizeInlineText).filter(Boolean) if (normalizedTags.length > 0) { - lines.push(`Tags: ${normalizedTags.join(", ")}`) + metaLines.push(`Tags: ${normalizedTags.join(", ")}`) } } - if (lines.length > 0) { + if (metaLines.length > 0) { lines.push("") + lines.push(...metaLines) } - lines.push(normalizedContent) return lines.join("\n") } diff --git a/core/tests/unit/core-rag-chunking.test.ts b/core/tests/unit/core-rag-chunking.test.ts index e050239b4d9..766e9ace621 100644 --- a/core/tests/unit/core-rag-chunking.test.ts +++ b/core/tests/unit/core-rag-chunking.test.ts @@ -305,7 +305,7 @@ describe("core/rag/chunking — pairwise test suite", () => { chunkContent: "Body text", settings: { use_section_headings: true, use_tags: true }, }) - expect(text).toBe("Section: Intro\nTags: a, b\n\nBody text") + expect(text).toBe("Body text\n\nSection: Intro\nTags: a, b") }) it("H2: heading=null → строка Section: опущена", () => { @@ -315,7 +315,7 @@ describe("core/rag/chunking — pairwise test suite", () => { chunkContent: "Body text", settings: { use_section_headings: true, use_tags: true }, }) - expect(text).toBe("Tags: a\n\nBody text") + expect(text).toBe("Body text\n\nTags: a") expect(text).not.toContain("Section:") })