diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b398497de..27b330aa9 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -20,8 +20,8 @@ jobs: close-pr-message: 'This PR has been automatically closed due to inactivity.' days-before-stale: 30 # Days of inactivity before marking as stale days-before-close: 7 # Days of inactivity before closing stale issues/PRs - stale-issue-label: 'stale' - stale-pr-label: 'stale' - exempt-issue-labels: 'do not close' - exempt-pr-labels: 'do not close' + stale-issue-label: 'status:stale' + stale-pr-label: 'status:stale' + exempt-issue-labels: 'status:do not close' + exempt-pr-labels: 'status:do not close' remove-stale-when-updated: true diff --git a/apps/memos-local-plugin/core/config/defaults.ts b/apps/memos-local-plugin/core/config/defaults.ts index 4ab4c4240..cbcfd9d1f 100644 --- a/apps/memos-local-plugin/core/config/defaults.ts +++ b/apps/memos-local-plugin/core/config/defaults.ts @@ -53,6 +53,9 @@ export const DEFAULT_CONFIG: ResolvedConfig = { temperature: 0, timeoutMs: 60_000, }, + storage: { + ftsTokenizer: "trigram", + }, algorithm: { lightweightMemory: { enabled: true, diff --git a/apps/memos-local-plugin/core/config/schema.ts b/apps/memos-local-plugin/core/config/schema.ts index 92479373e..2fb3ab4d0 100644 --- a/apps/memos-local-plugin/core/config/schema.ts +++ b/apps/memos-local-plugin/core/config/schema.ts @@ -91,6 +91,18 @@ const SkillEvolverSchema = Type.Object({ timeoutMs: NumberInRange(60_000, 1_000), }, { default: {} }); +const StorageSchema = Type.Object({ + /** + * Keyword tokenizer mode used when compiling FTS5 MATCH expressions. + * `trigram` preserves the historical SQLite trigram behavior; `cjk` + * keeps short Chinese words and mixed ASCII+CJK tokens searchable. + */ + ftsTokenizer: Type.Union([ + Type.Literal("trigram"), + Type.Literal("cjk"), + ], { default: "trigram" }), +}, { default: {} }); + const AlgorithmSchema = Type.Object({ lightweightMemory: Type.Object({ /** @@ -580,6 +592,7 @@ export const ConfigSchema = Type.Object({ embedding: EmbeddingSchema, llm: LlmSchema, skillEvolver: SkillEvolverSchema, + storage: StorageSchema, algorithm: AlgorithmSchema, hub: HubSchema, telemetry: TelemetrySchema, diff --git a/apps/memos-local-plugin/core/pipeline/deps.ts b/apps/memos-local-plugin/core/pipeline/deps.ts index 7bd23d681..f28909c94 100644 --- a/apps/memos-local-plugin/core/pipeline/deps.ts +++ b/apps/memos-local-plugin/core/pipeline/deps.ts @@ -148,6 +148,7 @@ export function extractAlgorithmConfig( minTraceSim: alg.retrieval.minTraceSim, tagFilter: alg.retrieval.tagFilter, keywordTopK: alg.retrieval.keywordTopK, + ftsTokenizer: deps.config.storage.ftsTokenizer, relativeThresholdFloor: alg.retrieval.relativeThresholdFloor, skillEtaBlend: alg.retrieval.skillEtaBlend, smartSeed: alg.retrieval.smartSeed, diff --git a/apps/memos-local-plugin/core/retrieval/query-builder.ts b/apps/memos-local-plugin/core/retrieval/query-builder.ts index b63819ad5..940edae15 100644 --- a/apps/memos-local-plugin/core/retrieval/query-builder.ts +++ b/apps/memos-local-plugin/core/retrieval/query-builder.ts @@ -10,11 +10,19 @@ */ import { extractErrorSignatures } from "../capture/error-signature.js"; -import { extractPatternTerms, prepareFtsMatch } from "../storage/keyword.js"; +import { + extractPatternTerms, + prepareFtsMatch, + type FtsTokenizerMode, +} from "../storage/keyword.js"; import type { RetrievalCtx } from "./types.js"; const MAX_QUERY_CHARS = 1_500; +export interface BuildQueryOptions { + ftsTokenizer?: FtsTokenizerMode; +} + /** Public tag list kept in sync with `capture/tagger.ts#KEYWORD_TAGS`. */ const KEYWORD_TAGS: ReadonlyArray<{ re: RegExp; tag: string }> = [ { re: /\bdocker\b|\bcontainer\b/i, tag: "docker" }, @@ -68,36 +76,36 @@ export interface CompiledQuery { * Build a `CompiledQuery` from a retrieval context. Behavior varies per * reason so that e.g. `decision_repair` biases toward the failing tool name. */ -export function buildQuery(ctx: RetrievalCtx): CompiledQuery { +export function buildQuery(ctx: RetrievalCtx, opts: BuildQueryOptions = {}): CompiledQuery { switch (ctx.reason) { case "turn_start": { const hintText = hintToText(ctx.contextHints); const parts = [ctx.userText?.trim() ?? ""]; if (hintText) parts.push(hintText); - return finalize(parts.join("\n")); + return finalize(parts.join("\n"), opts); } case "tool_driven": { if (typeof ctx.args?.query === "string" && ctx.args.query.trim()) { const rest = { ...ctx.args }; delete rest.query; const restText = Object.keys(rest).length > 0 ? renderArgs(rest) : ""; - return finalize([ctx.args.query.trim(), restText].filter(Boolean).join("\n")); + return finalize([ctx.args.query.trim(), restText].filter(Boolean).join("\n"), opts); } const args = renderArgs(ctx.args); - return finalize(`tool:${ctx.tool}\n${args}`); + return finalize(`tool:${ctx.tool}\n${args}`, opts); } case "skill_invoke": { const head = ctx.skillId ? `skill:${ctx.skillId}\n` : ""; - return finalize(head + (ctx.query ?? "")); + return finalize(head + (ctx.query ?? ""), opts); } case "sub_agent": { const profile = ctx.profile ? `profile:${ctx.profile}\n` : ""; - return finalize(profile + (ctx.mission ?? "")); + return finalize(profile + (ctx.mission ?? ""), opts); } case "decision_repair": { const head = `failing_tool:${ctx.failingTool}\nfailures:${ctx.failureCount}\n`; const tail = ctx.lastErrorCode ? `error:${ctx.lastErrorCode}` : ""; - return finalize(head + tail); + return finalize(head + tail, opts); } default: { // Exhaustiveness — compile-time check. @@ -126,7 +134,7 @@ export function extractTags(text: string): string[] { // ─── Helpers ──────────────────────────────────────────────────────────────── -function finalize(raw: string): CompiledQuery { +function finalize(raw: string, opts: BuildQueryOptions): CompiledQuery { const trimmed = (raw ?? "").trim(); if (!trimmed) { return { @@ -149,7 +157,7 @@ function finalize(raw: string): CompiledQuery { // Keyword channels — derived from the original text *before* truncation // so we don't lose tail content. The actual queries are bounded by the // helpers themselves. - const ftsMatch = prepareFtsMatch(trimmed); + const ftsMatch = prepareFtsMatch(trimmed, { tokenizer: opts.ftsTokenizer }); const patternTerms = extractPatternTerms(trimmed); if (trimmed.length <= MAX_QUERY_CHARS) { return { diff --git a/apps/memos-local-plugin/core/retrieval/retrieve.ts b/apps/memos-local-plugin/core/retrieval/retrieve.ts index c8f656a1b..da4604589 100644 --- a/apps/memos-local-plugin/core/retrieval/retrieve.ts +++ b/apps/memos-local-plugin/core/retrieval/retrieve.ts @@ -236,7 +236,7 @@ async function runAll( const episodeId = (ctx as { episodeId?: EpisodeId }).episodeId; const ts = deps.now(); - const compiled = buildQuery(ctx); + const compiled = buildQuery(ctx, { ftsTokenizer: deps.config.ftsTokenizer }); opts.events?.emit({ kind: "retrieval.started", reason: ctx.reason, diff --git a/apps/memos-local-plugin/core/retrieval/types.ts b/apps/memos-local-plugin/core/retrieval/types.ts index f83f74620..19dca2f43 100644 --- a/apps/memos-local-plugin/core/retrieval/types.ts +++ b/apps/memos-local-plugin/core/retrieval/types.ts @@ -234,6 +234,8 @@ export interface RetrievalConfig { /** Per-tier keyword (FTS + pattern) channel size. Default 20. */ keywordTopK?: number; + /** Keyword tokenizer mode used when compiling FTS5 MATCH expressions. */ + ftsTokenizer?: "trigram" | "cjk"; /** * Drop candidates with `relevance < topRelevance · this`. 0 disables * the relative cutoff. Default 0.4. diff --git a/apps/memos-local-plugin/core/storage/keyword.ts b/apps/memos-local-plugin/core/storage/keyword.ts index 446278fd3..0b14fdf49 100644 --- a/apps/memos-local-plugin/core/storage/keyword.ts +++ b/apps/memos-local-plugin/core/storage/keyword.ts @@ -4,9 +4,9 @@ * Two utilities live here: * * 1. `prepareFtsMatch(query)` — sanitise a free-form user query for an - * FTS5 MATCH clause. We split on whitespace, drop tokens shorter - * than the trigram window where useful, escape internal quotes and - * AND the resulting phrases. + * FTS5 MATCH clause. We split on whitespace, apply the configured + * keyword token mode, escape internal quotes and AND the resulting + * phrases. * * 2. `extractPatternTerms(query)` — return short tokens (length 2) * and CJK bigrams (sliding 2-char windows over CJK runs). These @@ -25,22 +25,37 @@ const PATTERN_MIN = 2; const MAX_FTS_TOKENS = 12; const MAX_PATTERN_TERMS = 16; +export type FtsTokenizerMode = "trigram" | "cjk"; + +export interface PrepareFtsMatchOptions { + tokenizer?: FtsTokenizerMode; +} + /** * Sanitised FTS5 MATCH expression. * * Returns `null` when no usable token is left (caller should skip the * FTS channel rather than issue an empty MATCH). */ -export function prepareFtsMatch(query: string): string | null { +export function prepareFtsMatch( + query: string, + opts: PrepareFtsMatchOptions = {}, +): string | null { if (!query) return null; const cleaned = String(query).replace(PUNCT, " ").trim(); if (!cleaned) return null; + const tokenizer = opts.tokenizer ?? "trigram"; // Split on whitespace AND on CJK boundaries: a CJK run becomes its // own token so we don't end up with 50-character "phrases". const rough = cleaned.split(/\s+/).filter(Boolean); const expanded: string[] = []; for (const tok of rough) { + if (tokenizer === "cjk") { + expanded.push(...expandCjkToken(tok)); + continue; + } + // If the token has both ASCII and CJK, split out CJK runs as their // own tokens; trigram handles each well. const cjkRuns = tok.match(CJK_RUN) ?? []; @@ -62,6 +77,34 @@ export function prepareFtsMatch(query: string): string | null { return safe.join(" "); } +function expandCjkToken(token: string): string[] { + const out: string[] = []; + const cjkRuns = token.match(CJK_RUN) ?? []; + let stripped = token; + for (const run of cjkRuns) { + stripped = stripped.replace(run, " "); + if (run.length === 1) { + out.push(run); + continue; + } + for (let i = 0; i <= run.length - PATTERN_MIN; i++) { + out.push(run.slice(i, i + PATTERN_MIN)); + } + } + + for (const sub of stripped.split(/\s+/).filter(Boolean)) { + if (sub.length >= PATTERN_MIN) out.push(sub); + } + + const hasNonCjk = /[^\u4e00-\u9fff]/.test(token); + const hasCjk = /[\u4e00-\u9fff\u3400-\u4dbf\uF900-\uFAFF]/.test(token); + if (token.length >= PATTERN_MIN && hasNonCjk && hasCjk) { + out.push(token); + } + + return out; +} + /** * Pattern-channel terms — what the trigram FTS can't catch on its own. * diff --git a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md index 014e3deea..67915a397 100644 --- a/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md +++ b/apps/memos-local-plugin/docs/CONFIG-ADVANCED.md @@ -47,6 +47,16 @@ llm: maxRetries: 3 ``` +### `storage` +```yaml +storage: + ftsTokenizer: trigram # trigram | cjk +``` + +`trigram` preserves the historical FTS5 behavior. `cjk` keeps short Chinese +words and mixed ASCII+CJK terms searchable in the keyword channel, which helps +queries such as `早报`, `配置`, and `API配置`. + ### `algorithm` Direct mapping to the V7 spec (γ, support, gain, top-K, etc.). Change only if you know what you're doing — defaults are calibrated for the paper. diff --git a/apps/memos-local-plugin/templates/config.hermes.yaml b/apps/memos-local-plugin/templates/config.hermes.yaml index 191580f85..d08b320db 100644 --- a/apps/memos-local-plugin/templates/config.hermes.yaml +++ b/apps/memos-local-plugin/templates/config.hermes.yaml @@ -27,6 +27,9 @@ llm: apiKey: "" # REQUIRED — fill in before running model: "" # blank = provider default (e.g. gpt-4o-mini for openai_compatible) +storage: + ftsTokenizer: trigram # trigram | cjk; cjk improves short Chinese keyword recall + algorithm: lightweightMemory: enabled: true # true = low-cost summaries only; false = memory self-evolution with tasks/experiences/world models/skills diff --git a/apps/memos-local-plugin/templates/config.openclaw.yaml b/apps/memos-local-plugin/templates/config.openclaw.yaml index e263dd3c2..1a84a55dc 100644 --- a/apps/memos-local-plugin/templates/config.openclaw.yaml +++ b/apps/memos-local-plugin/templates/config.openclaw.yaml @@ -26,6 +26,9 @@ llm: apiKey: "" # required except for provider=host | local_only model: "" # blank = let the provider pick its default +storage: + ftsTokenizer: trigram # trigram | cjk; cjk improves short Chinese keyword recall + algorithm: lightweightMemory: enabled: true # true = low-cost summaries only; false = memory self-evolution with tasks/experiences/world models/skills diff --git a/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts b/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts index f2efde86c..f5b55cf37 100644 --- a/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts +++ b/apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts @@ -37,6 +37,24 @@ describe("retrieval/query-builder", () => { expect(cq.tags).toContain("docker"); }); + it("passes CJK tokenizer mode to keyword query compilation", () => { + const cq = buildQuery( + { + reason: "tool_driven", + agent: "openclaw", + sessionId: "s1" as unknown as never, + tool: "memos_search", + args: { query: "早报 API配置 C盘" }, + ts: NOW, + }, + { ftsTokenizer: "cjk" }, + ); + expect(cq.ftsMatch).toContain('"早报"'); + expect(cq.ftsMatch).toContain('"API"'); + expect(cq.ftsMatch).toContain('"配置"'); + expect(cq.ftsMatch).toContain('"C盘"'); + }); + it("skill_invoke prepends skill id when provided", () => { const cq = buildQuery({ reason: "skill_invoke", diff --git a/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts b/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts index bcc9ba686..62bf20c2e 100644 --- a/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts +++ b/apps/memos-local-plugin/tests/unit/storage/keyword.test.ts @@ -32,6 +32,14 @@ describe("storage/keyword.prepareFtsMatch", () => { expect(prepareFtsMatch("唐波")).toBeNull(); // 2-char CJK only }); + it("keeps short CJK and mixed ASCII+CJK terms in cjk mode", () => { + const out = prepareFtsMatch("早报 API配置 C盘", { tokenizer: "cjk" }); + expect(out).toContain('"早报"'); + expect(out).toContain('"API"'); + expect(out).toContain('"配置"'); + expect(out).toContain('"C盘"'); + }); + it("escapes quotes inside tokens for FTS5 phrase syntax", () => { const out = prepareFtsMatch('check "quoted" word'); // FTS5 phrase syntax doubles the inner quote. diff --git a/docs/cn/open_source/modules/dream.md b/docs/cn/open_source/modules/dream.md index a83beab42..55cd9e2cf 100644 --- a/docs/cn/open_source/modules/dream.md +++ b/docs/cn/open_source/modules/dream.md @@ -79,7 +79,7 @@ Dream 模拟这一点：从*未完成的内在动机*出发，而不是从原始 ```json { "motive_id": "motive:dream_memory_strategy_alignment", - "description": "Several conversations failed for the same hidden reason: weekly reporting, future planning, and filter design were treated as separate tasks, while the user needed a shared strategic narrative.", + "description": "几次对话失败，背后是同一个隐藏原因：周报、未来规划和 filter 设计被当成三个独立任务，而用户需要的是一条统一的战略叙事。", "memory_ids": ["weekly_report_thread", "future_planning_thread", "filter_design_thread"] } ```