feat: add configurable CJK keyword tokenization

.github/workflows/stale.yml

@@ -20,8 +20,8 @@ jobs:
           close-pr-message: 'This PR has been automatically closed due to inactivity.'
           days-before-stale: 30  # Days of inactivity before marking as stale
           days-before-close: 7  # Days of inactivity before closing stale issues/PRs
-          stale-issue-label: 'stale'
-          stale-pr-label: 'stale'
-          exempt-issue-labels: 'do not close'
-          exempt-pr-labels: 'do not close'
+          stale-issue-label: 'status:stale'
+          stale-pr-label: 'status:stale'
+          exempt-issue-labels: 'status:do not close'
+          exempt-pr-labels: 'status:do not close'
           remove-stale-when-updated: true

apps/memos-local-plugin/core/config/defaults.ts

@@ -53,6 +53,9 @@ export const DEFAULT_CONFIG: ResolvedConfig = {
     temperature: 0,
     timeoutMs: 60_000,
   },
+  storage: {
+    ftsTokenizer: "trigram",
+  },
   algorithm: {
     lightweightMemory: {
       enabled: true,

apps/memos-local-plugin/core/config/schema.ts

@@ -91,6 +91,18 @@ const SkillEvolverSchema = Type.Object({
   timeoutMs: NumberInRange(60_000, 1_000),
 }, { default: {} });
 
+const StorageSchema = Type.Object({
+  /**
+   * Keyword tokenizer mode used when compiling FTS5 MATCH expressions.
+   * `trigram` preserves the historical SQLite trigram behavior; `cjk`
+   * keeps short Chinese words and mixed ASCII+CJK tokens searchable.
+   */
+  ftsTokenizer: Type.Union([
+    Type.Literal("trigram"),
+    Type.Literal("cjk"),
+  ], { default: "trigram" }),
+}, { default: {} });
+
 const AlgorithmSchema = Type.Object({
   lightweightMemory: Type.Object({
     /**
@@ -580,6 +592,7 @@ export const ConfigSchema = Type.Object({
   embedding: EmbeddingSchema,
   llm: LlmSchema,
   skillEvolver: SkillEvolverSchema,
+  storage: StorageSchema,
   algorithm: AlgorithmSchema,
   hub: HubSchema,
   telemetry: TelemetrySchema,

apps/memos-local-plugin/core/pipeline/deps.ts

@@ -148,6 +148,7 @@ export function extractAlgorithmConfig(
       minTraceSim: alg.retrieval.minTraceSim,
       tagFilter: alg.retrieval.tagFilter,
       keywordTopK: alg.retrieval.keywordTopK,
+      ftsTokenizer: deps.config.storage.ftsTokenizer,
       relativeThresholdFloor: alg.retrieval.relativeThresholdFloor,
       skillEtaBlend: alg.retrieval.skillEtaBlend,
       smartSeed: alg.retrieval.smartSeed,

apps/memos-local-plugin/core/retrieval/query-builder.ts

@@ -10,11 +10,19 @@
  */
 
 import { extractErrorSignatures } from "../capture/error-signature.js";
-import { extractPatternTerms, prepareFtsMatch } from "../storage/keyword.js";
+import {
+  extractPatternTerms,
+  prepareFtsMatch,
+  type FtsTokenizerMode,
+} from "../storage/keyword.js";
 import type { RetrievalCtx } from "./types.js";
 
 const MAX_QUERY_CHARS = 1_500;
 
+export interface BuildQueryOptions {
+  ftsTokenizer?: FtsTokenizerMode;
+}
+
 /** Public tag list kept in sync with `capture/tagger.ts#KEYWORD_TAGS`. */
 const KEYWORD_TAGS: ReadonlyArray<{ re: RegExp; tag: string }> = [
   { re: /\bdocker\b|\bcontainer\b/i, tag: "docker" },
@@ -68,36 +76,36 @@ export interface CompiledQuery {
  * Build a `CompiledQuery` from a retrieval context. Behavior varies per
  * reason so that e.g. `decision_repair` biases toward the failing tool name.
  */
-export function buildQuery(ctx: RetrievalCtx): CompiledQuery {
+export function buildQuery(ctx: RetrievalCtx, opts: BuildQueryOptions = {}): CompiledQuery {
   switch (ctx.reason) {
     case "turn_start": {
       const hintText = hintToText(ctx.contextHints);
       const parts = [ctx.userText?.trim() ?? ""];
       if (hintText) parts.push(hintText);
-      return finalize(parts.join("\n"));
+      return finalize(parts.join("\n"), opts);
     }
     case "tool_driven": {
       if (typeof ctx.args?.query === "string" && ctx.args.query.trim()) {
         const rest = { ...ctx.args };
         delete rest.query;
         const restText = Object.keys(rest).length > 0 ? renderArgs(rest) : "";
-        return finalize([ctx.args.query.trim(), restText].filter(Boolean).join("\n"));
+        return finalize([ctx.args.query.trim(), restText].filter(Boolean).join("\n"), opts);
       }
       const args = renderArgs(ctx.args);
-      return finalize(`tool:${ctx.tool}\n${args}`);
+      return finalize(`tool:${ctx.tool}\n${args}`, opts);
     }
     case "skill_invoke": {
       const head = ctx.skillId ? `skill:${ctx.skillId}\n` : "";
-      return finalize(head + (ctx.query ?? ""));
+      return finalize(head + (ctx.query ?? ""), opts);
     }
     case "sub_agent": {
       const profile = ctx.profile ? `profile:${ctx.profile}\n` : "";
-      return finalize(profile + (ctx.mission ?? ""));
+      return finalize(profile + (ctx.mission ?? ""), opts);
     }
     case "decision_repair": {
       const head = `failing_tool:${ctx.failingTool}\nfailures:${ctx.failureCount}\n`;
       const tail = ctx.lastErrorCode ? `error:${ctx.lastErrorCode}` : "";
-      return finalize(head + tail);
+      return finalize(head + tail, opts);
     }
     default: {
       // Exhaustiveness — compile-time check.
@@ -126,7 +134,7 @@ export function extractTags(text: string): string[] {
 
 // ─── Helpers ────────────────────────────────────────────────────────────────
 
-function finalize(raw: string): CompiledQuery {
+function finalize(raw: string, opts: BuildQueryOptions): CompiledQuery {
   const trimmed = (raw ?? "").trim();
   if (!trimmed) {
     return {
@@ -149,7 +157,7 @@ function finalize(raw: string): CompiledQuery {
   // Keyword channels — derived from the original text *before* truncation
   // so we don't lose tail content. The actual queries are bounded by the
   // helpers themselves.
-  const ftsMatch = prepareFtsMatch(trimmed);
+  const ftsMatch = prepareFtsMatch(trimmed, { tokenizer: opts.ftsTokenizer });
   const patternTerms = extractPatternTerms(trimmed);
   if (trimmed.length <= MAX_QUERY_CHARS) {
     return {

apps/memos-local-plugin/core/retrieval/retrieve.ts

@@ -236,7 +236,7 @@ async function runAll(
   const episodeId = (ctx as { episodeId?: EpisodeId }).episodeId;
   const ts = deps.now();
 
-  const compiled = buildQuery(ctx);
+  const compiled = buildQuery(ctx, { ftsTokenizer: deps.config.ftsTokenizer });
   opts.events?.emit({
     kind: "retrieval.started",
     reason: ctx.reason,

apps/memos-local-plugin/core/retrieval/types.ts

@@ -234,6 +234,8 @@ export interface RetrievalConfig {
 
   /** Per-tier keyword (FTS + pattern) channel size. Default 20. */
   keywordTopK?: number;
+  /** Keyword tokenizer mode used when compiling FTS5 MATCH expressions. */
+  ftsTokenizer?: "trigram" | "cjk";
   /**
    * Drop candidates with `relevance < topRelevance · this`. 0 disables
    * the relative cutoff. Default 0.4.

apps/memos-local-plugin/core/storage/keyword.ts

@@ -4,9 +4,9 @@
  * Two utilities live here:
  *
  *   1. `prepareFtsMatch(query)` — sanitise a free-form user query for an
- *      FTS5 MATCH clause. We split on whitespace, drop tokens shorter
- *      than the trigram window where useful, escape internal quotes and
- *      AND the resulting phrases.
+ *      FTS5 MATCH clause. We split on whitespace, apply the configured
+ *      keyword token mode, escape internal quotes and AND the resulting
+ *      phrases.
  *
  *   2. `extractPatternTerms(query)` — return short tokens (length 2)
  *      and CJK bigrams (sliding 2-char windows over CJK runs). These
@@ -25,22 +25,37 @@ const PATTERN_MIN = 2;
 const MAX_FTS_TOKENS = 12;
 const MAX_PATTERN_TERMS = 16;
 
+export type FtsTokenizerMode = "trigram" | "cjk";
+
+export interface PrepareFtsMatchOptions {
+  tokenizer?: FtsTokenizerMode;
+}
+
 /**
  * Sanitised FTS5 MATCH expression.
  *
  * Returns `null` when no usable token is left (caller should skip the
  * FTS channel rather than issue an empty MATCH).
  */
-export function prepareFtsMatch(query: string): string | null {
+export function prepareFtsMatch(
+  query: string,
+  opts: PrepareFtsMatchOptions = {},
+): string | null {
   if (!query) return null;
   const cleaned = String(query).replace(PUNCT, " ").trim();
   if (!cleaned) return null;
+  const tokenizer = opts.tokenizer ?? "trigram";
 
   // Split on whitespace AND on CJK boundaries: a CJK run becomes its
   // own token so we don't end up with 50-character "phrases".
   const rough = cleaned.split(/\s+/).filter(Boolean);
   const expanded: string[] = [];
   for (const tok of rough) {
+    if (tokenizer === "cjk") {
+      expanded.push(...expandCjkToken(tok));
+      continue;
+    }
+
     // If the token has both ASCII and CJK, split out CJK runs as their
     // own tokens; trigram handles each well.
     const cjkRuns = tok.match(CJK_RUN) ?? [];
@@ -62,6 +77,34 @@ export function prepareFtsMatch(query: string): string | null {
   return safe.join(" ");
 }
 
+function expandCjkToken(token: string): string[] {
+  const out: string[] = [];
+  const cjkRuns = token.match(CJK_RUN) ?? [];
+  let stripped = token;
+  for (const run of cjkRuns) {
+    stripped = stripped.replace(run, " ");
+    if (run.length === 1) {
+      out.push(run);
+      continue;
+    }
+    for (let i = 0; i <= run.length - PATTERN_MIN; i++) {
+      out.push(run.slice(i, i + PATTERN_MIN));
+    }
+  }
+
+  for (const sub of stripped.split(/\s+/).filter(Boolean)) {
+    if (sub.length >= PATTERN_MIN) out.push(sub);
+  }
+
+  const hasNonCjk = /[^\u4e00-\u9fff]/.test(token);
+  const hasCjk = /[\u4e00-\u9fff\u3400-\u4dbf\uF900-\uFAFF]/.test(token);
+  if (token.length >= PATTERN_MIN && hasNonCjk && hasCjk) {
+    out.push(token);
+  }
+
+  return out;
+}
+
 /**
  * Pattern-channel terms — what the trigram FTS can't catch on its own.
  *

apps/memos-local-plugin/docs/CONFIG-ADVANCED.md

@@ -47,6 +47,16 @@ llm:
   maxRetries: 3
 ```
 
+### `storage`
+```yaml
+storage:
+  ftsTokenizer: trigram      # trigram | cjk
+```
+
+`trigram` preserves the historical FTS5 behavior. `cjk` keeps short Chinese
+words and mixed ASCII+CJK terms searchable in the keyword channel, which helps
+queries such as `早报`, `配置`, and `API配置`.
+
 ### `algorithm`
 Direct mapping to the V7 spec (γ, support, gain, top-K, etc.). Change only
 if you know what you're doing — defaults are calibrated for the paper.

apps/memos-local-plugin/templates/config.hermes.yaml

@@ -27,6 +27,9 @@ llm:
   apiKey: ""              # REQUIRED — fill in before running
   model: ""               # blank = provider default (e.g. gpt-4o-mini for openai_compatible)
 
+storage:
+  ftsTokenizer: trigram    # trigram | cjk; cjk improves short Chinese keyword recall
+
 algorithm:
   lightweightMemory:
     enabled: true          # true = low-cost summaries only; false = memory self-evolution with tasks/experiences/world models/skills

apps/memos-local-plugin/templates/config.openclaw.yaml

@@ -26,6 +26,9 @@ llm:
   apiKey: ""              # required except for provider=host | local_only
   model: ""               # blank = let the provider pick its default
 
+storage:
+  ftsTokenizer: trigram    # trigram | cjk; cjk improves short Chinese keyword recall
+
 algorithm:
   lightweightMemory:
     enabled: true          # true = low-cost summaries only; false = memory self-evolution with tasks/experiences/world models/skills

apps/memos-local-plugin/tests/unit/retrieval/query-builder.test.ts

@@ -37,6 +37,24 @@ describe("retrieval/query-builder", () => {
     expect(cq.tags).toContain("docker");
   });
 
+  it("passes CJK tokenizer mode to keyword query compilation", () => {
+    const cq = buildQuery(
+      {
+        reason: "tool_driven",
+        agent: "openclaw",
+        sessionId: "s1" as unknown as never,
+        tool: "memos_search",
+        args: { query: "早报 API配置 C盘" },
+        ts: NOW,
+      },
+      { ftsTokenizer: "cjk" },
+    );
+    expect(cq.ftsMatch).toContain('"早报"');
+    expect(cq.ftsMatch).toContain('"API"');
+    expect(cq.ftsMatch).toContain('"配置"');
+    expect(cq.ftsMatch).toContain('"C盘"');
+  });
+
   it("skill_invoke prepends skill id when provided", () => {
     const cq = buildQuery({
       reason: "skill_invoke",

apps/memos-local-plugin/tests/unit/storage/keyword.test.ts

@@ -32,6 +32,14 @@ describe("storage/keyword.prepareFtsMatch", () => {
     expect(prepareFtsMatch("唐波")).toBeNull(); // 2-char CJK only
   });
 
+  it("keeps short CJK and mixed ASCII+CJK terms in cjk mode", () => {
+    const out = prepareFtsMatch("早报 API配置 C盘", { tokenizer: "cjk" });
+    expect(out).toContain('"早报"');
+    expect(out).toContain('"API"');
+    expect(out).toContain('"配置"');
+    expect(out).toContain('"C盘"');
+  });
+
   it("escapes quotes inside tokens for FTS5 phrase syntax", () => {
     const out = prepareFtsMatch('check "quoted" word');
     // FTS5 phrase syntax doubles the inner quote.

docs/cn/open_source/modules/dream.md

@@ -79,7 +79,7 @@ Dream 模拟这一点：从*未完成的内在动机*出发，而不是从原始
 ```json
 {
   "motive_id": "motive:dream_memory_strategy_alignment",
-  "description": "Several conversations failed for the same hidden reason: weekly reporting, future planning, and filter design were treated as separate tasks, while the user needed a shared strategic narrative.",
+  "description": "几次对话失败，背后是同一个隐藏原因：周报、未来规划和 filter 设计被当成三个独立任务，而用户需要的是一条统一的战略叙事。",
   "memory_ids": ["weekly_report_thread", "future_planning_thread", "filter_design_thread"]
 }
 ```