diff --git a/CHANGELOG.md b/CHANGELOG.md index 63ead98..d0fdf2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Fixes + +- Summaries: inputs larger than the model's context window are now truncated to fit (head + tail, keeping the document's opening and its conclusion) instead of aborting the run with an `Input token count exceeds model input limit` error. A degraded summary of a truncated document beats no summary. The same head+tail strategy now backs `--max-extract-characters` so the closing section survives an explicit content budget. + ## 0.14.1 - 2026-04-26 ### Features diff --git a/src/content/link-preview/content/cleaner.ts b/src/content/link-preview/content/cleaner.ts index 20c8d72..46411f9 100644 --- a/src/content/link-preview/content/cleaner.ts +++ b/src/content/link-preview/content/cleaner.ts @@ -70,13 +70,71 @@ export function clipAtSentenceBoundary(input: string, maxLength: number): string return slice; } +// Marker left where the middle of an over-budget document is dropped. Kept +// short so it costs little of the budget, and explicit so the model — and +// anyone reading --json output — can tell the input was clipped, not complete. +const HEAD_TAIL_OMISSION_MARKER = "\n\n[… middle truncated to fit length budget …]\n\n"; + +// Below this budget a head+tail split leaves windows too small to be useful, so +// a single head-only clip reads better. Sized to comfortably clear the marker. +const MIN_HEAD_TAIL_BUDGET = HEAD_TAIL_OMISSION_MARKER.length + 400; + +/** + * Clip `input` to at most `maxLength` characters while keeping BOTH a head and + * a tail window, dropping the middle. For summarization the closing material — + * conclusions, results, recommendations — is often the highest-value part of a + * document, so removing the middle preserves more signal than a head-only cut. + * + * - Returns `input` unchanged when it already fits. + * - The result never exceeds `maxLength` characters (omission marker included). + * - Head and tail cuts snap to a nearby sentence/line boundary when one is + * close, so windows don't begin or end mid-sentence. + * - Falls back to a head-only {@link clipAtSentenceBoundary} when `maxLength` is + * too small to hold a meaningful head + marker + tail. + */ +export function clipHeadAndTail(input: string, maxLength: number): string { + if (input.length <= maxLength) { + return input; + } + if (maxLength <= MIN_HEAD_TAIL_BUDGET) { + return clipAtSentenceBoundary(input, maxLength); + } + + const windowBudget = maxLength - HEAD_TAIL_OMISSION_MARKER.length; + const headBudget = Math.floor(windowBudget * 0.75); + const tailBudget = windowBudget - headBudget; + + let head = input.slice(0, headBudget); + const headBreak = Math.max( + head.lastIndexOf(". "), + head.lastIndexOf("! "), + head.lastIndexOf("? "), + head.lastIndexOf("\n"), + ); + if (headBreak > headBudget * 0.5) { + head = head.slice(0, headBreak + 1); + } + + let tail = input.slice(input.length - tailBudget); + const tailBreak = Math.min( + ...[tail.indexOf("\n"), tail.indexOf(". "), tail.indexOf("! "), tail.indexOf("? ")] + .filter((index) => index >= 0) + .concat(tailBudget), + ); + if (tailBreak < tailBudget * 0.5) { + tail = tail.slice(tailBreak + 1); + } + + return `${head.trimEnd()}${HEAD_TAIL_OMISSION_MARKER}${tail.trimStart()}`; +} + export function applyContentBudget( baseContent: string, maxCharacters: number, ): ContentBudgetResult { const totalCharacters = baseContent.length; const truncated = totalCharacters > maxCharacters; - const clipped = truncated ? clipAtSentenceBoundary(baseContent, maxCharacters) : baseContent; + const clipped = truncated ? clipHeadAndTail(baseContent, maxCharacters) : baseContent; const content = clipped.trim(); const wordCount = content.length > 0 ? compact(content.split(WORD_SPLIT_PATTERN)).length : 0; return { content, truncated, totalCharacters, wordCount }; diff --git a/src/run/summary-engine.ts b/src/run/summary-engine.ts index c50203f..7c478da 100644 --- a/src/run/summary-engine.ts +++ b/src/run/summary-engine.ts @@ -1,6 +1,7 @@ import { countTokens } from "gpt-tokenizer"; import { createMarkdownStreamer, render as renderMarkdownAnsi } from "markdansi"; import type { CliProvider } from "../config.js"; +import { clipHeadAndTail } from "../content/link-preview/content/cleaner.js"; import { isCliDisabled, runCliModel } from "../llm/cli.js"; import { streamTextWithModelId } from "../llm/generate-text.js"; import { parseGatewayStyleModelId } from "../llm/model-id.js"; @@ -21,6 +22,31 @@ import { resolveModelIdForLlmCall, summarizeWithModelId } from "./summary-llm.js import { isRichTty, markdownRenderWidth, supportsColor } from "./terminal.js"; import type { ModelAttempt, ModelMeta } from "./types.js"; +// Shrink `userText` until it fits the model's input-token budget, keeping a +// head+tail window so the closing instructions and the document's conclusion +// survive (see clipHeadAndTail). gpt-tokenizer only approximates the target +// model's tokenizer, so we aim a little under budget and re-check the real +// count, tightening a few times if the estimate ran long. Used in place of a +// hard error when an assembled prompt is too large: a summary of a truncated +// document beats no summary at all. +export function fitUserTextToInputTokenBudget(userText: string, maxInputTokens: number): string { + const currentTokens = countTokens(userText); + if (currentTokens <= maxInputTokens) { + return userText; + } + const targetTokens = Math.max(1, Math.floor(maxInputTokens * 0.95)); + let charBudget = Math.max( + 1, + Math.floor(userText.length * (targetTokens / Math.max(1, currentTokens))), + ); + let clipped = clipHeadAndTail(userText, charBudget); + for (let guard = 0; guard < 12 && countTokens(clipped) > maxInputTokens; guard += 1) { + charBudget = Math.max(1, Math.floor(charBudget * 0.9)); + clipped = clipHeadAndTail(userText, charBudget); + } + return clipped; +} + export type SummaryEngineDeps = { env: Record; envForRun: Record; @@ -295,9 +321,21 @@ export function createSummaryEngine(deps: SummaryEngineDeps) { ) { const tokenCount = countTokens(prompt.userText); if (tokenCount > maxInputTokensForCall) { - throw new Error( - `Input token count (${formatCompactCount(tokenCount)}) exceeds model input limit (${formatCompactCount(maxInputTokensForCall)}). Tokenized with GPT tokenizer; prompt included.`, + // Too large for the model's context window. Rather than failing the + // run, clip the prompt to fit (keeping a head+tail window) so we still + // produce a summary of the available content. + const fittedUserText = fitUserTextToInputTokenBudget( + prompt.userText, + maxInputTokensForCall, + ); + writeVerbose( + deps.stderr, + deps.verbose, + `Input token count (${formatCompactCount(tokenCount)}) exceeds model input limit (${formatCompactCount(maxInputTokensForCall)}); truncated to ${formatCompactCount(countTokens(fittedUserText))} tokens (head+tail) to fit.`, + deps.verboseColor, + deps.envForRun, ); + prompt = { ...prompt, userText: fittedUserText }; } } diff --git a/tests/cleaner.test.ts b/tests/cleaner.test.ts index bbcb854..0605f73 100644 --- a/tests/cleaner.test.ts +++ b/tests/cleaner.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { applyContentBudget, clipAtSentenceBoundary, + clipHeadAndTail, decodeHtmlEntities, normalizeCandidate, normalizeForPrompt, @@ -61,4 +62,39 @@ describe("content cleaner utilities", () => { expect(empty.content).toBe(""); expect(empty.wordCount).toBe(0); }); + + it("drops the middle when applying a generous content budget", () => { + const input = `HEAD_START ${"x".repeat(3000)} MIDDLE_MARKER ${"y".repeat(3000)} TAIL_END`; + const result = applyContentBudget(input, 2000); + expect(result.truncated).toBe(true); + expect(result.totalCharacters).toBe(input.length); + expect(result.content.length).toBeLessThanOrEqual(2000); + expect(result.content).toContain("HEAD_START"); + expect(result.content).toContain("TAIL_END"); + expect(result.content).not.toContain("MIDDLE_MARKER"); + }); +}); + +describe("clipHeadAndTail", () => { + it("returns input unchanged when within budget", () => { + expect(clipHeadAndTail("short content", 100)).toBe("short content"); + }); + + it("keeps the head and tail and drops the middle for oversized input", () => { + const input = `HEAD_START ${"x".repeat(3000)} MIDDLE_MARKER ${"y".repeat(3000)} TAIL_END`; + const out = clipHeadAndTail(input, 2000); + expect(out.length).toBeLessThanOrEqual(2000); + expect(out).toContain("HEAD_START"); + expect(out).toContain("TAIL_END"); + expect(out).toContain("truncated"); // omission marker is present + expect(out).not.toContain("MIDDLE_MARKER"); + }); + + it("falls back to a head-only clip when the budget is too small for a tail", () => { + const input = "First sentence. Second sentence. Third sentence. Fourth sentence."; + const out = clipHeadAndTail(input, 30); + expect(out.length).toBeLessThanOrEqual(30); + expect(out).not.toContain("truncated"); // no omission marker + expect(input.startsWith(out)).toBe(true); // a pure head-only prefix + }); }); diff --git a/tests/cli.asset.local-file.test.ts b/tests/cli.asset.local-file.test.ts index 46cb7c5..bbf8bbe 100644 --- a/tests/cli.asset.local-file.test.ts +++ b/tests/cli.asset.local-file.test.ts @@ -229,7 +229,7 @@ describe("cli asset inputs (local file)", () => { globalFetchSpy.mockRestore(); }); - it("rejects local text files that exceed the input token limit", async () => { + it("truncates local text files that exceed the input token limit to fit", async () => { mocks.streamSimple.mockClear(); const root = mkdtempSync(join(tmpdir(), "summarize-asset-local-token-limit-")); @@ -263,20 +263,21 @@ describe("cli asset inputs (local file)", () => { const stdout = collectStream(); const stderr = collectStream(); - await expect( - runCli( - ["--model", "openai/gpt-5.2", "--timeout", "2s", "--stream", "on", "--plain", txtPath], - { - env: { HOME: root, OPENAI_API_KEY: "test" }, - fetch: vi.fn(async () => { - throw new Error("unexpected fetch"); - }) as unknown as typeof fetch, - stdout: stdout.stream, - stderr: stderr.stream, - }, - ), - ).rejects.toThrow(/Input token count/i); - expect(mocks.streamSimple).toHaveBeenCalledTimes(0); + await runCli( + ["--model", "openai/gpt-5.2", "--timeout", "2s", "--stream", "on", "--plain", txtPath], + { + env: { HOME: root, OPENAI_API_KEY: "test" }, + fetch: vi.fn(async () => { + throw new Error("unexpected fetch"); + }) as unknown as typeof fetch, + stdout: stdout.stream, + stderr: stderr.stream, + }, + ); + // Oversized input is truncated to fit (head+tail) and still summarized, + // rather than failing the run. + expect(mocks.streamSimple).toHaveBeenCalledTimes(1); + expect(stdout.getText()).toContain("OK"); globalFetchSpy.mockRestore(); }); @@ -375,8 +376,10 @@ describe("cli asset inputs (local file)", () => { expect(mocks.streamSimple).toHaveBeenCalledTimes(0); }); - it("errors when a text file exceeds the model input token limit", async () => { + it("truncates a text file that exceeds the model input token limit to fit", async () => { mocks.streamSimple.mockClear(); + mocks.completeSimple.mockReset(); + mocks.completeSimple.mockImplementation(async () => makeAssistantMessage({ text: "OK" })); const root = mkdtempSync(join(tmpdir(), "summarize-asset-local-tokens-")); const cacheDir = join(root, ".summarize", "cache"); @@ -406,20 +409,25 @@ describe("cli asset inputs (local file)", () => { const txtPath = join(root, "tokens.txt"); writeFileSync(txtPath, "hello ".repeat(50), "utf8"); + const stdout = collectStream(); const run = () => runCli(["--model", "openai/gpt-5.2", "--timeout", "2s", txtPath], { env: { HOME: root, OPENAI_API_KEY: "test" }, fetch: vi.fn(async () => { throw new Error("unexpected fetch"); }) as unknown as typeof fetch, - stdout: collectStream().stream, + stdout: stdout.stream, stderr: collectStream().stream, }); - await expect(run()).rejects.toThrow(/token count/i); - await expect(run()).rejects.toThrow(/input limit/i); + await run(); + // Oversized input is truncated to fit (head+tail) and still summarized via + // the non-streaming path, rather than failing the run. + expect(mocks.completeSimple).toHaveBeenCalledTimes(1); expect(mocks.streamSimple).toHaveBeenCalledTimes(0); + expect(stdout.getText()).toContain("OK"); + mocks.completeSimple.mockReset(); globalFetchSpy.mockRestore(); }); }); diff --git a/tests/cli.input-limit.test.ts b/tests/cli.input-limit.test.ts index 77e4a5d..39aac41 100644 --- a/tests/cli.input-limit.test.ts +++ b/tests/cli.input-limit.test.ts @@ -28,7 +28,8 @@ vi.mock("@mariozechner/pi-ai", () => ({ })); describe("cli input token limits", () => { - it("rejects large URL inputs before LLM calls", async () => { + it("truncates large URL inputs to fit the model input limit", async () => { + mocks.completeSimple.mockReset(); mocks.completeSimple.mockImplementation(async (model: MockModel) => makeAssistantMessage({ text: "OK", @@ -37,7 +38,6 @@ describe("cli input token limits", () => { api: model.api, }), ); - mocks.completeSimple.mockReset(); const root = mkdtempSync(join(tmpdir(), "summarize-input-limit-")); const cacheDir = join(root, ".summarize", "cache"); @@ -81,14 +81,13 @@ describe("cli input token limits", () => { }, }); - await expect( - runCli(["--model", "openai/gpt-5.2", "--timeout", "10s", "https://example.com"], { - env: { HOME: root, OPENAI_API_KEY: "test" }, - fetch: fetchMock as unknown as typeof fetch, - stdout, - stderr, - }), - ).rejects.toThrow(/Input token count/i); - expect(mocks.completeSimple).toHaveBeenCalledTimes(0); + await runCli(["--model", "openai/gpt-5.2", "--timeout", "10s", "https://example.com"], { + env: { HOME: root, OPENAI_API_KEY: "test" }, + fetch: fetchMock as unknown as typeof fetch, + stdout, + stderr, + }); + // Oversized input is truncated to fit (head+tail) and summarized, not rejected. + expect(mocks.completeSimple).toHaveBeenCalledTimes(1); }); }); diff --git a/tests/summary-engine-fit.test.ts b/tests/summary-engine-fit.test.ts new file mode 100644 index 0000000..854df54 --- /dev/null +++ b/tests/summary-engine-fit.test.ts @@ -0,0 +1,22 @@ +import { countTokens } from "gpt-tokenizer"; +import { describe, expect, it } from "vitest"; +import { fitUserTextToInputTokenBudget } from "../src/run/summary-engine.js"; + +describe("fitUserTextToInputTokenBudget", () => { + it("returns text unchanged when within the token budget", () => { + const text = "A short prompt that fits comfortably."; + expect(fitUserTextToInputTokenBudget(text, 1000)).toBe(text); + }); + + it("clips oversized text to fit the budget while keeping head and tail", () => { + const text = `HEAD_START ${"word ".repeat(20000)}TAIL_END`; + const budget = 500; + expect(countTokens(text)).toBeGreaterThan(budget); + + const out = fitUserTextToInputTokenBudget(text, budget); + + expect(countTokens(out)).toBeLessThanOrEqual(budget); + expect(out).toContain("HEAD_START"); + expect(out).toContain("TAIL_END"); + }); +});