Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/gateway/src/chat/tools/estimate-tokens-from-content.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,8 @@
* Estimates tokens from content length using simple division
*/
export function estimateTokensFromContent(content: string): number {
if (!content) {
return 0;
}
return Math.max(1, Math.round(content.length / 4));
}
9 changes: 6 additions & 3 deletions apps/gateway/src/lib/prompt-tokens.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,19 @@ describe("Prompt token calculation", () => {
describe("estimateTokensFromContent", () => {
it("should estimate tokens from content length", () => {
expect(estimateTokensFromContent("Hello world")).toBe(3); // 11 chars / 4 = 2.75, rounded to 3
expect(estimateTokensFromContent("")).toBe(1); // Always at least 1
expect(estimateTokensFromContent("")).toBe(0); // Empty content = 0 tokens
expect(
estimateTokensFromContent(
"A very long message that should result in more tokens",
),
).toBe(13); // 53 chars / 4 = 13.25, rounded to 13
});

it("should always return at least 1 token", () => {
expect(estimateTokensFromContent("")).toBe(1);
it("should return 0 for empty content", () => {
expect(estimateTokensFromContent("")).toBe(0);
});

it("should return at least 1 token for non-empty content", () => {
expect(estimateTokensFromContent("A")).toBe(1);
});
});
Expand Down
Loading