Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions packages/cli/src/ui/utils/InlineMarkdownRenderer.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
*/

import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import stripAnsi from 'strip-ansi';
import { renderWithProviders } from '../../test-utils/render.js';
import { getPlainTextLength, RenderInline } from './InlineMarkdownRenderer.js';
import { HYPERLINK_ENV_KEYS } from './osc8.js';
Expand Down Expand Up @@ -248,6 +249,45 @@ math then literal: $x^2\$$`;
expect(out).not.toContain(`\x1b]8;;${url}.\x07`);
});

it('stops the bare-URL hyperlink at glued-on full-width CJK punctuation', () => {
enableHyperlinks();
const url = 'https://github.com/QwenLM/qwen-code/pull/8742';
const { lastFrame } = renderWithProviders(
<RenderInline text={`PR:${url}(2 commits,等 CI)`} />,
);

const out = lastFrame() ?? '';
// OSC 8 target is exactly the URL — the (2 run is not swallowed into
// the link (https://github.com/QwenLM/qwen-code/issues/8750).
expect(out).toContain(`\x1b]8;;${url}\x07`);
expect(out).not.toContain(`\x1b]8;;${url}(`);
// The glued-on punctuation renders as plain text after the link.
expect(out.replace(/\s+/g, ' ')).toContain('(2 commits,等 CI)');
});

it('does not treat underscores around a later URL as emphasis', () => {
enableHyperlinks();
const firstUrl = 'https://a.com/x';
const secondUrl = 'https://b.com/y';
const { lastFrame } = renderWithProviders(
<RenderInline text={`见 ${firstUrl}(说明_1)和 ${secondUrl}_。`} />,
);

const out = lastFrame() ?? '';
expect(out).toContain(`\x1b]8;;${firstUrl}\x07`);
expect(out).toContain(`\x1b]8;;${secondUrl}\x07`);
expect(out.replace(/\s+/g, ' ')).toContain('(说明_1)和');
expect(stripAnsi(out)).toContain(`${secondUrl}_`);
});

it('preserves dunder identifiers as visible text', () => {
const { lastFrame } = renderWithProviders(
<RenderInline text="Python 的 __init__ 方法" />,
);

expect(stripAnsi(lastFrame() ?? '')).toContain('__init__');
});

it('leaves bare URLs unwrapped when unsupported', () => {
const url = 'https://example.com/plain';
const { lastFrame } = renderWithProviders(
Expand Down
10 changes: 7 additions & 3 deletions packages/cli/src/ui/utils/InlineMarkdownRenderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import stringWidth from 'string-width';
import { createDebugLogger } from '@qwen-code/qwen-code-core';
import { renderInlineLatex } from './latexRenderer.js';
import {
BARE_URL_PATTERN,
MD_LINK_CAPTURE,
MD_LINK_PATTERN,
isSafeOscScheme,
Expand All @@ -37,8 +38,8 @@ const INLINE_CODE_MARKER_LENGTH = 1; // For "`"
const UNDERLINE_TAG_START_LENGTH = 3; // For "<u>"
const UNDERLINE_TAG_END_LENGTH = 4; // For "</u>"
const INLINE_MARKDOWN_REGEX = new RegExp(
String.raw`(\*\*.*?\*\*|\*.*?\*|_.*?_|~~.*?~~|${MD_LINK_PATTERN}|` +
String.raw`${INLINE_CODE_SPAN_PATTERN_SOURCE}|<u>.*?<\/u>|https?:\/\/\S+)`,
String.raw`(\*\*.*?\*\*|\*.*?\*|(?<![\w\u3400-\u9fff])_(?!_)[^_]*_(?![\w\u3400-\u9fff])|~~.*?~~|${MD_LINK_PATTERN}|` +
String.raw`${INLINE_CODE_SPAN_PATTERN_SOURCE}|<u>.*?<\/u>|${BARE_URL_PATTERN})`,
Comment thread
yiliang114 marked this conversation as resolved.
Comment thread
yiliang114 marked this conversation as resolved.
'g',
);

Expand Down Expand Up @@ -249,7 +250,10 @@ const RenderInlineInternal: React.FC<RenderInlineProps> = ({
// alternative is anchored on `https?://`, so `isSafeOscScheme` is
// redundant but kept as a cheap defense-in-depth assertion.
const trimmedUrl = canHyperlink
? trimTrailingUrlPunctuation(fullMatch)
? trimTrailingUrlPunctuation(
fullMatch,
text[index + fullMatch.length],
)
: fullMatch;
const wrapOsc8 = canHyperlink && isSafeOscScheme(trimmedUrl);
renderedNode = (
Expand Down
32 changes: 32 additions & 0 deletions packages/cli/src/ui/utils/TableRenderer.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -649,6 +649,38 @@ describe('<TableRenderer />', () => {
expectAllLinesToHaveSameVisibleWidth(output);
});

it('stops a bare URL before glued-on CJK punctuation', () => {
enableHyperlinks();
const url = 'https://github.com/QwenLM/qwen-code/pull/8742';
const suffix = '(2 commits,等 CI)';
const output = renderTable(['PR'], [[`PR:${url}${suffix}`]], 100);
expect(output).toContain(`\x1b]8;;${url}\x07`);
expect(output).not.toContain(`\x1b]8;;${url}(`);
expect(stripAnsi(output)).toContain(suffix);
});

it('does not treat underscores around a later URL as emphasis', () => {
enableHyperlinks();
const firstUrl = 'https://a.com/x';
const secondUrl = 'https://b.com/y';
const output = renderTable(
['PR'],
[[`见 ${firstUrl}(说明_1)和 ${secondUrl}_。`]],
100,
);

expect(output).toContain(`\x1b]8;;${firstUrl}\x07`);
expect(output).toContain(`\x1b]8;;${secondUrl}\x07`);
expect(stripAnsi(output)).toContain('(说明_1)和');
expect(stripAnsi(output)).toContain(`${secondUrl}_。`);
});

it('preserves dunder identifiers as visible text', () => {
const output = renderTable(['Value'], [['Python 的 __init__ 方法']], 60);

expect(stripAnsi(output)).toContain('__init__');
});

it('falls back to legacy `label (url)` in cells on unsupported terminals', () => {
// isTTY=false from the suite-wide beforeEach disables hyperlinks.
const url = 'https://example.com/page';
Expand Down
10 changes: 7 additions & 3 deletions packages/cli/src/ui/utils/TableRenderer.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
unescapeMarkdownDollars,
} from './inline-math.js';
import {
BARE_URL_PATTERN,
MD_LINK_CAPTURE,
MD_LINK_PATTERN,
isSafeOscScheme,
Expand Down Expand Up @@ -49,8 +50,8 @@ const ABSOLUTE_MIN_HORIZONTAL_TABLE_WIDTH = 24;
const SAFETY_MARGIN = 4;

const INLINE_MARKDOWN_REGEX = new RegExp(
Comment thread
yiliang114 marked this conversation as resolved.
String.raw`(\*\*.*?\*\*|\*.*?\*|_.*?_|~~.*?~~|${MD_LINK_PATTERN}|` +
String.raw`${INLINE_CODE_SPAN_PATTERN_SOURCE}|<u>.*?<\/u>|https?:\/\/\S+)`,
String.raw`(\*\*.*?\*\*|\*.*?\*|(?<![\w\u3400-\u9fff])_(?!_)[^_]*_(?![\w\u3400-\u9fff])|~~.*?~~|${MD_LINK_PATTERN}|` +
String.raw`${INLINE_CODE_SPAN_PATTERN_SOURCE}|<u>.*?<\/u>|${BARE_URL_PATTERN})`,
Comment thread
yiliang114 marked this conversation as resolved.
'g',
);

Expand Down Expand Up @@ -371,7 +372,10 @@ function renderMarkdownToAnsi(text: string, enableInlineMath = false): string {
} else if (/^https?:\/\//.test(fullMatch)) {
const visible = applyColor(fullMatch, theme.text.link);
if (canHyperlink) {
const trimmedUrl = trimTrailingUrlPunctuation(fullMatch);
const trimmedUrl = trimTrailingUrlPunctuation(
fullMatch,
text[index + fullMatch.length],
);
rendered = isSafeOscScheme(trimmedUrl)
? `${osc8Open(trimmedUrl)}${visible}${osc8Close()}`
: visible;
Expand Down
41 changes: 41 additions & 0 deletions packages/cli/src/ui/utils/osc8.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
BARE_URL_PATTERN,
HYPERLINK_ENV_KEYS,
isSafeOscScheme,
labelMayDeceive,
Expand Down Expand Up @@ -294,6 +295,46 @@ describe('osc8 helpers', () => {
});
});

describe('BARE_URL_PATTERN', () => {
it.each([
'https://ja.wikipedia.org/wiki/人々',
Comment thread
yiliang114 marked this conversation as resolved.
'https://ja.wikipedia.org/wiki/〆切',
'https://example.com/二〇二六年報',
'https://ja.example.com/〼道と〻次',
'https://example.com/〱〢',
'https://example.com/a︳b︴c',
'https://zh.wikipedia.org/wiki/北京',
])('keeps word-forming CJK characters in %s', (url) => {
expect(new RegExp(BARE_URL_PATTERN).exec(url)?.[0]).toBe(url);
});

it.each([
'https://en.wikipedia.org/wiki/Mexico–United_States_border',
'https://example.com/it’s',
'https://example.com/thing…',
])('keeps typographic punctuation inside %s', (url) => {
expect(new RegExp(BARE_URL_PATTERN).exec(url)?.[0]).toBe(url);
});

it.each([
['https://example.com/page。下一步', 'https://example.com/page'],
['https://example.com/page、続き', 'https://example.com/page'],
['https://example.com/page:説明', 'https://example.com/page'],
['https://example.com/page?質問', 'https://example.com/page'],
['https://example.com/page!注意', 'https://example.com/page'],
])('stops before CJK punctuation in %s', (input, expected) => {
expect(new RegExp(BARE_URL_PATTERN).exec(input)?.[0]).toBe(expected);
});

it.each([
['https://example.com/x﹙備註﹚', 'https://example.com/x'],
['https://example.com/a︒後続', 'https://example.com/a'],
['https://example.com/p﹖説明', 'https://example.com/p'],
])('stops before CJK compat/vertical forms in %s', (input, expected) => {
expect(new RegExp(BARE_URL_PATTERN).exec(input)?.[0]).toBe(expected);
});
});

describe('supportsHyperlinks', () => {
function setTTY(value: boolean) {
Object.defineProperty(process.stdout, 'isTTY', {
Expand Down
37 changes: 33 additions & 4 deletions packages/cli/src/ui/utils/osc8.ts
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,11 @@ export function isSafeOscScheme(url: string): boolean {
return SAFE_OSC8_SCHEMES.has(match[1]!.toLowerCase());
}

const BARE_URL_BREAK_CHARACTERS = String.raw`\u3001-\u3004\u3008-\u3020\u302e-\u3030\u3036-\u3037\u303d-\u303f\uff01-\uff0f\uff1a-\uff20\uff3b-\uff40\uff5b-\uff65\ufe10-\ufe1f\ufe30-\ufe32\ufe35-\ufe6f`;
Comment thread
yiliang114 marked this conversation as resolved.
// The escaped CJK ranges do not contain literal combining characters.
// eslint-disable-next-line no-misleading-character-class
const BARE_URL_BREAK_PATTERN = new RegExp(`[${BARE_URL_BREAK_CHARACTERS}]`);

/**
* Trim trailing sentence punctuation off a bare URL run before it becomes
* an OSC 8 target. Models routinely produce `see https://example.com.` and
Expand All @@ -101,10 +106,13 @@ export function isSafeOscScheme(url: string): boolean {
* the URL so URLs that legitimately end with `)` (Wikipedia disambiguation,
* MSDN) aren't truncated.
*/
export function trimTrailingUrlPunctuation(url: string): string {
// Count `( [ {` opens once up-front; we then decrement running `)`/`]`/`}`
// close counts as we trim, keeping the whole trim O(n) instead of O(n²)
// for adversarial inputs like `https://x.com))))…`.
export function trimTrailingUrlPunctuation(
url: string,
nextCharacter = '',
Comment thread
yiliang114 marked this conversation as resolved.
): string {
// Count `( [ {` opens once up-front; we then
// decrement running `)`/`]`/`}` close counts as we trim, keeping the whole
// trim O(n) instead of O(n²) for adversarial inputs like `https://x.com))))…`.
Comment thread
yiliang114 marked this conversation as resolved.
let openParen = 0;
let openBracket = 0;
let openBrace = 0;
Expand All @@ -122,6 +130,12 @@ export function trimTrailingUrlPunctuation(url: string): string {
}

let end = url.length;
if (
url.charCodeAt(end - 1) === 0x5f &&
Comment thread
yiliang114 marked this conversation as resolved.
BARE_URL_BREAK_PATTERN.test(nextCharacter)
) {
end--;
}
Comment thread
yiliang114 marked this conversation as resolved.
while (end > 0) {
const c = url.charCodeAt(end - 1);
// .,;:!?'"`> — `>` covers CommonMark autolinks (`<https://x.com>`)
Expand Down Expand Up @@ -178,6 +192,21 @@ export const MD_LINK_PATTERN = String.raw`\[.*?\]\((?:[^()]|\([^()]*\))*\)`;
*/
export const MD_LINK_CAPTURE = /^\[(.*?)\]\(((?:[^()]|\([^()]*\))*)\)$/;

/**
* Bare-URL pattern shared between the React and ANSI renderers. Unlike a
* plain `\S+` run it stops at CJK / full-width punctuation: Chinese prose
* routinely glues `(…)`/`。` onto a URL with no space
* (`https://x.com(2 commits)`), and `\S` swallows the punctuation plus
* everything up to the next ASCII space, turning the OSC 8 target into a
* 404. Raw CJK ideographs, U+3005 々, U+3006 〆, and U+3007 〇 stay in the
* match because they are word-forming IRI characters. The exclusion also
* covers punctuation in CJK Compatibility Forms and Vertical Forms while
* preserving their word-forming repeat marks and vertical low lines. ASCII
* and other typographic punctuation stays matched and is left to
* `trimTrailingUrlPunctuation`.
*/
export const BARE_URL_PATTERN = String.raw`https?:\/\/[^\s${BARE_URL_BREAK_CHARACTERS}]+`;

Comment thread
yiliang114 marked this conversation as resolved.
/**
* Should the markdown renderers wrap a `[label](url)` token in an OSC 8
* envelope? Returns true only when (a) the host terminal advertises OSC 8,
Expand Down
Loading