From 9ea8cb07e7b177e3c5f806dccd70f5c58129cc44 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Fri, 16 Oct 2026 08:22:15 +0100 Subject: [PATCH 1/3] URL: Make `cleanForSlug` match WordPress core's `sanitize_title()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `cleanForSlug()` approximated `sanitize_title_with_dashes()`, but stripped every character that isn't a letter or a number, while core keeps any non-ASCII character (percent-encoded). So the editor's slug preview and the Slug input dropped emoji and other symbols that the published slug keeps. Port core's rules, in core's order, working on decoded characters: - Remove accents with the `$chars` table of core's `remove_accents()` (without its locale-specific rules), after NFC normalization, instead of the `remove-accents` package, whose table differs in about 100 letters. - Strip HTML tags and comments like `strip_tags()`. - Keep percent-encoded octets, and remove other percent signs. - Convert  , non-breaking hyphens, dashes, forward slashes and wide spaces to hyphens, and `×` to `x`; strip soft hyphens, inverted marks, angle and curly quotes, bullets, ©, ®, °, …, ™, accents and zero-width characters. - Only strip ASCII characters that aren't letters, numbers, spaces, underscores or hyphens, keeping emoji and other non-ASCII characters. Co-Authored-By: Claude Opus 5.5 --- package-lock.json | 3 - packages/url/CHANGELOG.md | 4 + packages/url/README.md | 4 +- packages/url/package.json | 3 - packages/url/src/clean-for-slug.ts | 109 +++++-- packages/url/src/remove-accents.ts | 354 ++++++++++++++++++++++ packages/url/src/test/index.jsdom.test.ts | 46 +++ 7 files changed, 497 insertions(+), 26 deletions(-) create mode 100644 packages/url/src/remove-accents.ts diff --git a/package-lock.json b/package-lock.json index 485973605da1ef..cd8eec21f10771 100644 --- a/package-lock.json +++ b/package-lock.json @@ -48848,9 +48848,6 @@ "name": "@wordpress/url", "version": "4.57.0", "license": "GPL-2.0-or-later", - "dependencies": { - "remove-accents": "^0.5.0" - }, "devDependencies": { "vitest": "^5.0.3" }, diff --git a/packages/url/CHANGELOG.md b/packages/url/CHANGELOG.md index 93e87f094f9c4f..7ff6258a15dc23 100644 --- a/packages/url/CHANGELOG.md +++ b/packages/url/CHANGELOG.md @@ -6,6 +6,10 @@ - Raise the minimum required Node.js version to `>=22.13.0`, for stable `require(esm)` support ([#84313](https://github.com/WordPress/gutenberg/pull/84313)). +### Enhancements + +- `cleanForSlug`: Match what `sanitize_title()` returns in WordPress core when saving a post. Keep emoji, symbols and other non-ASCII characters, remove accents with the table of core's `remove_accents()`, strip the punctuation and invisible characters core strips, convert wide spaces and dashes to hyphens and `×` to `x`, strip HTML tags and comments like `strip_tags()`, and keep percent-encoded octets. Drop the `remove-accents` dependency. + ### Internal - Convert the tests to TypeScript, and split tsconfig into a build project and a default dev project so the tests are type checked without publishing their declarations ([#84916](https://github.com/WordPress/gutenberg/pull/84916)). diff --git a/packages/url/README.md b/packages/url/README.md index 99a4ea5ae9165d..dd9b74c312156c 100644 --- a/packages/url/README.md +++ b/packages/url/README.md @@ -68,9 +68,9 @@ _Returns_ Performs some basic cleanup of a string for use as a post slug. -This replicates some of what `sanitize_title_with_dashes()` does in WordPress core, but is only designed to approximate what the slug will be. +This replicates what `sanitize_title()` does in WordPress core when saving a post, without the `sanitize_title` filters and the locale-specific transliterations of `remove_accents()`, so it can only approximate what the slug will be. It works on decoded characters, while WordPress returns the slug percent-encoded, and keeps percent-encoded octets as they are. -Converts Latin-1 Supplement and Latin Extended-A letters to basic Latin letters. Removes combining diacritical marks. Converts whitespace, periods, and forward slashes to hyphens. Removes any remaining non-word characters except hyphens. Converts remaining string to lowercase. It does not account for octets, HTML entities, or other encoded characters. +Converts accented letters to basic Latin letters, and strips HTML tags and entities. Converts spaces, periods, forward slashes, dashes and wide spaces to hyphens. Strips the punctuation and invisible characters WordPress strips, and any other ASCII character that isn't a letter, a number, an underscore or a hyphen. Keeps other characters, such as letters of other scripts and emoji. Converts the string to lowercase. _Parameters_ diff --git a/packages/url/package.json b/packages/url/package.json index fc6d92a65d54da..f2c5d9c6f8b5de 100644 --- a/packages/url/package.json +++ b/packages/url/package.json @@ -40,9 +40,6 @@ "wpScript": true, "types": "build-types", "sideEffects": false, - "dependencies": { - "remove-accents": "^0.5.0" - }, "devDependencies": { "vitest": "^5.0.3" }, diff --git a/packages/url/src/clean-for-slug.ts b/packages/url/src/clean-for-slug.ts index 2a7b1c801f8439..93686dfe4ab6f6 100644 --- a/packages/url/src/clean-for-slug.ts +++ b/packages/url/src/clean-for-slug.ts @@ -1,16 +1,70 @@ -import removeAccents from 'remove-accents'; +import { removeAccents } from './remove-accents'; + +/** + * Strips HTML tags like PHP's `strip_tags()`: a `<` not followed by ASCII + * whitespace opens a tag, even an unclosed one, nested `<` need as many `>`, + * a `>` inside quotes doesn't close a tag, and a comment ends at `-->`. + * + * @param string The string to process. + * + * @return The string without HTML tags. + */ +function stripTags( string: string ): string { + let result = ''; + let depth = 0; + let quote = ''; + for ( let i = 0; i < string.length; i++ ) { + const char = string[ i ]; + if ( depth === 0 ) { + // Like PHP's `isspace()`, only ASCII whitespace keeps a `<`. + if ( + char === '<' && + ! /[ \t\n\v\f\r]/.test( string[ i + 1 ] ?? '' ) + ) { + // A comment only ends at `-->`, or at the end of the string. + if ( string.startsWith( '', i + 2 ); + if ( end === -1 ) { + break; + } + i = end + 2; + } else { + depth = 1; + } + } else { + result += char; + } + } else if ( quote ) { + if ( char === quote ) { + quote = ''; + } + } else if ( char === '"' || char === "'" ) { + quote = char; + } else if ( char === '<' ) { + depth++; + } else if ( char === '>' ) { + depth--; + } + } + return result; +} /** * Performs some basic cleanup of a string for use as a post slug. * - * This replicates some of what `sanitize_title_with_dashes()` does in WordPress core, but - * is only designed to approximate what the slug will be. + * This replicates what `sanitize_title()` does in WordPress core when saving + * a post, without the `sanitize_title` filters and the locale-specific + * transliterations of `remove_accents()`, so it can only approximate what the + * slug will be. It works on decoded characters, while WordPress returns the + * slug percent-encoded, and keeps percent-encoded octets as they are. * - * Converts Latin-1 Supplement and Latin Extended-A letters to basic Latin - * letters. Removes combining diacritical marks. Converts whitespace, periods, - * and forward slashes to hyphens. Removes any remaining non-word characters - * except hyphens. Converts remaining string to lowercase. It does not account - * for octets, HTML entities, or other encoded characters. + * Converts accented letters to basic Latin letters, and strips HTML tags and + * entities. Converts spaces, periods, forward slashes, dashes and wide spaces + * to hyphens. Strips the punctuation + * and invisible characters WordPress strips, and any other ASCII character + * that isn't a letter, a number, an underscore or a hyphen. Keeps other + * characters, such as letters of other scripts and emoji. Converts the + * string to lowercase. * * @param string Title or slug to be processed. * @@ -21,17 +75,36 @@ export function cleanForSlug( string: string ): string { return ''; } return ( - removeAccents( string ) - // Convert  , &ndash, and &mdash to hyphens. - .replace( /( |–|—)/g, '-' ) - // Convert each group of whitespace, periods, and forward slashes to a hyphen. - .replace( /[\s\./]+/g, '-' ) - // Remove all HTML entities. - .replace( /&\S+?;/g, '' ) - // Remove anything that's not a letter, number, underscore or hyphen. - .replace( /[^\p{L}\p{N}_-]+/gu, '' ) - // Convert to lowercase + stripTags( removeAccents( string ) ) + // Remove percent signs that aren't part of an octet. + .replace( /%(?![0-9a-fA-F]{2})/g, '' ) .toLowerCase() + // Convert  , non-breaking hyphens, &ndash, &mdash and their HTML + // entities, and forward slashes to hyphens. + .replace( /[\u00a0\u2011\u2013\u2014/]/g, '-' ) + .replace( + / |‑| |–|–|—|—/g, + '-' + ) + // Strip soft hyphens, ¡ and ¿, angle and curly quotes, + // bullets, ©, ®, °, &hellip, &trade, accents, and + // invisible characters without a width. + .replace( + /[\u00ad\u00a1\u00bf\u00ab\u00bb\u2039\u203a\u2018-\u201f\u2022\u00a9\u00ae\u00b0\u2026\u2122\u00b4\u02ca\u0301\u0341\u0300\u0304\u030c\u200b-\u200f\u202a-\u202e\ufeff\ufffc]/g, + '' + ) + // Convert invisible characters with a width to hyphens. + .replace( /[\u2000-\u200a\u2028\u2029\u202f]/g, '-' ) + // Convert × to 'x'. + .replace( /\u00d7/g, 'x' ) + // Remove HTML entities. + .replace( /&.+?;/g, '' ) + .replace( /\./g, '-' ) + // Remove any ASCII character that isn't a letter, number, space, + // underscore, hyphen or the percent sign of an octet. + .replace( /[^%a-z0-9 _\-\u0080-\uffff]/g, '' ) + // Convert spaces to hyphens. + .replace( / +/g, '-' ) // Replace multiple hyphens with a single one. .replace( /-+/g, '-' ) // Remove any remaining leading or trailing hyphens. diff --git a/packages/url/src/remove-accents.ts b/packages/url/src/remove-accents.ts new file mode 100644 index 00000000000000..e03e67856346dd --- /dev/null +++ b/packages/url/src/remove-accents.ts @@ -0,0 +1,354 @@ +/** + * The letters `remove_accents()` in WordPress core converts to basic Latin, + * copied from its `$chars` table. The locale-specific rules (German, Danish, + * Catalan, Serbian and Bosnian) are left out, since they depend on the site + * locale. + */ +const CHARS: Record< string, string > = { + // Decompositions for Latin-1 Supplement. + '\u00aa': 'a', + '\u00ba': 'o', + '\u00c0': 'A', + '\u00c1': 'A', + '\u00c2': 'A', + '\u00c3': 'A', + '\u00c4': 'A', + '\u00c5': 'A', + '\u00c6': 'AE', + '\u00c7': 'C', + '\u00c8': 'E', + '\u00c9': 'E', + '\u00ca': 'E', + '\u00cb': 'E', + '\u00cc': 'I', + '\u00cd': 'I', + '\u00ce': 'I', + '\u00cf': 'I', + '\u00d0': 'D', + '\u00d1': 'N', + '\u00d2': 'O', + '\u00d3': 'O', + '\u00d4': 'O', + '\u00d5': 'O', + '\u00d6': 'O', + '\u00d9': 'U', + '\u00da': 'U', + '\u00db': 'U', + '\u00dc': 'U', + '\u00dd': 'Y', + '\u00de': 'TH', + '\u00df': 's', + '\u00e0': 'a', + '\u00e1': 'a', + '\u00e2': 'a', + '\u00e3': 'a', + '\u00e4': 'a', + '\u00e5': 'a', + '\u00e6': 'ae', + '\u00e7': 'c', + '\u00e8': 'e', + '\u00e9': 'e', + '\u00ea': 'e', + '\u00eb': 'e', + '\u00ec': 'i', + '\u00ed': 'i', + '\u00ee': 'i', + '\u00ef': 'i', + '\u00f0': 'd', + '\u00f1': 'n', + '\u00f2': 'o', + '\u00f3': 'o', + '\u00f4': 'o', + '\u00f5': 'o', + '\u00f6': 'o', + '\u00f8': 'o', + '\u00f9': 'u', + '\u00fa': 'u', + '\u00fb': 'u', + '\u00fc': 'u', + '\u00fd': 'y', + '\u00fe': 'th', + '\u00ff': 'y', + '\u00d8': 'O', + // Decompositions for Latin Extended-A. + '\u0100': 'A', + '\u0101': 'a', + '\u0102': 'A', + '\u0103': 'a', + '\u0104': 'A', + '\u0105': 'a', + '\u0106': 'C', + '\u0107': 'c', + '\u0108': 'C', + '\u0109': 'c', + '\u010a': 'C', + '\u010b': 'c', + '\u010c': 'C', + '\u010d': 'c', + '\u010e': 'D', + '\u010f': 'd', + '\u0110': 'D', + '\u0111': 'd', + '\u0112': 'E', + '\u0113': 'e', + '\u0114': 'E', + '\u0115': 'e', + '\u0116': 'E', + '\u0117': 'e', + '\u0118': 'E', + '\u0119': 'e', + '\u011a': 'E', + '\u011b': 'e', + '\u011c': 'G', + '\u011d': 'g', + '\u011e': 'G', + '\u011f': 'g', + '\u0120': 'G', + '\u0121': 'g', + '\u0122': 'G', + '\u0123': 'g', + '\u0124': 'H', + '\u0125': 'h', + '\u0126': 'H', + '\u0127': 'h', + '\u0128': 'I', + '\u0129': 'i', + '\u012a': 'I', + '\u012b': 'i', + '\u012c': 'I', + '\u012d': 'i', + '\u012e': 'I', + '\u012f': 'i', + '\u0130': 'I', + '\u0131': 'i', + '\u0132': 'IJ', + '\u0133': 'ij', + '\u0134': 'J', + '\u0135': 'j', + '\u0136': 'K', + '\u0137': 'k', + '\u0138': 'k', + '\u0139': 'L', + '\u013a': 'l', + '\u013b': 'L', + '\u013c': 'l', + '\u013d': 'L', + '\u013e': 'l', + '\u013f': 'L', + '\u0140': 'l', + '\u0141': 'L', + '\u0142': 'l', + '\u0143': 'N', + '\u0144': 'n', + '\u0145': 'N', + '\u0146': 'n', + '\u0147': 'N', + '\u0148': 'n', + '\u0149': 'n', + '\u014a': 'N', + '\u014b': 'n', + '\u014c': 'O', + '\u014d': 'o', + '\u014e': 'O', + '\u014f': 'o', + '\u0150': 'O', + '\u0151': 'o', + '\u0152': 'OE', + '\u0153': 'oe', + '\u0154': 'R', + '\u0155': 'r', + '\u0156': 'R', + '\u0157': 'r', + '\u0158': 'R', + '\u0159': 'r', + '\u015a': 'S', + '\u015b': 's', + '\u015c': 'S', + '\u015d': 's', + '\u015e': 'S', + '\u015f': 's', + '\u0160': 'S', + '\u0161': 's', + '\u0162': 'T', + '\u0163': 't', + '\u0164': 'T', + '\u0165': 't', + '\u0166': 'T', + '\u0167': 't', + '\u0168': 'U', + '\u0169': 'u', + '\u016a': 'U', + '\u016b': 'u', + '\u016c': 'U', + '\u016d': 'u', + '\u016e': 'U', + '\u016f': 'u', + '\u0170': 'U', + '\u0171': 'u', + '\u0172': 'U', + '\u0173': 'u', + '\u0174': 'W', + '\u0175': 'w', + '\u0176': 'Y', + '\u0177': 'y', + '\u0178': 'Y', + '\u0179': 'Z', + '\u017a': 'z', + '\u017b': 'Z', + '\u017c': 'z', + '\u017d': 'Z', + '\u017e': 'z', + '\u017f': 's', + // Decompositions for Latin Extended-B. + '\u018f': 'E', + '\u01dd': 'e', + '\u0218': 'S', + '\u0219': 's', + '\u021a': 'T', + '\u021b': 't', + // Euro sign. + '\u20ac': 'E', + // GBP (Pound) sign. + '\u00a3': '', + // Vowels with diacritic (Vietnamese). Unmarked. + '\u01a0': 'O', + '\u01a1': 'o', + '\u01af': 'U', + '\u01b0': 'u', + // Grave accent. + '\u1ea6': 'A', + '\u1ea7': 'a', + '\u1eb0': 'A', + '\u1eb1': 'a', + '\u1ec0': 'E', + '\u1ec1': 'e', + '\u1ed2': 'O', + '\u1ed3': 'o', + '\u1edc': 'O', + '\u1edd': 'o', + '\u1eea': 'U', + '\u1eeb': 'u', + '\u1ef2': 'Y', + '\u1ef3': 'y', + // Hook. + '\u1ea2': 'A', + '\u1ea3': 'a', + '\u1ea8': 'A', + '\u1ea9': 'a', + '\u1eb2': 'A', + '\u1eb3': 'a', + '\u1eba': 'E', + '\u1ebb': 'e', + '\u1ec2': 'E', + '\u1ec3': 'e', + '\u1ec8': 'I', + '\u1ec9': 'i', + '\u1ece': 'O', + '\u1ecf': 'o', + '\u1ed4': 'O', + '\u1ed5': 'o', + '\u1ede': 'O', + '\u1edf': 'o', + '\u1ee6': 'U', + '\u1ee7': 'u', + '\u1eec': 'U', + '\u1eed': 'u', + '\u1ef6': 'Y', + '\u1ef7': 'y', + // Tilde. + '\u1eaa': 'A', + '\u1eab': 'a', + '\u1eb4': 'A', + '\u1eb5': 'a', + '\u1ebc': 'E', + '\u1ebd': 'e', + '\u1ec4': 'E', + '\u1ec5': 'e', + '\u1ed6': 'O', + '\u1ed7': 'o', + '\u1ee0': 'O', + '\u1ee1': 'o', + '\u1eee': 'U', + '\u1eef': 'u', + '\u1ef8': 'Y', + '\u1ef9': 'y', + // Acute accent. + '\u1ea4': 'A', + '\u1ea5': 'a', + '\u1eae': 'A', + '\u1eaf': 'a', + '\u1ebe': 'E', + '\u1ebf': 'e', + '\u1ed0': 'O', + '\u1ed1': 'o', + '\u1eda': 'O', + '\u1edb': 'o', + '\u1ee8': 'U', + '\u1ee9': 'u', + // Dot below. + '\u1ea0': 'A', + '\u1ea1': 'a', + '\u1eac': 'A', + '\u1ead': 'a', + '\u1eb6': 'A', + '\u1eb7': 'a', + '\u1eb8': 'E', + '\u1eb9': 'e', + '\u1ec6': 'E', + '\u1ec7': 'e', + '\u1eca': 'I', + '\u1ecb': 'i', + '\u1ecc': 'O', + '\u1ecd': 'o', + '\u1ed8': 'O', + '\u1ed9': 'o', + '\u1ee2': 'O', + '\u1ee3': 'o', + '\u1ee4': 'U', + '\u1ee5': 'u', + '\u1ef0': 'U', + '\u1ef1': 'u', + '\u1ef4': 'Y', + '\u1ef5': 'y', + // Vowels with diacritic (Chinese, Hanyu Pinyin). + '\u0251': 'a', + // Macron. + '\u01d5': 'U', + '\u01d6': 'u', + // Acute accent. + '\u01d7': 'U', + '\u01d8': 'u', + // Caron. + '\u01cd': 'A', + '\u01ce': 'a', + '\u01cf': 'I', + '\u01d0': 'i', + '\u01d1': 'O', + '\u01d2': 'o', + '\u01d3': 'U', + '\u01d4': 'u', + '\u01d9': 'U', + '\u01da': 'u', + // Grave accent. + '\u01db': 'U', + '\u01dc': 'u', +}; + +const CHARS_REGEXP = new RegExp( + `[${ Object.keys( CHARS ).join( '' ) }]`, + 'g' +); + +/** + * Converts accented letters to basic Latin letters, like `remove_accents()` + * in WordPress core without a locale. + * + * @param string The string to process. + * + * @return The processed string. + */ +export function removeAccents( string: string ): string { + return string + .normalize( 'NFC' ) + .replace( CHARS_REGEXP, ( char ) => CHARS[ char ] ); +} diff --git a/packages/url/src/test/index.jsdom.test.ts b/packages/url/src/test/index.jsdom.test.ts index 442158ae10a44d..a08bd6554ce00f 100644 --- a/packages/url/src/test/index.jsdom.test.ts +++ b/packages/url/src/test/index.jsdom.test.ts @@ -1350,6 +1350,52 @@ describe( 'cleanForSlug', () => { cleanForSlug( 'No   Entities> – Here —<' ) ).toBe( 'no-entities-here' ); } ); + + // The expected values are what `sanitize_title()` returns in WordPress + // core, decoded. + it( 'Should keep emoji and other symbols, like WordPress', () => { + expect( cleanForSlug( 'Hello ❤️' ) ).toBe( 'hello-❤️' ); + expect( cleanForSlug( 'Emoji 🎉🚀 party' ) ).toBe( 'emoji-🎉🚀-party' ); + expect( cleanForSlug( '→ arrows ←' ) ).toBe( '→-arrows-←' ); + expect( cleanForSlug( 'x² + y³' ) ).toBe( 'x²-y³' ); + } ); + + it( 'Should strip the punctuation WordPress strips', () => { + expect( cleanForSlug( "C'est l’été « déjà »" ) ).toBe( + 'cest-lete-deja' + ); + expect( cleanForSlug( '¿Qué tal? ¡Hola!' ) ).toBe( 'que-tal-hola' ); + expect( cleanForSlug( 'Copyright © 2026 ™' ) ).toBe( 'copyright-2026' ); + expect( cleanForSlug( 'Ça va • bien…' ) ).toBe( 'ca-va-bien' ); + expect( cleanForSlug( 'zero\u200bwidth' ) ).toBe( 'zerowidth' ); + } ); + + it( 'Should convert dashes, wide spaces and the multiplication sign like WordPress', () => { + expect( cleanForSlug( 'a – b — c' ) ).toBe( 'a-b-c' ); + expect( cleanForSlug( 'thin\u2009space' ) ).toBe( 'thin-space' ); + expect( cleanForSlug( '1 × 2' ) ).toBe( '1-x-2' ); + } ); + + it( 'Should remove accents like WordPress', () => { + expect( cleanForSlug( 'Ünïcödé Straße' ) ).toBe( 'unicode-strase' ); + expect( cleanForSlug( 'Price €5' ) ).toBe( 'price-e5' ); + // WordPress keeps the accents of letters missing from its table. + expect( cleanForSlug( 'ǧ ȁ' ) ).toBe( 'ǧ-ȁ' ); + // Decomposed accents are composed first. + expect( cleanForSlug( 'cafe\u0301' ) ).toBe( 'cafe' ); + } ); + + it( 'Should strip HTML tags like WordPress', () => { + expect( cleanForSlug( 'Bold title' ) ).toBe( 'bold-title' ); + expect( cleanForSlug( '1<2' ) ).toBe( '1' ); + expect( cleanForSlug( '1 < 2' ) ).toBe( '1-2' ); + expect( cleanForSlug( 'ab' ) ).toBe( 'ab' ); + } ); + + it( 'Should keep percent-encoded octets and remove other percent signs', () => { + expect( cleanForSlug( '100% sure' ) ).toBe( '100-sure' ); + expect( cleanForSlug( 'a%20b' ) ).toBe( 'a%20b' ); + } ); } ); describe( 'normalizePath', () => { From d5d78b1b4689be92d99cc0e4baf3b68ce3b78844 Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Fri, 16 Oct 2026 08:23:01 +0100 Subject: [PATCH 2/3] URL: Link the cleanForSlug changelog entry to its PR Co-Authored-By: Claude Opus 5.5 --- packages/url/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/url/CHANGELOG.md b/packages/url/CHANGELOG.md index 7ff6258a15dc23..6060f6065871ad 100644 --- a/packages/url/CHANGELOG.md +++ b/packages/url/CHANGELOG.md @@ -8,7 +8,7 @@ ### Enhancements -- `cleanForSlug`: Match what `sanitize_title()` returns in WordPress core when saving a post. Keep emoji, symbols and other non-ASCII characters, remove accents with the table of core's `remove_accents()`, strip the punctuation and invisible characters core strips, convert wide spaces and dashes to hyphens and `×` to `x`, strip HTML tags and comments like `strip_tags()`, and keep percent-encoded octets. Drop the `remove-accents` dependency. +- `cleanForSlug`: Match what `sanitize_title()` returns in WordPress core when saving a post. Keep emoji, symbols and other non-ASCII characters, remove accents with the table of core's `remove_accents()`, strip the punctuation and invisible characters core strips, convert wide spaces and dashes to hyphens and `×` to `x`, strip HTML tags and comments like `strip_tags()`, and keep percent-encoded octets. Drop the `remove-accents` dependency ([#84896](https://github.com/WordPress/gutenberg/pull/84896)). ### Internal From 15b78eb8cb2eb1386665f17597b91b9f48a1087d Mon Sep 17 00:00:00 2001 From: Riad Benguella Date: Sun, 18 Oct 2026 13:25:43 +0100 Subject: [PATCH 3/3] URL: Truncate `cleanForSlug` to 200 encoded characters like WordPress `sanitize_title()` keeps what fits in 200 percent-encoded characters, the length of the `post_name` column, via `utf8_uri_encode( $title, 200 )`. Long non-Latin titles, whose characters take nine encoded characters each, were previewed in full and published truncated. Truncate at the same point of the pipeline, counting characters the same way. Co-Authored-By: Claude Opus 5.5 --- packages/url/CHANGELOG.md | 2 +- packages/url/README.md | 2 +- packages/url/src/clean-for-slug.ts | 47 ++++++++++++++++++++--- packages/url/src/test/index.jsdom.test.ts | 15 ++++++++ 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/packages/url/CHANGELOG.md b/packages/url/CHANGELOG.md index 6060f6065871ad..145dc3eef6e17f 100644 --- a/packages/url/CHANGELOG.md +++ b/packages/url/CHANGELOG.md @@ -8,7 +8,7 @@ ### Enhancements -- `cleanForSlug`: Match what `sanitize_title()` returns in WordPress core when saving a post. Keep emoji, symbols and other non-ASCII characters, remove accents with the table of core's `remove_accents()`, strip the punctuation and invisible characters core strips, convert wide spaces and dashes to hyphens and `×` to `x`, strip HTML tags and comments like `strip_tags()`, and keep percent-encoded octets. Drop the `remove-accents` dependency ([#84896](https://github.com/WordPress/gutenberg/pull/84896)). +- `cleanForSlug`: Match what `sanitize_title()` returns in WordPress core when saving a post. Keep emoji, symbols and other non-ASCII characters, remove accents with the table of core's `remove_accents()`, strip the punctuation and invisible characters core strips, convert wide spaces and dashes to hyphens and `×` to `x`, strip HTML tags and comments like `strip_tags()`, keep percent-encoded octets, and keep what fits in 200 percent-encoded characters. Drop the `remove-accents` dependency ([#84896](https://github.com/WordPress/gutenberg/pull/84896)). ### Internal diff --git a/packages/url/README.md b/packages/url/README.md index dd9b74c312156c..dbd9bfc538928a 100644 --- a/packages/url/README.md +++ b/packages/url/README.md @@ -68,7 +68,7 @@ _Returns_ Performs some basic cleanup of a string for use as a post slug. -This replicates what `sanitize_title()` does in WordPress core when saving a post, without the `sanitize_title` filters and the locale-specific transliterations of `remove_accents()`, so it can only approximate what the slug will be. It works on decoded characters, while WordPress returns the slug percent-encoded, and keeps percent-encoded octets as they are. +This replicates what `sanitize_title()` does in WordPress core when saving a post, without the `sanitize_title` filters and the locale-specific transliterations of `remove_accents()`, so it can only approximate what the slug will be. It works on decoded characters, while WordPress returns the slug percent-encoded, and keeps percent-encoded octets as they are. Like WordPress, it keeps what fits in 200 percent-encoded characters. Converts accented letters to basic Latin letters, and strips HTML tags and entities. Converts spaces, periods, forward slashes, dashes and wide spaces to hyphens. Strips the punctuation and invisible characters WordPress strips, and any other ASCII character that isn't a letter, a number, an underscore or a hyphen. Keeps other characters, such as letters of other scripts and emoji. Converts the string to lowercase. diff --git a/packages/url/src/clean-for-slug.ts b/packages/url/src/clean-for-slug.ts index 93686dfe4ab6f6..a79cd885906061 100644 --- a/packages/url/src/clean-for-slug.ts +++ b/packages/url/src/clean-for-slug.ts @@ -49,6 +49,39 @@ function stripTags( string: string ): string { return result; } +/** + * Truncates a string like PHP's `utf8_uri_encode()` with a length: keeps the + * characters whose percent-encoded form fits in the length, counting each + * ASCII character as one and any other character as three per UTF-8 byte, + * without splitting a character. + * + * @param string The string to process. + * @param length The maximum length of the percent-encoded string. + * + * @return The truncated string. + */ +function truncateEncoded( string: string, length: number ): string { + let result = ''; + let encodedLength = 0; + for ( const char of string ) { + const codePoint = char.codePointAt( 0 ) ?? 0; + let charLength = 12; + if ( codePoint < 0x80 ) { + charLength = 1; + } else if ( codePoint < 0x800 ) { + charLength = 6; + } else if ( codePoint < 0x10000 ) { + charLength = 9; + } + if ( encodedLength + charLength > length ) { + break; + } + result += char; + encodedLength += charLength; + } + return result; +} + /** * Performs some basic cleanup of a string for use as a post slug. * @@ -56,7 +89,8 @@ function stripTags( string: string ): string { * a post, without the `sanitize_title` filters and the locale-specific * transliterations of `remove_accents()`, so it can only approximate what the * slug will be. It works on decoded characters, while WordPress returns the - * slug percent-encoded, and keeps percent-encoded octets as they are. + * slug percent-encoded, and keeps percent-encoded octets as they are. Like + * WordPress, it keeps what fits in 200 percent-encoded characters. * * Converts accented letters to basic Latin letters, and strips HTML tags and * entities. Converts spaces, periods, forward slashes, dashes and wide spaces @@ -74,11 +108,14 @@ export function cleanForSlug( string: string ): string { if ( ! string ) { return ''; } + const lowercase = stripTags( removeAccents( string ) ) + // Remove percent signs that aren't part of an octet. + .replace( /%(?![0-9a-fA-F]{2})/g, '' ) + .toLowerCase(); return ( - stripTags( removeAccents( string ) ) - // Remove percent signs that aren't part of an octet. - .replace( /%(?![0-9a-fA-F]{2})/g, '' ) - .toLowerCase() + // Like WordPress, keep what fits in 200 percent-encoded characters, + // the length of the slug column. + truncateEncoded( lowercase, 200 ) // Convert  , non-breaking hyphens, &ndash, &mdash and their HTML // entities, and forward slashes to hyphens. .replace( /[\u00a0\u2011\u2013\u2014/]/g, '-' ) diff --git a/packages/url/src/test/index.jsdom.test.ts b/packages/url/src/test/index.jsdom.test.ts index a08bd6554ce00f..6dca967930c167 100644 --- a/packages/url/src/test/index.jsdom.test.ts +++ b/packages/url/src/test/index.jsdom.test.ts @@ -1392,6 +1392,21 @@ describe( 'cleanForSlug', () => { expect( cleanForSlug( 'ab' ) ).toBe( 'ab' ); } ); + it( 'Should keep what fits in 200 percent-encoded characters, like WordPress', () => { + expect( cleanForSlug( 'a'.repeat( 250 ) ) ).toBe( 'a'.repeat( 200 ) ); + // Each Devanagari character takes nine percent-encoded characters. + expect( cleanForSlug( 'अलग-अलग-प्रकार-के-प्लंबिंग-ट्रैप्स' ) ).toBe( + 'अलग-अलग-प्रकार-के-प्लंबिं' + ); + // A character that doesn't fit isn't split. + expect( cleanForSlug( 'a'.repeat( 198 ) + 'é' ) ).toBe( + 'a'.repeat( 198 ) + 'e' + ); + expect( cleanForSlug( 'a'.repeat( 198 ) + 'ψ' ) ).toBe( + 'a'.repeat( 198 ) + ); + } ); + it( 'Should keep percent-encoded octets and remove other percent signs', () => { expect( cleanForSlug( '100% sure' ) ).toBe( '100-sure' ); expect( cleanForSlug( 'a%20b' ) ).toBe( 'a%20b' );