regression: combined picker emojis split into separate symbols

apps/meteor/app/emoji-native/lib/generateEmojiData.ts

@@ -1,4 +1,3 @@
-import type { Emoji } from 'emojibase';
 import data from 'emojibase-data/en/data.json';
 import shortcodes from 'emojibase-data/en/shortcodes/emojibase.json';
 
@@ -57,8 +56,9 @@ function buildEmojiData() {
 		flags: [],
 	};
 	const toneList: Record<string, number> = {};
+	const bareAliases: [string, string][] = [];
 
-	for (const emojiData of data as Emoji[]) {
+	for (const emojiData of data) {
 		// Skip component group (skin tones, hair styles)
 		if (emojiData.group === 2) continue;
 
@@ -72,6 +72,7 @@ function buildEmojiData() {
 		const primaryShortcode = codes[0];
 		const altShortcodes = codes.slice(1).map((s) => `:${s}:`);
 		const hex = hexFromEmoji(emojiData.emoji);
+		const bare = emojiData.emoji.replace(/\uFE0F/g, '');
 
 		const entry: EmojiEntry = {
 			name: primaryShortcode,
@@ -88,6 +89,10 @@ function buildEmojiData() {
 		const key = `:${primaryShortcode}:`;
 		emojiList[key] = entry;
 
+		if (emojiData.type === 1 && bare !== emojiData.emoji) {
+			bareAliases.push([bare, key]);
+		}
+
 		// Only add to category if it's NOT a skin tone variant
 		if (!emojiData.tone && !isRegional) {
 			emojisByCategory[category].push(primaryShortcode);
@@ -130,7 +135,7 @@ function buildEmojiData() {
 		}
 	}
 
-	return { emojiList, emojisByCategory, toneList };
+	return { emojiList, emojisByCategory, toneList, bareAliases };
 }
 
 // Build once and cache

apps/meteor/app/emoji-native/lib/getEmojiConfig.spec.ts

@@ -52,4 +52,27 @@ describe('native emoji render', () => {
 
 		expect(render(':smiley:')).toContain('<span class="emoji" title=":smiley:">');
 	});
+
+	it('renders bare (non-VS16) emoji-default characters', () => {
+		const { render } = getEmojiConfig(buildEmojiPackages(false));
+
+		expect(render('🐕')).toBe('<span class="emoji" title=":dog:">🐕</span>');
+		expect(render('⭐')).toBe('<span class="emoji" title=":star:">⭐</span>');
+		expect(render('👍')).toBe('<span class="emoji" title=":+1:">👍</span>');
+	});
+
+	it('resolves the VS16-qualified form to the same shortcode', () => {
+		const { render } = getEmojiConfig(buildEmojiPackages(false));
+
+		expect(render('🐕\u{FE0F}')).toBe('<span class="emoji" title=":dog:">🐕\u{FE0F}</span>');
+		expect(render('⭐\u{FE0F}')).toBe('<span class="emoji" title=":star:">⭐\u{FE0F}</span>');
+	});
+
+	it('only treats text-default symbols as emoji when VS16 is present', () => {
+		const { render } = getEmojiConfig(buildEmojiPackages(false));
+
+		expect(render('©')).toBe('©');
+		expect(render('™')).toBe('™');
+		expect(render('©\u{FE0F}')).toBe('<span class="emoji" title=":copyright:">©\u{FE0F}</span>');
+	});
 });

apps/meteor/app/emoji-native/lib/getEmojiConfig.ts

@@ -24,7 +24,7 @@ let emojiRegex: RegExp | null = null;
 
 function getUnicodeToShortcodeMap(): Map<string, string> {
 	if (!unicodeToShortcodeMap) {
-		const { emojiList } = getEmojiData();
+		const { emojiList, bareAliases } = getEmojiData();
 		unicodeToShortcodeMap = new Map();
 
 		for (const [shortcode, entry] of Object.entries(emojiList)) {
@@ -33,6 +33,12 @@ function getUnicodeToShortcodeMap(): Map<string, string> {
 				unicodeToShortcodeMap.set(emojiEntry.unicode, shortcode);
 			}
 		}
+
+		for (const [bare, shortcode] of bareAliases) {
+			if (!unicodeToShortcodeMap.has(bare)) {
+				unicodeToShortcodeMap.set(bare, shortcode);
+			}
+		}
 	}
 	return unicodeToShortcodeMap;
 }

packages/message-parser/src/grammar.pegjs

@@ -795,6 +795,7 @@ EmoticonBackslash
 /* Unicode emojis */
 UnicodeEmoji
   = UnicodeEmojiTagSequence
+  / UnicodeEmojiKeycapSequence
   / $(
     (UnicodeEmojiZwjComponent [\u200D])*
     UnicodeEmojiZwjComponent
@@ -804,36 +805,58 @@ UnicodeEmoji
   / UnicodeEmojiMiscellaneousTechnical
   / UnicodeEmojiMiscellaneousSymbols
   / UnicodeEmojiDingbats
+  / UnicodeEmojiGeometricSquares
+  / UnicodeEmojiEnclosedBadges
+  / UnicodeEmojiTextPresentation
   / UnicodeEmojiFlags
 
-UnicodeEmojiEmoticon = $([\uD83D] [\uDE00-\uDE4F])
+UnicodeEmojiEmoticon = $([\uD83D] [\uDE00-\uDE4F] [︀-️]?)
 
 UnicodeEmojiSupplementalSymbolsAndPictographs = $([\uD83E] [\uDD00-\uDFFF])
 
 UnicodeEmojiZwjComponent
   = ( UnicodeEmojiSupplementalSymbolsAndPictographs
     / UnicodeEmojiMiscellaneousSymbolsAndPictographs
     / UnicodeEmojiEmoticon
+    / UnicodeEmojiTransportAndMapSymbols
     / UnicodeEmojiDingbats
     / UnicodeEmojiMiscellaneousSymbols
+    / UnicodeEmojiArrows
+    / UnicodeEmojiGeometricSquares
     ) UnicodeEmojiMiscellaneousSymbolsAndPictographsFitzpatrickModifiers?
 
 /* Emoji tag sequence: Black Flag + tag characters (U+E0020-U+E007E) + Cancel Tag (U+E007F), e.g. England/Scotland/Wales flags */
 UnicodeEmojiTagSequence = $([\uD83C] [\uDFF4] ([\uDB40] [\uDC20-\uDC7E])+ [\uDB40] [\uDC7F])
 
+UnicodeEmojiKeycapSequence = $([0-9#*] [️]? [⃣])
+
 UnicodeEmojiMiscellaneousSymbolsAndPictographs = $([\uD83C] [\uDF00-\uDFFF] [\uFE00-\uFE0F]?) / $([\uD83D] [\uDC00-\uDDFF] [\uFE00-\uFE0F]?)
 
 UnicodeEmojiMiscellaneousSymbolsAndPictographsFitzpatrickModifiers = $([\uD83C] [\uDFFB-\uDFFF])
 
-UnicodeEmojiTransportAndMapSymbols = $([\uD83D] [\uDE80-\uDEFA])
+UnicodeEmojiTransportAndMapSymbols = $([\uD83D] [\uDE80-\uDEFF] [︀-️]?)
 
 UnicodeEmojiMiscellaneousTechnical = $([\u2300-\u23FF] [\uFE00-\uFE0F]?)
 
 UnicodeEmojiMiscellaneousSymbols = $([\u2600-\u26FF] [\uFE00-\uFE0F]?)
 
 UnicodeEmojiDingbats = $([\u2700-\u27BF] [\uFE00-\uFE0F]?)
 
-UnicodeEmojiFlags = $([\uD83C] [\uDD00-\uDDFF] [\uD83C] [\uDD00-\uDDFF])
+/* U+2194/U+2195 only; kept narrow so bare prose arrows (U+2190..U+2193, U+21D2) aren't matched */
+UnicodeEmojiArrows = $([\u2194-\u2195] [\uFE00-\uFE0F]?)
+
+UnicodeEmojiGeometricSquares = $([\u2B1B-\u2B1C] [\uFE00-\uFE0F]?) / $([\uD83D] ([\uDFE0-\uDFEB] / [\uDFF0]) [\uFE00-\uFE0F]?)
+
+/* Tight ranges — these enclosed-alphanumeric/ideographic and playing-card blocks are mostly non-emoji */
+UnicodeEmojiEnclosedBadges = $([\uD83C] ([\uDCCF] / [\uDD8E] / [\uDD91-\uDD9A] / [\uDE01] / [\uDE32-\uDE3A] / [\uDE50-\uDE51]) [︀-️]?)
+
+/* Default-text chars that are emoji ONLY with a required trailing VS16, so a bare U+00A9/U+2122/U+25B6 in prose stays text */
+UnicodeEmojiTextPresentation
+  = $([©®‼⁉™ℹ↖-↙↩-↪Ⓜ▪-▫▶◀◻-◾⤴-⤵⬅-⬇⭐⭕〰〽㊗㊙] [️])
+  / $([\uD83C] ([\uDC04] / [\uDD70-\uDD71] / [\uDD7E-\uDD7F] / [\uDE02] / [\uDE1A] / [\uDE2F]) [️])
+
+/* Two regional indicators combine into a flag (e.g. U + S = US flag); a single one alone is not an emoji. Narrowed from U+1F100-1F1FF so squared badges aren't mis-grouped as flags. */
+UnicodeEmojiFlags = $([\uD83C] [\uDDE6-\uDDFF] [\uD83C] [\uDDE6-\uDDFF])
 
 /**
  *

packages/message-parser/tests/emoji.test.ts

@@ -1,4 +1,5 @@
 import { parse } from '../src';
+import { ALL_EMOJI, EMOJIBASE_VERSION } from './fixtures/allEmoji';
 import { emoji, bigEmoji, paragraph, plain, emojiUnicode } from './helpers';
 
 test.each([
@@ -124,3 +125,36 @@ test.each([
 ])('parses %p', (input, output) => {
 	expect(parse(input)).toEqual(output);
 });
+
+// KNOWN_UNSUPPORTED: the 26 regional indicators. Alone they're just letters, not emojis; combined
+// in a pair they form a flag (🇺 + 🇸 = 🇺🇸). Real letter emojis (🅰️ Ⓜ️) are supported. Stale check
+// below keeps the list honest.
+// prettier-ignore
+const KNOWN_UNSUPPORTED = new Set('🇦 🇧 🇨 🇩 🇪 🇫 🇬 🇭 🇮 🇯 🇰 🇱 🇲 🇳 🇴 🇵 🇶 🇷 🇸 🇹 🇺 🇻 🇼 🇽 🇾 🇿'.split(' '));
+
+function parsesWhole(input: string): boolean {
+	const nodes = parse(input);
+
+	if (!Array.isArray(nodes) || nodes.length !== 1) {
+		return false;
+	}
+
+	const [node] = nodes;
+	if (node.type !== 'BIG_EMOJI' || node.value.length !== 1) {
+		return false;
+	}
+
+	const [child] = node.value;
+	if (child.type !== 'EMOJI' || !('unicode' in child)) {
+		return false;
+	}
+
+	return child.unicode === input;
+}
+
+test(`every emoji parses as one token, except documented components (emojibase@${EMOJIBASE_VERSION})`, () => {
+	expect(ALL_EMOJI.length).toBeGreaterThan(3000);
+	const inFixture = new Set(ALL_EMOJI);
+	expect(ALL_EMOJI.filter((e) => !KNOWN_UNSUPPORTED.has(e) && !parsesWhole(e))).toEqual([]);
+	expect([...KNOWN_UNSUPPORTED].filter((e) => !inFixture.has(e) || parsesWhole(e))).toEqual([]);
+});

scripts/emoji/README.md

@@ -0,0 +1,19 @@
+# emoji
+
+Generates the emoji test fixture used by `@rocket.chat/message-parser`'s coverage test.
+
+## `generateEmojiFixture.mjs`
+
+Writes `packages/message-parser/tests/fixtures/allEmoji.ts` — the full list of every emoji `emojibase-data` ships. The message-parser coverage test parses each one and asserts it is recognized as a single emoji.
+
+### When to run it
+
+After bumping the `emojibase-data` dependency to a newer Unicode version:
+
+```bash
+node scripts/emoji/generateEmojiFixture.mjs
+```
+
+### Why
+
+The fixture is a committed snapshot, not generated at test time. Regenerating it after a bump adds the newly-introduced emojis to the sweep, so any that the parser grammar doesn't yet recognize fail the test and flag exactly what needs a new grammar rule. Without regenerating, new emojis simply wouldn't be covered.

scripts/emoji/generateEmojiFixture.mjs

@@ -0,0 +1,41 @@
+// Regenerates the emoji test fixture (packages/message-parser/tests/fixtures/allEmoji.ts) from
+// emojibase-data — the canonical set the coverage test parses. Snapshot: rerun after bumping the
+// dependency. Run: node scripts/emoji/generateEmojiFixture.mjs
+
+import { writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+
+const require = createRequire(import.meta.url);
+const data = require('emojibase-data/en/data.json');
+const { version } = require('emojibase-data/package.json');
+
+// Every base emoji + skin-tone variant, de-duplicated.
+const set = new Set();
+for (const e of data) {
+	if (e.emoji) {
+		set.add(e.emoji);
+	}
+
+	for (const s of e.skins ?? []) {
+		if (s.emoji) {
+			set.add(s.emoji);
+		}
+	}
+}
+const emojis = [...set];
+
+// One space-separated string split at runtime — a single compact line, not thousands. No emoji
+// contains a space, and prettier-ignore keeps it off Prettier's long-line wrapping.
+const out = `// Auto-generated by scripts/emoji/generateEmojiFixture.mjs — do not edit by hand.
+// Full emoji set from emojibase-data@${version} (${emojis.length} entries: base + skin-tone variants).
+// Regenerate after bumping emojibase-data: node scripts/emoji/generateEmojiFixture.mjs
+
+export const EMOJIBASE_VERSION = '${version}';
+
+// prettier-ignore
+export const ALL_EMOJI: readonly string[] = '${emojis.join(' ')}'.split(' ');
+`;
+
+const target = new URL('../../packages/message-parser/tests/fixtures/allEmoji.ts', import.meta.url);
+writeFileSync(target, out);
+console.log(`Wrote ${emojis.length} emojis (emojibase-data@${version}) to ${target.pathname}`);