feature: Rework shortcodes to align with the community and support localization.

bin/buildFilesizeTable.js

@@ -15,9 +15,10 @@ function calculatePackage(packageName) {
       return;
     }
 
+    // Load, sort, and stat files
     const rows = files
-      .filter(file => !file.includes('tests') && !file.includes('raw'))
-      .map(file => {
+      .filter((file) => !file.includes('tests') && !file.includes('raw'))
+      .map((file) => {
         const data = fs.readFileSync(file, 'utf8');
 
         return {
@@ -29,18 +30,45 @@ function calculatePackage(packageName) {
 
     rows.sort((a, b) => a.size - b.size);
 
-    console.log('| File | Size | Gzipped |');
-    console.log('| --- | --- | --- |');
+    // Group by type
+    const groups = [
+      [], // All
+      [], // Data
+      [], // Data: compact
+      [], // Shortcodes
+    ];
 
-    rows.forEach(row => {
+    rows.forEach((row) => {
       if (row.file === 'package.json' || row.file === 'tsconfig.json') {
         return;
       }
 
-      console.log(`| ${row.file} | ${size(row.size)} | ${size(row.gzip)} |`);
+      if (row.file.endsWith('compact.json')) {
+        groups[2].push(row);
+      } else if (row.file.endsWith('data.json')) {
+        groups[1].push(row);
+      } else if (row.file.includes('shortcodes/')) {
+        groups[3].push(row);
+      } else {
+        groups[0].push(row);
+      }
     });
 
-    console.log('');
+    // Print groups
+    groups.forEach((items) => {
+      if (items.length === 0) {
+        return;
+      }
+
+      console.log('| File | Size | Gzipped |');
+      console.log('| --- | --- | --- |');
+
+      items.forEach((row) => {
+        console.log(`| ${row.file} | ${size(row.size)} | ${size(row.gzip)} |`);
+      });
+
+      console.log('');
+    });
   });
 }
 

package.json

@@ -9,7 +9,8 @@
     "clean:cache": "rm -rf ./cache",
     "coverage": "yarn run jest --coverage",
     "lint": "beemo eslint",
-    "generate": "node ./bin/generateEmoji.js && node ./bin/buildFilesizeTable.js && cp ./packages/data/en/data.json ./packages/test-utils/test-data.json",
+    "generate": "yarn run generate:emojis && yarn run generate:shortcodes && node ./bin/buildFilesizeTable.js && cp ./packages/data/en/data.json ./packages/test-utils/test-data.json",
+    "generate:emojis": "node ./bin/generateEmoji.js",
     "generate:shortcodes": "node ./bin/generateShortcodes.js",
     "generate:types": "node ./bin/generateTypes.js",
     "jest": "beemo jest",
@@ -27,6 +28,7 @@
   "devDependencies": {
     "@milesj/build-tools": "^2.10.1",
     "conventional-changelog-beemo": "^2.0.0",
+    "fetch-mock-jest": "^1.3.0",
     "filesize": "^6.1.0",
     "glob": "^7.1.6",
     "gzip-size": "^5.1.1",

packages/core/src/appendSkinToneIndex.ts

@@ -0,0 +1,8 @@
+import { SkinTone } from './types';
+
+export default function appendSkinToneIndex(
+  shortcode: string,
+  emoji: { tone?: SkinTone | SkinTone[] },
+): string {
+  return `${shortcode}_${Array.isArray(emoji.tone) ? emoji.tone.join('-') : emoji.tone}`;
+}

packages/core/src/constants.ts

@@ -85,6 +85,8 @@ export const SUPPORTED_LOCALES = [
   'zh-hant', // Chinese (Traditional)
 ];
 
+export const NON_LATIN_LOCALES = ['ja', 'ko', 'ru', 'th', 'uk', 'zh', 'zh-hant'];
+
 // Special options for emoticon permutations.
 
 export const EMOTICON_OPTIONS: { [emoticon: string]: PermutationOptions } = {

packages/core/src/fetchEmojis.ts

@@ -0,0 +1,72 @@
+import {
+  ShortcodePreset,
+  CompactEmoji,
+  Emoji,
+  ShortcodesDataset,
+  FlatEmoji,
+  FlatCompactEmoji,
+} from './types';
+import fetchFromCDN, { FetchFromCDNOptions } from './fetchFromCDN';
+import fetchShortcodes from './fetchShortcodes';
+import flattenEmojiData from './flattenEmojiData';
+import joinShortcodes from './joinShortcodes';
+
+export interface FetchEmojisOptions extends FetchFromCDNOptions {
+  compact?: boolean;
+  flat?: boolean;
+  shortcodes?: (string | ShortcodePreset)[];
+}
+
+// Full
+async function fetchEmojis(
+  locale: string,
+  options?: FetchEmojisOptions & { compact?: false; flat?: false },
+): Promise<Emoji[]>;
+
+async function fetchEmojis(
+  locale: string,
+  options: FetchEmojisOptions & { compact?: false; flat: true },
+): Promise<FlatEmoji[]>;
+
+// Compact
+async function fetchEmojis(
+  locale: string,
+  options: FetchEmojisOptions & { compact: true; flat?: false },
+): Promise<CompactEmoji[]>;
+
+async function fetchEmojis(
+  locale: string,
+  options: FetchEmojisOptions & { compact: true; flat: true },
+): Promise<FlatCompactEmoji[]>;
+
+async function fetchEmojis(locale: string, options: FetchEmojisOptions = {}): Promise<unknown[]> {
+  const { compact = false, flat = false, shortcodes: presets = [], ...opts } = options;
+  const emojis = await fetchFromCDN<Emoji[]>(
+    `${locale}/${compact ? 'compact' : 'data'}.json`,
+    opts,
+  );
+  let shortcodes: ShortcodesDataset[] = [];
+
+  if (presets.length > 0) {
+    shortcodes = await Promise.all(
+      presets.map((preset) => {
+        let promise: Promise<ShortcodesDataset>;
+
+        if (preset.includes('/')) {
+          const [customLocale, customPreset] = preset.split('/');
+
+          promise = fetchShortcodes(customLocale, customPreset as ShortcodePreset, opts);
+        } else {
+          promise = fetchShortcodes(locale, preset as ShortcodePreset, opts);
+        }
+
+        // Ignore as the primary dataset should still load
+        return promise.catch(() => ({}));
+      }),
+    );
+  }
+
+  return flat ? flattenEmojiData(emojis, shortcodes) : joinShortcodes(emojis, shortcodes);
+}
+
+export default fetchEmojis;

packages/core/src/fetchFromCDN.ts

@@ -1,12 +1,14 @@
 export interface FetchFromCDNOptions extends RequestInit {
   local?: boolean;
+  version?: string;
 }
 
 export default function fetchFromCDN<T>(
   path: string,
-  version: string = 'latest',
   options: FetchFromCDNOptions = {},
-): Promise<T[]> {
+): Promise<T> {
+  const { local = false, version = 'latest', ...opts } = options;
+
   if (__DEV__) {
     if (!path || path.slice(-5) !== '.json') {
       throw new Error('A valid JSON dataset is required to fetch.');
@@ -17,18 +19,13 @@ export default function fetchFromCDN<T>(
     }
   }
 
-  const { local = false, ...opts } = options;
   const storage = local ? localStorage : sessionStorage;
   const cacheKey = `emojibase/${version}/${path}`;
   const cachedData = storage.getItem(cacheKey);
 
   // Check the cache first
   if (cachedData) {
-    try {
-      return Promise.resolve(JSON.parse(cachedData));
-    } catch (error) {
-      return Promise.resolve([]);
-    }
+    return Promise.resolve(JSON.parse(cachedData));
   }
 
   return fetch(`https://cdn.jsdelivr.net/npm/emojibase-data@${version}/${path}`, {

packages/core/src/fetchShortcodes.ts

@@ -0,0 +1,20 @@
+import { ShortcodePreset, ShortcodesDataset } from './types';
+import fetchFromCDN, { FetchFromCDNOptions } from './fetchFromCDN';
+import { NON_LATIN_LOCALES } from './constants';
+
+const ALIASES: Partial<Record<ShortcodePreset, string>> = {
+  discord: 'iamcal',
+  slack: 'iamcal',
+};
+
+export default function fetchShortcodes(
+  locale: string,
+  preset: ShortcodePreset,
+  options?: FetchFromCDNOptions,
+): Promise<ShortcodesDataset> {
+  if (preset === 'cldr-native' && !NON_LATIN_LOCALES.includes(locale)) {
+    return Promise.resolve({});
+  }
+
+  return fetchFromCDN(`${locale}/shortcodes/${ALIASES[preset] || preset}.json`, options);
+}

packages/core/src/flattenEmojiData.ts

@@ -1,30 +1,43 @@
-import { Emoji } from './types';
+import { Emoji, ShortcodesDataset, CompactEmoji, MaybeEmoji } from './types';
+import joinShortcodesToEmoji from './joinShortcodesToEmoji';
 
-export default function flattenEmojiData(data: Emoji[]): Emoji[] {
+function flattenEmojiData(data: Emoji[], shortcodeDatasets?: ShortcodesDataset[]): Emoji[];
+
+function flattenEmojiData(
+  data: CompactEmoji[],
+  shortcodeDatasets?: ShortcodesDataset[],
+): CompactEmoji[];
+
+function flattenEmojiData(
+  data: MaybeEmoji[],
+  shortcodeDatasets: ShortcodesDataset[] = [],
+): MaybeEmoji[] {
   const emojis: Emoji[] = [];
 
-  data.forEach((emoji) => {
+  (data as Emoji[]).forEach((emoji) => {
     if (emoji.skins) {
-      const { skins, ...restEmoji } = emoji;
-
       // Dont include nested skins array
-      emojis.push(restEmoji);
+      const { skins, ...baseEmoji } = emoji;
+
+      emojis.push(joinShortcodesToEmoji(baseEmoji, shortcodeDatasets));
 
       // Push each skin modification into the root list
       skins.forEach((skin) => {
         const skinEmoji = { ...skin };
 
         // Inherit tags from parent if they exist
-        if (emoji.tags) {
-          skinEmoji.tags = [...emoji.tags];
+        if (baseEmoji.tags) {
+          skinEmoji.tags = [...baseEmoji.tags];
         }
 
-        emojis.push(skinEmoji);
+        emojis.push(joinShortcodesToEmoji(skinEmoji, shortcodeDatasets));
       });
     } else {
-      emojis.push(emoji);
+      emojis.push(joinShortcodesToEmoji(emoji, shortcodeDatasets));
     }
   });
 
   return emojis;
 }
+
+export default flattenEmojiData;

packages/core/src/index.ts

@@ -3,21 +3,31 @@
  * @license     https://opensource.org/licenses/MIT
  */
 
+import appendSkinToneIndex from './appendSkinToneIndex';
+import fetchEmojis from './fetchEmojis';
 import fetchFromCDN from './fetchFromCDN';
+import fetchShortcodes from './fetchShortcodes';
 import flattenEmojiData from './flattenEmojiData';
 import fromCodepointToUnicode from './fromCodepointToUnicode';
 import fromHexcodeToCodepoint from './fromHexcodeToCodepoint';
 import fromUnicodeToHexcode from './fromUnicodeToHexcode';
 import generateEmoticonPermutations from './generateEmoticonPermutations';
+import joinShortcodes from './joinShortcodes';
+import joinShortcodesToEmoji from './joinShortcodesToEmoji';
 import stripHexcode from './stripHexcode';
 
 export {
+  appendSkinToneIndex,
+  fetchEmojis,
   fetchFromCDN,
+  fetchShortcodes,
   flattenEmojiData,
   fromCodepointToUnicode,
   fromHexcodeToCodepoint,
   fromUnicodeToHexcode,
   generateEmoticonPermutations,
+  joinShortcodes,
+  joinShortcodesToEmoji,
   stripHexcode,
 };
 

packages/core/src/joinShortcodes.ts

@@ -0,0 +1,26 @@
+import { Emoji, CompactEmoji, ShortcodesDataset, MaybeEmoji } from './types';
+import joinShortcodesToEmoji from './joinShortcodesToEmoji';
+
+function joinShortcodes(emojis: Emoji[], shortcodeDatasets: ShortcodesDataset[]): Emoji[];
+
+function joinShortcodes(
+  emojis: CompactEmoji[],
+  shortcodeDatasets: ShortcodesDataset[],
+): CompactEmoji[];
+
+function joinShortcodes(
+  emojis: MaybeEmoji[],
+  shortcodeDatasets: ShortcodesDataset[],
+): MaybeEmoji[] {
+  if (shortcodeDatasets.length === 0) {
+    return emojis;
+  }
+
+  emojis.forEach((emoji) => {
+    joinShortcodesToEmoji(emoji as Emoji, shortcodeDatasets);
+  });
+
+  return emojis;
+}
+
+export default joinShortcodes;

packages/core/src/joinShortcodesToEmoji.ts

@@ -0,0 +1,41 @@
+import { Emoji, CompactEmoji, ShortcodesDataset, MaybeEmoji } from './types';
+
+function joinShortcodesToEmoji(emoji: Emoji, shortcodeDatasets: ShortcodesDataset[]): Emoji;
+
+function joinShortcodesToEmoji(
+  emoji: CompactEmoji,
+  shortcodeDatasets: ShortcodesDataset[],
+): CompactEmoji;
+
+function joinShortcodesToEmoji(
+  emoji: MaybeEmoji,
+  shortcodeDatasets: ShortcodesDataset[],
+): MaybeEmoji {
+  if (shortcodeDatasets.length === 0) {
+    return emoji;
+  }
+
+  const list = new Set(emoji.shortcodes);
+
+  shortcodeDatasets.forEach((dataset) => {
+    const shortcodes = dataset[emoji.hexcode];
+
+    if (Array.isArray(shortcodes)) {
+      shortcodes.forEach((code) => list.add(code));
+    } else if (shortcodes) {
+      list.add(shortcodes);
+    }
+  });
+
+  emoji.shortcodes = Array.from(list);
+
+  if (emoji.skins) {
+    emoji.skins.forEach((skin) => {
+      joinShortcodesToEmoji(skin as Emoji, shortcodeDatasets);
+    });
+  }
+
+  return emoji;
+}
+
+export default joinShortcodesToEmoji;