Merge pull request #9 from MyriaCore/shortcode-search

Shortcode search

EmojiSpider.py

@@ -3,6 +3,7 @@
 import re
 import scrapy
 import requests
+import lxml.html
 import sqlite3
 import shutil
 import base64
@@ -27,14 +28,31 @@ def cleanup():
 def setup_db():
     conn = sqlite3.connect('emoji.sqlite', check_same_thread=False)
     conn.executescript('''
-        CREATE TABLE emoji (name VARCHAR PRIMARY KEY, code VARCHAR,
-                            icon_apple VARCHAR, icon_twemoji VARCHAR,
-                            icon_noto VARCHAR, icon_blobmoji VARCHAR,
-                            keywords VARCHAR, name_search VARCHAR);
-        CREATE TABLE skin_tone (name VARCHAR, code VARCHAR, tone VARCHAR,
-                                icon_apple VARCHAR, icon_twemoji VARCHAR,
-                                icon_noto VARCHAR, icon_blobmoji VARCHAR);
-        CREATE INDEX name_idx ON skin_tone (name);''')
+        CREATE TABLE emoji (
+            name VARCHAR PRIMARY KEY, 
+            code VARCHAR,
+            icon_apple VARCHAR, 
+            icon_twemoji VARCHAR,
+            icon_noto VARCHAR, 
+            icon_blobmoji VARCHAR,
+            keywords VARCHAR, 
+            name_search VARCHAR
+        );
+        CREATE TABLE skin_tone (
+            name VARCHAR, 
+            code VARCHAR, 
+            tone VARCHAR,
+            icon_apple VARCHAR, 
+            icon_twemoji VARCHAR,
+            icon_noto VARCHAR, 
+            icon_blobmoji VARCHAR
+        );
+        CREATE TABLE shortcode (
+            name VARCHAR,
+            code VARCHAR
+        );
+        CREATE INDEX name_idx ON skin_tone (name);
+        ''')
     conn.row_factory = sqlite3.Row
 
     return conn
@@ -75,17 +93,36 @@ def codepoint_to_url(codepoint, style):
         return 'https://github.com/C1710/blobmoji/raw/master/png/128/emoji_u%s.png' \
                 % base.replace(' ', '_')
 
+def name_to_shortcodes(shortname):
+    """
+    Given an emoji's CLDR Shortname (e.g. 'grinning face with smiling eyes'), returns a list
+    of common shortcodes used for that emoji.
+    
+    NOTE: These shortcodes will NOT have colons at the beginning and end, even if they normally
+          would.
+    """
+    url = 'https://emojipedia.org/%s/' % re.sub(r'[^a-z0-9 ]', '', shortname.lower()).replace(' ', '-')
+    response = requests.get(url, stream=True)
+    response.raw.decode_content = True
+    html = lxml.html.parse(response.raw) if response.ok else None
+    shortcode_nodes = html.xpath('//ul[@class="shortcodes"]/li/span[@class="shortcode"]') if html else []
+    return [s.text for s in shortcode_nodes]
+
 cleanup()
 conn = setup_db()
 
 class EmojiSpider(scrapy.Spider):
     name = 'emojispider'
-    start_urls = ['http://unicode.org/emoji/charts/emoji-list.html']
+    start_urls = ['http://unicode.org/emoji/charts/emoji-list.html',
+                  'http://unicode.org/emoji/charts/full-emoji-modifiers.html']
     # start_urls = ['http://172.17.0.1:8000/list2.html']
 
     def parse(self, response):
         icon = 0
-        for tr in response.xpath('//tr[.//td[@class="code"]]'):
+        emoji_nodes = response.xpath('//tr[.//td[@class="code"]]')
+        for i in range(0, len(emoji_nodes)):
+            # Scrape Data from unicode.org
+            tr = emoji_nodes[i]
             code = tr.css('.code a::text').extract_first()
             encoded_code = str_to_unicode_emoji(code)
             name = ''.join(tr.xpath('(.//td[@class="name"])[1]//text()').extract())
@@ -98,22 +135,28 @@ def parse(self, response):
             if found:
                 skin_tone = found.group('skin_tone')
                 name = name.replace(': %s skin tone' % skin_tone, '')
+            shortcodes = name_to_shortcodes(name)
 
+            # Prepare emoji data to be inserted into DB
+            print("Fetching %i/%i: %s %s" % (i+1, len(emoji_nodes), encoded_code, name))
             record = {
                 'name': name,
                 'code': encoded_code,
+                'shortcodes': ' '.join(shortcodes),
                 'keywords': keywords,
                 'tone': skin_tone,
                 'name_search': ' '.join(set(
-                    [kw.strip() for kw in ('%s %s' % (name, keywords)).split(' ')]
+                    [s[1:-1] for s in shortcodes] + [kw.strip() for kw in ('%s %s' % (name, keywords)).split(' ')]
                 )),
-                # Icons Styles 
+                # Merge icon styles into record
                 **{ 'icon_%s' % style: '%s/%s.png' \
                         % (ICONS_PATH(style), icon_name) \
                         for style in EMOJI_STYLES \
                 }
             }
-
+
+            # Download Icons for each emoji
+            print("🖼  Downloading Icons...")
             supported_styles = []
             for style in EMOJI_STYLES:
                 if style == 'apple':
@@ -123,13 +166,14 @@ def parse(self, response):
                     link = codepoint_to_url(code, style)
                     resp = requests.get(link)
                     icon_data = resp.content if resp.ok else None
-                    print('[%s] %s' % ('OK' if resp.ok else 'BAD', link))
-
+                    print('- %s: %s' % ('✅' if resp.ok else '❎', style))
+                
                 if icon_data:
                     with open(record['icon_%s' % style], 'wb') as f:
                         f.write(icon_data)
                     supported_styles += [style] 
 
+            # Prepare emoji insertion query
             supported_styles = ['icon_%s' % style for style in supported_styles]
             if skin_tone:
                 query = '''INSERT INTO skin_tone (name, code, tone, ''' + ', '.join(supported_styles) + ''')
@@ -140,8 +184,11 @@ def parse(self, response):
                            VALUES (:name, :code, ''' + ', '.join([':%s' % s for s in supported_styles]) + ''',
                                    :keywords, :name_search)'''
 
+            # Insert emoji & associated shortcodes into DB
             conn.execute(query, record)
-
-            yield record
+            for sc in shortcodes:
+                squery = '''INSERT INTO shortcode (name, code)
+                            VALUES (:name, "%s")''' % sc
+                conn.execute(squery, record)
 
         conn.commit()

README.md

@@ -1,9 +1,23 @@
 # Emoji Extension
 
-<img aligh="center" src="screenshots/search.png">
+<table>
+  <tr>
+    <td><img src="screenshots/search.png"></td>
+    <td><img src="screenshots/shortcode-search.png"></td>
+  </tr>
+</table>
 
-<img aligh="center" src="screenshots/preferences.png">
+## Features
 
-### Credits
+- Supports Twemoji, Blobmoji, and Noto emoji preview renders
+- Search by emoji name, *or* by shortcode by beginning the search with `:`
+- Support for multiple skin tones via settings
 
-[twemoji](https://github.com/twitter/twemoji), [noto-emoji](https://github.com/googlefonts/noto-emoji), and [blobmoji](https://github.com/C1710/blobmoji) for emoji styles :heart:.
+### Settings
+
+![](screenshots/preferences.png)
+
+## Credits
+
+- [emojipedia.org](https://emojipedia.org/) for emoji shortcode data
+- [twemoji](https://github.com/twitter/twemoji), [noto-emoji](https://github.com/googlefonts/noto-emoji), and [blobmoji](https://github.com/C1710/blobmoji) for emoji styles :heart:.