Make Pygments work correctly with QSyntaxHighlighter

This work was done by @randyheydon in

https://bitbucket.org/spyder-ide/spyderlib/pull-requests/103

and now we're moving it to GitHub

spyder/utils/syntaxhighlighters.py

@@ -63,7 +63,6 @@
 CUSTOM_EXTENSION_LEXER = {'.ipynb': 'json',
                           '.txt': 'text',
                           '.nt': 'bat',
-                          '.scss': 'css',
                           '.m': 'matlab',
                           ('.properties', '.session', '.inf', '.reg', '.url',
                            '.cfg', '.cnf', '.aut', '.iss'): 'ini'}
@@ -917,6 +916,7 @@ class PygmentsSH(BaseSH):
     # Store the language name and a ref to the lexer
     _lang_name = None
     _lexer = None
+    _charlist = []
     # Syntax highlighting states (from one text block to another):
     NORMAL = 0
     def __init__(self, parent, font=None, color_scheme=None):
@@ -939,34 +939,55 @@ def __init__(self, parent, font=None, color_scheme=None):
         # Load Pygments' Lexer
         if self._lang_name is not None:
             self._lexer = get_lexer_by_name(self._lang_name)
+        # Connect document updates to a re-lexing of the document.
+        parent.contentsChange.connect(self._make_charlist)
         BaseSH.__init__(self, parent, font, color_scheme)
 
-    def get_fmt(self, typ):
-        """ Get the format code for this type """
+    def _get_fmt(self, typ):
+        """Get the Spyder format code for the given Pygments token type."""
         # Exact matches first
-        for key in self._tokmap:
-            if typ is key:
-                return self._tokmap[key]            
+        if typ in self._tokmap:
+            return self._tokmap[typ]
         # Partial (parent-> child) matches
-        for key in self._tokmap:
-            if typ in key.subtypes:
-                return self._tokmap[key]
+        for key, val in self._tokmap.items():
+            if typ in key: # Checks if typ is a subtype of key.
+                return val
         return 'normal'
 
+    def _make_charlist(self, *args):
+        """
+        Parses the complete text and stores format for each character.
+
+        Uses the attached lexer to parse into a list of tokens and Pygments
+        token types.  Then breaks tokens into individual letters, each with a
+        Spyder token type attached.  Stores this list as self._charlist.
+
+        It's attached to the contentsChange signal of the parent QTextDocument
+        so that the charlist is updated whenever the document changes.
+        """
+
+        text = to_text_string(self.document().toPlainText())
+        tokens = self._lexer.get_tokens(text)
+        charlist = []
+        for typ, token in tokens:
+            fmt = self.formats[self._get_fmt(typ)]
+            for letter in token:
+                charlist.append((fmt, letter))
+        self._charlist = charlist
+
     def highlightBlock(self, text):
-        """ Actually highlight the block """        
-        text = to_text_string(text)                
-        lextree = self._lexer.get_tokens(text)        
-        ct = 0
-        for item in lextree:            
-            typ, val = item            
-            key = self.get_fmt(typ)
-            start = ct
-            ct += len(val)        
-            self.setFormat(start, ct-start, self.formats[key])
+        """ Actually highlight the block"""
+        # Note that an undefined blockstate is equal to -1, so the first block
+        # will have the correct behaviour of starting at 0.
+        start = self.previousBlockState() + 1
+        end = start + len(text)
+        for i, (fmt, letter) in enumerate(self._charlist[start:end]):
+            self.setFormat(i, 1, fmt)
+        self.setCurrentBlockState(end)
 
         self.highlight_spaces(text)
 
+
 def guess_pygments_highlighter(filename):
     """Factory to generate syntax highlighter for the given filename.