switch syntax highlighting to Linguist w/ auto-detection

README.md

@@ -45,31 +45,40 @@ output to the next filter's input. So if you wanted to have content be
 filtered through Markdown and be syntax highlighted, you can create the
 following pipeline:
 
-```ruby
+<pre lang=rb>
 pipeline = HTML::Pipeline.new [
   HTML::Pipeline::MarkdownFilter,
   HTML::Pipeline::SyntaxHighlightFilter
 ]
-result = pipeline.call <<CODE
-This is *great*:
 
-``` ruby
-some_code(:first)
-```
+result = pipeline.call &lt;&lt;-MDOWN
+Language defined *explicitly*:
 
-CODE
-result[:output].to_s
+```ruby
+5.times { puts "Odelay!" }
 ```
 
+Language *auto-detected* from code:
+
+    5.times { puts "Odelay!" }
+
+MDOWN
+
+puts result[:output].to_s
+</pre>
+
 Prints:
 
 ```html
-<p>This is <em>great</em>:</p>
+<p>Language defined <em>explicitly</em>:</p>
 
-<div class="highlight">
-<pre><span class="n">some_code</span><span class="p">(</span><span class="ss">:first</span><span class="p">)</span>
-</pre>
-</div>
+<div class="highlight"><pre><span class="mi">5</span><span class="o">.</span><span class="n">times</span> <span class="p">{</span> <span class="nb">print</span> <span class="s2">"Odelay!"</span> <span class="p">}</span>
+</pre></div>
+
+<p>Language <em>auto-detected</em> from code:</p>
+
+<div class="highlight"><pre><span class="mi">5</span><span class="o">.</span><span class="nb">times</span> <span class="p">{</span> <span class="k">print</span> <span class="s">"Odelay!"</span> <span class="p">}</span>
+</pre></div>
 ```
 
 Some filters take an optional **context** and/or **result** hash. These are

html-pipeline.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency "nokogiri",        "~> 1.4"
   gem.add_dependency "github-markdown", "~> 0.5"
   gem.add_dependency "sanitize",        "~> 2.0"
-  gem.add_dependency "pygments.rb",     ">= 0.2.13"
+  gem.add_dependency "github-linguist", "~> 2.1"
   gem.add_dependency "rinku",           "~> 1.7"
   gem.add_dependency "escape_utils",    "~> 0.2"
   gem.add_dependency "activesupport",   ">= 2"

lib/html/pipeline/syntax_highlight_filter.rb

@@ -1,26 +1,69 @@
-require 'pygments'
+require 'linguist'
 
 module HTML
   class Pipeline
-    # HTML Filter that syntax highlights code blocks wrapped
-    # in <pre lang="...">.
+    # HTML Filter that syntax highlights code blocks wrapped in <pre> tags.
+    #
+    # If a <pre> has a "lang" attribute, it is taken as the language
+    # identifier. Otherwise, the language is auto-detected from the contents of
+    # the code block. Pass in `:detect_syntax => false` to disable this.
+    # You can also disable language detection per code block by assigning a
+    # value to the "lang" attribute such as "plain".
+    #
+    # Language detection is done with GitHub Linguist. Note that some popular
+    # languages that Linguist 2.3.4 isn't yet taught to detect are:
+    # ActionScript, C#, Common Lisp, CSS, Erlang, Haskell, HTML, Lua, SQL.
     class SyntaxHighlightFilter < Filter
       def call
-        doc.search('pre').each do |node|
-          next unless lang = node['lang']
-          next unless lexer = Pygments::Lexer[lang]
-          text = node.inner_text
+        doc.search('pre').each do |pre|
+          code = pre.inner_text
 
-          html = highlight_with_timeout_handling(lexer, text)
-          next if html.nil?
+          if language_name = language_name_from_node(pre)
+            language = lookup_language(language_name)
+          elsif detect_language?
+            detected = detect_languages(code).first
+            language = detected && lookup_language(detected[0])
+          end
 
-          node.replace(html)
+          if html = language && colorize(language, code)
+            pre.replace(html)
+          end
         end
         doc
       end
 
-      def highlight_with_timeout_handling(lexer, text)
-        lexer.highlight(text)
+      def detect_language?
+        context[:detect_syntax] != false
+      end
+
+      def language_name_from_node node
+        node['lang']
+      end
+
+      def lookup_language name
+        Linguist::Language[name]
+      end
+
+      def detect_languages code
+        Linguist::Classifier.classify(classifier_db, code, possible_languages)
+      end
+
+      def classifier_db() Linguist::Samples::DATA end
+
+      def possible_languages
+        popular_language_names & sampled_languages
+      end
+
+      def popular_language_names
+        Linguist::Language.popular.map {|lang| lang.name }
+      end
+
+      def sampled_languages
+        classifier_db['languages'].keys
+      end
+
+      def colorize language, code
+        language.colorize code
       rescue Timeout::Error => boom
         nil
       end

test/html/pipeline/syntax_highlight_filter_test.rb

@@ -8,18 +8,51 @@ def filter(*args)
   end
 
   def test_unchanged
-    html = "<pre>plain</pre>"
+    html = %(<pre lang="plain">I am a poem</pre>)
     assert_equal html, filter(html).to_s
   end
 
   def test_syntax_highlighting
     html = "<pre lang=rb>a = 1</pre>"
     assert_equal_html <<-RESULT, filter(html).to_s
-      <div class="highlight">
-        <pre>
+      <div class="highlight"><pre>
         <span class="n">a</span> <span class="o">=</span> <span class="mi">1</span>
-        </pre>
-      </div>
+      </pre></div>
     RESULT
   end
+
+  def test_explicit_lang_skips_detection
+    html = "<pre lang=rb>var a = null</pre>"
+    assert_equal_html <<-RESULT, filter(html).to_s
+      <div class="highlight"><pre>
+        <span class="n">var</span> <span class="n">a</span>
+        <span class="o">=</span> <span class="n">null</span>
+      </pre></div>
+    RESULT
+  end
+
+  def test_detects_ruby
+    html = "<pre>def foo; end</pre>"
+    assert_equal_html <<-RUBY, filter(html).to_s
+      <div class="highlight"><pre>
+        <span class="k">def</span> <span class="nf">foo</span><span class="p">;</span>
+        <span class="k">end</span>
+      </pre></div>
+    RUBY
+  end
+
+  def test_detects_javascript
+    html = "<pre>var a = null</pre>"
+    assert_equal_html <<-RESULT, filter(html).to_s
+      <div class="highlight"><pre>
+        <span class="kd">var</span> <span class="nx">a</span>
+        <span class="o">=</span> <span class="kc">null</span>
+      </pre></div>
+    RESULT
+  end
+
+  def test_disable_detect
+    html = "<pre>def foo; end</pre>"
+    assert_equal html, filter(html, :detect_syntax => false).to_s
+  end
 end