Set body string encoding from content type charset.

igrigorik · Oct 24, 2010 · 65dc4f5 · 65dc4f5
1 parent 52b308b
commit 65dc4f5
Show file tree

Hide file tree

Showing 2 changed files with 15 additions and 4 deletions.
diff --git a/lib/em-http/client.rb b/lib/em-http/client.rb
@@ -225,6 +225,7 @@ class HttpClient < Connection
     TRANSFER_ENCODING="TRANSFER_ENCODING"
     CONTENT_ENCODING="CONTENT_ENCODING"
     CONTENT_LENGTH="CONTENT_LENGTH"
+    CONTENT_TYPE="CONTENT_TYPE".freeze
     LAST_MODIFIED="LAST_MODIFIED"
     KEEP_ALIVE="CONNECTION"
     SET_COOKIE="SET_COOKIE"
@@ -249,6 +250,7 @@ def post_init
       @headers = nil
       @last_effective_url = nil
       @content_decoder = nil
+      @content_charset = nil
       @stream = nil
       @disconnect = nil
       @state = :response_header
@@ -472,6 +474,7 @@ def on_body_data(data)
     end
 
     def on_decoded_body_data(data)
+      data.force_encoding @content_charset  if @content_charset
       if @stream
         @stream.call(data)
       else
@@ -644,6 +647,10 @@ def parse_response_header
         end
       end
 
+      if ''.respond_to?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(response_header[CONTENT_TYPE])
+        @content_charset = Encoding.find $1
+      end
+
       true
     end
 

diff --git a/spec/mock_spec.rb b/spec/mock_spec.rb
@@ -44,7 +44,8 @@
       http.errback { fail }
       http.callback {
         http.response_header.status.should == 200
-        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca')).split("\r\n\r\n", 2).last
+        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca'), :encoding => 'ISO-8859-1').split("\r\n\r\n", 2).last
+        http.response.encoding.to_s.should == 'ISO-8859-1'
         EventMachine::HttpRequest.count('http://www.google.ca:80/', :get, {}).should == 1
         EventMachine.stop
       }
@@ -59,7 +60,8 @@
       http.errback { fail }
       http.callback {
         http.response_header.status.should == 200
-        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca')).split("\r\n\r\n", 2).last
+        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca'), :encoding => 'ISO-8859-1').split("\r\n\r\n", 2).last
+        http.response.encoding.to_s.should == 'ISO-8859-1'
         EventMachine::HttpRequest.count('http://www.google.ca:80/', :get, {}).should == 1
         EventMachine.stop
       }
@@ -130,7 +132,8 @@
       http.errback { fail }
       http.callback {
         http.response_header.status.should == 200
-        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca')).split("\r\n\r\n", 2).last
+        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca'), :encoding => 'ISO-8859-1').split("\r\n\r\n", 2).last
+        http.response.encoding.to_s.should == 'ISO-8859-1'
         EventMachine::HttpRequest.count('http://www.google.ca:80/', :get, {}).should == 1
         EventMachine::HttpRequest.count('http://www.google.ca:80/', :get, {:user_agent => 'BERT'}).should == 0
         EventMachine.stop
@@ -142,7 +145,8 @@
       http.errback { fail }
       http.callback {
         http.response_header.status.should == 200
-        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca')).split("\r\n\r\n", 2).last
+        http.response.should == File.read(File.join(File.dirname(__FILE__), 'fixtures', 'google.ca'), :encoding => 'ISO-8859-1').split("\r\n\r\n", 2).last
+        http.response.encoding.to_s.should == 'ISO-8859-1'
         EventMachine::HttpRequest.count('http://www.google.ca:80/', :get, {:user_agent => 'BERT'}).should == 1
         EventMachine.stop
       }