[fix] Addresses issues when using client.content() and dealing with mismatched encoding

Fix broken encoding (fixes #920)

Author: nickfloyd

lib/octokit/connection.rb

@@ -154,7 +154,7 @@ def request(method, path, data, options = {})
       end
 
       @last_response = response = agent.call(method, Addressable::URI.parse(path.to_s).normalize.to_s, data, options)
-      response.data
+      response_data_correctly_encoded(response)
     rescue Octokit::Error => e
       @last_response = nil
       raise e
@@ -206,5 +206,13 @@ def parse_query_and_convenience_headers(options)
 
       opts
     end
+
+    def response_data_correctly_encoded(response)
+      content_type = response.headers.fetch('content-type', '')
+      return response.data unless content_type.include?('charset') && response.data.is_a?(String)
+
+      reported_encoding = content_type.match(/charset=([^ ]+)/)[1]
+      response.data.force_encoding(reported_encoding)
+    end
   end
 end

spec/cassettes/Octokit_Client_Contents/_contents/returns_the_contents_of_a_file_properly_encoded_as_specified_by_the_response_headers.json

@@ -0,0 +1 @@
+{"http_interactions":[{"request":{"method":"get","uri":"https://api.github.com/repos/octokit/octokit.rb/contents/spec/fixtures/utf8.en.js","body":{"encoding":"US-ASCII","base64_string":""},"headers":{"Accept":["application/vnd.github.V3.raw"],"User-Agent":["Octokit Ruby Gem 4.14.0"],"Content-Type":["application/json"],"Authorization":["token <<ACCESS_TOKEN>>"],"Accept-Encoding":["gzip;q=1.0,deflate;q=0.6,identity;q=0.3"]}},"response":{"status":{"code":200,"message":"OK"},"headers":{"Date":["Sun, 28 Apr 2019 22:51:27 GMT"],"Content-Type":["application/vnd.github.V3.raw; charset=utf-8"],"Content-Length":["67"],"Server":["GitHub.com"],"Status":["200 OK"],"X-Ratelimit-Limit":["5000"],"X-Ratelimit-Remaining":["4978"],"X-Ratelimit-Reset":["1556493266"],"Cache-Control":["private, max-age=60, s-maxage=60"],"Vary":["Accept, Authorization, Cookie, X-GitHub-OTP","Accept-Encoding"],"Etag":["\"a03fe032178ccc868a9c36935d55199088523cec\""],"Last-Modified":["Sun, 28 Apr 2019 22:42:09 GMT"],"X-Oauth-Scopes":["repo, user"],"X-Accepted-Oauth-Scopes":[""],"X-Github-Media-Type":["github.v3; param=V3.raw"],"Access-Control-Expose-Headers":["ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type"],"Access-Control-Allow-Origin":["*"],"Strict-Transport-Security":["max-age=31536000; includeSubdomains; preload"],"X-Frame-Options":["deny"],"X-Content-Type-Options":["nosniff"],"X-Xss-Protection":["1; mode=block"],"Referrer-Policy":["origin-when-cross-origin, strict-origin-when-cross-origin"],"Content-Security-Policy":["default-src 'none'"],"X-Github-Request-Id":["D9A1:55E3:8DF93A:AC7A70:5CC62E6E"]},"body":{"encoding":"ASCII-8BIT","base64_string":"Ly8gQGZsb3cKZXhwb3J0IGRlZmF1bHQgewogIGtleTogIlNvbWV0aGluZ+KA\npiB3aXRoIHV0ZjggY2hhcnMiLAp9Cg==\n"},"http_version":null},"recorded_at":"Sun, 28 Apr 2019 22:51:27 GMT"}],"recorded_with":"VCR 4.0.0"}

spec/fixtures/utf8.en.js

@@ -0,0 +1,4 @@
+// @flow
+export default {
+  key: "Something… with utf8 chars",
+}

spec/octokit/client/contents_spec.rb

@@ -25,6 +25,13 @@
       expect(contents.type).to eq('file')
       assert_requested :get, github_url('/repos/octokit/octokit.rb/contents/lib/octokit.rb')
     end
+
+    it 'returns the contents of a file properly encoded as specified by the response headers' do
+      contents = @client.contents('octokit/octokit.rb', path: 'spec/fixtures/utf8.en.js', accept: 'application/vnd.github.V3.raw')
+      expect(contents).to eq(File.read('spec/fixtures/utf8.en.js'))
+      expect(contents.encoding).to eq(Encoding::UTF_8)
+      assert_requested :get, github_url('/repos/octokit/octokit.rb/contents/spec/fixtures/utf8.en.js')
+    end
   end # .contents
 
   describe '.archive_link', :vcr do