Fix: encode body before signing to match manticore adapter encoding.

lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb

@@ -12,6 +12,9 @@
 require 'manticore'
 require 'uri'
 
+java_import 'org.apache.http.util.EntityUtils'
+java_import 'org.apache.http.entity.StringEntity'
+
 module LogStash; module Outputs; class OpenSearch; class HttpClient;
   AWS_DEFAULT_PORT = 443
   AWS_DEFAULT_PROFILE = 'default'
@@ -181,8 +184,20 @@ def perform_request(url, method, path, params={}, body=nil)
       resp
     end
 
+    # from Manticore, https://github.com/cheald/manticore/blob/acc25cac2999f4658a77a0f39f60ddbca8fe14a4/lib/manticore/client.rb#L536
+    ISO_8859_1 = "ISO-8859-1".freeze
+
+    def minimum_encoding_for(string)
+      if string.ascii_only?
+        ISO_8859_1
+      else
+        string.encoding.to_s
+      end
+    end
+
     def sign_aws_request(request_uri, path, method, params)
       url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>AWS_DEFAULT_PORT.to_s, :path=>path})
+
       request = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase,
         :headers => params[:headers],:body => params[:body]})
 
@@ -191,7 +206,8 @@ def sign_aws_request(request_uri, path, method, params)
         http_method: request.http_method,
         url: url,
         headers: params[:headers],
-        body: params[:body]
+        # match encoding of the HTTP adapter, see https://github.com/opensearch-project/logstash-output-opensearch/issues/207
+        body: params[:body] ? EntityUtils.toString(StringEntity.new(params[:body], minimum_encoding_for(params[:body]))) : nil
       )
       params[:headers] = params[:headers].merge(signed_key.headers)
     end

spec/unit/outputs/opensearch/http_client/manticore_adapter_spec.rb

@@ -70,35 +70,80 @@
     } }
     subject { described_class.new(logger, options) }
     let(:uri) { ::LogStash::Util::SafeURI.new("http://localhost:9200") }
-    let(:sign_aws_request) {  }
 
-    it "should validate AWS IAM credentials initialization" do
-      expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
-      expect(subject.get_service_name).to eq("es") 
-    end
-
-    it "should validate AWS IAM service_name config" do
-      expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
-      expect(subject.get_service_name).to eq("svc_test") 
-    end
+    let(:expected_uri) {
+      expected_uri = uri.clone
+      expected_uri.path = "/"
+      expected_uri
+    }
 
-    it "should validate signing aws request" do
+    let(:resp) {
       resp = double("response")
       allow(resp).to receive(:call)
       allow(resp).to receive(:code).and_return(200)
-      allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+      resp
+    }
 
-      expected_uri = uri.clone
-      expected_uri.path = "/"
+    context 'with a signer' do
+      let(:sign_aws_request) {  }
 
-      expect(subject.manticore).to receive(:get).
-        with(expected_uri.to_s, {
-          :headers => {"content-type"=> "application/json"}
-        }
+      it "should validate AWS IAM credentials initialization" do
+        expect(subject.aws_iam_auth_initialization(options)).not_to be_nil
+        expect(subject.get_service_name).to eq("es")
+      end
+
+      it "should validate AWS IAM service_name config" do
+        expect(subject.aws_iam_auth_initialization(options_svc)).not_to be_nil
+        expect(subject.get_service_name).to eq("svc_test")
+      end
+
+      it "should validate signing aws request" do
+        allow(subject).to receive(:sign_aws_request).with(any_args).and_return(sign_aws_request)
+
+        expect(subject.manticore).to receive(:get).
+          with(expected_uri.to_s, {
+            :headers => {"content-type"=> "application/json"}
+          }
+          ).and_return resp
+
+        expect(subject).to receive(:sign_aws_request)
+        subject.perform_request(uri, :get, "/")
+      end
+    end
+
+    context 'sign_aws_request' do
+      it 'handles UTF-8' do
+        encoded_body = body = "boîte de réception"
+        expect_any_instance_of(Aws::Sigv4::Signer).to receive(:sign_request).with(hash_including({
+            body: body,
+        })).and_return(
+          double(headers: {})
+        )
+        expect(subject.manticore).to receive(:post).
+          with(expected_uri.to_s, {
+            :body => encoded_body,
+            :headers => {"content-type"=> "application/json"}
+          }
         ).and_return resp
+        subject.perform_request(uri, :post, "/", { body: encoded_body })
+      end
 
-      expect(subject).to receive(:sign_aws_request)
-      subject.perform_request(uri, :get, "/")
+      it 'encodes body before signing to match manticore adapter encoding' do
+        body = "boîte de réception"
+        encoded_body = body.encode("ISO-8859-1")
+        expect_any_instance_of(Aws::Sigv4::Signer).to receive(:sign_request).with(hash_including({
+            body: body,
+        })).and_return(
+          double(headers: {})
+        )
+        expect(subject.manticore).to receive(:post).
+          with(expected_uri.to_s, {
+            :body => encoded_body,
+            :headers => {"content-type"=> "application/json"}
+          }
+        ).and_return resp
+        subject.perform_request(uri, :post, "/", { body: encoded_body })
+      end
     end
   end