diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index c1f9cdd3..a0247f5c 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -5,23 +5,25 @@ | logstash-output-opensearch | Logstash OSS| | ------------- | ------------- | | 1.0.0+ | 7.13.2 | +| 2.0.0+ | 7.13.2 | ### Matrix for OpenSearch | logstash-output-opensearch | OpenSearch | | ------------- | ------------- | -| 1.0.0+ | 1.0.0 | - +| 1.0.0+ | 1.0.0 | +| 2.0.0+ | 1.0.0 | ### Matrix for ODFE | logstash-output-opensearch | ODFE | | ------------- | ------------- | | 1.0.0+ | 1.x - 1.13.2 | - +| 2.0.0+ | 1.x - 1.13.2 | ### Matrix for Elasticsearch OSS | logstash-output-opensearch | Elasticsearch OSS| | ------------- | ------------- | | 1.0.0+ | 7.x - 7.10.2 | +| 2.0.0+ | 7.x - 7.10.2 | diff --git a/README.md b/README.md index efad3317..47bc8cd2 100644 --- a/README.md +++ b/README.md @@ -44,17 +44,17 @@ output { To run the Logstash Output Opensearch plugin using aws_iam authentication, refer to the sample configuration shown below: ``` -output { - opensearch { - hosts => ["hostname:port"] - auth_type => { - type => 'aws_iam' - aws_access_key_id => 'ACCESS_KEY' - aws_secret_access_key => 'SECRET_KEY' - region => 'us-west-2' - } - index => "logstash-logs-%{+YYYY.MM.dd}" - } +output { + opensearch { + hosts => ["hostname:port"] + auth_type => { + type => 'aws_iam' + aws_access_key_id => 'ACCESS_KEY' + aws_secret_access_key => 'SECRET_KEY' + region => 'us-west-2' + } + index => "logstash-logs-%{+YYYY.MM.dd}" + } } ``` @@ -62,34 +62,49 @@ In addition to the existing authentication mechanisms, if we want to add new aut Example Configuration for basic authentication: ``` -output { - opensearch { - hosts => ["hostname:port"] - auth_type => { - type => 'basic' - user => 'admin' - password => 'admin' - } - index => "logstash-logs-%{+YYYY.MM.dd}" - } -} +output { + opensearch { + hosts => ["hostname:port"] + auth_type => { + type => 'basic' + user => 'admin' + password => 'admin' + } + index => "logstash-logs-%{+YYYY.MM.dd}" + } +} ``` To ingest data into a `data stream` through logstash, we need to create the data stream and specify the name of data stream and the `op_type` of `create` in the output configuration. The sample configuration is shown below: ```yml -output { - opensearch { - hosts => ["https://hostname:port"] - auth_type => { - type => 'basic' - user => 'admin' - password => 'admin' +output { + opensearch { + hosts => ["https://hostname:port"] + auth_type => { + type => 'basic' + user => 'admin' + password => 'admin' } index => "my-data-stream" action => "create" - } -} + } +} +``` + +Starting in 2.0.0, the aws sdk version is bumped to v3. In order for all other AWS plugins to work together, please remove pre-installed aws plugins and install logstash-integration-aws plugin as follows. See also https://github.com/logstash-plugins/logstash-mixin-aws/issues/38 +``` +# Remove existing logstash aws plugins and install logstash-integration-aws to keep sdk dependency the same +# https://github.com/logstash-plugins/logstash-mixin-aws/issues/38 +/usr/share/logstash/bin/logstash-plugin remove logstash-input-s3 +/usr/share/logstash/bin/logstash-plugin remove logstash-input-sqs +/usr/share/logstash/bin/logstash-plugin remove logstash-output-s3 +/usr/share/logstash/bin/logstash-plugin remove logstash-output-sns +/usr/share/logstash/bin/logstash-plugin remove logstash-output-sqs +/usr/share/logstash/bin/logstash-plugin remove logstash-output-cloudwatch + +/usr/share/logstash/bin/logstash-plugin install --version 0.1.0.pre logstash-integration-aws +bin/logstash-plugin install --version 2.0.0 logstash-output-opensearch ``` For more details refer to this [documentation](https://opensearch.org/docs/latest/clients/logstash/ship-to-opensearch/#opensearch-output-plugin) @@ -104,4 +119,4 @@ This project is licensed under the [Apache v2.0 License](LICENSE). ## Copyright -Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details. \ No newline at end of file +Copyright OpenSearch Contributors. See [NOTICE](NOTICE) for details. diff --git a/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb b/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb index 15292d8f..b0024c5b 100644 --- a/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb +++ b/lib/logstash/outputs/opensearch/http_client/manticore_adapter.rb @@ -57,7 +57,7 @@ def initialize(logger, options={}) if options[:proxy] options[:proxy] = manticore_proxy_hash(options[:proxy]) end - + @manticore = ::Manticore::Client.new(options) end @@ -105,7 +105,7 @@ def get_user() def get_password() @password end - + # Transform the proxy option to a hash. Manticore's support for non-hash # proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5 # but this is not yet released @@ -137,12 +137,12 @@ def perform_request(url, method, path, params={}, body=nil) params[:body] = body if body if url.user - params[:auth] = { + params[:auth] = { :user => CGI.unescape(url.user), # We have to unescape the password here since manticore won't do it # for us unless its part of the URL - :password => CGI.unescape(url.password), - :eager => true + :password => CGI.unescape(url.password), + :eager => true } elsif @type == BASIC_AUTH_TYPE add_basic_auth_to_params(params) @@ -174,11 +174,17 @@ def perform_request(url, method, path, params={}, body=nil) def sign_aws_request(request_uri, path, method, params) url = URI::HTTPS.build({:host=>URI(request_uri.to_s).host, :port=>AWS_DEFAULT_PORT.to_s, :path=>path}) - key = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase, - :headers => params[:headers],:body => params[:body]}) - aws_signer = Aws::Signers::V4.new(@credentials, AWS_SERVICE, get_aws_region ) - signed_key = aws_signer.sign(key) - params[:headers] = params[:headers].merge(signed_key.headers) + request = Seahorse::Client::Http::Request.new(options={:endpoint=>url, :http_method => method.to_s.upcase, + :headers => params[:headers],:body => params[:body]}) + + aws_signer = Aws::Sigv4::Signer.new(service: @service_name, region: @region, credentials_provider: @credentials) + signed_key = aws_signer.sign_request( + http_method: request.http_method, + url: url, + headers: params[:headers], + body: params[:body] + ) + params[:headers] = params[:headers].merge(signed_key.headers) end def add_basic_auth_to_params(params) @@ -192,7 +198,7 @@ def add_basic_auth_to_params(params) # Returned urls from this method should be checked for double escaping. def format_url(url, path_and_query=nil) request_uri = url.clone - + # We excise auth info from the URL in case manticore itself tries to stick # sensitive data in a thrown exception or log data request_uri.user = nil @@ -208,7 +214,7 @@ def format_url(url, path_and_query=nil) new_query_parts = [request_uri.query, parsed_path_and_query.query].select do |part| part && !part.empty? # Skip empty nil and "" end - + request_uri.query = new_query_parts.join("&") unless new_query_parts.empty? # use `raw_path`` as `path` will unescape any escaped '/' in the path diff --git a/logstash-output-opensearch.gemspec b/logstash-output-opensearch.gemspec index 5d36af8e..0484320d 100644 --- a/logstash-output-opensearch.gemspec +++ b/logstash-output-opensearch.gemspec @@ -11,7 +11,7 @@ signing_key_path = "gem-private_key.pem" Gem::Specification.new do |s| s.name = 'logstash-output-opensearch' - s.version = '1.3.0' + s.version = '2.0.0' s.licenses = ['Apache-2.0'] s.summary = "Stores logs in OpenSearch" @@ -45,7 +45,7 @@ Gem::Specification.new do |s| s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0'] s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~>1.0' - s.add_runtime_dependency 'aws-sdk', '>= 2.11.632', '~> 2' + s.add_runtime_dependency 'aws-sdk', '~> 3' s.add_runtime_dependency 'json', '>= 2.3.0', '~> 2' s.add_development_dependency 'logstash-codec-plain'