From 9c648bb4ab9a9b588cb465a027f45b3f19ba5bb5 Mon Sep 17 00:00:00 2001 From: Deep Datta Date: Thu, 28 Jul 2022 16:36:05 -0500 Subject: [PATCH 1/5] Fix the integration test to work with OpenSearch 2.x Don't run the parent_spec for opensearch >= 2.x Update index_spec.rb and compressed_indexing_spec.rb to not expect the _type field in the doc for Opensearch >= 2.0.0 Signed-off-by: Deep Datta --- spec/integration/outputs/compressed_indexing_spec.rb | 6 +++++- spec/integration/outputs/index_spec.rb | 6 +++++- spec/integration/outputs/parent_spec.rb | 2 +- spec/opensearch_spec_helper.rb | 10 ++++++++++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/spec/integration/outputs/compressed_indexing_spec.rb b/spec/integration/outputs/compressed_indexing_spec.rb index faf48cc..11cfdf3 100644 --- a/spec/integration/outputs/compressed_indexing_spec.rb +++ b/spec/integration/outputs/compressed_indexing_spec.rb @@ -59,7 +59,11 @@ response = http_client.get("#{index_url}/_search?q=*&size=1000") result = LogStash::Json.load(response.body) result["hits"]["hits"].each do |doc| - expect(doc["_type"]).to eq(type) + if OpenSearchHelper.check_version?("< 2") + expect(doc["_type"]).to eq(type) + else + expect(doc).not_to include("_type") + end expect(doc["_index"]).to eq(index) end end diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb index 83ed12f..48f65c4 100644 --- a/spec/integration/outputs/index_spec.rb +++ b/spec/integration/outputs/index_spec.rb @@ -88,7 +88,11 @@ response = http_client.get("#{index_url}/_search?q=*&size=1000") result = LogStash::Json.load(response.body) result["hits"]["hits"].each do |doc| - expect(doc["_type"]).to eq(type) + if OpenSearchHelper.check_version?("< 2") + expect(doc["_type"]).to eq(type) + else + expect(doc).not_to include("_type") + end expect(doc["_index"]).to eq(index) end end diff --git a/spec/integration/outputs/parent_spec.rb b/spec/integration/outputs/parent_spec.rb index f8c957d..4edd256 100644 --- a/spec/integration/outputs/parent_spec.rb +++ b/spec/integration/outputs/parent_spec.rb @@ -10,7 +10,7 @@ require_relative "../../../spec/opensearch_spec_helper" require "logstash/outputs/opensearch" -context "join field tests", :integration => true do +context "join field tests", :integration => true && OpenSearchHelper.check_version?("<2") do shared_examples "a join field based parent indexer" do let(:index) { 10.times.collect { rand(10).to_s }.join("") } diff --git a/spec/opensearch_spec_helper.rb b/spec/opensearch_spec_helper.rb index 6eb447d..6bcdb16 100644 --- a/spec/opensearch_spec_helper.rb +++ b/spec/opensearch_spec_helper.rb @@ -49,6 +49,16 @@ def self.version RSpec.configuration.filter[:version] end + def self.check_version?(*requirement) + version = self.version + if version.nil? || version.empty? + puts "version tag isn't set. Returning false from 'check_version?`." + return false + end + release_version = Gem::Version.new(version).release + Gem::Requirement.new(requirement).satisfied_by?(release_version) + end + RSpec::Matchers.define :have_hits do |expected| match do |actual| expected == actual['hits']['total']['value'] From 214f11e85d6def80eb93f17ced6351b9ebeab174 Mon Sep 17 00:00:00 2001 From: Deep Datta Date: Mon, 8 Aug 2022 16:00:21 -0500 Subject: [PATCH 2/5] Update the OpenSearch versions in .github/workflows/CI Signed-off-by: Deep Datta --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 6205329..a01dcdb 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -34,7 +34,7 @@ jobs: strategy: matrix: logstash: [ "7.16.3", "7.17.1" ] - opensearch: [ "1.2.1" ] + opensearch: [ "1.3.4", "2.1.0" ] secure: [ true, false ] name: Integration Test logstash-output-opensearch against OpenSearch @@ -78,4 +78,4 @@ jobs: - name: Run Integration tests against OpenDistro-${{ matrix.opendistro }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }} run: | - ./scripts/opendistro/docker-run.sh \ No newline at end of file + ./scripts/opendistro/docker-run.sh From b04cad6405d1f4a7a5215f52a09eacc00376e774 Mon Sep 17 00:00:00 2001 From: Deep Datta Date: Tue, 9 Aug 2022 00:26:29 -0500 Subject: [PATCH 3/5] Install python in the logstash container for integration test to prevent the version fetch from failing. For some logstash version e.g. 7.17.1, 8.3.2 the images are missing python leading to the following comand in scripts/logstash-run.sh to fail echo $(curl -s $SERVICE_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])") That is causing some integration test to fail which depends on the OpenSearch version number. Signed-off-by: Deep Datta --- scripts/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 34857fd..229baba 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -1,5 +1,7 @@ ARG LOGSTASH_VERSION FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION} +USER root +RUN apt-get update && apt-get install -y python USER logstash COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ @@ -10,4 +12,4 @@ RUN gem install bundler -v '< 2' WORKDIR /usr/share/plugins/plugin RUN bundle install --with test ci COPY --chown=logstash:logstash . /usr/share/plugins/plugin -RUN bundle exec rake vendor \ No newline at end of file +RUN bundle exec rake vendor From f6878322b599917c03de830089cb36012dbbb643 Mon Sep 17 00:00:00 2001 From: Deep Datta Date: Wed, 10 Aug 2022 12:01:40 -0500 Subject: [PATCH 4/5] Replace the python command to extract the version field with grep Signed-off-by: Deep Datta --- scripts/Dockerfile | 2 -- scripts/logstash-run.sh | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 229baba..5f726da 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -1,7 +1,5 @@ ARG LOGSTASH_VERSION FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION} -USER root -RUN apt-get update && apt-get install -y python USER logstash COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ diff --git a/scripts/logstash-run.sh b/scripts/logstash-run.sh index c6916a0..4b823f3 100755 --- a/scripts/logstash-run.sh +++ b/scripts/logstash-run.sh @@ -16,7 +16,7 @@ wait_for_es() { [[ $count -eq 0 ]] && exit 1 sleep 20 done - echo $(curl -s $SERVICE_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])") + echo $(curl -s $SERVICE_URL | grep -oP '"number"[^"]+"\K[^"]+') } if [[ "$SECURE_INTEGRATION" == "true" ]]; then From ba635708a0414265d67bab353dc31a0c694a3f6f Mon Sep 17 00:00:00 2001 From: Deep Datta Date: Wed, 10 Aug 2022 11:27:12 -0500 Subject: [PATCH 5/5] Modify version check to account for OpenDistro Signed-off-by: Deep Datta --- spec/integration/outputs/compressed_indexing_spec.rb | 4 +++- spec/integration/outputs/index_spec.rb | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/integration/outputs/compressed_indexing_spec.rb b/spec/integration/outputs/compressed_indexing_spec.rb index 11cfdf3..cc9703a 100644 --- a/spec/integration/outputs/compressed_indexing_spec.rb +++ b/spec/integration/outputs/compressed_indexing_spec.rb @@ -59,7 +59,9 @@ response = http_client.get("#{index_url}/_search?q=*&size=1000") result = LogStash::Json.load(response.body) result["hits"]["hits"].each do |doc| - if OpenSearchHelper.check_version?("< 2") + # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x + # need a cleaner way to check this. + if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7") expect(doc["_type"]).to eq(type) else expect(doc).not_to include("_type") diff --git a/spec/integration/outputs/index_spec.rb b/spec/integration/outputs/index_spec.rb index 48f65c4..5b8e025 100644 --- a/spec/integration/outputs/index_spec.rb +++ b/spec/integration/outputs/index_spec.rb @@ -88,7 +88,9 @@ response = http_client.get("#{index_url}/_search?q=*&size=1000") result = LogStash::Json.load(response.body) result["hits"]["hits"].each do |doc| - if OpenSearchHelper.check_version?("< 2") + # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x + # need a cleaner way to check this. + if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7") expect(doc["_type"]).to eq(type) else expect(doc).not_to include("_type")