Make ci compatible with os2.x (#153)

* Fix the integration test to work with OpenSearch 2.x
Don't run the parent_spec for opensearch >= 2.x
Update index_spec.rb and compressed_indexing_spec.rb to not expect the _type field in the doc for Opensearch >= 2.0.0

* Update the OpenSearch versions in the .github/workflows/CI test matrix

* Replace the python command in scripts/logstash-run.sh to extract the version field with grep
For some logstash version e.g. 7.17.1, 8.3.2 the images are missing python leading to the following command to fail
echo $(curl -s $SERVICE_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
That is causing some integration test to fail which depends on the OpenSearch version number.
installing python in the container failed as GitHub action is complaining about missing apt-get

Signed-off-by: Deep Datta <[email protected]>

Author: deepdatta

.github/workflows/CI.yml

@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         logstash: [ "7.16.3", "7.17.1", "8.3.2" ]
-        opensearch: [ "1.2.1" ]
+        opensearch: [ "1.3.4", "2.1.0" ]
         secure: [ true, false ]
 
     name: Integration Test logstash-output-opensearch against OpenSearch

scripts/Dockerfile

@@ -10,4 +10,4 @@ RUN gem install bundler -v '< 2'
 WORKDIR /usr/share/plugins/plugin
 RUN bundle install --with test ci
 COPY --chown=logstash:logstash . /usr/share/plugins/plugin
-RUN bundle exec rake vendor
+RUN bundle exec rake vendor

scripts/logstash-run.sh

@@ -16,7 +16,7 @@ wait_for_es() {
     [[ $count -eq 0 ]] && exit 1
     sleep 20
   done
-  echo $(curl -s $SERVICE_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
+  echo $(curl -s $SERVICE_URL | grep -oP '"number"[^"]+"\K[^"]+')
 }
 
 if [[ "$SECURE_INTEGRATION" == "true" ]]; then

spec/integration/outputs/compressed_indexing_spec.rb

@@ -59,7 +59,13 @@
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end

spec/integration/outputs/index_spec.rb

@@ -88,7 +88,13 @@
       response = http_client.get("#{index_url}/_search?q=*&size=1000")
       result = LogStash::Json.load(response.body)
       result["hits"]["hits"].each do |doc|
-        expect(doc["_type"]).to eq(type)
+        # FIXME This checks for OpenSearch 1.x or OpenDistro which has version 7.10.x
+        # need a cleaner way to check this.
+        if OpenSearchHelper.check_version?("< 2") || OpenSearchHelper.check_version?("> 7")
+          expect(doc["_type"]).to eq(type)
+        else
+          expect(doc).not_to include("_type")
+        end
         expect(doc["_index"]).to eq(index)
       end
     end

spec/integration/outputs/parent_spec.rb

@@ -10,7 +10,7 @@
 require_relative "../../../spec/opensearch_spec_helper"
 require "logstash/outputs/opensearch"
 
-context "join field tests", :integration => true do
+context "join field tests", :integration => true && OpenSearchHelper.check_version?("<2") do
 
   shared_examples "a join field based parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }

spec/opensearch_spec_helper.rb

@@ -49,6 +49,16 @@ def self.version
     RSpec.configuration.filter[:version]
   end
 
+  def self.check_version?(*requirement)
+    version = self.version
+    if version.nil? || version.empty?
+      puts "version tag isn't set. Returning false from 'check_version?`."
+      return false
+    end
+    release_version = Gem::Version.new(version).release
+    Gem::Requirement.new(requirement).satisfied_by?(release_version)
+  end
+
   RSpec::Matchers.define :have_hits do |expected|
     match do |actual|
       expected == actual['hits']['total']['value']