Update workflow to configure logstash version

.github/workflows/CI.yml

@@ -8,50 +8,74 @@ on:
       - "*"
 
 jobs:
-  Build:
-    name: Build and Test logstash-output-opensearch
+  Unit-test:
+    strategy:
+      matrix:
+        logstash: [ "7.13.2" ]
+    name: Unit Test logstash-output-opensearch
     runs-on: ubuntu-latest
     env:
-      LOGSTASH_VERSION: "7.13.2"
-      OPENDISTRO_VERSION: "1.13.2"
-      OPENSEARCH_VERSION: "1.0.0-rc1"
+      LOGSTASH_VERSION: ${{ matrix.logstash }}
 
     steps:
       - name: Checkout plugin
         uses: actions/checkout@v1
 
-      - name: Build and pull docker files for unit tests
+      - name: Build and pull docker files for unit tests with logstash ${{ matrix.logstash }}
         run: |
           ./scripts/unit-test/docker-setup.sh
 
       - name: Run unit tests
         run: |
           ./scripts/unit-test/docker-run.sh
 
-      - name: Build and pull docker files for OpenDistro
-        run: |
-          ./scripts/opendistro/docker-setup.sh
 
-      - name: Run Integration tests against OpenDistro
-        run: |
-          ./scripts/opendistro/docker-run.sh
+  Integration-Test-OpenSearch:
+    strategy:
+      matrix:
+        logstash: [ "7.13.2" ]
+        opensearch: [ "1.0.0" ]
+        secure: [ true, false ]
+
+    name: Integration Test logstash-output-opensearch against OpenSearch
+    runs-on: ubuntu-latest
+    env:
+      LOGSTASH_VERSION: ${{ matrix.logstash }}
+      OPENSEARCH_VERSION: ${{ matrix.opensearch }}
+      SECURE_INTEGRATION: ${{ matrix.secure }}
+    steps:
+      - name: Checkout plugin
+        uses: actions/checkout@v1
 
-      - name: Build and pull docker files for OpenSearch
+      - name: Build and pull docker files for OpenSearch-${{ matrix.opensearch }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }}
         run: |
           ./scripts/opensearch/docker-setup.sh
 
-      - name: Run Integration tests against OpenSearch
+      - name: Run Integration tests against OpenSearch-${{ matrix.opensearch }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }}
         run: |
           ./scripts/opensearch/docker-run.sh
 
-      - name: Build and pull docker files for OpenSearch with security
-        env:
-          SECURE_INTEGRATION: true
+  Integration-Test-OpenDistro:
+    strategy:
+      matrix:
+        logstash: [ "7.13.2" ]
+        opendistro: [ "1.13.2" ]
+        secure: [ true, false ]
+
+    name: Integration Test logstash-output-opensearch against OpenDistro
+    runs-on: ubuntu-latest
+    env:
+      LOGSTASH_VERSION: ${{ matrix.logstash }}
+      OPENDISTRO_VERSION: ${{ matrix.opendistro }}
+      SECURE_INTEGRATION: ${{ matrix.secure }}
+    steps:
+      - name: Checkout plugin
+        uses: actions/checkout@v1
+
+      - name: Build and pull docker files for OpenDistro-${{ matrix.opendistro }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }}
         run: |
-          ./scripts/opensearch/docker-setup.sh
+          ./scripts/opendistro/docker-setup.sh
 
-      - name: Run Security Integration tests against OpenSearch
-        env:
-          SECURE_INTEGRATION: true
+      - name: Run Integration tests against OpenDistro-${{ matrix.opendistro }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }}
         run: |
-          ./scripts/opensearch/docker-run.sh
+          ./scripts/opendistro/docker-run.sh

DEVELOPER_GUIDE.md

@@ -59,7 +59,7 @@ Perform the following from your project root directory eg: `~/workspace/logstash
 
 ```bash
 # Set up Environment variable for docker to pull your test environment
-export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified.
+export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified.
 
 # Set up docker ( this will build, install into Logstash )
 scripts/unit-test/docker-setup.sh
@@ -73,8 +73,8 @@ scripts/unit-test/docker-run.sh
 
 ```bash
 # Set up Environment variable for Docker to pull your test environment
-export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified.
-export OPENSEARCH_VERSION=1.0.0-rc1 # will use latest if not specified.
+export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified.
+export OPENSEARCH_VERSION=1.0.0 # will use latest if not specified.
 
 # Set up docker ( this will build, install into Logstash )
 scripts/opensearch/docker-setup.sh
@@ -87,8 +87,8 @@ scripts/opensearch/docker-run.sh
 
 ```bash
 # Set up Environment variable for Docker to pull your test environment
-export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified.
-export OPENSEARCH_VERSION=1.0.0-rc1 # will use latest if not specified.
+export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified.
+export OPENSEARCH_VERSION=1.0.0 # will use latest if not specified.
 export SECURE_INTEGRATION=true # to run against cluster with security plugin
 # Set up docker ( this will build, install into Logstash )
 scripts/opensearch/docker-setup.sh
@@ -101,7 +101,7 @@ scripts/opensearch/docker-run.sh
 
 ```bash
 # Set up Environment variable for docker to pull your test environment
-export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified.
+export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified.
 export OPENDISTRO_VERSION=1.13.2 # will use latest if not specified.
 
 # Set up docker ( this will build, install into Logstash )

scripts/Dockerfile

@@ -1,4 +1,5 @@
-FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION:-7.13.2}
+ARG LOGSTASH_VERSION
+FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION}
 USER logstash
 COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile
 COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/

scripts/opendistro/Dockerfile.opendistro

@@ -1,7 +1,7 @@
-FROM amazon/opendistro-for-elasticsearch:${OPENDISTRO_VERSION:-latest}
+ARG OPENDISTRO_VERSION
+FROM amazon/opendistro-for-elasticsearch:${OPENDISTRO_VERSION}
 
 ARG es_path=/usr/share/elasticsearch
-ARG SECURE_INTEGRATION
 
-RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security; fi
-COPY --chown=elasticsearch:elasticsearch scripts/opendistro/elasticsearch-run.sh $es_path/
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then $es_path/bin/elasticsearch-plugin remove opendistro_security; fi

scripts/opendistro/docker-compose.override.yml

@@ -15,7 +15,10 @@ services:
       args:
         - INTEGRATION=true
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
-    command: /usr/share/elasticsearch/elasticsearch-run.sh
+        - OPENDISTRO_VERSION=${OPENDISTRO_VERSION:-latest}
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
     ports:
       - "9200:9200"
     user: elasticsearch

scripts/opendistro/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     build:
       context: ../../
       dockerfile: scripts/Dockerfile
+      args:
+        - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2}
     command: scripts/run.sh
     env_file: ../docker.env
     environment:

scripts/opensearch/Dockerfile.opensearch

@@ -1,9 +1,8 @@
-FROM opensearchproject/opensearch:${OPENSEARCH_VERSION:-latest}
+ARG OPENSEARCH_VERSION
+FROM opensearchproject/opensearch:${OPENSEARCH_VERSION}
 
-ARG plugin_path=/usr/share/plugins/plugin
 ARG opensearch_path=/usr/share/opensearch
 ARG opensearch_yml=$opensearch_path/config/opensearch.yml
-ARG SECURE_INTEGRATION
 
-RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi
-COPY --chown=opensearch:opensearch scripts/opensearch/opensearch-run.sh $opensearch_path/
+ARG SECURE_INTEGRATION
+RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi

scripts/opensearch/docker-compose.override.yml

@@ -15,7 +15,10 @@ services:
       args:
         - INTEGRATION=true
         - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false}
-    command: /usr/share/opensearch/opensearch-run.sh
+        - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest}
+    environment:
+      - discovery.type=single-node
+      - bootstrap.memory_lock=true
     ports:
       - "9200:9200"
     user: opensearch

scripts/opensearch/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     build:
       context: ../../
       dockerfile: scripts/Dockerfile
+      args:
+        - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2}
     command: scripts/run.sh
     env_file: ../docker.env
     environment:

scripts/unit-test/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     build:
       context: ../../
       dockerfile: scripts/Dockerfile
+      args:
+        - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2}
     command: /usr/share/plugins/plugin/scripts/unit-test/logstash-run.sh
     env_file: ../docker.env
     environment: