Refactor unit tests and integration tests

.github/workflows/ci.yml

@@ -16,23 +16,25 @@ jobs:
       - name: Checkout plugin
         uses: actions/checkout@v1
 
-      - name: Build and pull docker files for opendistro
+      - name: Build and pull docker files for unit tests
         env:
           OPENDISTRO_VERSION: "1.13.2"
           LOGSTASH_VERSION: "7.13.2"
         run: |
-          ./scripts/opendistro/docker-setup.sh
+          ./scripts/unit-test/docker-setup.sh
 
-    # TODO: Refactor scripts into one unit test and multiple integration test
-      - name: Run unit tests for opendistro
-        env:
-          INTEGRATION: false
+      - name: Run unit tests
         run: |
-          ./scripts/opendistro/docker-run.sh
+          ./scripts/unit-test/docker-run.sh
 
-      - name: Run integration tests for opendistro
+      - name: Build and pull docker files for OpenDistro
         env:
-          INTEGRATION: true
+          OPENDISTRO_VERSION: "1.13.2"
+          LOGSTASH_VERSION: "7.13.2"
+        run: |
+          ./scripts/opendistro/docker-setup.sh
+
+      - name: Run Integration tests against OpenDistro
         run: |
           ./scripts/opendistro/docker-run.sh
 
@@ -43,14 +45,6 @@ jobs:
         run: |
           ./scripts/opensearch/docker-setup.sh
 
-      - name: Run unit tests for OpenSearch
-        env:
-          INTEGRATION: false
-        run: |
-          ./scripts/opensearch/docker-run.sh
-
-      - name: Run integration tests for OpenSearch
-        env:
-          INTEGRATION: true
+      - name: Run Integration tests against OpenSearch
         run: |
           ./scripts/opensearch/docker-run.sh

scripts/Dockerfile

@@ -5,8 +5,6 @@ COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/pl
 RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml
 ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin:/usr/share/logstash/jdk/bin"
 ENV LOGSTASH_SOURCE="1"
-ARG INTEGRATION
-ENV INTEGRATION=$INTEGRATION
 RUN gem install bundler -v '< 2'
 WORKDIR /usr/share/plugins/plugin
 RUN bundle install --with test ci

scripts/logstash-run.sh

@@ -15,11 +15,7 @@ wait_for_es() {
   echo $(curl -s $SERVICE_URL | python -c "import sys, json; print(json.load(sys.stdin)['version']['number'])")
 }
 
-if [[ "$INTEGRATION" != "true" ]]; then
-  bundle exec rspec -fd spec/unit -t ~integration
-else
-  echo "Waiting for cluster to respond..."
-  VERSION=$(wait_for_es)
-  echo "Integration test cluster $VERSION is Up!"
-  bundle exec rspec -fd --tag integration --tag update_tests:painless --tag version:$VERSION spec/integration
-fi
+echo "Waiting for cluster to respond..."
+VERSION=$(wait_for_es)
+echo "Integration test cluster $VERSION is Up!"
+bundle exec rspec -fd --tag integration --tag update_tests:painless --tag version:$VERSION spec/integration

scripts/opendistro/docker-compose.override.yml

@@ -5,15 +5,15 @@ services:
   logstash:
     command: /usr/share/plugins/plugin/scripts/logstash-run.sh
     environment:
-      - INTEGRATION=${INTEGRATION:-false}
+      - INTEGRATION=true
 
 
   integration:
     build:
       context: ../../
       dockerfile: scripts/opendistro/Dockerfile.opendistro
       args:
-        - INTEGRATION=${INTEGRATION:-false}
+        - INTEGRATION=true
     command: /usr/share/elasticsearch/elasticsearch-run.sh
     ports:
       - "9200:9200"

scripts/opendistro/docker-run.sh

@@ -5,8 +5,4 @@ set -ex
 
 cd scripts/opendistro
 
-if [ "$INTEGRATION" == "true" ]; then
-    docker-compose up --exit-code-from logstash
-else
-    docker-compose up --exit-code-from logstash logstash
-fi
+docker-compose up --exit-code-from logstash

scripts/opendistro/docker-setup.sh

@@ -9,10 +9,5 @@ if [ -f Gemfile.lock ]; then
 fi
 cd scripts/opendistro;
 
-if [ "$INTEGRATION" == "true" ]; then
-    docker-compose down
-    docker-compose build
-else
-    docker-compose down
-    docker-compose build logstash
-fi
+docker-compose down
+docker-compose build

scripts/opensearch/docker-compose.override.yml

@@ -5,15 +5,15 @@ services:
   logstash:
     command: /usr/share/plugins/plugin/scripts/logstash-run.sh
     environment:
-      - INTEGRATION=${INTEGRATION:-false}
+      - INTEGRATION=true
 
 
   integration:
     build:
       context: ../../
       dockerfile: scripts/opensearch/Dockerfile.opensearch
       args:
-        - INTEGRATION=${INTEGRATION:-false}
+        - INTEGRATION=true
     command: /usr/share/opensearch/opensearch-run.sh
     ports:
       - "9200:9200"

scripts/opensearch/docker-run.sh

@@ -5,8 +5,4 @@ set -ex
 
 cd scripts/opensearch
 
-if [ "$INTEGRATION" == "true" ]; then
-    docker-compose up --exit-code-from logstash
-else
-    docker-compose up --exit-code-from logstash logstash
-fi
+docker-compose up --exit-code-from logstash

scripts/opensearch/docker-setup.sh

@@ -9,10 +9,5 @@ if [ -f Gemfile.lock ]; then
 fi
 cd scripts/opensearch;
 
-if [ "$INTEGRATION" == "true" ]; then
-    docker-compose down
-    docker-compose build
-else
-    docker-compose down
-    docker-compose build logstash
-fi
+docker-compose down
+docker-compose build

scripts/unit-test/docker-compose.yml

@@ -0,0 +1,14 @@
+version: '3'
+
+services:
+
+  logstash:
+    build:
+      context: ../../
+      dockerfile: scripts/Dockerfile
+    command: /usr/share/plugins/plugin/scripts/unit-test/logstash-run.sh
+    env_file: ../docker.env
+    environment:
+      - SPEC_OPTS
+      - LOG_LEVEL # devutils (>= 2.0.4) reads the ENV and sets LS logging
+    tty: true

scripts/unit-test/docker-run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# This is intended to be run inside the docker container as the command of the docker-compose.
+set -ex
+
+cd scripts/unit-test;
+docker-compose up --exit-code-from logstash logstash

scripts/unit-test/docker-setup.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# This is intended to be run the plugin's root directory. `.scripts/opendistro/docker-setup.sh`
+# Ensure you have Docker installed locally and set the OPENDISTRO_VERSION environment variable.
+set -e
+
+if [ -f Gemfile.lock ]; then
+    rm Gemfile.lock
+fi
+
+cd scripts/unit-test;
+docker-compose down
+docker-compose build logstash

scripts/unit-test/logstash-run.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -ex
+
+export PATH=$BUILD_DIR/gradle/bin:$PATH
+
+bundle exec rspec -fd spec/unit -t ~integration