diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 80a5792..0c483f5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -8,19 +8,20 @@ on: - "*" jobs: - Build: - name: Build and Test logstash-output-opensearch + Unit-test: + strategy: + matrix: + logstash: [ "7.13.2" ] + name: Unit Test logstash-output-opensearch runs-on: ubuntu-latest env: - LOGSTASH_VERSION: "7.13.2" - OPENDISTRO_VERSION: "1.13.2" - OPENSEARCH_VERSION: "1.0.0-rc1" + LOGSTASH_VERSION: ${{ matrix.logstash }} steps: - name: Checkout plugin uses: actions/checkout@v1 - - name: Build and pull docker files for unit tests + - name: Build and pull docker files for unit tests with logstash ${{ matrix.logstash }} run: | ./scripts/unit-test/docker-setup.sh @@ -28,30 +29,53 @@ jobs: run: | ./scripts/unit-test/docker-run.sh - - name: Build and pull docker files for OpenDistro - run: | - ./scripts/opendistro/docker-setup.sh - - name: Run Integration tests against OpenDistro - run: | - ./scripts/opendistro/docker-run.sh + Integration-Test-OpenSearch: + strategy: + matrix: + logstash: [ "7.13.2" ] + opensearch: [ "1.0.0" ] + secure: [ true, false ] + + name: Integration Test logstash-output-opensearch against OpenSearch + runs-on: ubuntu-latest + env: + LOGSTASH_VERSION: ${{ matrix.logstash }} + OPENSEARCH_VERSION: ${{ matrix.opensearch }} + SECURE_INTEGRATION: ${{ matrix.secure }} + steps: + - name: Checkout plugin + uses: actions/checkout@v1 - - name: Build and pull docker files for OpenSearch + - name: Build and pull docker files for OpenSearch-${{ matrix.opensearch }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }} run: | ./scripts/opensearch/docker-setup.sh - - name: Run Integration tests against OpenSearch + - name: Run Integration tests against OpenSearch-${{ matrix.opensearch }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }} run: | ./scripts/opensearch/docker-run.sh - - name: Build and pull docker files for OpenSearch with security - env: - SECURE_INTEGRATION: true + Integration-Test-OpenDistro: + strategy: + matrix: + logstash: [ "7.13.2" ] + opendistro: [ "1.13.2" ] + secure: [ true, false ] + + name: Integration Test logstash-output-opensearch against OpenDistro + runs-on: ubuntu-latest + env: + LOGSTASH_VERSION: ${{ matrix.logstash }} + OPENDISTRO_VERSION: ${{ matrix.opendistro }} + SECURE_INTEGRATION: ${{ matrix.secure }} + steps: + - name: Checkout plugin + uses: actions/checkout@v1 + + - name: Build and pull docker files for OpenDistro-${{ matrix.opendistro }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }} run: | - ./scripts/opensearch/docker-setup.sh + ./scripts/opendistro/docker-setup.sh - - name: Run Security Integration tests against OpenSearch - env: - SECURE_INTEGRATION: true + - name: Run Integration tests against OpenDistro-${{ matrix.opendistro }}-security-${{ matrix.secure }} for logstash ${{ matrix.logstash }} run: | - ./scripts/opensearch/docker-run.sh + ./scripts/opendistro/docker-run.sh \ No newline at end of file diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md index 11d6aa8..e3d5c78 100644 --- a/DEVELOPER_GUIDE.md +++ b/DEVELOPER_GUIDE.md @@ -59,7 +59,7 @@ Perform the following from your project root directory eg: `~/workspace/logstash ```bash # Set up Environment variable for docker to pull your test environment -export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified. +export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified. # Set up docker ( this will build, install into Logstash ) scripts/unit-test/docker-setup.sh @@ -73,8 +73,8 @@ scripts/unit-test/docker-run.sh ```bash # Set up Environment variable for Docker to pull your test environment -export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified. -export OPENSEARCH_VERSION=1.0.0-rc1 # will use latest if not specified. +export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified. +export OPENSEARCH_VERSION=1.0.0 # will use latest if not specified. # Set up docker ( this will build, install into Logstash ) scripts/opensearch/docker-setup.sh @@ -87,8 +87,8 @@ scripts/opensearch/docker-run.sh ```bash # Set up Environment variable for Docker to pull your test environment -export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified. -export OPENSEARCH_VERSION=1.0.0-rc1 # will use latest if not specified. +export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified. +export OPENSEARCH_VERSION=1.0.0 # will use latest if not specified. export SECURE_INTEGRATION=true # to run against cluster with security plugin # Set up docker ( this will build, install into Logstash ) scripts/opensearch/docker-setup.sh @@ -101,7 +101,7 @@ scripts/opensearch/docker-run.sh ```bash # Set up Environment variable for docker to pull your test environment -export LOGSTASH_VERSION=7.13.2 # will use latest version if not specified. +export LOGSTASH_VERSION=7.13.2 # will use 7.13.2 version if not specified. export OPENDISTRO_VERSION=1.13.2 # will use latest if not specified. # Set up docker ( this will build, install into Logstash ) diff --git a/scripts/Dockerfile b/scripts/Dockerfile index 89051b3..34857fd 100644 --- a/scripts/Dockerfile +++ b/scripts/Dockerfile @@ -1,4 +1,5 @@ -FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION:-7.13.2} +ARG LOGSTASH_VERSION +FROM docker.elastic.co/logstash/logstash-oss:${LOGSTASH_VERSION} USER logstash COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ diff --git a/scripts/opendistro/Dockerfile.opendistro b/scripts/opendistro/Dockerfile.opendistro index b18da6d..05f3193 100644 --- a/scripts/opendistro/Dockerfile.opendistro +++ b/scripts/opendistro/Dockerfile.opendistro @@ -1,7 +1,7 @@ -FROM amazon/opendistro-for-elasticsearch:${OPENDISTRO_VERSION:-latest} +ARG OPENDISTRO_VERSION +FROM amazon/opendistro-for-elasticsearch:${OPENDISTRO_VERSION} ARG es_path=/usr/share/elasticsearch -ARG SECURE_INTEGRATION -RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security; fi -COPY --chown=elasticsearch:elasticsearch scripts/opendistro/elasticsearch-run.sh $es_path/ +ARG SECURE_INTEGRATION +RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then $es_path/bin/elasticsearch-plugin remove opendistro_security; fi diff --git a/scripts/opendistro/docker-compose.override.yml b/scripts/opendistro/docker-compose.override.yml index e149ec9..260a3e1 100644 --- a/scripts/opendistro/docker-compose.override.yml +++ b/scripts/opendistro/docker-compose.override.yml @@ -15,7 +15,10 @@ services: args: - INTEGRATION=true - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - command: /usr/share/elasticsearch/elasticsearch-run.sh + - OPENDISTRO_VERSION=${OPENDISTRO_VERSION:-latest} + environment: + - discovery.type=single-node + - bootstrap.memory_lock=true ports: - "9200:9200" user: elasticsearch diff --git a/scripts/opendistro/docker-compose.yml b/scripts/opendistro/docker-compose.yml index 9753ff1..3392923 100644 --- a/scripts/opendistro/docker-compose.yml +++ b/scripts/opendistro/docker-compose.yml @@ -6,6 +6,8 @@ services: build: context: ../../ dockerfile: scripts/Dockerfile + args: + - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2} command: scripts/run.sh env_file: ../docker.env environment: diff --git a/scripts/opendistro/elasticsearch-run.sh b/scripts/opendistro/elasticsearch-run.sh deleted file mode 100755 index 9df8c88..0000000 --- a/scripts/opendistro/elasticsearch-run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -ex - -/usr/share/elasticsearch/bin/elasticsearch -Ediscovery.type=single-node diff --git a/scripts/opensearch/Dockerfile.opensearch b/scripts/opensearch/Dockerfile.opensearch index e0f800a..44f221a 100755 --- a/scripts/opensearch/Dockerfile.opensearch +++ b/scripts/opensearch/Dockerfile.opensearch @@ -1,9 +1,8 @@ -FROM opensearchproject/opensearch:${OPENSEARCH_VERSION:-latest} +ARG OPENSEARCH_VERSION +FROM opensearchproject/opensearch:${OPENSEARCH_VERSION} -ARG plugin_path=/usr/share/plugins/plugin ARG opensearch_path=/usr/share/opensearch ARG opensearch_yml=$opensearch_path/config/opensearch.yml -ARG SECURE_INTEGRATION -RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi -COPY --chown=opensearch:opensearch scripts/opensearch/opensearch-run.sh $opensearch_path/ \ No newline at end of file +ARG SECURE_INTEGRATION +RUN if [ "$SECURE_INTEGRATION" != "true" ] ; then echo "plugins.security.disabled: true" >> $opensearch_yml; fi \ No newline at end of file diff --git a/scripts/opensearch/docker-compose.override.yml b/scripts/opensearch/docker-compose.override.yml index 8af81ae..9bf1eab 100755 --- a/scripts/opensearch/docker-compose.override.yml +++ b/scripts/opensearch/docker-compose.override.yml @@ -15,7 +15,10 @@ services: args: - INTEGRATION=true - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - command: /usr/share/opensearch/opensearch-run.sh + - OPENSEARCH_VERSION=${OPENSEARCH_VERSION:-latest} + environment: + - discovery.type=single-node + - bootstrap.memory_lock=true ports: - "9200:9200" user: opensearch diff --git a/scripts/opensearch/docker-compose.yml b/scripts/opensearch/docker-compose.yml index 9753ff1..3392923 100755 --- a/scripts/opensearch/docker-compose.yml +++ b/scripts/opensearch/docker-compose.yml @@ -6,6 +6,8 @@ services: build: context: ../../ dockerfile: scripts/Dockerfile + args: + - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2} command: scripts/run.sh env_file: ../docker.env environment: diff --git a/scripts/opensearch/opensearch-run.sh b/scripts/opensearch/opensearch-run.sh deleted file mode 100755 index 2f6d0d4..0000000 --- a/scripts/opensearch/opensearch-run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -set -ex - -/usr/share/opensearch/bin/opensearch -Ediscovery.type=single-node diff --git a/scripts/unit-test/docker-compose.yml b/scripts/unit-test/docker-compose.yml index d3ee0a0..fb48d16 100644 --- a/scripts/unit-test/docker-compose.yml +++ b/scripts/unit-test/docker-compose.yml @@ -6,6 +6,8 @@ services: build: context: ../../ dockerfile: scripts/Dockerfile + args: + - LOGSTASH_VERSION=${LOGSTASH_VERSION:-7.13.2} command: /usr/share/plugins/plugin/scripts/unit-test/logstash-run.sh env_file: ../docker.env environment: