diff --git a/.github/draft-release-notes-config.yml b/.github/draft-release-notes-config.yml new file mode 100644 index 0000000..d168727 --- /dev/null +++ b/.github/draft-release-notes-config.yml @@ -0,0 +1,40 @@ +# The overall template of the release notes +template: | + Compatible with OpenSearch (**set version here**). + $CHANGES + +# Setting the formatting and sorting for the release notes body +name-template: Version (set version here) +change-template: '* $TITLE (#$NUMBER)' +sort-by: merged_at +sort-direction: ascending +replacers: + - search: '##' + replace: '###' + +# Organizing the tagged PRs into categories +categories: + - title: 'Breaking Changes' + labels: + - 'Breaking Changes' + - title: 'Features' + labels: + - 'Features' + - title: 'Enhancements' + labels: + - 'Enhancements' + - title: 'Bug Fixes' + labels: + - 'Bug Fixes' + - title: 'Infrastructure' + labels: + - 'Infrastructure' + - title: 'Documentation' + labels: + - 'Documentation' + - title: 'Maintenance' + labels: + - 'Maintenance' + - title: 'Refactoring' + labels: + - 'Refactoring' \ No newline at end of file diff --git a/.github/workflows/draft-release-notes-workflow.yml b/.github/workflows/draft-release-notes-workflow.yml new file mode 100644 index 0000000..6b3d89c --- /dev/null +++ b/.github/workflows/draft-release-notes-workflow.yml @@ -0,0 +1,20 @@ +name: Release Drafter + +on: + push: + branches: + - main + +jobs: + update_release_draft: + name: Update draft release notes + runs-on: ubuntu-latest + steps: + - name: Update draft release notes + uses: release-drafter/release-drafter@v5 + with: + config-name: draft-release-notes-config.yml + name: Version (set here) + tag: (None) + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 865bc1e..17f1aec 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ vendor/ /lib/logstash/outputs/elasticsearch/templates/ecs-v* *.iml /.idea/ +/dockerfiles/bin/ +/dockerfiles/.env \ No newline at end of file diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile new file mode 100644 index 0000000..4fda8bd --- /dev/null +++ b/dockerfiles/Dockerfile @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Build arguments: +# VERSION: Optional. Specify the label for image. Defaults to 7.13.2 + +ARG VERSION +ARG ARCH +FROM docker.elastic.co/logstash/logstash-oss:${VERSION}-${ARCH} +USER logstash +COPY --chown=logstash:logstash dockerfiles/bin/logstash-output-opensearch-*.gem /tmp/logstash-output-opensearch.gem +COPY --chown=logstash:logstash dockerfiles/logstash-opensearch-sample.conf /usr/share/logstash/config/ +RUN /usr/share/logstash/bin/logstash-plugin install /tmp/logstash-output-opensearch.gem diff --git a/dockerfiles/build.sh b/dockerfiles/build.sh new file mode 100755 index 0000000..e425d3b --- /dev/null +++ b/dockerfiles/build.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + +# This is intended to be run the plugin's root directory. `dockerfiles/build.sh` +# Ensure you have Docker installed locally and set the VERSION and BUILD_DATE environment variable. +set -e + +if [ -d dockerfiles/bin ]; then + rm -rf dockerfiles/bin +fi + +mkdir -p dockerfiles/bin + +echo 'Building plugin' +gem build logstash-output-opensearch.gemspec + +echo "Moving gem to bin directory" +mv logstash-output-opensearch*.gem dockerfiles/bin/ diff --git a/dockerfiles/docker-compose.yml b/dockerfiles/docker-compose.yml new file mode 100644 index 0000000..cc536a4 --- /dev/null +++ b/dockerfiles/docker-compose.yml @@ -0,0 +1,22 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + +version: '3' + +services: + + logstash: + container_name: logstash-oss-${VERSION}${SUFFIX} + build: + context: ../ + dockerfile: dockerfiles/Dockerfile + args: + - VERSION=${LOGSTASH_VERSION:-7.13.2} + - ARCH=${ARCH} + image: "logstash-oss-with-opensearch-output-plugin:${VERSION}${SUFFIX}" diff --git a/dockerfiles/logstash-opensearch-sample.conf b/dockerfiles/logstash-opensearch-sample.conf new file mode 100644 index 0000000..fefca66 --- /dev/null +++ b/dockerfiles/logstash-opensearch-sample.conf @@ -0,0 +1,17 @@ +# Sample Logstash configuration for creating a simple +# Beats -> Logstash -> OpenSearch pipeline. + +input { + beats { + port => 5044 + } +} + +output { + opensearch { + hosts => ["http://localhost:9200"] + index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}" + #user => "admin" + #password => "admin" + } +} diff --git a/dockerfiles/run.sh b/dockerfiles/run.sh new file mode 100755 index 0000000..2a06527 --- /dev/null +++ b/dockerfiles/run.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + +# This is intended to be run the plugin's root directory. `dockerfiles/run.sh` +# Ensure you have Docker and docker-compose installed locally +set -e + +# Building plugin +./dockerfiles/build.sh + +# Identify Logstash version +version=${LOGSTASH_VERSION} +if [[ -z "$version" ]]; then + version=7.13.2 +fi +echo "VERSION=$version" > dockerfiles/.env + +# Identify Architecture +arch=`uname -m` +suffix='' +if [ $arch == 'arm64' ]; then + suffix='-arm64' +elif [ $arch == 'x86_64' ]; then + arch='amd64' + suffix='-x64' +else + echo "Unknown Architecture. Only amd64 and arm64 is supported" + exit 1 +fi +echo "ARCH=$arch" >> dockerfiles/.env +echo "SUFFIX=$suffix" >> dockerfiles/.env + +echo 'shutdown existing docker cluster' +docker-compose --file dockerfiles/docker-compose.yml down + +echo 'remove previous image to avoid conflict' +docker image rmi -f "logstash-oss-with-opensearch-output-plugin:$version$suffix" + +echo 'build new docker image with latest changes' +docker-compose --file dockerfiles/docker-compose.yml build + +#echo 'start docker cluster' +#docker-compose --file dockerfiles/docker-compose.yml up + +# steps to publish image to registry +# docker push ${DOCKER_NAMESPACE}/"logstash-oss-with-opensearch-output-plugin:$version$suffix" diff --git a/lib/logstash/outputs/opensearch/http_client/pool.rb b/lib/logstash/outputs/opensearch/http_client/pool.rb index cb16a21..dc180b9 100644 --- a/lib/logstash/outputs/opensearch/http_client/pool.rb +++ b/lib/logstash/outputs/opensearch/http_client/pool.rb @@ -432,7 +432,7 @@ def set_last_version(version, url) major = major_version(version) if @maximum_seen_major_version.nil? - @logger.info("OpenSearch version determined (#{version})", version: major) + @logger.info("Cluster version determined (#{version})", version: major) set_maximum_seen_major_version(major) elsif major > @maximum_seen_major_version warn_on_higher_major_version(major, url) @@ -441,7 +441,6 @@ def set_last_version(version, url) end def set_maximum_seen_major_version(major) - @logger.warn("the `type` event field won't be used to determine the document _type") @maximum_seen_major_version = major end diff --git a/logstash-output-opensearch.gemspec b/logstash-output-opensearch.gemspec index deaf439..167a949 100644 --- a/logstash-output-opensearch.gemspec +++ b/logstash-output-opensearch.gemspec @@ -19,7 +19,11 @@ Gem::Specification.new do |s| s.test_files = s.files.grep(%r{^(test|spec|features)/}) # Special flag to let us know this is actually a logstash plugin - s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" } + s.metadata = { + "logstash_plugin" => "true", + "logstash_group" => "output", + "source_code_uri" => "https://github.com/opensearch-project/logstash-output-opensearch" + } s.add_runtime_dependency "manticore", '>= 0.5.4', '< 1.0.0' s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']