Backport to 1.x from main

.github/draft-release-notes-config.yml

@@ -0,0 +1,40 @@
+# The overall template of the release notes
+template: |
+  Compatible with OpenSearch (**set version here**).
+  $CHANGES
+
+# Setting the formatting and sorting for the release notes body
+name-template: Version (set version here)
+change-template: '* $TITLE (#$NUMBER)'
+sort-by: merged_at
+sort-direction: ascending
+replacers:
+  - search: '##'
+    replace: '###'
+
+# Organizing the tagged PRs into categories
+categories:
+  - title: 'Breaking Changes'
+    labels:
+      - 'Breaking Changes'
+  - title: 'Features'
+    labels:
+      - 'Features'
+  - title: 'Enhancements'
+    labels:
+      - 'Enhancements'
+  - title: 'Bug Fixes'
+    labels:
+      - 'Bug Fixes'
+  - title: 'Infrastructure'
+    labels:
+      - 'Infrastructure'
+  - title: 'Documentation'
+    labels:
+      - 'Documentation'
+  - title: 'Maintenance'
+    labels:
+      - 'Maintenance'
+  - title: 'Refactoring'
+    labels:
+      - 'Refactoring'

.github/workflows/draft-release-notes-workflow.yml

@@ -0,0 +1,20 @@
+name: Release Drafter
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  update_release_draft:
+    name: Update draft release notes
+    runs-on: ubuntu-latest
+    steps:
+      - name: Update draft release notes
+        uses: release-drafter/release-drafter@v5
+        with:
+          config-name: draft-release-notes-config.yml
+          name: Version (set here)
+          tag: (None)
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -15,3 +15,5 @@ vendor/
 /lib/logstash/outputs/elasticsearch/templates/ecs-v*
 *.iml
 /.idea/
+/dockerfiles/bin/
+/dockerfiles/.env

DEVELOPER_GUIDE.md

@@ -201,7 +201,7 @@ Authentication to a secure OpenSearch cluster is possible by using username/pass
 
 ### Authorization
 
-Authorization to a secure OpenSearch cluster requires read permission at [index level](https://docs-beta.opensearch.org/security-plugin/access-control/default-action-groups/#index-level).
+Authorization to a secure OpenSearch cluster requires read permission at [index level](https://opensearch.org/docs/security-plugin/access-control/default-action-groups/#index-level).
 
 ## Submitting Changes
 

README.md

@@ -15,7 +15,7 @@
 ## Project Resources
 
 * [Project Website](https://opensearch.org/)
-* [Documentation](https://docs-beta.opensearch.org/)
+* [Documentation](https://opensearch.org/)
 * Need help? Try [Forums](https://discuss.opendistrocommunity.dev/)
 * [Project Principles](https://opensearch.org/#principles)
 * [Contributing to OpenSearch](CONTRIBUTING.md)

dockerfiles/Dockerfile

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+
+# Build arguments:
+#   VERSION: Optional. Specify the label for image. Defaults to 7.13.2
+
+ARG VERSION
+ARG ARCH
+FROM docker.elastic.co/logstash/logstash-oss:${VERSION}-${ARCH}
+USER logstash
+COPY --chown=logstash:logstash dockerfiles/bin/logstash-output-opensearch-*.gem /tmp/logstash-output-opensearch.gem
+COPY --chown=logstash:logstash dockerfiles/logstash-opensearch-sample.conf /usr/share/logstash/config/
+RUN /usr/share/logstash/bin/logstash-plugin install /tmp/logstash-output-opensearch.gem

dockerfiles/build.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+# This is intended to be run the plugin's root directory. `dockerfiles/build.sh`
+# Ensure you have Docker installed locally and set the VERSION and BUILD_DATE environment variable.
+set -e
+
+if [ -d dockerfiles/bin ]; then
+  rm -rf dockerfiles/bin
+fi
+
+mkdir -p dockerfiles/bin
+
+echo 'Building plugin'
+gem build logstash-output-opensearch.gemspec
+
+echo "Moving gem to bin directory"
+mv logstash-output-opensearch*.gem dockerfiles/bin/

dockerfiles/docker-compose.yml

@@ -0,0 +1,22 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+version: '3'
+
+services:
+
+  logstash:
+    container_name: logstash-oss-${VERSION}${SUFFIX}
+    build:
+      context: ../
+      dockerfile: dockerfiles/Dockerfile
+      args:
+        - VERSION=${LOGSTASH_VERSION:-7.13.2}
+        - ARCH=${ARCH}
+    image: "logstash-oss-with-opensearch-output-plugin:${VERSION}${SUFFIX}"

dockerfiles/logstash-opensearch-sample.conf

@@ -0,0 +1,17 @@
+# Sample Logstash configuration for creating a simple
+# Beats -> Logstash -> OpenSearch pipeline.
+
+input {
+  beats {
+    port => 5044
+  }
+}
+
+output {
+  opensearch {
+    hosts => ["http://localhost:9200"]
+    index => "%{[@metadata][beat]}-%{[@metadata][version]}-%{+YYYY.MM.dd}"
+    #user => "admin"
+    #password => "admin"
+  }
+}

dockerfiles/run.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+# Modifications Copyright OpenSearch Contributors. See
+# GitHub history for details.
+
+# This is intended to be run the plugin's root directory. `dockerfiles/run.sh`
+# Ensure you have Docker and docker-compose installed locally
+set -e
+
+# Building plugin
+./dockerfiles/build.sh
+
+# Identify Logstash version
+version=${LOGSTASH_VERSION}
+if [[ -z "$version" ]]; then
+  version=7.13.2
+fi
+echo "VERSION=$version" > dockerfiles/.env
+
+# Identify Architecture
+arch=`uname -m`
+suffix=''
+if [ $arch == 'arm64' ]; then
+  suffix='-arm64'
+elif [ $arch == 'x86_64' ]; then
+  arch='amd64'
+  suffix='-x64'
+else
+  echo "Unknown Architecture. Only amd64 and arm64 is supported"
+  exit 1
+fi
+echo "ARCH=$arch" >> dockerfiles/.env
+echo "SUFFIX=$suffix" >> dockerfiles/.env
+
+echo 'shutdown existing docker cluster'
+docker-compose  --file dockerfiles/docker-compose.yml down
+
+echo 'remove previous image to avoid conflict'
+docker image rmi -f "logstash-oss-with-opensearch-output-plugin:$version$suffix"
+
+echo 'build new docker image with latest changes'
+docker-compose  --file dockerfiles/docker-compose.yml build
+
+#echo 'start docker cluster'
+#docker-compose --file dockerfiles/docker-compose.yml up
+
+# steps to publish image to registry
+# docker push ${DOCKER_NAMESPACE}/"logstash-oss-with-opensearch-output-plugin:$version$suffix"

lib/logstash/outputs/opensearch.rb

@@ -68,6 +68,7 @@
 class LogStash::Outputs::OpenSearch < LogStash::Outputs::Base
   declare_threadsafe!
 
+  require "logstash/outputs/opensearch/distribution_checker"
   require "logstash/outputs/opensearch/http_client"
   require "logstash/outputs/opensearch/http_client_builder"
   require "logstash/plugin_mixins/opensearch/api_configs"
@@ -217,7 +218,7 @@ def register
 
     @logger.info("New OpenSearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
 
-    @client = build_client
+    @client = build_client(DistributionChecker.new(@logger))
 
     @after_successful_connection_thread = after_successful_connection do
       begin

lib/logstash/outputs/opensearch/distribution_checker.rb

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+#  The OpenSearch Contributors require contributions made to
+#  this file be licensed under the Apache-2.0 license or a
+#  compatible open source license.
+#
+#  Modifications Copyright OpenSearch Contributors. See
+#  GitHub history for details.
+#
+module LogStash; module Outputs; class OpenSearch
+  class DistributionChecker
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    # Checks whether connecting cluster is one of supported distribution or not
+    # @param pool
+    # @param url [LogStash::Util::SafeURI] OpenSearch node URL
+    # @param major_version OpenSearch major version number
+    # @return [Boolean] true if supported
+    def is_supported?(pool, url, major_version)
+      case get_distribution(pool, url)
+      when 'opensearch'
+        return true
+      when 'oss'
+        if major_version == 7
+          return true
+        end
+      end
+      log_incompatible_version(url)
+      false
+    end
+
+    def get_distribution(pool, url)
+      pool.get_distribution(url)
+    end
+
+    def log_incompatible_version(url)
+      @logger.error("Could not connect to cluster: incompatible version", url: url.sanitized.to_s)
+    end
+  end
+end; end; end

lib/logstash/outputs/opensearch/http_client.rb

@@ -332,6 +332,7 @@ def build_pool(options)
       adapter = build_adapter(options)
 
       pool_options = {
+        :distribution_checker => options[:distribution_checker],
         :sniffing => sniffing,
         :sniffer_delay => options[:sniffer_delay],
         :sniffing_path => options[:sniffing_path],

lib/logstash/outputs/opensearch/http_client/pool.rb

@@ -8,6 +8,7 @@
 #  GitHub history for details.
 
 require "concurrent/atomic/atomic_reference"
+require "logstash/plugin_mixins/opensearch/noop_distribution_checker"
 
 module LogStash; module Outputs; class OpenSearch; class HttpClient;
   class Pool
@@ -40,6 +41,7 @@ def message
     end
 
     attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :healthcheck_path, :sniffing_path, :bulk_path
+    attr_reader :distribution_checker
 
     ROOT_URI_PATH = '/'.freeze
 
@@ -78,6 +80,7 @@ def initialize(logger, adapter, initial_urls=[], options={})
       @stopping = false
 
       @last_version = Concurrent::AtomicReference.new
+      @distribution_checker = options[:distribution_checker] || LogStash::PluginMixins::OpenSearch::NoopDistributionChecker::INSTANCE
     end
 
     def start
@@ -236,7 +239,8 @@ def healthcheck!
           @state_mutex.synchronize do
             meta[:version] = version
             set_last_version(version, url)
-            meta[:state] = :alive
+            alive = @distribution_checker.is_supported?(self, url, @maximum_seen_major_version)
+            meta[:state] = alive ? :alive : :dead
           end
         rescue HostUnreachableError, BadResponseCodeError => e
           logger.warn("Attempted to resurrect connection to dead OpenSearch instance, but got an error", url: url.sanitized.to_s, exception: e.class, message: e.message)
@@ -411,9 +415,18 @@ def return_connection(url)
       end
     end
 
-    def get_version(url)
+    def get_version_map(url)
       request = perform_request_to_url(url, :get, ROOT_URI_PATH)
-      LogStash::Json.load(request.body)["version"]["number"] # e.g. "7.10.0"
+      LogStash::Json.load(request.body)['version']
+    end
+
+    def get_version(url)
+      get_version_map(url)['number'] # e.g. "7.10.0"
+    end
+
+    def get_distribution(url)
+      version_map = get_version_map(url)
+      version_map.has_key?('distribution') ? version_map['distribution'] : version_map['build_flavor'] # e.g. "opensearch or oss"
     end
 
     def last_version
@@ -432,7 +445,7 @@ def set_last_version(version, url)
 
       major = major_version(version)
       if @maximum_seen_major_version.nil?
-        @logger.info("OpenSearch version determined (#{version})", version: major)
+        @logger.info("Cluster version determined (#{version})", version: major)
         set_maximum_seen_major_version(major)
       elsif major > @maximum_seen_major_version
         warn_on_higher_major_version(major, url)
@@ -441,7 +454,6 @@ def set_last_version(version, url)
     end
 
     def set_maximum_seen_major_version(major)
-      @logger.warn("the `type` event field won't be used to determine the document _type")
       @maximum_seen_major_version = major
     end