diff --git a/.ci/Dockerfile.elasticsearch b/.ci/Dockerfile.elasticsearch deleted file mode 100644 index 95cbc47..0000000 --- a/.ci/Dockerfile.elasticsearch +++ /dev/null @@ -1,21 +0,0 @@ -ARG ELASTIC_STACK_VERSION -ARG DISTRIBUTION_SUFFIX -FROM docker.elastic.co/elasticsearch/elasticsearch$DISTRIBUTION_SUFFIX:$ELASTIC_STACK_VERSION - -ARG plugin_path=/usr/share/plugins/plugin -ARG es_path=/usr/share/elasticsearch -ARG es_yml=$es_path/config/elasticsearch.yml -ARG SECURE_INTEGRATION - -RUN rm -f $es_path/config/scripts - -COPY --chown=elasticsearch:elasticsearch spec/fixtures/scripts/groovy/* $es_path/config/scripts/ -COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/config/test_certs/ -COPY --chown=elasticsearch:elasticsearch .ci/elasticsearch-run.sh $es_path/ - -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.enabled: $SECURE_INTEGRATION" >> $es_yml; fi -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/test.key" >> $es_yml; fi -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/test.crt" >> $es_yml; fi -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd simpleuser -p abc123 -r superuser; fi -RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then $es_path/bin/elasticsearch-users useradd 'f@ncyuser' -p 'ab%12#' -r superuser; fi diff --git a/.ci/docker-compose.override.yml b/.ci/docker-compose.override.yml deleted file mode 100644 index 4283210..0000000 --- a/.ci/docker-compose.override.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: '3' - -services: - - logstash: - command: /usr/share/plugins/plugin/.ci/logstash-run.sh - build: - args: - - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION - - DISTRIBUTION=${DISTRIBUTION:-default} - - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX} - environment: - - DISTRIBUTION=${DISTRIBUTION:-default} - - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX} - - INTEGRATION=${INTEGRATION:-false} - - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - - elasticsearch: - build: - context: ../ - dockerfile: .ci/Dockerfile.elasticsearch - args: - - ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION - - INTEGRATION=${INTEGRATION:-false} - - SECURE_INTEGRATION=${SECURE_INTEGRATION:-false} - - DISTRIBUTION_SUFFIX=${DISTRIBUTION_SUFFIX} - command: /usr/share/elasticsearch/elasticsearch-run.sh - tty: true - ports: - - "9200:9200" - user: elasticsearch - diff --git a/.ci/docker-setup.sh b/.ci/docker-setup.sh deleted file mode 100755 index 7dd3361..0000000 --- a/.ci/docker-setup.sh +++ /dev/null @@ -1,67 +0,0 @@ -#!/bin/bash - -# This is intended to be run the plugin's root directory. `.ci/docker-setup.sh` -# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. -set -e - -VERSION_URL="https://raw.githubusercontent.com/elastic/logstash/master/ci/logstash_releases.json" - - -pull_docker_snapshot() { - project="${1?project name required}" - local docker_image="docker.elastic.co/${project}/${project}${DISTRIBUTION_SUFFIX}:${ELASTIC_STACK_VERSION}" - echo "Pulling $docker_image" - docker pull "$docker_image" -} - -if [ "$ELASTIC_STACK_VERSION" ]; then - echo "Fetching versions from $VERSION_URL" - VERSIONS=$(curl --silent $VERSION_URL) - if [[ "$SNAPSHOT" = "true" ]]; then - ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.snapshots."'"$ELASTIC_STACK_VERSION"'"') - echo $ELASTIC_STACK_RETRIEVED_VERSION - else - ELASTIC_STACK_RETRIEVED_VERSION=$(echo $VERSIONS | jq '.releases."'"$ELASTIC_STACK_VERSION"'"') - fi - if [[ "$ELASTIC_STACK_RETRIEVED_VERSION" != "null" ]]; then - # remove starting and trailing double quotes - ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION%\"}" - ELASTIC_STACK_RETRIEVED_VERSION="${ELASTIC_STACK_RETRIEVED_VERSION#\"}" - echo "Translated $ELASTIC_STACK_VERSION to ${ELASTIC_STACK_RETRIEVED_VERSION}" - export ELASTIC_STACK_VERSION=$ELASTIC_STACK_RETRIEVED_VERSION - fi - - case "${DISTRIBUTION}" in - default) DISTRIBUTION_SUFFIX="" ;; # empty string when explicit "default" is given - *) DISTRIBUTION_SUFFIX="${DISTRIBUTION/*/-}${DISTRIBUTION}" ;; - esac - export DISTRIBUTION_SUFFIX - - echo "Testing against version: $ELASTIC_STACK_VERSION (distribution: ${DISTRIBUTION:-'default'})" - - if [[ "$ELASTIC_STACK_VERSION" = *"-SNAPSHOT" ]]; then - pull_docker_snapshot "logstash" - if [ "$INTEGRATION" == "true" ]; then - pull_docker_snapshot "elasticsearch" - fi - fi - - if [ -f Gemfile.lock ]; then - rm Gemfile.lock - fi - - cd .ci - - if [ "$INTEGRATION" == "true" ]; then - docker-compose down - docker-compose build - else - docker-compose down - docker-compose build logstash - fi -else - echo "Please set the ELASTIC_STACK_VERSION environment variable" - echo "For example: export ELASTIC_STACK_VERSION=6.2.4" - exit 1 -fi - diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md deleted file mode 100644 index d325328..0000000 --- a/.github/CONTRIBUTING.md +++ /dev/null @@ -1,65 +0,0 @@ -# Contributing to Logstash - -All contributions are welcome: ideas, patches, documentation, bug reports, -complaints, etc! - -Programming is not a required skill, and there are many ways to help out! -It is more important to us that you are able to contribute. - -That said, some basic guidelines, which you are free to ignore :) - -## Want to learn? - -Want to lurk about and see what others are doing with Logstash? - -* The irc channel (#logstash on irc.freenode.org) is a good place for this -* The [forum](https://discuss.elastic.co/c/logstash) is also - great for learning from others. - -## Got Questions? - -Have a problem you want Logstash to solve for you? - -* You can ask a question in the [forum](https://discuss.elastic.co/c/logstash) -* Alternately, you are welcome to join the IRC channel #logstash on -irc.freenode.org and ask for help there! - -## Have an Idea or Feature Request? - -* File a ticket on [GitHub](https://github.com/elastic/logstash/issues). Please remember that GitHub is used only for issues and feature requests. If you have a general question, the [forum](https://discuss.elastic.co/c/logstash) or IRC would be the best place to ask. - -## Something Not Working? Found a Bug? - -If you think you found a bug, it probably is a bug. - -* If it is a general Logstash or a pipeline issue, file it in [Logstash GitHub](https://github.com/elasticsearch/logstash/issues) -* If it is specific to a plugin, please file it in the respective repository under [logstash-plugins](https://github.com/logstash-plugins) -* or ask the [forum](https://discuss.elastic.co/c/logstash). - -# Contributing Documentation and Code Changes - -If you have a bugfix or new feature that you would like to contribute to -logstash, and you think it will take more than a few minutes to produce the fix -(ie; write code), it is worth discussing the change with the Logstash users and developers first! You can reach us via [GitHub](https://github.com/elastic/logstash/issues), the [forum](https://discuss.elastic.co/c/logstash), or via IRC (#logstash on freenode irc) -Please note that Pull Requests without tests will not be merged. If you would like to contribute but do not have experience with writing tests, please ping us on IRC/forum or create a PR and ask our help. - -## Contributing to plugins - -Check our [documentation](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) on how to contribute to plugins or write your own! It is super easy! - -## Contribution Steps - -1. Test your changes! [Run](https://github.com/elastic/logstash#testing) the test suite -2. Please make sure you have signed our [Contributor License - Agreement](https://www.elastic.co/contributor-agreement/). We are not - asking you to assign copyright to us, but to give us the right to distribute - your code without restriction. We ask this of all contributors in order to - assure our users of the origin and continuing existence of the code. You - only need to sign the CLA once. -3. Send a pull request! Push your changes to your fork of the repository and - [submit a pull - request](https://help.github.com/articles/using-pull-requests). In the pull - request, describe what your changes do and mention any bugs/issues related - to the pull request. - - diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index c3cc91d..0000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,9 +0,0 @@ -Please post all product and debugging questions on our [forum](https://discuss.elastic.co/c/logstash). Your questions will reach our wider community members there, and if we confirm that there is a bug, then we can open a new issue here. - -For all general issues, please provide the following details for fast resolution: - -- Version: -- Operating System: -- Config File (if you have sensitive info, please remove it): -- Sample Data: -- Steps to Reproduce: diff --git a/.github/ISSUE_TEMPLATE/bug_template.md b/.github/ISSUE_TEMPLATE/bug_template.md new file mode 100644 index 0000000..c49a9d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_template.md @@ -0,0 +1,33 @@ +--- +name: ๐Ÿ› Bug report +about: Create a report to help us improve +title: "[BUG]" +labels: 'bug, untriaged, beta' +assignees: '' +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Plugins** +Please list all plugins currently enabled. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Host/Environment (please complete the following information):** + - OS: [e.g. iOS] + - Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..a8199a1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,7 @@ +contact_links: + - name: OpenSearch Community Support + url: https://discuss.opendistrocommunity.dev/ + about: Please ask and answer questions here. + - name: AWS/Amazon Security + url: https://aws.amazon.com/security/vulnerability-reporting/ + about: Please report security vulnerabilities here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/feature_request_template.md b/.github/ISSUE_TEMPLATE/feature_request_template.md new file mode 100644 index 0000000..2791b80 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request_template.md @@ -0,0 +1,19 @@ +--- +name: ๐ŸŽ† Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/proposal.md b/.github/ISSUE_TEMPLATE/proposal.md new file mode 100644 index 0000000..7d571f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/proposal.md @@ -0,0 +1,31 @@ + + +## Requirement - what kind of business use case are you trying to solve? + + + +## Problem - what blocks you from solving the requirement? + + + + +## Proposal - what do you suggest to solve the problem or improve the existing situation? + + + +## Assumptions - are there certain things you assume the user is doing before this? + + + +## Any open questions to address + + diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index a153827..0000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1 +0,0 @@ -Thanks for contributing to Logstash! If you haven't already signed our CLA, here's a handy link: https://www.elastic.co/contributor-agreement/ diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..22f2d7e --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,15 @@ +### Description +[Describe what this change achieves] + +### Issues Resolved +[List any issues this PR will resolve] + +### Check List +- [ ] New functionality includes testing. + - [ ] All tests pass +- [ ] New functionality has been documented. + - [ ] New functionality has documentation added +- [ ] Commits are signed as per the DCO using --signoff + +By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. +For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin). \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6b537d3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: Build and Test logstash-output-opensearch plugin +on: + push: + branches: + - "*" + pull_request: + branches: + - "*" + +jobs: + Build: + name: Build and Test logstash-output-opensearch + runs-on: ubuntu-latest + + steps: + - name: Checkout plugin + uses: actions/checkout@v1 + + - name: Build and pull docker files + env: + OPENDISTRO_VERSION: "1.13.2" + LOGSTASH_VERSION: "7.13.2" + run: | + ./scripts/docker-setup.sh + + - name: Run unit tests + run: | + ./scripts/docker-run.sh + + - name: Run integration tests + env: + INTEGRATION: true + run: | + ./scripts/docker-run.sh diff --git a/.gitignore b/.gitignore index bb81a88..865bc1e 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,5 @@ vendor/ /spec/fixtures/server.key /spec/fixtures/server.crt /lib/logstash/outputs/elasticsearch/templates/ecs-v* +*.iml +/.idea/ diff --git a/ADMINS.md b/ADMINS.md new file mode 100644 index 0000000..28ca2fb --- /dev/null +++ b/ADMINS.md @@ -0,0 +1,30 @@ +## Overview + +This document explains who the admins are (see below), what they do in this repo, and how they should be doing it. If you're interested in becoming a maintainer, see [MAINTAINERS](MAINTAINERS.md). If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md). + +## Current Admins + +| Admin | GitHub ID | Affiliation | +| -------------------------| --------------------------------------- | ----------- | +| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon | +| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | +| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | + + +## Admin Responsibilities + +As an admin you own stewartship of the repository and its settings. Admins have [admin-level permissions on a repository](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization). Use those privileges to serve the community and protect the repository as follows. + +### Prioritize Security + +Security is your number one priority. Manage security keys and safeguard access to the repository. + +Note that this repository is monitored and supported 24/7 by Amazon Security, see [Reporting a Vulnerability](SECURITY.md) for details. + +### Enforce Code of Conduct + +Act on [CODE_OF_CONDUCT](CODE_OF_CONDUCT.md) violations by revoking access, and blocking malicious actors. + +### Adopt Organizational Best Practices + +Adopt organizational best practices, work in the open, and collaborate with other admins by opening issues before making process changes. Prefer consistency, and avoid diverging from practices in the opensearch-project organization. \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index c50682d..0000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,483 +0,0 @@ -## 11.0.2 - - Validate that required functionality in Elasticsearch is available upon initial connection [#1015](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1015) - -## 11.0.1 - - Fix: DLQ regression shipped in 11.0.0 [#1012](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1012) - - [DOC] Fixed broken link in list item [#1011](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1011) - -## 11.0.0 - - Feat: Data stream support [#988](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/988) - - Refactor: reviewed logging format + restored ES (initial) setup error logging - - Feat: always check ES license [#1005](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1005) - - Since Elasticsearch no longer provides an OSS artifact the plugin will no longer skip the license check on OSS Logstash. - -## 10.8.6 - - Fixed an issue where a single over-size event being rejected by Elasticsearch would cause the entire entire batch to be retried indefinitely. The oversize event will still be retried on its own and logging has been improved to include payload sizes in this situation [#972](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/972) - - Fixed an issue with `http_compression => true` where a well-compressed payload could fit under our outbound 20MB limit but expand beyond Elasticsearch's 100MB limit, causing bulk failures. Bulk grouping is now determined entirely by the decompressed payload size [#823](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/823) - - Improved debug-level logging about bulk requests. - -## 10.8.5 - - Feat: assert returned item count from _bulk [#997](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/997) - -## 10.8.4 - - Fixed an issue where a retried request would drop "update" parameters [#800](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/800) - -## 10.8.3 - - Avoid to implicitly set deprecated type to `_doc` when connects to Elasticsearch version 7.x [#994](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/994) - -## 10.8.2 - - [DOC] Update links to use shared attributes [#985](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/985) - -## 10.8.1 - - Fixed an issue when assigning the no-op license checker [#984](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/984) - -## 10.8.0 - - Refactored configuration options into specific and shared in PluginMixins namespace [#973](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/973) - - Refactored common methods into specific and shared in PluginMixins namespace [#976](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/976) - -## 10.7.3 - - Added composable index template support for elasticsearch version 8 [#980](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/980) - -## 10.7.2 - - [DOC] Fixed links to restructured Logstash-to-cloud docs [#975](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/975) - -## 10.7.1 - - [DOC] Document the permissions required in secured clusters [#969](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/969) - -## 10.7.0 - - Changed: don't set the pipeline parameter if the value resolves to an empty string [#962](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/962) - -## 10.6.2 - - [DOC] Added clarifying info on http compression settings and behaviors [#943](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/943) - - [DOC] Fixed entry for ilm_policy default value[#956](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/956) - -## 10.6.1 - - Fixed an issue introduced in 10.6.0 that broke Logstash Core's monitoring feature when this plugin is run in Logstash 7.7-7.8. [#953](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/953) - -## 10.6.0 - - Added `ecs_compatiblity` mode, for managing ECS-compatable templates [#952](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/952) - -## 10.5.1 - - [DOC] Removed outdated compatibility notices, reworked cloud notice, and fixed formatting for `hosts` examples [#938](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/938) - -## 10.5.0 - - Added api_key support [#934](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/934) - -## 10.4.1 - - [DOC] Added note about `_type` setting change from `doc` to `_doc` [#884](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/884) - -## 10.4.0 - - Fixed default index value [#927](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/927) - -## 10.3.3 - - [DOC] Replaced link to Elastic Cloud trial with attribute, and fixed a comma splice [#926](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/926) - -## 10.3.2 - - [DOC] Replaced setting name with correct value [#919](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/919) - - Fixed integration tests for Elasticsearch 7.6+ [#922](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/922) - - Fixed integration tests for Elasticsearch API `7.5.0` [#923](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/923) - -## 10.3.1 - - Fix: handle proxy => '' as if none was set [#912](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/912) - -## 10.3.0 - - Feat: Added support for cloud_id and cloud_auth [#906](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/906) - -## 10.2.3 - - Opened type removal logic for extension. This allows X-Pack Elasticsearch output to continue using types for special case `/_monitoring` bulk endpoint, enabling a fix for LogStash #11312. [#900](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/900) - -## 10.2.2 - - Fixed 8.x type removal compatibility issue [#892](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/892) - -## 10.2.1 - - Fixed wording and corrected option in documentation [#881](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/881) [#883](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/883) - -## 10.2.0 - - Deprecation: Added warning about connecting a default Distribution of Logstash with an OSS version of ES [#875](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/875) - - Added template for connecting to ES 8.x [#871](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/871) - - Added sniffing support for ES 8.x [#878](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/878) - -## 10.1.0 - - Added cluster id tracking through the plugin metadata registry [#857](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/857) - -## 10.0.2 - - Fixed bug where index patterns in custom templates could be erroneously overwritten [#861](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/861) - -## 10.0.1 - - Reverted `document_type` obsoletion [#844](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/844) - -## 10.0.0 (yanked due to issues with document_type obsoletion) - - Changed deprecated `document_type` option to obsolete [#824](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/824) - - Remove support for parent child (still support join data type) since we don't support multiple document types any more - - Removed obsolete `flush_size` and `idle_flush_time` - - Switched default setting for ilm_enabled to 'auto' [#838](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/838) - -## 9.4.0 - - Added 'auto' setting for ilm_enabled with default of 'false' - -## 9.3.2 - - Fixed sniffing support for 7.x [#827](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/827) - -## 9.3.1 - - Fixed issue with escaping index names which was causing writing aliases for ILM to fail [#831](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/831) - -## 9.3.0 - - Adds support for Index Lifecycle Management for Elasticsearch 6.6.0 and above, running with at least a Basic License(Beta) [#805](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/805) - -## 9.2.4 - - Fixed support for Elasticsearch 7.x [#812](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/812) - -## 9.2.3 - - Tweaked logging statements to reduce verbosity - -## 9.2.2 - - Fixed numerous issues relating to builds on Travis [#799](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/799) - -## 9.2.1 - - Added text offering hosted Elasticsearch service - -## 9.2.0 - - Added support for customizing HTTP headers [#782](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/782) - -## 9.1.4 - - Log an error -- not a warning -- when ES raises an invalid\_index\_name\_exception. - -## 9.1.3 - - Improve plugin behavior when Elasticsearch is down on startup #758 - -## 9.1.2 - - No user facing changes, removed unnecessary test dep. - -## 9.1.1 - - Docs: Set the default_codec doc attribute. - -## 9.1.0 - - Set number_of_shards to 1 and document_type to '_doc' for es 7.x clusters #741 #747 - - Fix usage of upsert and script when update action is interpolated #239 - - Add metrics to track bulk level and document level responses #585 - -## 9.0.3 - - Ignore master-only nodes when using sniffing - -## 9.0.2 - - Ignore event's type field for the purpose of setting document `_type` if cluster is es 6.x or above - -## 9.0.1 - - Update gemspec summary - -### 9.0.0 - - Change default document type to 'doc' from 'logs' to align with beats and reflect the generic nature of logstash. - - Deprecate 'document_type' option - -### 8.2.2 - - Use `#response_body` instead of `#body` when debugging response from the server #679 - -## 8.2.1 - - Docs: Add DLQ policy section - -## 8.2.0 - - Improved Elasticsearch version handling - - Improved event error logging when DLQ is disabled in Logstash - -## 8.1.1 - - Retry all non-200 responses of the bulk API indefinitely - - Improve documentation on retry codes - -## 8.1.0 - - Support Elasticsearch 6.x join field type -## 8.0.2 - - Fix bug where logging errors for bad response codes would raise an unhandled exception - -## 8.0.1 - - Fix some documentation issues - -## 8.0.0 - - Breaking: make deprecated options :flush_size and :idle_flush_time obsolete - - Remove obsolete options :max_retries and :retry_max_items - - Fix: handling of initial single big event - - Fix: typo was enabling http compression by default this returns it back to false - -## 7.3.7 - - Properly support characters needing escaping in users / passwords across multiple SafeURI implementions (pre/post LS 5.5.1) - - Logstash 5.5.0 does NOT work with this release as it has a broken SafeURI implementation - -## 7.3.6 - - Bump for doc gen - -## 7.3.5 - - Fix incorrect variable reference when DLQing events - -## 7.3.4 - - Fix incorrect handling of bulk_path containing ?s - -## 7.3.3 - - Fix JRuby 9k incompatibilities and use new URI class that is JRuby 9k compatible - -## 7.3.2 - - Fix error where a 429 would cause this output to crash - - Wait for all inflight requests to complete before stopping - -## 7.3.1 - - Fix the backwards compatibility layer used for detecting DLQ capabilities in logstash core - -## 7.3.0 - - Log 429 errors as debug instead of error. These aren't actual errors and cause users undue concern. - This status code is triggered when ES wants LS to backoff, which it does correctly (exponentially) - -## 7.2.2 - - Docs: Add requirement to use version 6.2.5 or higher to support sending Content-Type headers. - -## 7.2.1 - - Expose a `#post` method in the http client class to be use by other modules - -## 7.2.0 - - Support 6.0.0-alpha1 version of Elasticsearch by adding a separate 6x template - - Note: This version is backwards compatible w.r.t. config, but for ES 6.0.0, `_all` has been - removed. This BWC issue only affects ES version 6.x; older versions - can be used with this plugin as is. - -## 7.1.0 - - Add support to compress requests using the new `http_compression` option. - -## 7.0.0 -- introduce customization of bulk, healthcheck and sniffing paths with the behaviour: - - if not set: the default value will be used - - if not set and path is also set: the default is appended to path - - if set: the set value will be used, ignoring the default and path setting -- removes absolute_healthcheck_path and query_parameters - -## 6.2.6 -- Fixed: Change how the healthcheck_path is treated: either append it to any existing path (default) or replace any existing path - Also ensures that the healthcheck url contains no query parameters regarless of hosts urls contains them or query_params being set. #554 - -## 6.2.5 -- Send the Content-Type: application/json header that proper ES clients should send - -## 6.2.4 -- Fix bug where using escaped characters in the password field would attempt to show a warning but instead crash. - The warning was also not necessary since escaped characters never worked there before. - -## 6.2.3 -- Fixed a bug introduced in 6.2.2 where passwords needing escapes were not actually sent to ES properly - encoded. - -## 6.2.2 -- Fixed a bug that forced users to URL encode the `password` option. - If you are currently manually escaping your passwords upgrading to this version - will break authentication. You should unescape your password if you have implemented - this workaround as it will otherwise be doubly encoded. - URL escaping is STILL required for passwords inline with URLs in the `hosts` option. - -## 6.2.1 -- When an HTTP error is encountered, log the response body instead of the request. - The request body will still be logged at debug level. - -## 6.2.0 -- Add version number / version conflict support - -## 6.1.0 -- Add option to use an absolute healthcheck path - -## 6.0.0 -- Proxies requiring auth now always work when a URL is specified -- It is no longer possible to specify a proxy as a hash due to security reasons -- Fix URL normalization logic to correctly apply all settings to sniffed hosts -- Proxies requiring auth now always work when a URL is specified -- Switch internals to new LogStash::Util::SafeURI type for more defensive approach to logging credentials - -## 5.4.1 -- Correctly sniff against ES 5.x clusters - -## 5.4.0 -- Perform healthcheck against hosts right after startup / sniffing -- Add support for custom query parameters - -## 5.3.5 -- Docs: Remove mention of using the elasticsearch_java output plugin because it is no longer supported - -## 5.3.4 -- Add `sprintf` or event dependent configuration when specifying ingest pipeline - -## 5.3.3 -- Hide user/password in connection pool - -## 5.3.2 -- Use byte size, not char count for bulk operation size checks - -## 5.3.1 -- depends on Adressable ~> 2.3.0 to satisfy development dependency of the core ([logstash/#6204](https://github.com/elastic/logstash/issues/6204)) - -## 5.3.0 -- Bulk operations will now target 20MB chunks at a time to reduce heap usage - -## 5.2.0 -- Change default lang for scripts to be painless, inline with ES 5.0. Earlier there was no default. - -## 5.1.2 -- Hide credentials in exceptions and log messages ([#482](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/482)) -- [internal] Remove dependency on longshoreman project - -## 5.1.1 -- Hide user and password from the URL logged during sniffing process. - -## 5.1.0 -- Add check_connection_timeout parameter (default 10m) -- Set default timeout to 60s - -## 5.0.0 -- Breaking Change: Index template for 5.0 has been changed to reflect Elasticsearch's mapping changes. Most importantly, -the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default behavior. ([#386](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/386)) - -**Users installing ES 5.x and LS 5.x** -This change will not affect you and you will continue to use the ES defaults. - -**Users upgrading from LS 2.x to LS 5.x with ES 5.x** -LS will not force upgrade the template, if `logstash` template already exists. This means you will still use -`.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after -the new template is installed. - -## 4.1.3 -- Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99 - -## 4.1.2 - -- Added a configuration called failure_type_logging_whitelist which takes a list of strings, that are error types from elasticsearch, so we prevent logging WARN if elasticsearch fails with that action. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/423 - -## 4.1.1 -- Fix bug where setting credentials would cause fatal errors. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/441 - -## 4.1.0 -- breaking,config: Removed obsolete config `host` and `port`. Please use the `hosts` config with the `[host:port]` syntax. -- breaking,config: Removed obsolete config `index_type`. Please use `document_type` instead. -- breaking,config: Set config `max_retries` and `retry_max_items` as obsolete - -## 4.0.0 - - Make this plugin threadsafe. Workers no longer needed or supported - - Add pool_max and pool_max_per_route options - -## 3.0.2 - - Fix issues where URI based paths in 'hosts' would not function correctly - -## 3.0.1 - - Republish all the gems under jruby. - -## 3.0.0 - - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141 - -## 2.7.0 - - Add `pipeline` configuration option for setting an ingest pipeline to run upon indexing - - -## 2.6.2 - - Fix bug where update index actions would not work with events with 'data' field - -## 2.6.1 - - Add 'retry_on_conflict' configuration option which should have been here from the beginning - -## 2.5.2 - - Fix bug with update document with doc_as_upsert and scripting (#364, #359) - - Make error messages more verbose and easier to parse by humans - - Retryable failures are now logged at the info level instead of warning. (issue #372) - -## 2.5.1 - - Fix bug where SSL would sometimes not be enabled - -## 2.5.0 - - Host settings now are more robust to bad input - - Host settings can now take full URLs - -## 2.4.2 - - Make flush_size actually cap the batch size in LS 2.2+ - -## 2.4.1 - - Used debug level instead of info when emitting flush log message - - Updated docs about template - -## 2.4.0 - - Scripted update support courtesy of @Da-Wei - -## 2.3.2 - - Fix bug where max_retry_interval was not respected for HTTP error codes - -## 2.3.1 - - Bump manticore dependenvy to 0.5.2 - -## 2.3.0 - - Now retry too busy and service unavailable errors infinitely. - - Never retry conflict errors - - Fix broken delete verb that would fail due to sending body with verb - -## 2.2.0 - - Serialize access to the connection pool in es-ruby client - - Add support for parent relationship - -## 2.1.5 - - Sprintf style 'action' parameters no longer raise a LogStash::ConfigurationError - -## 2.1.4 - - Improved the default template to disable fielddata on analyzed string fields. #309 - - Dependend on logstash-core 2.0.0 released version, rather than RC1 - -## 2.1.3 - - Improved the default template to use doc_values wherever possible. - - Template contains example mappings for every numeric type. You must map your - own fields to make use of anything other than long and double. - -## 2.1.2 - - Fixed dependencies (#280) - - Fixed an RSpec test (#281) - -## 2.1.1 - - Made host config obsolete. - -## 2.1.0 - - New setting: timeout. This lets you control the behavior of a slow/stuck - request to Elasticsearch that could be, for example, caused by network, - firewall, or load balancer issues. - -## 2.0.0 - - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, - instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895 - - Dependency on logstash-core update to 2.0 - -## 2.0.0-beta2 - - Massive internal refactor of client handling - - Background HTTP sniffing support - - Reduced bulk request size to 500 from 5000 (better memory utilization) - - Removed 'host' config option. Now use 'hosts' - -## 2.0.0-beta - - Only support HTTP Protocol - - Removed support for node and transport protocols (now in logstash-output-elasticsearch_java) - -## 1.0.7 - - Add update API support - -## 1.0.6 - - Fix warning about Concurrent lib deprecation - -## 1.0.4 - - Update to Elasticsearch 1.7 - -## 1.0.3 - - Add HTTP proxy support - -## 1.0.2 - - Upgrade Manticore HTTP Client - -## 1.0.1 - - Allow client certificates - -## 0.2.9 - - Add 'path' parameter for ES HTTP hosts behind a proxy on a subpath - -## 0.2.8 (June 12, 2015) - - Add option to enable and disable SSL certificate verification during handshake (#160) - - Doc improvements for clarifying round robin behavior using hosts config - -## 0.2.7 (May 28, 2015) - - Bump es-ruby version to 1.0.10 - -## 0.2.6 (May 28, 2015) - - Disable timeouts when using http protocol which would cause bulk requests to fail (#103) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..997bae6 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,25 @@ + +This code of conduct applies to all spaces provided by the OpenSource project including in code, documentation, issue trackers, mailing lists, chat channels, wikis, blogs, social media and any other communication channels used by the project. + + +**Our open source communities endeavor to:** + +* Be Inclusive: We are committed to being a community where everyone can join and contribute. This means using inclusive and welcoming language. +* Be Welcoming: We are committed to maintaining a safe space for everyone to be able to contribute. +* Be Respectful: We are committed to encouraging differing viewpoints, accepting constructive criticism and work collaboratively towards decisions that help the project grow. Disrespectful and unacceptable behavior will not be tolerated. +* Be Collaborative: We are committed to supporting what is best for our community and users. When we build anything for the benefit of the project, we should document the work we do and communicate to others on how this affects their work. + + +**Our Responsibility. As contributors, members, or bystanders we each individually have the responsibility to behave professionally and respectfully at all times. Disrespectful and unacceptable behaviors include, but are not limited to:** + +* The use of violent threats, abusive, discriminatory, or derogatory language; +* Offensive comments related to gender, gender identity and expression, sexual orientation, disability, mental illness, race, political or religious affiliation; +* Posting of sexually explicit or violent content; +* The use of sexualized language and unwelcome sexual attention or advances; +* Public or private harassment of any kind; +* Publishing private information, such as physical or electronic address, without permission; +* Other conduct which could reasonably be considered inappropriate in a professional setting; +* Advocating for or encouraging any of the above behaviors. +* Enforcement and Reporting Code of Conduct Issues: + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported. [Contact us](mailto:opensource-codeofconduct@amazon.com). All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1510fe2 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,99 @@ +- [Contributing to OpenSearch](#contributing-to-logstash-output-opensearch) +- [First Things First](#first-things-first) +- [Ways to Contribute](#ways-to-contribute) + - [Bug Reports](#bug-reports) + - [Feature Requests](#feature-requests) + - [Documentation Changes](#documentation-changes) + - [Contributing Code](#contributing-code) +- [Developer Certificate of Origin](#developer-certificate-of-origin) +- [Review Process](#review-process) + +## Contributing to logstash-output-opensearch + +logstash-output-opensearch is a community project that is built and maintained by people just like **you**. We're glad you're interested in helping out. There are several different ways you can do it, but before we talk about that, let's talk about how to get started. + +## First Things First + +1. **When in doubt, open an issue** - For almost any type of contribution, the first step is opening an issue. Even if you think you already know what the solution is, writing down a description of the problem you're trying to solve will help everyone get context when they review your pull request. If it's truly a trivial change (e.g. spelling error), you can skip this step -- but as the subject says, when it doubt, [open an issue]1. **When in doubt, open an issue** - For almost any type of contribution, the first step is opening an issue. Even if you think you already know what the solution is, writing down a description of the problem you're trying to solve will help everyone get context when they review your pull request. If it's truly a trivial change (e.g. spelling error), you can skip this step -- but as the subject says, when it doubt, [open an issue](https://github.com/opensearch-project/logstash-output-opensearch/issues). + +2. **Only submit your own work** (or work you have sufficient rights to submit) - Please make sure that any code or documentation you submit is your work or you have the rights to submit. We respect the intellectual property rights of others, and as part of contributing, we'll ask you to sign your contribution with a "Developer Certificate of Origin" (DCO) that states you have the rights to submit this work and you understand we'll + use your contribution. There's more information about this topic in the [DCO section](#developer-certificate-of-origin). +## Ways to Contribute + +**Please note:** logstash-output-opensearch is a fork of [logstash-output-elasticsearch 11.0.2](https://github.com/logstash-plugins/logstash-output-elasticsearch), and is currently in a pre-alpha state, so it's still very much a work in progress. If you do find references to Elasticsearch (outside of attributions and copyrights!) please [open an issue](https://github.com/opensearch-project/OpenSearch/issues) + +### Bug Reports + +Ugh! Bugs! + +A bug is when software behaves in a way that you didn't expect and the developer didn't intend. To help us understand what's going on, we first want to make sure you're working from the latest version. + +Once you've confirmed that the bug still exists in the latest version, you'll want to check to make sure it's not something we already know about on the [open issues GitHub page](https://github.com/opensearch-project/logstash-output-opensearch/issues). + +If you've upgraded to the latest version and you can't find it in our open issues list, then you'll need to tell us how to reproduce it Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. +### Feature Requests + +If you've thought of a way that logstash-output-opensearch could be better, we want to hear about it. We track feature requests using GitHub, so please feel free to open an issue which describes the feature you would like to see, why you need it, and how it should work. + +### Documentation Changes + +//TODO + +### Contributing Code + +As with other types of contributions, the first step is to [**open an issue on GitHub**](https://github.com/opensearch-project/logstash-output-opensearch/issues/new/choose). Opening an issue before you make changes makes sure that someone else isn't already working on that particular problem. It also lets us all work together to find the right approach before you spend a bunch of time on a PR. So again, when in doubt, open an issue. + +## Developer Certificate of Origin + +logstash-output-opensearch is an open source product released under the Apache 2.0 license (see either [the Apache site](https://www.apache.org/licenses/LICENSE-2.0) or the [LICENSE.txt file](./LICENSE.txt)). The Apache 2.0 license allows you to freely use, modify, distribute, and sell your own products that include Apache 2.0 licensed software. + +We respect intellectual property rights of others and we want to make sure all incoming contributions are correctly attributed and licensed. A Developer Certificate of Origin (DCO) is a lightweight mechanism to do that. + +The DCO is a declaration attached to every contribution made by every developer. In the commit message of the contribution, the developer simply adds a `Signed-off-by` statement and thereby agrees to the DCO, which you can find below or at [DeveloperCertificate.org](http://developercertificate.org/). + +``` +Developer's Certificate of Origin 1.1 + +By making a contribution to this project, I certify that: + +(a) The contribution was created in whole or in part by me and I + have the right to submit it under the open source license + indicated in the file; or + +(b) The contribution is based upon previous work that, to the + best of my knowledge, is covered under an appropriate open + source license and I have the right under that license to + submit that work with modifications, whether created in whole + or in part by me, under the same open source license (unless + I am permitted to submit under a different license), as + Indicated in the file; or + +(c) The contribution was provided directly to me by some other + person who certified (a), (b) or (c) and I have not modified + it. + +(d) I understand and agree that this project and the contribution + are public and that a record of the contribution (including + all personal information I submit with it, including my + sign-off) is maintained indefinitely and may be redistributed + consistent with this project or the open source license(s) + involved. + ``` +We require that every contribution to logstash-output-opensearch is signed with a Developer Certificate of Origin. Additionally, please use your real name. We do not accept anonymous contributors nor those utilizing pseudonyms. + +Each commit must include a DCO which looks like this + +``` +Signed-off-by: Jane Smith +``` +You may type this line on your own when writing your commit messages. However, if your user.name and user.email are set in your git configs, you can use `-s` or `โ€“ โ€“ signoff` to add the `Signed-off-by` line to the end of the commit message. + +## Review Process + +We deeply appreciate everyone who takes the time to make a contribution. We will review all contributions as quickly as possible. As a reminder, [opening an issue](https://github.com/opensearch-project/logstash-output-opensearch/issues/new/choose) discussing your change before you make it is the best way to smooth the PR process. This will prevent a rejection because someone else is already working on the problem, or because the solution is incompatible with the architectural direction. + +During the PR process, expect that there will be some back-and-forth. Please try to respond to comments in a timely fashion, and if you don't wish to continue with the PR, let us know. If a PR takes too many iterations for its complexity or size, we may reject it. Additionally, if you stop responding we may close the PR as abandoned. In either case, if you feel this was done in error, please add a comment on the PR. + +If we accept the PR, a [maintainer](MAINTAINERS.md) will merge your change and usually take care of backporting it to appropriate branches ourselves. + +If we reject the PR, we will close the pull request with a comment explaining why. This decision isn't always final: if you feel we have misunderstood your intended change or otherwise think that we should reconsider then please continue the conversation with a comment on the PR and we'll do our best to address any further points you raise. diff --git a/CONTRIBUTORS b/CONTRIBUTORS deleted file mode 100644 index 5baf4ea..0000000 --- a/CONTRIBUTORS +++ /dev/null @@ -1,34 +0,0 @@ -The following is a list of people who have contributed ideas, code, bug -reports, or in general have helped logstash along its way. - -Contributors: -* Aaron Mildenstein (untergeek) -* Bob Corsaro (dokipen) -* Colin Surprenant (colinsurprenant) -* Dmitry Koprov (dkoprov) -* Graham Bleach (bleach) -* Hao Chen (haoch) -* Ivan Babrou (bobrik) -* James Turnbull (jamtur01) -* John E. Vincent (lusis) -* Jordan Sissel (jordansissel) -* Joรฃo Duarte (jsvd) -* Kurt Hurtado (kurtado) -* Miah Johnson (miah) -* Pere Urbรณn (purbon) -* Pete Fritchman (fetep) -* Pier-Hugues Pellerin (ph) -* Raymond Feng (raymondfeng) -* Richard Pijnenburg (electrical) -* Spenser Jones (SpenserJ) -* Suyog Rao (suyograo) -* Tal Levy (talevy) -* Tom Hodder (tolland) -* jimmyjones2 -* Gabriel Moskovicz (gmoskovicz) -* Luca Belluccini (lucabelluccini) - -Note: If you've sent us patches, bug reports, or otherwise contributed to -Logstash, and you aren't on the list above and want to be, please let us know -and we'll make sure you're here. Contributions from folks like you are what make -open source awesome. diff --git a/MAINTAINERS.md b/MAINTAINERS.md new file mode 100644 index 0000000..91481d0 --- /dev/null +++ b/MAINTAINERS.md @@ -0,0 +1,71 @@ +- [Overview](#overview) +- [Current Maintainers](#current-maintainers) +- [Maintainer Responsibilities](#maintainer-responsibilities) + - [Uphold Code of Conduct](#uphold-code-of-conduct) + - [Prioritize Security](#prioritize-security) + - [Review Pull Requests](#review-pull-requests) + - [Triage Open Issues](#triage-open-issues) + - [Be Responsive](#be-responsive) + - [Maintain Overall Health of the Repo](#maintain-overall-health-of-the-repo) + - [Use Semver](#use-semver) + - [Release Frequently](#release-frequently) + - [Promote Other Maintainers](#promote-other-maintainers) + +## Overview + +This document explains who the maintainers are (see below), what they do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md). + +## Current Maintainers + +| Maintainer | GitHub ID | Affiliation | +| ------------------------ | --------------------------------------- | ----------- | +| Jack Mazanec | [jmazanec15](https://github.com/jmazanec15) | Amazon | +| Vamshi Vijay Nakkirtha | [vamshin](https://github.com/vamshin) | Amazon | +| Vijayan Balasubramanian | [VijayanB](https://github.com/VijayanB) | Amazon | + +## Maintainer Responsibilities + +Maintainers are active and visible members of the community, and have [maintain-level permissions on a repository](https://docs.github.com/en/organizations/managing-access-to-your-organizations-repositories/repository-permission-levels-for-an-organization). Use those privileges to serve the community and evolve code as follows. + +### Uphold Code of Conduct + +Model the behavior set forward by the [Code of Conduct](CODE_OF_CONDUCT.md) and raise any violations to other maintainers and admins. + +### Prioritize Security + +Security is your number one priority. Maintainer's Github keys must be password protected securely and any reported security vulnerabilities are addressed before features or bugs. + +Note that this repository is monitored and supported 24/7 by Amazon Security, see [Reporting a Vulnerability](SECURITY.md) for details. + +### Review Pull Requests + +Review pull requests regularly, comment, suggest, reject, merge and close. Accept only high quality pull-requests. Provide code reviews and guidance on incomming pull requests. Don't let PRs be stale and do your best to be helpful to contributors. + +### Triage Open Issues + +Manage labels, review issues regularly, and triage by labelling them. + +All repositories in this organization have a standard set of labels, including `bug`, `documentation`, `duplicate`, `enhancement`, `good first issue`, `help wanted`, `blocker`, `invalid`, `question`, `wontfix`, and `untriaged`, along with release labels, such as `v1.0.0`, `v1.1.0` and `v2.0.0`, and `backport`. + +Use labels to target an issue or a PR for a given release, add `help wanted` to good issues for new community members, and `blocker` for issues that scare you or need immediate attention. Request for more information from a submitter if an issue is not clear. Create new labels as needed by the project. + +### Be Responsive + +Respond to enhancement requests, and forum posts. Allocate time to reviewing and commenting on issues and conversations as they come in. + +### Maintain Overall Health of the Repo + +Keep the `main` branch at production quality at all times. Backport features as needed. Cut release branches and tags to enable future patches. + +### Use Semver + +Use and enforce [semantic versioning](https://semver.org/) and do not let breaking changes be made outside of major releases. + +### Release Frequently + +Make frequent project releases to the community. + +### Promote Other Maintainers + +Assist, add, and remove [MAINTAINERS](MAINTAINERS.md). Exercise good judgement, and propose high quality contributors to become co-maintainers. + diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..cd3abbd --- /dev/null +++ b/NOTICE @@ -0,0 +1,2 @@ +logstash-output-opensearch +Copyright 2021 OpenSearch CLI Contributors diff --git a/NOTICE.TXT b/NOTICE.TXT deleted file mode 100644 index 0b8a947..0000000 --- a/NOTICE.TXT +++ /dev/null @@ -1,5 +0,0 @@ -Elasticsearch -Copyright 2012-2015 Elasticsearch - -This product includes software developed by The Apache Software -Foundation (http://www.apache.org/). \ No newline at end of file diff --git a/README.md b/README.md index 0b54e9e..6babf8d 100644 --- a/README.md +++ b/README.md @@ -1,106 +1,35 @@ -# Logstash Plugin +# Logstash plugin -[![Travis Build Status](https://travis-ci.com/logstash-plugins/logstash-output-elasticsearch.svg)](https://travis-ci.com/logstash-plugins/logstash-output-elasticsearch) +- [Welcome!](#welcome) +- [Project Resources](#project-resources) +- [Code of Conduct](#code-of-conduct) +- [License](#license) +- [Copyright](#copyright) -This is a plugin for [Logstash](https://github.com/elastic/logstash). +## Welcome! -It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way. +**logstash-output-opensearch** is a community-driven, open source fork logstash-output-elasticsearch licensed under the [Apache v2.0 License](LICENSE.txt). For more information, see [opensearch.org](https://opensearch.org/). -## Documentation +## Project Resources -Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elastic.co/guide/en/logstash/current/). +* [Project Website](https://opensearch.org/) +* [Documentation](https://docs-beta.opensearch.org/) +* Need help? Try [Forums](https://discuss.opendistrocommunity.dev/) +* [Project Principles](https://opensearch.org/#principles) +* [Contributing to OpenSearch](CONTRIBUTING.md) +* [Maintainer Responsibilities](MAINTAINERS.md) +* [Release Management](RELEASING.md) +* [Admin Responsibilities](ADMINS.md) +* [Security](SECURITY.md) -- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive -- For more asciidoc formatting tips, see the excellent reference here https://github.com/elastic/docs#asciidoc-guide +## Code of Conduct -## Need Help? +This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUCT.md). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq), or contact [opensource-codeofconduct@amazon.com](mailto:opensource-codeofconduct@amazon.com) with any additional questions or comments. -Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum. +## License -## Developing +This project is licensed under the [Apache v2.0 License](LICENSE.txt). -### 1. Plugin Developement and Testing +## Copyright -#### Code -- To get started, you'll need JRuby with the Bundler gem installed. - -- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example). - -- Install dependencies -```sh -bundle install -``` - -#### Test - -- Update your dependencies - -```sh -bundle install -``` - -- Run unit tests - -```sh -bundle exec rspec -``` - -- Run integration tests - -```sh -export INTEGRATION=true -export ES_VERSION=5.1.1 -./travis-run.sh -``` - -### 2. Running your unpublished Plugin in Logstash - -#### 2.1 Run in a local Logstash clone - -- Edit Logstash `Gemfile` and add the local plugin path, for example: -```ruby -gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome" -``` -- Install plugin -```sh -# Logstash 2.3 and higher -bin/logstash-plugin install --no-verify - -# Prior to Logstash 2.3 -bin/plugin install --no-verify - -``` -- Run Logstash with your plugin -```sh -bin/logstash -e 'filter {awesome {}}' -``` -At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash. - -#### 2.2 Run in an installed Logstash - -You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using: - -- Build your plugin gem -```sh -gem build logstash-filter-awesome.gemspec -``` -- Install the plugin from the Logstash home -```sh -# Logstash 2.3 and higher -bin/logstash-plugin install --no-verify - -# Prior to Logstash 2.3 -bin/plugin install --no-verify - -``` -- Start Logstash and proceed to test the plugin - -## Contributing - -All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin. - -Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here. - -It is more important to the community that you are able to contribute. - -For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file. +Copyright 2020-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..aa99d9e --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,36 @@ +- [Overview](#overview) +- [Branching](#branching) + - [Release Branching](#release-branching) + - [Feature Branches](#feature-branches) +- [Release Labels](#release-labels) +- [Releasing](#releasing) + +## Overview + +This document explains the release strategy for artifacts in this organization. + +## Branching + +### Release Branching + +Given the current major release of 1.0, projects in this organization maintain the following active branches. + +* **main**: The next _major_ release. This is the branch where all merges take place and code moves fast. +* **1.x**: The next _minor_ release. Once a change is merged into `main`, decide whether to backport it to `1.x`. +* **1.0**: The _current_ release. In between minor releases, only hotfixes (e.g. security) are backported to `1.0`. + +Label PRs with the next major version label (e.g. `2.0.0`) and merge changes into `main`. Label PRs that you believe need to be backported as `1.x` and `1.0`. Backport PRs by checking out the versioned branch, cherry-pick changes and open a PR against each target backport branch. + +### Feature Branches + +Do not creating branches in the upstream repo, use your fork, for the exception of long lasting feature branches that require active collaboration from multiple developers. Name feature branches `feature/`. Once the work is merged to `main`, please make sure to delete the feature branch. + +## Release Labels + +Repositories create consistent release labels, such as `v1.0.0`, `v1.1.0` and `v2.0.0`, as well as `backport`. Use release labels to target an issue or a PR for a given release. See [MAINTAINERS](MAINTAINERS.md#triage-open-issues) for more information on triaging issues. + +## Releasing + +The release process is standard across repositories in this org and is run by a release manager volunteering from amongst [MAINTAINERS](MAINTAINERS.md). + +TODO diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0b85ca0 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,3 @@ +## Reporting a Vulnerability + +If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue. \ No newline at end of file diff --git a/lib/logstash/outputs/elasticsearch.rb b/lib/logstash/outputs/elasticsearch.rb index 84e87dd..a5110e5 100644 --- a/lib/logstash/outputs/elasticsearch.rb +++ b/lib/logstash/outputs/elasticsearch.rb @@ -235,25 +235,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base # here like `pipeline => "%{INGEST_PIPELINE}"` config :pipeline, :validate => :string, :default => nil - # ----- - # ILM configurations (beta) - # ----- - # Flag for enabling Index Lifecycle Management integration. - config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto' - - # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index - config :ilm_rollover_alias, :validate => :string - - # appends โ€œ{now/d}-000001โ€ by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. โ€œ000002โ€ - # {now/d} is date math, and will insert the appropriate value automatically. - config :ilm_pattern, :validate => :string, :default => '{now/d}-000001' - - # ILM policy to use, if undefined the default policy will be used. - config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY - attr_reader :client attr_reader :default_index - attr_reader :default_ilm_rollover_alias attr_reader :default_template_name def initialize(*params) @@ -307,7 +290,6 @@ def finish_register assert_es_version_supports_data_streams if data_stream_config? discover_cluster_uuid install_template - setup_ilm if ilm_in_use? super end @@ -502,18 +484,14 @@ def setup_ecs_compatibility_related_defaults case ecs_compatibility when :disabled @default_index = "logstash-%{+yyyy.MM.dd}" - @default_ilm_rollover_alias = "logstash" @default_template_name = 'logstash' when :v1 @default_index = "ecs-logstash-%{+yyyy.MM.dd}" - @default_ilm_rollover_alias = "ecs-logstash" @default_template_name = 'ecs-logstash' else fail("unsupported ECS Compatibility `#{ecs_compatibility}`") end - @index ||= default_index - @ilm_rollover_alias ||= default_ilm_rollover_alias @template_name ||= default_template_name end diff --git a/lib/logstash/outputs/elasticsearch/default-ilm-policy.json b/lib/logstash/outputs/elasticsearch/default-ilm-policy.json deleted file mode 100644 index f2ac4da..0000000 --- a/lib/logstash/outputs/elasticsearch/default-ilm-policy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "policy" : { - "phases": { - "hot" : { - "actions" : { - "rollover" : { - "max_size" : "50gb", - "max_age":"30d" - } - } - } - } - } -} \ No newline at end of file diff --git a/lib/logstash/outputs/elasticsearch/http_client.rb b/lib/logstash/outputs/elasticsearch/http_client.rb index dbcb96e..9e69cab 100644 --- a/lib/logstash/outputs/elasticsearch/http_client.rb +++ b/lib/logstash/outputs/elasticsearch/http_client.rb @@ -394,8 +394,6 @@ def template_endpoint maximum_seen_major_version < 8 ? '_template' : '_index_template' end - # ILM methods - # check whether rollover alias already exists def rollover_alias_exists?(name) exists?(name) @@ -416,25 +414,6 @@ def rollover_alias_put(alias_name, alias_definition) end end - def get_xpack_info - get("/_xpack") - end - - def get_ilm_endpoint - @pool.get("/_ilm/policy") - end - - def ilm_policy_exists?(name) - exists?("/_ilm/policy/#{name}", true) - end - - def ilm_policy_put(name, policy) - path = "_ilm/policy/#{name}" - logger.info("Installing ILM policy #{policy}", name: name) - @pool.put(path, nil, LogStash::Json.dump(policy)) - end - - # Build a bulk item for an elasticsearch update action def update_action_builder(args, source) args = args.clone() diff --git a/lib/logstash/outputs/elasticsearch/ilm.rb b/lib/logstash/outputs/elasticsearch/ilm.rb index af0b179..264001b 100644 --- a/lib/logstash/outputs/elasticsearch/ilm.rb +++ b/lib/logstash/outputs/elasticsearch/ilm.rb @@ -1,116 +1,7 @@ module LogStash; module Outputs; class ElasticSearch module Ilm - - ILM_POLICY_PATH = "default-ilm-policy.json" - - def setup_ilm - logger.warn("Overwriting supplied index #{@index} with rollover alias #{@ilm_rollover_alias}") unless default_index?(@index) - @index = @ilm_rollover_alias - maybe_create_rollover_alias - maybe_create_ilm_policy - end - def ilm_in_use? - return @ilm_actually_enabled if defined?(@ilm_actually_enabled) - @ilm_actually_enabled = - begin - if @ilm_enabled == 'auto' - if ilm_on_by_default? - ilm_ready, error = ilm_ready? - if !ilm_ready - @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - #{error}") - false - else - ilm_alias_set? - end - else - @logger.info("Index Lifecycle Management is set to 'auto', but will be disabled - Your Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management") - false - end - elsif @ilm_enabled.to_s == 'true' - ilm_ready, error = ilm_ready? - raise LogStash::ConfigurationError,"Index Lifecycle Management is set to enabled in Logstash, but cannot be used - #{error}" unless ilm_ready - ilm_alias_set? - else - false - end - end - end - - private - - def ilm_alias_set? - default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias) - end - - def ilm_on_by_default? - maximum_seen_major_version >= 7 - end - - def ilm_ready? - # Check the Elasticsearch instance for ILM readiness - this means that the version has to be a non-OSS release, with ILM feature - # available and enabled. - begin - xpack = client.get_xpack_info - features = xpack.nil? || xpack.empty? ? nil : xpack["features"] - ilm = features.nil? ? nil : features["ilm"] - return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" if features.nil? || ilm.nil? - return false, "Index Lifecycle management is not available in your Elasticsearch cluster" unless ilm['available'] - return false, "Index Lifecycle management is not enabled in your Elasticsearch cluster" unless ilm['enabled'] - return true, nil - rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e - # Check xpack endpoint: If no xpack endpoint, then this version of Elasticsearch is not compatible - if e.response_code == 404 - return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" - elsif e.response_code == 400 - return false, "Index Lifecycle management is not installed on your Elasticsearch cluster" - else - raise e - end - end - end - - def default_index?(index) - index == @default_index - end - - def default_rollover_alias?(rollover_alias) - rollover_alias == default_ilm_rollover_alias - end - - def ilm_policy_default? - ilm_policy == LogStash::Outputs::ElasticSearch::DEFAULT_POLICY - end - - def maybe_create_ilm_policy - if ilm_policy_default? - client.ilm_policy_put(ilm_policy, policy_payload) unless client.ilm_policy_exists?(ilm_policy) - else - raise LogStash::ConfigurationError, "The specified ILM policy #{ilm_policy} does not exist on your Elasticsearch instance" unless client.ilm_policy_exists?(ilm_policy) - end - end - - def maybe_create_rollover_alias - client.rollover_alias_put(rollover_alias_target, rollover_alias_payload) unless client.rollover_alias_exists?(ilm_rollover_alias) - end - - def rollover_alias_target - "<#{ilm_rollover_alias}-#{ilm_pattern}>" - end - - def rollover_alias_payload - { - 'aliases' => { - ilm_rollover_alias =>{ - 'is_write_index' => true - } - } - } - end - - def policy_payload - policy_path = ::File.expand_path(ILM_POLICY_PATH, ::File.dirname(__FILE__)) - LogStash::Json.load(::IO.read(policy_path)) + false end end end; end; end \ No newline at end of file diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index f2c837a..ee7f6fd 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -12,7 +12,6 @@ def self.install_template(plugin) template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility) end - add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use? plugin.logger.debug("Attempting to install template", template: template) install(plugin.client, template_name(plugin), template, plugin.template_overwrite) end @@ -29,27 +28,12 @@ def self.install(client, template_name, template, template_overwrite) client.template_install(template_name, template, template_overwrite) end - def self.add_ilm_settings_to_template(plugin, template) - # Overwrite any index patterns, and use the rollover alias. Use 'index_patterns' rather than 'template' for pattern - # definition - remove any existing definition of 'template' - template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8 - template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" - settings = template_settings(plugin, template) - if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias']) - plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled") - end - settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) - end - def self.template_settings(plugin, template) - plugin.maximum_seen_major_version < 8 ? template['settings']: template['template']['settings'] + template['settings'] end - # Template name - if template_name set, use it - # if not and ILM is enabled, use the rollover alias - # else use the default value of template_name def self.template_name(plugin) - plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name + plugin.template_name end def self.default_template_path(es_major_version, ecs_compatibility=:disabled) diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index 13422e2..21e9967 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -3,7 +3,7 @@ Gem::Specification.new do |s| s.version = '11.0.2' s.licenses = ['apache-2.0'] - s.summary = "Stores logs in Elasticsearch" + s.summary = "Stores logs in OpenSearch" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' @@ -13,7 +13,7 @@ Gem::Specification.new do |s| s.platform = RUBY_PLATFORM # Files - s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] + s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","Gemfile","LICENSE","NOTICE", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"] # Tests s.test_files = s.files.grep(%r{^(test|spec|features)/}) @@ -30,6 +30,4 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'flores' s.add_development_dependency 'cabin', ['~> 0.6'] - # Still used in some specs, we should remove this ASAP - s.add_development_dependency 'elasticsearch' end diff --git a/.ci/Dockerfile b/scripts/Dockerfile similarity index 55% rename from .ci/Dockerfile rename to scripts/Dockerfile index d0445e9..56a82c9 100644 --- a/.ci/Dockerfile +++ b/scripts/Dockerfile @@ -1,23 +1,15 @@ -ARG ELASTIC_STACK_VERSION -# TODO: refactor this to be implicitly resolved by logstash-plugins/.ci/Dockerfile -ARG DISTRIBUTION_SUFFIX -FROM docker.elastic.co/logstash/logstash$DISTRIBUTION_SUFFIX:$ELASTIC_STACK_VERSION +ARG LOGSTASH_VERSION +FROM docker.elastic.co/logstash/logstash-oss:$LOGSTASH_VERSION USER logstash COPY --chown=logstash:logstash Gemfile /usr/share/plugins/plugin/Gemfile COPY --chown=logstash:logstash *.gemspec VERSION* version* /usr/share/plugins/plugin/ RUN cp /usr/share/logstash/logstash-core/versions-gem-copy.yml /usr/share/logstash/versions.yml ENV PATH="${PATH}:/usr/share/logstash/vendor/jruby/bin:/usr/share/logstash/jdk/bin" ENV LOGSTASH_SOURCE="1" -ENV ELASTIC_STACK_VERSION=$ELASTIC_STACK_VERSION -# DISTRIBUTION="default" (by default) or "oss" -ARG DISTRIBUTION -ENV DISTRIBUTION=$DISTRIBUTION -# INTEGRATION="true" while integration testing (false-y by default) ARG INTEGRATION ENV INTEGRATION=$INTEGRATION RUN gem install bundler -v '< 2' WORKDIR /usr/share/plugins/plugin RUN bundle install --with test ci COPY --chown=logstash:logstash . /usr/share/plugins/plugin -RUN bundle exec rake vendor -RUN .ci/setup.sh +RUN bundle exec rake vendor \ No newline at end of file diff --git a/scripts/Dockerfile.opendistro b/scripts/Dockerfile.opendistro new file mode 100644 index 0000000..a2db289 --- /dev/null +++ b/scripts/Dockerfile.opendistro @@ -0,0 +1,11 @@ +FROM amazon/opendistro-for-elasticsearch:1.13.2 + +ARG plugin_path=/usr/share/plugins/plugin +ARG es_path=/usr/share/elasticsearch +ARG es_yml=$es_path/config/elasticsearch.yml + +RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security +COPY --chown=elasticsearch:elasticsearch spec/fixtures/scripts/groovy/* $es_path/config/scripts/ +COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/config/test_certs/ +COPY --chown=elasticsearch:elasticsearch scripts/elasticsearch-run.sh $es_path/ + diff --git a/scripts/docker-compose.override.yml b/scripts/docker-compose.override.yml new file mode 100644 index 0000000..bc9bca9 --- /dev/null +++ b/scripts/docker-compose.override.yml @@ -0,0 +1,25 @@ +version: '3' + +services: + + logstash: + command: /usr/share/plugins/plugin/scripts/logstash-run.sh + build: + args: + - LOGSTASH_VERSION=$LOGSTASH_VERSION + environment: + - INTEGRATION=${INTEGRATION:-false} + + + elasticsearch: + build: + context: ../ + dockerfile: scripts/Dockerfile.opendistro + args: + - OPENDISTRO_VERSION=$OPENDISTRO_VERSION + - INTEGRATION=${INTEGRATION:-false} + command: /usr/share/elasticsearch/elasticsearch-run.sh + ports: + - "9200:9200" + user: elasticsearch + diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml new file mode 100644 index 0000000..1781416 --- /dev/null +++ b/scripts/docker-compose.yml @@ -0,0 +1,14 @@ +version: '3' + +services: + + logstash: + build: + context: ../ + dockerfile: scripts/Dockerfile + command: scripts/run.sh + env_file: docker.env + environment: + - SPEC_OPTS + - LOG_LEVEL # devutils (>= 2.0.4) reads the ENV and sets LS logging + tty: true \ No newline at end of file diff --git a/.ci/docker-run.sh b/scripts/docker-run.sh similarity index 96% rename from .ci/docker-run.sh rename to scripts/docker-run.sh index 02f0c42..303afa8 100755 --- a/.ci/docker-run.sh +++ b/scripts/docker-run.sh @@ -3,7 +3,7 @@ # This is intended to be run inside the docker container as the command of the docker-compose. set -ex -cd .ci +cd scripts if [ "$INTEGRATION" == "true" ]; then docker-compose up --exit-code-from logstash diff --git a/scripts/docker-setup.sh b/scripts/docker-setup.sh new file mode 100755 index 0000000..964ff2c --- /dev/null +++ b/scripts/docker-setup.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +# This is intended to be run the plugin's root directory. `.ci/docker-setup.sh` +# Ensure you have Docker installed locally and set the ELASTIC_STACK_VERSION environment variable. +set -e + +if [ -f Gemfile.lock ]; then + rm Gemfile.lock +fi +cd scripts + +if [ "$INTEGRATION" == "true" ]; then + docker-compose down + docker-compose build +else + docker-compose down + docker-compose build logstash +fi diff --git a/scripts/docker.env b/scripts/docker.env new file mode 100644 index 0000000..e54794e --- /dev/null +++ b/scripts/docker.env @@ -0,0 +1,5 @@ +#LS_JAVA_OPTS="-Xms256m -Xmx256m -XX:MaxMetaspaceSize=256m" +# - `-Xregexp.interruptible=true -Xcompile.invokedynamic=true -Xjit.threshold=0` LS base-line +# - `-XX:+UseParallelGC` do not use G1 (default) on Java 11 +# - `-v -W1` print JRuby version but do not go verbose +JRUBY_OPTS=-Xregexp.interruptible=true -Xcompile.invokedynamic=true -Xjit.threshold=0 -J-XX:+UseParallelGC -J-XX:+PrintCommandLineFlags -v -W1 \ No newline at end of file diff --git a/.ci/elasticsearch-run.sh b/scripts/elasticsearch-run.sh similarity index 63% rename from .ci/elasticsearch-run.sh rename to scripts/elasticsearch-run.sh index c270707..9df8c88 100755 --- a/.ci/elasticsearch-run.sh +++ b/scripts/elasticsearch-run.sh @@ -1,4 +1,4 @@ #!/bin/bash set -ex -/usr/share/elasticsearch/bin/elasticsearch -Ediscovery.type=single-node -Eaction.destructive_requires_name=false +/usr/share/elasticsearch/bin/elasticsearch -Ediscovery.type=single-node diff --git a/.ci/logstash-run.sh b/scripts/logstash-run.sh similarity index 55% rename from .ci/logstash-run.sh rename to scripts/logstash-run.sh index 39d0a3d..1ddbe6f 100755 --- a/.ci/logstash-run.sh +++ b/scripts/logstash-run.sh @@ -3,11 +3,7 @@ set -ex export PATH=$BUILD_DIR/gradle/bin:$PATH -if [[ "$SECURE_INTEGRATION" == "true" ]]; then - ES_URL="https://elasticsearch:9200 -k" -else - ES_URL="http://elasticsearch:9200" -fi +ES_URL="http://elasticsearch:9200" wait_for_es() { count=120 @@ -22,18 +18,8 @@ wait_for_es() { if [[ "$INTEGRATION" != "true" ]]; then bundle exec rspec -fd spec/unit -t ~integration -t ~secure_integration else - - if [[ "$SECURE_INTEGRATION" == "true" ]]; then - extra_tag_args="--tag secure_integration" - else - extra_tag_args="--tag ~secure_integration --tag integration" - fi - - if [[ "$DISTRIBUTION" == "oss" ]]; then - extra_tag_args="$extra_tag_args --tag distribution:oss --tag ~distribution:xpack" - elif [[ "$DISTRIBUTION" == "default" ]]; then - extra_tag_args="$extra_tag_args --tag ~distribution:oss --tag distribution:xpack" - fi + extra_tag_args="--tag ~secure_integration --tag integration" + extra_tag_args="$extra_tag_args --tag distribution:oss --tag ~distribution:xpack" echo "Waiting for elasticsearch to respond..." ES_VERSION=$(wait_for_es) echo "Elasticsearch $ES_VERSION is Up!" diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100644 index 0000000..f34baf0 --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# This is intended to be run inside the docker container as the command of the docker-compose. + +env + +set -ex + +jruby -rbundler/setup -S rspec -fd \ No newline at end of file diff --git a/spec/es_spec_helper.rb b/spec/es_spec_helper.rb index 44dfcb7..da3346e 100644 --- a/spec/es_spec_helper.rb +++ b/spec/es_spec_helper.rb @@ -24,9 +24,7 @@ def get_client end def doc_type - if ESHelper.es_version_satisfies?(">=8") - nil - elsif ESHelper.es_version_satisfies?(">=7") + if ESHelper.es_version_satisfies?(">=7") "_doc" else "doc" @@ -61,7 +59,7 @@ def routing_field_name end def self.es_version - RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION'] + RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] end RSpec::Matchers.define :have_hits do |expected| @@ -82,7 +80,7 @@ def self.es_version end def self.es_version_satisfies?(*requirement) - es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION'] || ENV['ELASTIC_STACK_VERSION'] + es_version = RSpec.configuration.filter[:es_version] if es_version.nil? puts "Info: ES_VERSION, ELASTIC_STACK_VERSION or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call." return false @@ -96,7 +94,6 @@ def clean(client) client.indices.delete_index_template(:name => "logstash*") rescue nil # This can fail if there are no indexes, ignore failure. client.indices.delete(:index => "*") rescue nil - clean_ilm(client) if supports_ilm?(client) end def set_cluster_settings(client, cluster_settings) @@ -108,14 +105,6 @@ def get_cluster_settings(client) client.cluster.get_settings end - def get_policy(client, policy_name) - client.get_ilm_policy(name: policy_name) - end - - def put_policy(client, policy_name, policy) - client.put_ilm_policy({:name => policy_name, :body=> policy}) - end - def put_alias(client, the_alias, index) body = { "aliases" => { @@ -127,19 +116,6 @@ def put_alias(client, the_alias, index) client.put_alias({name: the_alias, body: body}) end - def clean_ilm(client) - client.get_ilm_policy.each_key { |key| client.delete_ilm_policy(name: key) if key =~ /logstash-policy/ } - end - - def supports_ilm?(client) - begin - client.get_ilm_policy - true - rescue - false - end - end - def max_docs_policy(max_docs) { "policy" => { @@ -173,27 +149,16 @@ def max_age_policy(max_age) end def get_template(client, name) - if ESHelper.es_version_satisfies?(">=8") - t = client.indices.get_index_template(name: name) - t['index_templates'][0]['index_template'] - else - t = client.indices.get_template(name: name) - t[name] - end + t = client.indices.get_template(name: name) + t[name] end def get_template_settings(template) - if ESHelper.es_version_satisfies?(">=8") - template['template']['settings'] - else - template['settings'] - end + template['settings'] end def get_template_mappings(template) - if ESHelper.es_version_satisfies?(">=8") - template['template']['mappings'] - elsif ESHelper.es_version_satisfies?(">=7") + if ESHelper.es_version_satisfies?(">=7") template['mappings'] else template['mappings']["_default_"] diff --git a/spec/unit/outputs/elasticsearch/template_manager_spec.rb b/spec/unit/outputs/elasticsearch/template_manager_spec.rb index a47f77e..a51244c 100644 --- a/spec/unit/outputs/elasticsearch/template_manager_spec.rb +++ b/spec/unit/outputs/elasticsearch/template_manager_spec.rb @@ -26,35 +26,4 @@ expect(described_class.default_template_path(7, :v1)).to end_with("/templates/ecs-v1/elasticsearch-7x.json") end end - - describe "index template with ilm settings" do - let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} } - let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) } - - describe "in version 8+" do - let(:file_path) { described_class.default_template_path(8) } - let(:template) { described_class.read_template_file(file_path)} - - it "should update settings" do - expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) - described_class.add_ilm_settings_to_template(plugin, template) - expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil) - expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil) - expect(template.include?('settings')).to be_falsey - end - end - - describe "in version < 8" do - let(:file_path) { described_class.default_template_path(7) } - let(:template) { described_class.read_template_file(file_path)} - - it "should update settings" do - expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) - described_class.add_ilm_settings_to_template(plugin, template) - expect(template['settings']['index.lifecycle.name']).not_to eq(nil) - expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil) - expect(template.include?('template')).to be_falsey - end - end - end end