From 5537129a91399abfcc014c1d6f9f4b3da42737af Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Thu, 12 Aug 2021 13:39:56 +0200 Subject: [PATCH 1/3] Use es-index-cleaner golang implementation Signed-off-by: Pavol Loffay --- .github/workflows/ci-elasticsearch.yml | 14 +++ Makefile | 15 ++- cmd/es-index-cleaner/Dockerfile | 6 + cmd/es-index-cleaner/app/flags.go | 2 +- cmd/es-index-cleaner/app/index_filter.go | 15 +-- cmd/es-index-cleaner/app/index_filter_test.go | 12 +- docker/Makefile | 6 +- plugin/storage/es/Dockerfile | 11 -- plugin/storage/es/esCleaner.py | 117 ------------------ .../integration/es_index_cleaner_test.go | 4 +- scripts/build-upload-docker-images.sh | 2 +- 11 files changed, 50 insertions(+), 154 deletions(-) create mode 100644 cmd/es-index-cleaner/Dockerfile delete mode 100644 plugin/storage/es/Dockerfile delete mode 100755 plugin/storage/es/esCleaner.py mode change 100644 => 100755 scripts/build-upload-docker-images.sh diff --git a/.github/workflows/ci-elasticsearch.yml b/.github/workflows/ci-elasticsearch.yml index 65b7f458064..9d27a83e83d 100644 --- a/.github/workflows/ci-elasticsearch.yml +++ b/.github/workflows/ci-elasticsearch.yml @@ -9,6 +9,11 @@ on: jobs: elasticsearch: runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 strategy: matrix: version: @@ -41,5 +46,14 @@ jobs: - name: Install tools run: make install-ci + - uses: docker/setup-qemu-action@v1 + + - uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + - name: Run elasticsearch integration tests run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.image }} + env: + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} diff --git a/Makefile b/Makefile index f2b943806b9..d8f0003d096 100644 --- a/Makefile +++ b/Makefile @@ -115,7 +115,6 @@ grpc-plugin-storage-integration-test: .PHONY: test-compile-es-scripts test-compile-es-scripts: docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py - docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py .PHONY: index-cleaner-integration-test index-cleaner-integration-test: docker-images-elastic @@ -216,6 +215,10 @@ build-esmapping-generator: build-esmapping-generator-linux: GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go +.PHONY: build-es-index-cleaner +build-es-index-cleaner: + $(GOBUILD) -o ./cmd/es-index-cleaner/es-index-cleaner-$(GOOS)-$(GOARCH) ./cmd/es-index-cleaner/main.go + .PHONY: docker-hotrod docker-hotrod: GOOS=linux $(MAKE) build-examples @@ -306,7 +309,8 @@ build-platform-binaries: build-agent \ build-examples \ build-tracegen \ build-anonymizer \ - build-esmapping-generator + build-esmapping-generator \ + build-es-index-cleaner .PHONY: build-all-platforms build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le @@ -316,10 +320,13 @@ docker-images-cassandra: docker build -t $(DOCKER_NAMESPACE)/jaeger-cassandra-schema:${DOCKER_TAG} plugin/storage/cassandra/ @echo "Finished building jaeger-cassandra-schema ==============" +docker-images-elastic: TARGET = release + .PHONY: docker-images-elastic -docker-images-elastic: +docker-images-elastic: create-baseimg GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator - docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es + GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner + docker build --target $(TARGET) -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH) @echo "Finished building jaeger-es-indices-clean ==============" diff --git a/cmd/es-index-cleaner/Dockerfile b/cmd/es-index-cleaner/Dockerfile new file mode 100644 index 00000000000..66caa9539fc --- /dev/null +++ b/cmd/es-index-cleaner/Dockerfile @@ -0,0 +1,6 @@ +ARG base_image + +FROM $base_image AS release +ARG TARGETARCH +COPY es-index-cleaner-linux-$TARGETARCH /go/bin/es-index-cleaner-linux +ENTRYPOINT ["/go/bin/es-index-cleaner-linux"] diff --git a/cmd/es-index-cleaner/app/flags.go b/cmd/es-index-cleaner/app/flags.go index 7dfef44c240..53e4d06ea9a 100644 --- a/cmd/es-index-cleaner/app/flags.go +++ b/cmd/es-index-cleaner/app/flags.go @@ -45,7 +45,7 @@ type Config struct { // AddFlags adds flags for TLS to the FlagSet. func (c *Config) AddFlags(flags *flag.FlagSet) { flags.String(indexPrefix, "", "Index prefix") - flags.Bool(archive, false, "Whether to remove archive indices") + flags.Bool(archive, false, "Whether to remove archive indices. It works only for rollover") flags.Bool(rollover, false, "Whether to remove indices created by rollover") flags.Int(timeout, 120, "Number of seconds to wait for master node response") flags.String(indexDateSeparator, "-", "Index date separator") diff --git a/cmd/es-index-cleaner/app/index_filter.go b/cmd/es-index-cleaner/app/index_filter.go index e173ae6dd12..a01582730d6 100644 --- a/cmd/es-index-cleaner/app/index_filter.go +++ b/cmd/es-index-cleaner/app/index_filter.go @@ -42,18 +42,13 @@ func (i *IndexFilter) Filter(indices []Index) []Index { func (i *IndexFilter) filter(indices []Index) []Index { var reg *regexp.Regexp - if !i.Rollover && !i.Archive { - // daily indices - reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator)) - } else if !i.Rollover && i.Archive { - // daily archive - reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive", i.IndexPrefix)) - } else if i.Rollover && !i.Archive { - // rollover + if i.Archive { + // archive works only for rollover + reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix)) + } else if i.Rollover { reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service)-\\d{6}", i.IndexPrefix)) } else { - // rollover archive - reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix)) + reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator)) } var filtered []Index diff --git a/cmd/es-index-cleaner/app/index_filter_test.go b/cmd/es-index-cleaner/app/index_filter_test.go index da280e6ba68..0f2a58920f5 100644 --- a/cmd/es-index-cleaner/app/index_filter_test.go +++ b/cmd/es-index-cleaner/app/index_filter_test.go @@ -229,19 +229,21 @@ func testIndexFilter(t *testing.T, prefix string) { }, }, { - name: "archive indices, remove older 2 days", + name: "archive indices, remove older 1 days - archive works only for rollover", filter: &IndexFilter{ IndexPrefix: prefix, IndexDateSeparator: "-", Archive: true, Rollover: false, - DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(2)), + DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(1)), }, expected: []Index{ { - Index: prefix + "jaeger-span-archive", - CreationTime: time.Date(2020, time.August, 0, 15, 0, 0, 0, time.UTC), - Aliases: map[string]bool{}, + Index: prefix + "jaeger-span-archive-000001", + CreationTime: time.Date(2020, time.August, 5, 15, 0, 0, 0, time.UTC), + Aliases: map[string]bool{ + prefix + "jaeger-span-archive-read": true, + }, }, }, }, diff --git a/docker/Makefile b/docker/Makefile index 6db950eb614..815c24bd35b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -3,9 +3,9 @@ ROOT_IMAGE ?= alpine:3.13 CERT_IMAGE := $(ROOT_IMAGE) GOLANG_IMAGE := golang:1.16-alpine -BASE_IMAGE := localhost:5000/baseimg_alpine:latest -DEBUG_IMAGE := localhost:5000/debugimg_alpine:latest -PLATFORMS := linux/amd64,linux/s390x,linux/ppc64le,linux/arm64 +BASE_IMAGE ?= localhost:5000/baseimg_alpine:latest +DEBUG_IMAGE ?= localhost:5000/debugimg_alpine:latest +PLATFORMS ?= linux/amd64,linux/s390x,linux/ppc64le,linux/arm64 create-baseimg-debugimg: create-baseimg create-debugimg diff --git a/plugin/storage/es/Dockerfile b/plugin/storage/es/Dockerfile deleted file mode 100644 index 4fab7e98012..00000000000 --- a/plugin/storage/es/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM python:3-alpine3.11 - -# Temporary fix for https://github.com/jaegertracing/jaeger/issues/1494 -RUN pip install urllib3==1.24.3 -# Temporary fix for https://github.com/jaegertracing/jaeger/issues/3179 -RUN pip install elasticsearch==7.13.0 - -RUN pip install elasticsearch elasticsearch-curator -COPY esCleaner.py /es-index-cleaner/ - -ENTRYPOINT ["python3", "/es-index-cleaner/esCleaner.py"] diff --git a/plugin/storage/es/esCleaner.py b/plugin/storage/es/esCleaner.py deleted file mode 100755 index ace05fcd92f..00000000000 --- a/plugin/storage/es/esCleaner.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/usr/bin/env python3 - -import curator -import elasticsearch -import os -import ssl -import sys - -TIMEOUT=120 - -def main(): - if len(sys.argv) != 3: - print('USAGE: [INDEX_PREFIX=(default "")] [ARCHIVE=(default false)] ... {} NUM_OF_DAYS http://HOSTNAME[:PORT]'.format(sys.argv[0])) - print('NUM_OF_DAYS ... delete indices that are older than the given number of days.') - print('HOSTNAME ... specifies which Elasticsearch hosts URL to search and delete indices from.') - print('TIMEOUT ... number of seconds to wait for master node response (default {}).'.format(TIMEOUT)) - print('INDEX_PREFIX ... specifies index prefix.') - print('INDEX_DATE_SEPARATOR ... specifies index date separator.') - print('ARCHIVE ... specifies whether to remove archive indices (only works for rollover) (default false).') - print('ROLLOVER ... specifies whether to remove indices created by rollover (default false).') - print('ES_USERNAME ... The username required by Elasticsearch.') - print('ES_PASSWORD ... The password required by Elasticsearch.') - print('ES_TLS ... enable TLS (default false).') - print('ES_TLS_CA ... Path to TLS CA file.') - print('ES_TLS_CERT ... Path to TLS certificate file.') - print('ES_TLS_KEY ... Path to TLS key file.') - print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.') - sys.exit(1) - - client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false'))) - ilo = curator.IndexList(client) - empty_list(ilo, 'Elasticsearch has no indices') - - prefix = os.getenv("INDEX_PREFIX", '') - if prefix != '': - prefix += '-' - separator = os.getenv("INDEX_DATE_SEPARATOR", '-') - - if str2bool(os.getenv("ARCHIVE", 'false')): - filter_archive_indices_rollover(ilo, prefix) - else: - if str2bool(os.getenv("ROLLOVER", 'false')): - filter_main_indices_rollover(ilo, prefix) - else: - filter_main_indices(ilo, prefix, separator) - - empty_list(ilo, 'No indices to delete') - - for index in ilo.working_list(): - print("Removing", index) - timeout = int(os.getenv("TIMEOUT", TIMEOUT)) - delete_indices = curator.DeleteIndices(ilo, master_timeout=timeout) - delete_indices.do_action() - - -def filter_main_indices(ilo, prefix, separator): - date_regex = "\d{4}" + separator + "\d{2}" + separator + "\d{2}" - time_string = "%Y" + separator + "%m" + separator + "%d" - - ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-(span|service|dependencies)-" + date_regex) - empty_list(ilo, "No indices to delete") - # This excludes archive index as we use source='name' - # source `creation_date` would include archive index - ilo.filter_by_age(source='name', direction='older', timestring=time_string, unit='days', unit_count=int(sys.argv[1])) - - -def filter_main_indices_rollover(ilo, prefix): - ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-(span|service)-\d{6}") - empty_list(ilo, "No indices to delete") - # do not remove active write indices - ilo.filter_by_alias(aliases=[prefix + 'jaeger-span-write'], exclude=True) - empty_list(ilo, "No indices to delete") - ilo.filter_by_alias(aliases=[prefix + 'jaeger-service-write'], exclude=True) - empty_list(ilo, "No indices to delete") - ilo.filter_by_age(source='creation_date', direction='older', unit='days', unit_count=int(sys.argv[1])) - - -def filter_archive_indices_rollover(ilo, prefix): - # Remove only rollover archive indices - # Do not remove active write archive index - ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-span-archive-\d{6}") - empty_list(ilo, "No indices to delete") - ilo.filter_by_alias(aliases=[prefix + 'jaeger-span-archive-write'], exclude=True) - empty_list(ilo, "No indices to delete") - ilo.filter_by_age(source='creation_date', direction='older', unit='days', unit_count=int(sys.argv[1])) - - -def empty_list(ilo, error_msg): - try: - ilo.empty_list_check() - except curator.NoIndices: - print(error_msg) - sys.exit(0) - - -def str2bool(v): - return v.lower() in ('true', '1') - - -def create_client(username, password, tls, ca, cert, key, skipHostVerify): - context = ssl.create_default_context() - if ca is not None: - context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca) - elif skipHostVerify: - context.check_hostname = False - context.verify_mode = ssl.CERT_NONE - if username is not None and password is not None: - return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context) - elif tls: - context.load_cert_chain(certfile=cert, keyfile=key) - return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) - else: - return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context) - - -if __name__ == "__main__": - main() diff --git a/plugin/storage/integration/es_index_cleaner_test.go b/plugin/storage/integration/es_index_cleaner_test.go index 1eec135f032..b9a19b38dfe 100644 --- a/plugin/storage/integration/es_index_cleaner_test.go +++ b/plugin/storage/integration/es_index_cleaner_test.go @@ -114,10 +114,10 @@ func TestIndexCleaner(t *testing.T) { }, } for _, test := range tests { - t.Run(fmt.Sprintf("%s_no_prefix", test.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_no_prefix, %s", test.name, test.envVars), func(t *testing.T) { runIndexCleanerTest(t, client, "", test.expectedIndices, test.envVars) }) - t.Run(fmt.Sprintf("%s_prefix", test.name), func(t *testing.T) { + t.Run(fmt.Sprintf("%s_prefix, %s", test.name, test.envVars), func(t *testing.T) { runIndexCleanerTest(t, client, indexPrefix, test.expectedIndices, append(test.envVars, "INDEX_PREFIX="+indexPrefix)) }) } diff --git a/scripts/build-upload-docker-images.sh b/scripts/build-upload-docker-images.sh old mode 100644 new mode 100755 index fd28807be97..f32793c2eb3 --- a/scripts/build-upload-docker-images.sh +++ b/scripts/build-upload-docker-images.sh @@ -20,7 +20,7 @@ do done # build/upload images for jaeger-es-index-cleaner and jaeger-es-rollover -bash scripts/build-upload-a-docker-image.sh -c jaeger-es-index-cleaner -d plugin/storage/es -p "${platforms}" +bash scripts/build-upload-a-docker-image.sh -b -c jaeger-es-index-cleaner -d cmd/es-index-cleaner -p "${platforms}" -t release bash scripts/build-upload-a-docker-image.sh -c jaeger-es-rollover -d plugin/storage/es -f Dockerfile.rollover -p "${platforms}" # build/upload images for jaeger-tracegen and jaeger-anonymizer From 248359d80dd185afa431ab38ad0464323e96ff97 Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Wed, 18 Aug 2021 12:51:52 +0200 Subject: [PATCH 2/3] Fix Signed-off-by: Pavol Loffay --- .github/workflows/ci-elasticsearch.yml | 12 ------------ docker/Makefile | 6 +++--- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-elasticsearch.yml b/.github/workflows/ci-elasticsearch.yml index 9d27a83e83d..6eec0cd9eb9 100644 --- a/.github/workflows/ci-elasticsearch.yml +++ b/.github/workflows/ci-elasticsearch.yml @@ -9,11 +9,6 @@ on: jobs: elasticsearch: runs-on: ubuntu-latest - services: - registry: - image: registry:2 - ports: - - 5000:5000 strategy: matrix: version: @@ -48,12 +43,5 @@ jobs: - uses: docker/setup-qemu-action@v1 - - uses: docker/setup-buildx-action@v1 - with: - driver-opts: network=host - - name: Run elasticsearch integration tests run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.image }} - env: - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - QUAY_TOKEN: ${{ secrets.QUAY_TOKEN }} diff --git a/docker/Makefile b/docker/Makefile index 815c24bd35b..6db950eb614 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -3,9 +3,9 @@ ROOT_IMAGE ?= alpine:3.13 CERT_IMAGE := $(ROOT_IMAGE) GOLANG_IMAGE := golang:1.16-alpine -BASE_IMAGE ?= localhost:5000/baseimg_alpine:latest -DEBUG_IMAGE ?= localhost:5000/debugimg_alpine:latest -PLATFORMS ?= linux/amd64,linux/s390x,linux/ppc64le,linux/arm64 +BASE_IMAGE := localhost:5000/baseimg_alpine:latest +DEBUG_IMAGE := localhost:5000/debugimg_alpine:latest +PLATFORMS := linux/amd64,linux/s390x,linux/ppc64le,linux/arm64 create-baseimg-debugimg: create-baseimg create-debugimg From 10284ca7a6aca23b30d00cba6ab6067feb80b57c Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Wed, 18 Aug 2021 19:39:26 +0200 Subject: [PATCH 3/3] Fix Signed-off-by: Pavol Loffay --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d8f0003d096..5ee96071b30 100644 --- a/Makefile +++ b/Makefile @@ -320,13 +320,11 @@ docker-images-cassandra: docker build -t $(DOCKER_NAMESPACE)/jaeger-cassandra-schema:${DOCKER_TAG} plugin/storage/cassandra/ @echo "Finished building jaeger-cassandra-schema ==============" -docker-images-elastic: TARGET = release - .PHONY: docker-images-elastic docker-images-elastic: create-baseimg GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner - docker build --target $(TARGET) -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner + docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH) @echo "Finished building jaeger-es-indices-clean =============="