Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use es-index-cleaner golang implementation #3204

Merged
merged 3 commits into from
Aug 19, 2021
Merged

Use es-index-cleaner golang implementation #3204

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5537129
Use es-index-cleaner golang implementation
pavolloffay Aug 12, 2021
248359d
Fix
pavolloffay Aug 18, 2021
10284ca
Fix
pavolloffay Aug 18, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci-elasticsearch.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,5 +41,7 @@ jobs:
- name: Install tools
run: make install-ci

- uses: docker/setup-qemu-action@v1

- name: Run elasticsearch integration tests
run: bash scripts/es-integration-test.sh ${{ matrix.version.distribution }} ${{ matrix.version.image }}
15 changes: 11 additions & 4 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,6 @@ grpc-plugin-storage-integration-test:
.PHONY: test-compile-es-scripts
test-compile-es-scripts:
docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esRollover.py
docker run --rm -v ${PWD}:/tmp/jaeger python:3-alpine3.11 /usr/local/bin/python -m py_compile /tmp/jaeger/plugin/storage/es/esCleaner.py

.PHONY: index-cleaner-integration-test
index-cleaner-integration-test: docker-images-elastic
Expand Down Expand Up @@ -216,6 +215,10 @@ build-esmapping-generator:
build-esmapping-generator-linux:
GOOS=linux GOARCH=amd64 $(GOBUILD) -o ./plugin/storage/es/esmapping-generator ./cmd/esmapping-generator/main.go

.PHONY: build-es-index-cleaner
build-es-index-cleaner:
$(GOBUILD) -o ./cmd/es-index-cleaner/es-index-cleaner-$(GOOS)-$(GOARCH) ./cmd/es-index-cleaner/main.go

.PHONY: docker-hotrod
docker-hotrod:
GOOS=linux $(MAKE) build-examples
Expand Down Expand Up @@ -306,7 +309,8 @@ build-platform-binaries: build-agent \
build-examples \
build-tracegen \
build-anonymizer \
build-esmapping-generator
build-esmapping-generator \
build-es-index-cleaner

.PHONY: build-all-platforms
build-all-platforms: build-binaries-linux build-binaries-windows build-binaries-darwin build-binaries-s390x build-binaries-arm64 build-binaries-ppc64le
Expand All @@ -316,10 +320,13 @@ docker-images-cassandra:
docker build -t $(DOCKER_NAMESPACE)/jaeger-cassandra-schema:${DOCKER_TAG} plugin/storage/cassandra/
@echo "Finished building jaeger-cassandra-schema =============="

docker-images-elastic: TARGET = release
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I thought --target would be useful if a Dockerfile contained multiple build stages like debug and release. However, from what I see, there is only a release build stage in cmd/es-index-cleaner/Dockerfile. Would it be useful to still include the target param in this case?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

True, this can be removed. Done.


.PHONY: docker-images-elastic
docker-images-elastic:
docker-images-elastic: create-baseimg
GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-esmapping-generator
docker build -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} plugin/storage/es
GOOS=linux GOARCH=$(GOARCH) $(MAKE) build-es-index-cleaner
docker build --target $(TARGET) -t $(DOCKER_NAMESPACE)/jaeger-es-index-cleaner:${DOCKER_TAG} --build-arg base_image=$(BASE_IMAGE) --build-arg TARGETARCH=$(GOARCH) cmd/es-index-cleaner
docker build -t $(DOCKER_NAMESPACE)/jaeger-es-rollover:${DOCKER_TAG} plugin/storage/es -f plugin/storage/es/Dockerfile.rollover --build-arg TARGETARCH=$(GOARCH)
@echo "Finished building jaeger-es-indices-clean =============="

Expand Down
6 changes: 6 additions & 0 deletions cmd/es-index-cleaner/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
ARG base_image
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Ashmita152, are you interested in reviewing this?


FROM $base_image AS release
ARG TARGETARCH
COPY es-index-cleaner-linux-$TARGETARCH /go/bin/es-index-cleaner-linux
ENTRYPOINT ["/go/bin/es-index-cleaner-linux"]
2 changes: 1 addition & 1 deletion cmd/es-index-cleaner/app/flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ type Config struct {
// AddFlags adds flags for TLS to the FlagSet.
func (c *Config) AddFlags(flags *flag.FlagSet) {
flags.String(indexPrefix, "", "Index prefix")
flags.Bool(archive, false, "Whether to remove archive indices")
flags.Bool(archive, false, "Whether to remove archive indices. It works only for rollover")
flags.Bool(rollover, false, "Whether to remove indices created by rollover")
flags.Int(timeout, 120, "Number of seconds to wait for master node response")
flags.String(indexDateSeparator, "-", "Index date separator")
Expand Down
15 changes: 5 additions & 10 deletions cmd/es-index-cleaner/app/index_filter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,18 +42,13 @@ func (i *IndexFilter) Filter(indices []Index) []Index {

func (i *IndexFilter) filter(indices []Index) []Index {
var reg *regexp.Regexp
if !i.Rollover && !i.Archive {
// daily indices
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator))
} else if !i.Rollover && i.Archive {
// daily archive
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive", i.IndexPrefix))
} else if i.Rollover && !i.Archive {
// rollover
if i.Archive {
// archive works only for rollover
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix))
pavolloffay marked this conversation as resolved.
Show resolved Hide resolved
} else if i.Rollover {
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service)-\\d{6}", i.IndexPrefix))
} else {
// rollover archive
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-span-archive-\\d{6}", i.IndexPrefix))
reg, _ = regexp.Compile(fmt.Sprintf("^%sjaeger-(span|service|dependencies)-\\d{4}%s\\d{2}%s\\d{2}", i.IndexPrefix, i.IndexDateSeparator, i.IndexDateSeparator))
}

var filtered []Index
Expand Down
12 changes: 7 additions & 5 deletions cmd/es-index-cleaner/app/index_filter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,19 +229,21 @@ func testIndexFilter(t *testing.T, prefix string) {
},
},
{
name: "archive indices, remove older 2 days",
name: "archive indices, remove older 1 days - archive works only for rollover",
filter: &IndexFilter{
IndexPrefix: prefix,
IndexDateSeparator: "-",
Archive: true,
Rollover: false,
DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(2)),
DeleteBeforeThisDate: time20200807.Add(-time.Hour * 24 * time.Duration(1)),
},
expected: []Index{
{
Index: prefix + "jaeger-span-archive",
CreationTime: time.Date(2020, time.August, 0, 15, 0, 0, 0, time.UTC),
Aliases: map[string]bool{},
Index: prefix + "jaeger-span-archive-000001",
CreationTime: time.Date(2020, time.August, 5, 15, 0, 0, 0, time.UTC),
Aliases: map[string]bool{
prefix + "jaeger-span-archive-read": true,
},
},
},
},
Expand Down
11 changes: 0 additions & 11 deletions plugin/storage/es/Dockerfile
View file
Open in desktop

This file was deleted.

117 changes: 0 additions & 117 deletions plugin/storage/es/esCleaner.py
View file
Open in desktop

This file was deleted.

4 changes: 2 additions & 2 deletions plugin/storage/integration/es_index_cleaner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,10 @@ func TestIndexCleaner(t *testing.T) {
},
}
for _, test := range tests {
t.Run(fmt.Sprintf("%s_no_prefix", test.name), func(t *testing.T) {
t.Run(fmt.Sprintf("%s_no_prefix, %s", test.name, test.envVars), func(t *testing.T) {
runIndexCleanerTest(t, client, "", test.expectedIndices, test.envVars)
})
t.Run(fmt.Sprintf("%s_prefix", test.name), func(t *testing.T) {
t.Run(fmt.Sprintf("%s_prefix, %s", test.name, test.envVars), func(t *testing.T) {
runIndexCleanerTest(t, client, indexPrefix, test.expectedIndices, append(test.envVars, "INDEX_PREFIX="+indexPrefix))
})
}
Expand Down
2 changes: 1 addition & 1 deletion scripts/build-upload-docker-images.sh
100644 → 100755
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ do
done

# build/upload images for jaeger-es-index-cleaner and jaeger-es-rollover
bash scripts/build-upload-a-docker-image.sh -c jaeger-es-index-cleaner -d plugin/storage/es -p "${platforms}"
bash scripts/build-upload-a-docker-image.sh -b -c jaeger-es-index-cleaner -d cmd/es-index-cleaner -p "${platforms}" -t release
bash scripts/build-upload-a-docker-image.sh -c jaeger-es-rollover -d plugin/storage/es -f Dockerfile.rollover -p "${platforms}"

# build/upload images for jaeger-tracegen and jaeger-anonymizer
Expand Down