diff --git a/.github/workflows/helm-ci.yml b/.github/workflows/helm-ci.yml new file mode 100644 index 00000000000..10fb765d251 --- /dev/null +++ b/.github/workflows/helm-ci.yml @@ -0,0 +1,14 @@ +name: helm-ci + +on: pull_request + +jobs: + call-lint: + uses: grafana/helm-charts/.github/workflows/linter.yml@main + with: + filter_regex_include: .*operations/helm/.* + call-lint-test: + uses: grafana/helm-charts/.github/workflows/lint-test.yaml@main + with: + ct_configfile: operations/helm/ct.yaml + ct_check_version_increment: false diff --git a/.github/workflows/helm-release.yaml b/.github/workflows/helm-release.yaml new file mode 100644 index 00000000000..19c9f40c775 --- /dev/null +++ b/.github/workflows/helm-release.yaml @@ -0,0 +1,19 @@ +name: helm-release + +on: + push: + branches: + - main + - "release-[0-9]+.[0-9]+" + +jobs: + call-update-helm-repo: + uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main + with: + charts_dir: operations/helm/charts + cr_configfile: operations/helm/cr.yaml + ct_configfile: operations/helm/ct.yaml + secrets: + helm_repo_token: ${{ secrets.GH_BOT_ACCESS_TOKEN }} + # See https://github.com/grafana/helm-charts/blob/main/INTERNAL.md about this key + gpg_key_base64: ${{ secrets.HELM_SIGN_KEY_BASE64 }} diff --git a/.github/workflows/publish-technical-documentation-next.yml b/.github/workflows/publish-technical-documentation-next.yml new file mode 100644 index 00000000000..2d6516914e2 --- /dev/null +++ b/.github/workflows/publish-technical-documentation-next.yml @@ -0,0 +1,39 @@ +name: "publish-technical-documentation-next" + +on: + push: + branches: + - "main" + paths: + - "docs/sources/**" + workflow_dispatch: +jobs: + test: + runs-on: "ubuntu-latest" + steps: + - name: "Check out code" + uses: "actions/checkout@v3" + - name: "Build website" + run: | + docker run -v ${PWD}/docs/sources:/hugo/content/docs/mimir/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo' + + sync: + runs-on: "ubuntu-latest" + needs: "test" + steps: + - name: "Check out code" + uses: "actions/checkout@v3" + + - name: "Clone website-sync Action" + run: "git clone --single-branch --no-tags --depth 1 -b master https://grafanabot:${{ secrets.GH_BOT_ACCESS_TOKEN }}@github.com/grafana/website-sync ./.github/actions/website-sync" + + - name: "Publish to website repository (next)" + uses: "./.github/actions/website-sync" + id: "publish-next" + with: + repository: "grafana/website" + branch: "master" + host: "github.com" + github_pat: "${{ secrets.GH_BOT_ACCESS_TOKEN }}" + source_folder: "docs/sources" + target_folder: "content/docs/mimir/next" diff --git a/.github/workflows/publish-technical-documentation.yml b/.github/workflows/publish-technical-documentation-release.yml similarity index 56% rename from .github/workflows/publish-technical-documentation.yml rename to .github/workflows/publish-technical-documentation-release.yml index 06140ef15f9..1214a8e4d4d 100644 --- a/.github/workflows/publish-technical-documentation.yml +++ b/.github/workflows/publish-technical-documentation-release.yml @@ -1,9 +1,8 @@ -name: "publish-technical-documentation" +name: "publish-technical-documentation-release" on: push: branches: - - "main" - "release-*" tags: - "mimir-[0-9]+.[0-9]+.[0-9]+" @@ -38,49 +37,29 @@ jobs: - name: "Install Actions from library" run: "npm install --production --prefix ./actions" - - name: "Determine technical documentation version" - uses: "./actions/docs-target" - id: "target" + - name: "Determine if there is a matching release tag" + id: "has-matching-release-tag" + uses: "./actions/has-matching-release-tag" with: ref_name: "${{ github.ref_name }}" + release_tag_regexp: "^mimir-(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$" + release_branch_regexp: "^release-(0|[1-9]\\d*)\\.(0|[1-9]\\d*)$" - - name: "Determine latest release tag" - id: "latest-released" - shell: "bash" - run: | - tag="$(git tag --sort=-v:refname | grep -E '^mimir-(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$' | head -n1)" - echo "::set-output name=tag::${tag}" - - - name: "Determine latest release tag target" + - name: "Determine technical documentation version" + if: "steps.has-matching-release-tag.outputs.bool == 'true'" uses: "./actions/docs-target" - id: "latest-released-target" + id: "target" with: - ref_name: "${{ steps.latest-released.outputs.tag }}" + ref_name: "${{ github.ref_name }}" - name: "Clone website-sync Action" + if: "steps.has-matching-release-tag.outputs.bool == 'true'" run: "git clone --single-branch --no-tags --depth 1 -b master https://grafanabot:${{ secrets.GH_BOT_ACCESS_TOKEN }}@github.com/grafana/website-sync ./.github/actions/website-sync" - - name: "Publish to website repository (next)" - uses: "./.github/actions/website-sync" - id: "publish-next" - # Only publish on main branch. - if: "github.ref == 'refs/heads/main'" - with: - repository: "grafana/website" - branch: "master" - host: "github.com" - github_pat: "${{ secrets.GH_BOT_ACCESS_TOKEN }}" - source_folder: "docs/sources" - target_folder: "content/docs/mimir/next" - - name: "Publish to website repository (release)" + if: "steps.has-matching-release-tag.outputs.bool == 'true'" uses: "./.github/actions/website-sync" id: "publish-release" - # Only publish if the target of the latest release tag matches the target of this reference. - # For release tags the tag and reference are the same. - # For release branches, this guard prevents publishing documentation before a tag - # has been pushed that has officially "released" that documentation. - if: "steps.latest-released-target.outputs.target == steps.target.outputs.target" with: repository: "grafana/website" branch: "master" diff --git a/.github/workflows/scripts/build-images.sh b/.github/workflows/scripts/build-images.sh new file mode 100755 index 00000000000..c120eaf3522 --- /dev/null +++ b/.github/workflows/scripts/build-images.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only + +set -o errexit +set -o nounset +set -o pipefail + +OUTPUT="$1" +shift + +mkdir "$OUTPUT" + +# Build all supplied targets (will be prefixed with push-multiarch- prefix) +# shellcheck disable=SC2068 +for target in $@ +do + DIRNAME="$(dirname "$target")" + NAME="$(basename "$DIRNAME")" + make BUILD_IN_CONTAINER=false PUSH_MULTIARCH_TARGET="type=oci,dest=$OUTPUT/$NAME.oci" push-multiarch-$target +done diff --git a/.github/workflows/scripts/install-docker.sh b/.github/workflows/scripts/install-docker.sh index fa1bf70d69b..66f17a385f1 100755 --- a/.github/workflows/scripts/install-docker.sh +++ b/.github/workflows/scripts/install-docker.sh @@ -1,7 +1,8 @@ #!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only set -x -VER="18.06.3-ce" +VER="20.10.14" curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$VER.tgz tar -xz -C /tmp -f /tmp/docker-$VER.tgz mv /tmp/docker/* /usr/bin diff --git a/.github/workflows/scripts/push-images.sh b/.github/workflows/scripts/push-images.sh new file mode 100755 index 00000000000..48d94e90daa --- /dev/null +++ b/.github/workflows/scripts/push-images.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only + +set -o errexit +set -o nounset +set -o pipefail + +IMAGES_DIR="$1" +IMAGE_PREFIX="$2" +TAG="$3" + +# If image is the latest stable git tag, also push :latest image. +# Do not tag with latest any release candidate (tag ends with "-rc.*"). +EXTRA_TAG="" +if [[ "$(git tag | grep -E '^mimir-[0-9]+\.[0-9]+\.[0-9]+$' | sort -V | tail -n 1)" == "mimir-${TAG}" ]]; then + EXTRA_TAG="latest" +fi + +# Push images from OCI archives to docker registry. +for image in "$IMAGES_DIR"/* +do + NAME=$(basename ${image%%.oci}) + # --all uploads all platform images from OCI + echo + echo "Uploading ${IMAGE_PREFIX}${NAME}:${TAG}" + echo + skopeo copy --all --retry-times 3 oci-archive:${image} "docker://${IMAGE_PREFIX}${NAME}:${TAG}" + + if [[ -n "${EXTRA_TAG}" ]]; then + echo "Tagging with ${EXTRA_TAG}" + skopeo copy --all --retry-times 3 "docker://${IMAGE_PREFIX}${NAME}:${TAG}" "docker://${IMAGE_PREFIX}${NAME}:${EXTRA_TAG}" + fi +done diff --git a/.github/workflows/scripts/run-integration-tests-group.sh b/.github/workflows/scripts/run-integration-tests-group.sh index 3d6db75519c..10850ec4c62 100755 --- a/.github/workflows/scripts/run-integration-tests-group.sh +++ b/.github/workflows/scripts/run-integration-tests-group.sh @@ -1,4 +1,5 @@ #!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only SCRIPT_DIR=$(cd `dirname $0` && pwd) INTEGRATION_DIR=$(realpath "${SCRIPT_DIR}/../../../integration/") diff --git a/.github/workflows/scripts/run-unit-tests-group.sh b/.github/workflows/scripts/run-unit-tests-group.sh index 5ecb3d7127f..c1cd10e2c2b 100755 --- a/.github/workflows/scripts/run-unit-tests-group.sh +++ b/.github/workflows/scripts/run-unit-tests-group.sh @@ -1,4 +1,5 @@ #!/bin/bash +# SPDX-License-Identifier: AGPL-3.0-only SCRIPT_DIR=$(cd `dirname $0` && pwd) MIMIR_DIR=$(realpath "${SCRIPT_DIR}/../../../") diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index f5b34189c3a..4a1617a7e6e 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -18,9 +18,9 @@ concurrency: jobs: lint: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest container: - image: grafana/mimir-build-image:update-go-1.17.8-8a996bb57 + image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -48,9 +48,9 @@ jobs: run: make BUILD_IN_CONTAINER=false check-tsdb-blocks-storage-s3-docker-compose-yaml lint-jsonnet: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest container: - image: grafana/mimir-build-image:update-go-1.17.8-8a996bb57 + image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -71,8 +71,28 @@ jobs: - name: Check Jsonnet Tests run: make BUILD_IN_CONTAINER=false check-jsonnet-tests + lint-helm: + runs-on: ubuntu-latest + container: + image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d + steps: + - name: Checkout Repo + uses: actions/checkout@v2 + # Commands in the Makefile are hardcoded with an assumed file structure of the CI container + # Symlink ensures paths specified in previous commands don’t break + - name: Symlink Expected Path to Workspace + run: | + mkdir -p /go/src/github.com/grafana/mimir + ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir + - name: Set up Helm + uses: azure/setup-helm@v1 + with: + version: v3.5.2 + - name: Check Helm Tests + run: make BUILD_IN_CONTAINER=false check-helm-tests + test: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest strategy: # Do not abort other groups when one fails. fail-fast: false @@ -81,7 +101,7 @@ jobs: test_group_id: [0, 1, 2, 3] test_group_total: [4] container: - image: grafana/mimir-build-image:update-go-1.17.8-8a996bb57 + image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -104,68 +124,40 @@ jobs: run: | docker run -v ${PWD}/docs/sources:/hugo/content/docs/mimir/latest -e HUGO_REFLINKSERRORLEVEL=ERROR --rm grafana/docs-base:latest /bin/bash -c 'make hugo' - build-mimir: - runs-on: ubuntu-20.04 - container: - image: grafana/mimir-build-image:update-go-1.17.8-8a996bb57 - steps: - - name: Checkout Repo - uses: actions/checkout@v2 - - name: Install Docker Client - run: ./.github/workflows/scripts/install-docker.sh - - name: Symlink Expected Path to Workspace - run: | - mkdir -p /go/src/github.com/grafana/mimir - ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir - - name: Build Image - run: | - make BUILD_IN_CONTAINER=false ./cmd/mimir/.uptodate - - name: Save Images - run: | - mkdir /tmp/images - ln -s /tmp/images ./docker-images - make BUILD_IN_CONTAINER=false save-images - - name: Create Docker Images Archive - run: tar -cvf mimir-images.tar /tmp/images - - name: Upload Docker Images Artifact - uses: actions/upload-artifact@v2 - with: - name: Docker Images Mimir - path: ./mimir-images.tar - - build-tools: - runs-on: ubuntu-20.04 + build: + runs-on: ubuntu-latest container: - image: grafana/mimir-build-image:update-go-1.17.8-8a996bb57 + image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d steps: - name: Checkout Repo uses: actions/checkout@v2 - name: Install Docker Client run: ./.github/workflows/scripts/install-docker.sh + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1 - name: Symlink Expected Path to Workspace run: | mkdir -p /go/src/github.com/grafana/mimir ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir - - name: Build Images - # Build everything except mimir (run by build-mimir job) and build image (not managed by CI). + - name: Build Multiarch Docker Images Locally + # Ignore mimir-build-image and mimir-rules-action. run: | - make list-image-targets | grep -v -E '/mimir-build-image/|/cmd/mimir/|/mimir-mixin-tools/' | xargs -I {} make BUILD_IN_CONTAINER=false {} - - name: Save Images + ./.github/workflows/scripts/build-images.sh /tmp/images $(make list-image-targets | grep -v -E '/mimir-build-image/|/mimir-rules-action/') + - name: Build Archive With Docker Images run: | - mkdir /tmp/images - ln -s /tmp/images ./docker-images - make BUILD_IN_CONTAINER=false save-images - - name: Create Docker Images Archive - run: tar -cvf tools-images.tar /tmp/images - - name: Upload Docker Image Artifact + tar cvf images.tar /tmp/images + - name: Upload Archive with Docker Images uses: actions/upload-artifact@v2 with: - name: Docker Images Tools - path: ./tools-images.tar + name: Docker Images + path: ./images.tar integration: - needs: build-mimir - runs-on: ubuntu-20.04 + needs: build + runs-on: ubuntu-latest strategy: # Do not abort other groups when one fails. fail-fast: false @@ -186,16 +178,19 @@ jobs: run: | sudo mkdir -p /go/src/github.com/grafana/mimir sudo ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir - - name: Download Docker Images Mimir Artifacts + - name: Download Archive with Docker Images uses: actions/download-artifact@v2 with: - name: Docker Images Mimir - - name: Extract Docker Image Archive - run: tar -xvf mimir-images.tar -C / - - name: Load Docker Images + name: Docker Images + - name: Extract Docker Images from Archive + run: tar xvf images.tar -C / + - name: Load Mimir Image into Docker run: | - ln -s /tmp/images ./docker-images - make BUILD_IN_CONTAINER=false load-images + export IMAGE_TAG=$(make image-tag) + # skopeo will by default load system-specific version of the image (linux/amd64). + skopeo copy oci-archive:/tmp/images/mimir.oci "docker-daemon:grafana/mimir:$IMAGE_TAG" + # Print Mimir version and architecture loaded to Docker. + docker run "grafana/mimir:$IMAGE_TAG" --version - name: Preload Images # We download docker images used by integration tests so that all images are available # locally and the download time doesn't account in the test execution time, which is subject @@ -211,12 +206,12 @@ jobs: ./.github/workflows/scripts/run-integration-tests-group.sh --index ${{ matrix.test_group_id }} --total ${{ matrix.test_group_total }} deploy: - needs: [build-mimir, build-tools, test, lint, integration] + needs: [build, test, lint, integration] # Only deploy images on pushes to the grafana/mimir repo, which either are tag pushes or weekly release branch pushes. if: (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/r') ) && github.event_name == 'push' && github.repository == 'grafana/mimir' - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest container: - image: grafana/mimir-build-image:update-go-1.17.8-8a996bb57 + image: grafana/mimir-build-image:publish-multiarch-images-7a4b40a6d steps: - name: Checkout Repo uses: actions/checkout@v2 @@ -226,29 +221,18 @@ jobs: run: | mkdir -p /go/src/github.com/grafana/mimir ln -s $GITHUB_WORKSPACE/* /go/src/github.com/grafana/mimir - - name: Download Docker Tool Images Artifacts + - name: Download Archive with Docker Images uses: actions/download-artifact@v2 with: - name: Docker Images Tools - - name: Download Docker Mimir Image Artifacts - uses: actions/download-artifact@v2 - with: - name: Docker Images Mimir - - name: Extract Docker Images - run: | - tar -xvf mimir-images.tar -C / - tar -xvf tools-images.tar -C / - - name: Load Images - run: | - ln -s /tmp/images ./docker-images - make BUILD_IN_CONTAINER=false load-images + name: Docker Images + - name: Extract Docker Images from Archive + run: tar xvf images.tar -C / - name: Deploy run: | if [ -n "$DOCKER_PASSWORD" ]; then - printenv DOCKER_PASSWORD | docker login -u "$DOCKER_USERNAME" --password-stdin + printenv DOCKER_PASSWORD | skopeo login -u "$DOCKER_USERNAME" --password-stdin docker.io fi - export IMAGE_TAG=$(make image-tag) - ./push-images + ./.github/workflows/scripts/push-images.sh /tmp/images grafana/ $(make image-tag) env: DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.gitignore b/.gitignore index 19578645a63..c1981871785 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,19 @@ cmd/mimirtool/mimirtool +cmd/mimirtool/mimirtool_linux_amd64 +cmd/mimirtool/mimirtool_linux_arm64 cmd/test-exporter/test-exporter cmd/mimir/mimir +cmd/mimir/mimir_linux_amd64 +cmd/mimir/mimir_linux_arm64 cmd/query-tee/query-tee +cmd/query-tee/query-tee_linux_amd64 +cmd/query-tee/query-tee_linux_arm64 cmd/metaconvert/metaconvert +cmd/metaconvert/metaconvert_linux_amd64 +cmd/metaconvert/metaconvert_linux_arm64 cmd/mimir-continuous-test/mimir-continuous-test +cmd/mimir-continuous-test/mimir-continuous-test_linux_amd64 +cmd/mimir-continuous-test/mimir-continuous-test_linux_arm64 .uptodate .pkg .cache diff --git a/.prettierignore b/.prettierignore index e2d7eb7e6a2..862d9ab5c92 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,3 +1,5 @@ CHANGELOG.md vendor .*/ +# The README.md of helm chart is generated, but tables are formatted differently (helm-docs 1.8.1) +operations/helm/charts/mimir-distributed/README.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 4adc26a6650..c5366a58616 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,27 +2,137 @@ ## Grafana Mimir - main / unreleased +### Grafana Mimir + +* [CHANGE] Increased default configuration for `-server.grpc-max-recv-msg-size-bytes` and `-server.grpc-max-send-msg-size-bytes` from 4MB to 100MB. #1883 +* [CHANGE] Default values have changed for the following settings. This improves query performance for recent data (within 12h) by only reading from ingesters: #1909 #1921 + - `-blocks-storage.bucket-store.ignore-blocks-within` now defaults to `10h` (previously `0`) + - `-querier.query-store-after` now defaults to `12h` (previously `0`) + - `-querier.shuffle-sharding-ingesters-lookback-period` now defaults to `13h` (previously `0`) +* [CHANGE] The following settings are now classified as advanced because the defaults should work for most users and tuning them requires in-depth knowledge of how the read path works: #1929 + - `-querier.query-ingesters-within` + - `-querier.query-store-after` +* [CHANGE] Config flag category overrides can be set dynamically at runtime. #1934 +* [CHANGE] Ingester: deprecated `-ingester.ring.join-after`. Mimir now behaves as this setting is always set to 0s. This configuration option will be removed in Mimir 2.4.0. #1965 +* [CHANGE] Blocks uploaded by ingester no longer contain `__org_id__` label. Compactor now ignores this label and will compact blocks with and without this label together. `mimirconvert` tool will remove the label from blocks as "unknown" label. #1972 +* [ENHANCEMENT] Distributor: Added limit to prevent tenants from sending excessive number of requests: #1843 + * The following CLI flags (and their respective YAML config options) have been added: + * `-distributor.request-rate-limit` + * `-distributor.request-burst-limit` + * The following metric is exposed to tell how many requests have been rejected: + * `cortex_discarded_requests_total` +* [ENHANCEMENT] Store-gateway: Add the experimental ability to run requests in a dedicated OS thread pool. This feature can be configured using `-store-gateway.thread-pool-size` and is disabled by default. Replaces the ability to run index header operations in a dedicated thread pool. #1660 #1812 +* [ENHANCEMENT] Improved error messages to make them easier to understand; each now have a unique, global identifier that you can use to look up in the runbooks for more information. #1907 #1919 #1888 #1939 #1984 #2009 +* [ENHANCEMENT] Memberlist KV: incoming messages are now processed on per-key goroutine. This may reduce loss of "maintanance" packets in busy memberlist installations, but use more CPU. New `memberlist_client_received_broadcasts_dropped_total` counter tracks number of dropped per-key messages. #1912 +* [ENHANCEMENT] Blocks Storage, Alertmanager, Ruler: add support a prefix to the bucket store (`*_storage.storage_prefix`). This enables using the same bucket for the three components. #1686 #1951 +* [ENHANCEMENT] Upgrade Docker base images to `alpine:3.16.0`. #2028 +* [ENHANCEMENT] Store-gateway: Add experimental configuration option for the store-gateway to attempt to pre-populate the file system cache when memory-mapping index-header files. Enabled with `-blocks-storage.bucket-store.index-header.map-populate-enabled=true`. Note this flag only has an effect when running on Linux. #2019 #2054 +* [ENHANCEMENT] Chunk Mapper: reduce memory usage of async chunk mapper. #2043 +* [BUGFIX] Fix regexp parsing panic for regexp label matchers with start/end quantifiers. #1883 +* [BUGFIX] Ingester: fixed deceiving error log "failed to update cached shipped blocks after shipper initialisation", occurring for each new tenant in the ingester. #1893 +* [BUGFIX] Ring: fix bug where instances may appear unhealthy in the hash ring web UI even though they are not. #1933 +* [BUGFIX] API: gzip is now enforced when identity encoding is explicitly rejected. #1864 +* [BUGFIX] Fix panic at startup when Mimir is running in monolithic mode and query sharding is enabled. #2036 +* [BUGFIX] Ruler: report failed evaluation metric for any 5xx status code returned by the query-frontend when remote operational mode is enabled. #2053 + +### Mixin + +* [CHANGE] Split `mimir_queries` rules group into `mimir_queries` and `mimir_ingester_queries` to keep number of rules per group within the default per-tenant limit. #1885 +* [CHANGE] Dashboards: Expose full image tag in "Mimir / Rollout progress" dashboard's "Pod per version panel." #1932 +* [CHANGE] Dashboards: Disabled gateway panels by default, because most users don't have a gateway exposing the metrics expected by Mimir dashboards. You can re-enable it setting `gateway_enabled: true` in the mixin config and recompiling the mixin running `make build-mixin`. #1954 +* [CHANGE] Alerts: adapt `MimirFrontendQueriesStuck` and `MimirSchedulerQueriesStuck` to consider ruler query path components. #1949 +* [ENHANCEMENT] Dashboards: Add config option `datasource_regex` to customise the regular expression used to select valid datasources for Mimir dashboards. #1802 +* [ENHANCEMENT] Dashboards: Added "Mimir / Remote ruler reads" and "Mimir / Remote ruler reads resources" dashboards. #1911 #1937 +* [ENHANCEMENT] Dashboards: Make networking panels work for pods created by the mimir-distributed helm chart. #1927 +* [ENHANCEMENT] Alerts: Add `MimirStoreGatewayNoSyncedTenants` alert that fires when there is a store-gateway owning no tenants. #1882 +* [BUGFIX] Fix `container_memory_usage_bytes:sum` recording rule #1865 +* [BUGFIX] Fix `MimirGossipMembersMismatch` alerts if Mimir alertmanager is activated #1870 +* [BUGFIX] Fix `MimirRulerMissedEvaluations` to show % of missed alerts as a value between 0 and 100 instead of 0 and 1. #1895 +* [BUGFIX] Fix `MimirCompactorHasNotUploadedBlocks` alert false positive when Mimir is deployed in monolithic mode. #1901 +* [BUGFIX] Fix `MimirGossipMembersMismatch` to make it less sensitive during rollouts and fire one alert per installation, not per job. #1926 +* [BUGFIX] Do not trigger `MimirAllocatingTooMuchMemory` alerts if no container limits are supplied. #1905 +* [BUGFIX] Dashboards: Remove empty "Chunks per query" panel from `Mimir / Queries` dashboard. #1928 +* [BUGFIX] Dashboards: Use Grafana's `$__rate_interval` for rate queries in dashboards to support scrape intervals of >15s. #2011 + +### Jsonnet + +* [CHANGE] Remove use of `-querier.query-store-after`, `-querier.shuffle-sharding-ingesters-lookback-period`, `-blocks-storage.bucket-store.ignore-blocks-within`, and `-blocks-storage.tsdb.close-idle-tsdb-timeout` CLI flags since the values now match defaults. #1915 #1921 +* [CHANGE] Change default value for `-blocks-storage.bucket-store.chunks-cache.memcached.timeout` to `450ms` to increase use of cached data. #2035 +* [FEATURE] Added querier autoscaling support. It requires [KEDA](https://keda.sh) installed in the Kubernetes cluster and query-scheduler enabled in the Mimir cluster. Querier autoscaler can be enabled and configure through the following options in the jsonnet config: #2013 #2023 + * `autoscaling_querier_enabled`: `true` to enable autoscaling. + * `autoscaling_querier_min_replicas`: minimum number of querier replicas. + * `autoscaling_querier_max_replicas`: maximum number of querier replicas. + * `autoscaling_prometheus_url`: Prometheus base URL from which to scrape Mimir metrics (e.g. `http://prometheus.default:9090/prometheus`). +* [ENHANCEMENT] Added `compactor` service, that can be used to route requests directly to compactor (e.g. admin UI). #2063 + +### Mimirtool + +* [BUGFIX] mimirtool analyze: Fix dashboard JSON unmarshalling errors (#1840). #1973 + +### Mimir Continuous Test + +* [ENHANCEMENT] Added the `-tests.smoke-test` flag to run the `mimir-continuous-test` suite once and immediately exit. #2047 + +### Documentation + +* [ENHANCEMENT] Published Grafana Mimir runbooks as part of documentation. #1970 +* [ENHANCEMENT] Improved ruler's "remote operational mode" documentation. #1906 +* [ENHANCEMENT] Recommend fast disks for ingesters and store-gateways in production tips. #1903 +* [ENHANCEMENT] Explain the runtime override of active series matchers. #1868 +* [ENHANCEMENT] Clarify "Set rule group" API specification. #1869 +* [BUGFIX] Fixed ruler configuration used in the getting started guide. #2052 + +## 2.1.0 +### Grafana Mimir * [CHANGE] Compactor: No longer upload debug meta files to object storage. #1257 * [CHANGE] Default values have changed for the following settings: #1547 - `-alertmanager.alertmanager-client.grpc-max-recv-msg-size` now defaults to 100 MiB (previously was not configurable and set to 16 MiB) - `-alertmanager.alertmanager-client.grpc-max-send-msg-size` now defaults to 100 MiB (previously was not configurable and set to 4 MiB) - `-alertmanager.max-recv-msg-size` now defaults to 100 MiB (previously was 16 MiB) * [CHANGE] Ingester: Add `user` label to metrics `cortex_ingester_ingested_samples_total` and `cortex_ingester_ingested_samples_failures_total`. #1533 -* [CHANGE] Ingester: Changed `-blocks-storage.tsdb.isolation-enabled` default from `true` to `false`. The config option has also been deprecated and will be removed in 2 minor version. +* [CHANGE] Ingester: Changed `-blocks-storage.tsdb.isolation-enabled` default from `true` to `false`. The config option has also been deprecated and will be removed in 2 minor version. #1655 * [CHANGE] Query-frontend: results cache keys are now versioned, this will cause cache to be re-filled when rolling out this version. #1631 * [CHANGE] Store-gateway: enabled attributes in-memory cache by default. New default configuration is `-blocks-storage.bucket-store.chunks-cache.attributes-in-memory-max-items=50000`. #1727 * [CHANGE] Compactor: Removed the metric `cortex_compactor_garbage_collected_blocks_total` since it duplicates `cortex_compactor_blocks_marked_for_deletion_total`. #1728 -* [CHANGE] All: Logs that used the`org_id` label now use `user` label. #1634 +* [CHANGE] All: Logs that used the`org_id` label now use `user` label. #1634 #1758 +* [CHANGE] Alertmanager: the following metrics are not exported for a given `user` and `integration` when the metric value is zero: #1783 + * `cortex_alertmanager_notifications_total` + * `cortex_alertmanager_notifications_failed_total` + * `cortex_alertmanager_notification_requests_total` + * `cortex_alertmanager_notification_requests_failed_total` + * `cortex_alertmanager_notification_rate_limited_total` +* [CHANGE] Removed the following metrics exposed by the Mimir hash rings: #1791 + * `cortex_member_ring_tokens_owned` + * `cortex_member_ring_tokens_to_own` + * `cortex_ring_tokens_owned` + * `cortex_ring_member_ownership_percent` +* [CHANGE] Querier / Ruler: removed the following metrics tracking number of query requests send to each ingester. You can use `cortex_request_duration_seconds_count{route=~"/cortex.Ingester/(QueryStream|QueryExemplars)"}` instead. #1797 + * `cortex_distributor_ingester_queries_total` + * `cortex_distributor_ingester_query_failures_total` +* [CHANGE] Distributor: removed the following metrics tracking the number of requests from a distributor to ingesters: #1799 + * `cortex_distributor_ingester_appends_total` + * `cortex_distributor_ingester_append_failures_total` +* [CHANGE] Distributor / Ruler: deprecated `-distributor.extend-writes`. Now Mimir always behaves as if this setting was set to `false`, which we expect to be safe for every Mimir cluster setup. #1856 +* [FEATURE] Querier: Added support for [streaming remote read](https://prometheus.io/blog/2019/10/10/remote-read-meets-streaming/). Should be noted that benefits of chunking the response are partial here, since in a typical `query-frontend` setup responses will be buffered until they've been completed. #1735 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 -* [FEATURE] Ruler: Added support for expression remote evaluation. #1536 +* [FEATURE] Ruler: Added support for expression remote evaluation. #1536 #1818 * The following CLI flags (and their respective YAML config options) have been added: * `-ruler.query-frontend.address` - * `-ruler.query-frontend.tls-enabled` - * `-ruler.query-frontend.tls-ca-path` - * `-ruler.query-frontend.tls-cert-path` - * `-ruler.query-frontend.tls-key-path` - * `-ruler.query-frontend.tls-server-name` - * `-ruler.query-frontend.tls-insecure-skip-verify` + * `-ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size` + * `-ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size` + * `-ruler.query-frontend.grpc-client-config.grpc-compression` + * `-ruler.query-frontend.grpc-client-config.grpc-client-rate-limit` + * `-ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst` + * `-ruler.query-frontend.grpc-client-config.backoff-on-ratelimits` + * `-ruler.query-frontend.grpc-client-config.backoff-min-period` + * `-ruler.query-frontend.grpc-client-config.backoff-max-period` + * `-ruler.query-frontend.grpc-client-config.backoff-retries` + * `-ruler.query-frontend.grpc-client-config.tls-enabled` + * `-ruler.query-frontend.grpc-client-config.tls-ca-path` + * `-ruler.query-frontend.grpc-client-config.tls-cert-path` + * `-ruler.query-frontend.grpc-client-config.tls-key-path` + * `-ruler.query-frontend.grpc-client-config.tls-server-name` + * `-ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify` * [FEATURE] Distributor: Added the ability to forward specifics metrics to alternative remote_write API endpoints. #1052 * [FEATURE] Ingester: Active series custom trackers now supports runtime tenant-specific overrides. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 * [ENHANCEMENT] Alertmanager API: Concurrency limit for GET requests is now configurable using `-alertmanager.max-concurrent-get-requests-per-tenant`. #1547 @@ -37,21 +147,48 @@ - `-alertmanager.alertmanager-client.grpc-max-recv-msg-size` - `-alertmanager.alertmanager-client.grpc-max-send-msg-size` * [ENHANCEMENT] Ruler: Add more detailed query information to ruler query stats logging. #1411 -* [ENHANCEMENT] Admin: Admin API now has some styling. #1482 #1549 +* [ENHANCEMENT] Admin: Admin API now has some styling. #1482 #1549 #1821 #1824 * [ENHANCEMENT] Alertmanager: added `insight=true` field to alertmanager dispatch logs. #1379 * [ENHANCEMENT] Store-gateway: Add the experimental ability to run index header operations in a dedicated thread pool. This feature can be configured using `-blocks-storage.bucket-store.index-header-thread-pool-size` and is disabled by default. #1660 +* [ENHANCEMENT] Store-gateway: don't drop all blocks if instance finds itself as unhealthy or missing in the ring. #1806 #1823 +* [ENHANCEMENT] Querier: wait until inflight queries are completed when shutting down queriers. #1756 #1767 * [BUGFIX] Query-frontend: do not shard queries with a subquery unless the subquery is inside a shardable aggregation function call. #1542 * [BUGFIX] Query-frontend: added `component=query-frontend` label to results cache memcached metrics to fix a panic when Mimir is running in single binary mode and results cache is enabled. #1704 * [BUGFIX] Mimir: services' status content-type is now correctly set to `text/html`. #1575 * [BUGFIX] Multikv: Fix panic when using using runtime config to set primary KV store used by `multi` KV. #1587 * [BUGFIX] Multikv: Fix watching for runtime config changes in `multi` KV store in ruler and querier. #1665 * [BUGFIX] Memcached: allow to use CNAME DNS records for the memcached backend addresses. #1654 +* [BUGFIX] Querier: fixed temporary partial query results when shuffle sharding is enabled and hash ring backend storage is flushed / reset. #1829 +* [BUGFIX] Alertmanager: prevent more file traversal cases related to template names. #1833 +* [BUGFUX] Alertmanager: Allow usage with `-alertmanager-storage.backend=local`. Note that when using this storage type, the Alertmanager is not able persist state remotely, so it not recommended for production use. #1836 +* [BUGFIX] Alertmanager: Do not validate alertmanager configuration if it's not running. #1835 ### Mixin * [CHANGE] Dashboards: Remove per-user series legends from Tenants dashboard. #1605 * [CHANGE] Dashboards: Show in-memory series and the per-user series limit on Tenants dashboard. #1613 * [CHANGE] Dashboards: Slow-queries dashboard now uses `user` label from logs instead of `org_id`. #1634 +* [CHANGE] Dashboards: changed all Grafana dashboards UIDs to not conflict with Cortex ones, to let people install both while migrating from Cortex to Mimir: #1801 #1808 + * Alertmanager from `a76bee5913c97c918d9e56a3cc88cc28` to `b0d38d318bbddd80476246d4930f9e55` + * Alertmanager Resources from `68b66aed90ccab448009089544a8d6c6` to `a6883fb22799ac74479c7db872451092` + * Compactor from `9c408e1d55681ecb8a22c9fab46875cc` to `1b3443aea86db629e6efdb7d05c53823` + * Compactor Resources from `df9added6f1f4332f95848cca48ebd99` to `09a5c49e9cdb2f2b24c6d184574a07fd` + * Config from `61bb048ced9817b2d3e07677fb1c6290` to `5d9d0b4724c0f80d68467088ec61e003` + * Object Store from `d5a3a4489d57c733b5677fb55370a723` to `e1324ee2a434f4158c00a9ee279d3292` + * Overrides from `b5c95fee2e5e7c4b5930826ff6e89a12` to `1e2c358600ac53f09faea133f811b5bb` + * Queries from `d9931b1054053c8b972d320774bb8f1d` to `b3abe8d5c040395cc36615cb4334c92d` + * Reads from `8d6ba60eccc4b6eedfa329b24b1bd339` to `e327503188913dc38ad571c647eef643` + * Reads Networking from `c0464f0d8bd026f776c9006b05910000` to `54b2a0a4748b3bd1aefa92ce5559a1c2` + * Reads Resources from `2fd2cda9eea8d8af9fbc0a5960425120` to `cc86fd5aa9301c6528986572ad974db9` + * Rollout Progress from `7544a3a62b1be6ffd919fc990ab8ba8f` to `7f0b5567d543a1698e695b530eb7f5de` + * Ruler from `44d12bcb1f95661c6ab6bc946dfc3473` to `631e15d5d85afb2ca8e35d62984eeaa0` + * Scaling from `88c041017b96856c9176e07cf557bdcf` to `64bbad83507b7289b514725658e10352` + * Slow queries from `e6f3091e29d2636e3b8393447e925668` to `6089e1ce1e678788f46312a0a1e647e6` + * Tenants from `35fa247ce651ba189debf33d7ae41611` to `35fa247ce651ba189debf33d7ae41611` + * Top Tenants from `bc6e12d4fe540e4a1785b9d3ca0ffdd9` to `bc6e12d4fe540e4a1785b9d3ca0ffdd9` + * Writes from `0156f6d15aa234d452a33a4f13c838e3` to `8280707b8f16e7b87b840fc1cc92d4c5` + * Writes Networking from `681cd62b680b7154811fe73af55dcfd4` to `978c1cb452585c96697a238eaac7fe2d` + * Writes Resources from `c0464f0d8bd026f776c9006b0591bb0b` to `bc9160e50b52e89e0e49c840fea3d379` * [FEATURE] Alerts: added the following alerts on `mimir-continuous-test` tool: #1676 - `MimirContinuousTestNotRunningOnWrites` - `MimirContinuousTestNotRunningOnReads` @@ -60,12 +197,16 @@ * [ENHANCEMENT] Dashboards: Show QPS and latency of the Alertmanager Distributor. #1696 * [ENHANCEMENT] Playbooks: Add Alertmanager suggestions for `MimirRequestErrors` and `MimirRequestLatency` #1702 * [ENHANCEMENT] Dashboards: Allow custom datasources. #1749 +* [ENHANCEMENT] Dashboards: Add config option `gateway_enabled` (defaults to `true`) to disable gateway panels from dashboards. #1761 +* [ENHANCEMENT] Dashboards: Extend Top tenants dashboard with queries for tenants with highest sample rate, discard rate, and discard rate growth. #1842 +* [ENHANCEMENT] Dashboards: Show ingestion rate limit and rule group limit on Tenants dashboard. #1845 +* [ENHANCEMENT] Dashboards: Add "last successful run" panel to compactor dashboard. #1628 * [BUGFIX] Dashboards: Fix "Failed evaluation rate" panel on Tenants dashboard. #1629 * [BUGFIX] Honor the configured `per_instance_label` in all dashboards and alerts. #1697 ### Jsonnet -* [FEATURE] Added support for `mimir-continuous-test`. To deploy `mimir-continuous-test` you can use the following configuration: #1675 +* [FEATURE] Added support for `mimir-continuous-test`. To deploy `mimir-continuous-test` you can use the following configuration: #1675 #1850 ```jsonnet _config+: { continuous_test_enabled: true, @@ -78,16 +219,17 @@ * [ENHANCEMENT] Added `node_selector` configuration option to select Kubernetes nodes where Mimir should run. #1596 * [ENHANCEMENT] Alertmanager: Added a `PodDisruptionBudget` of `withMaxUnavailable = 1`, to ensure we maintain quorum during rollouts. #1683 * [ENHANCEMENT] Store-gateway anti-affinity can now be enabled/disabled using `store_gateway_allow_multiple_replicas_on_same_node` configuration key. #1730 +* [ENHANCEMENT] Added `store_gateway_zone_a_args`, `store_gateway_zone_b_args` and `store_gateway_zone_c_args` configuration options. #1807 * [BUGFIX] Pass primary and secondary multikv stores via CLI flags. Introduced new `multikv_switch_primary_secondary` config option to flip primary and secondary in runtime config. ### Mimirtool -* [BUGFIX] `config convert`: Retain Cortex defaults for `blocks_storage.backend`, `ruler_storage.backend`, and `alertmanager_storage.backend`. #1626 +* [BUGFIX] `config convert`: Retain Cortex defaults for `blocks_storage.backend`, `ruler_storage.backend`, `alertmanager_storage.backend`, `auth.type`, `activity_tracker.filepath`, `alertmanager.data_dir`, `blocks_storage.filesystem.dir`, `compactor.data_dir`, `ruler.rule_path`, `ruler_storage.filesystem.dir`, and `graphite.querier.schemas.backend`. #1626 #1762 ### Tools * [FEATURE] Added a `markblocks` tool that creates `no-compact` and `delete` marks for the blocks. #1551 -* [FEATURE] Added `mimir-continuous-test` tool to continuously run smoke tests on live Mimir clusters. #1535 #1540 #1653 #1603 #1630 #1691 #1675 #1676 #1692 #1706 #1709 +* [FEATURE] Added `mimir-continuous-test` tool to continuously run smoke tests on live Mimir clusters. #1535 #1540 #1653 #1603 #1630 #1691 #1675 #1676 #1692 #1706 #1709 #1775 #1777 #1778 #1795 * [FEATURE] Added `mimir-rules-action` GitHub action, located at `operations/mimir-rules-action/`, used to lint, prepare, verify, diff, and sync rules to a Mimir cluster. #1723 ## 2.0.0 diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 37473b95d60..6d8332dbc11 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -59,6 +59,7 @@ The current team members are: - Peter Štibraný — [@pstibrany](https://github.com/pstibrany) ([Grafana Labs](https://grafana.com/)) - Steve Simpson - [@stevesg](https://github.com/stevesg) ([Grafana Labs](https://grafana.com/)) - Tyler Reid — [@treid314](https://github.com/treid314) ([Grafana Labs](https://grafana.com/)) +- Patrick Oyarzun - [@Logiraptor](https://github.com/Logiraptor) ([Grafana Labs](https://grafana.com/)) Previous team members: diff --git a/MAINTAINERS.md b/MAINTAINERS.md index 573bb47cb00..4aabcfb019d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -11,6 +11,7 @@ The following are the main/default maintainers: - Peter Štibraný — [@pstibrany](https://github.com/pstibrany) ([Grafana Labs](https://grafana.com/)) - Steve Simpson - [@stevesg](https://github.com/stevesg) ([Grafana Labs](https://grafana.com/)) - Tyler Reid — [@treid314](https://github.com/treid314) ([Grafana Labs](https://grafana.com/)) +- Patrick Oyarzun — [@Logiraptor](https://github.com/Logiraptor) ([Grafana Labs](https://grafana.com/)) Some parts of the codebase have other maintainers, the package paths also include all sub-packages: diff --git a/Makefile b/Makefile index 3dde46ca888..dd04a798935 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ help: # WARNING: do not commit to a repository! -include Makefile.local -.PHONY: all test test-with-race integration-tests cover clean images protos exes dist doc clean-doc check-doc push-multiarch-build-image license check-license format check-mixin check-mixin-jb check-mixin-mixtool checkin-mixin-playbook build-mixin format-mixin check-jsonnet-manifests format-jsonnet-manifests push-multiarch-mimir list-image-targets check-jsonnet-getting-started mixin-screenshots +.PHONY: all test test-with-race integration-tests cover clean images protos exes dist doc clean-doc check-doc push-multiarch-build-image license check-license format check-mixin check-mixin-jb check-mixin-mixtool check-mixin-runbooks build-mixin format-mixin check-jsonnet-manifests format-jsonnet-manifests push-multiarch-mimir list-image-targets check-jsonnet-getting-started mixin-screenshots .DEFAULT_GOAL := all # Version number @@ -96,14 +96,24 @@ SED ?= $(shell which gsed 2>/dev/null || which sed) @echo @touch $@ +# This variable controls where result of building of multiarch image should be sent. Default is registry. +# Other options are documented in https://docs.docker.com/engine/reference/commandline/buildx_build/#output. +# CI workflow uses PUSH_MULTIARCH_TARGET="type=oci,dest=file.oci" to store images locally for next steps in the pipeline. +PUSH_MULTIARCH_TARGET ?= type=registry + # This target compiles mimir for linux/amd64 and linux/arm64 and then builds and pushes a multiarch image to the target repository. -# We don't separate building of single-platform and multiplatform images here (as we do for push-multiarch-build-image), as -# Mimir's Dockerfile is not doing much, and is unlikely to fail. -push-multiarch-mimir: - @echo - $(MAKE) GOOS=linux GOARCH=amd64 BINARY_SUFFIX=_linux_amd64 cmd/mimir/mimir - $(MAKE) GOOS=linux GOARCH=arm64 BINARY_SUFFIX=_linux_arm64 cmd/mimir/mimir - $(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) --build-arg=USE_BINARY_SUFFIX=true -t $(IMAGE_PREFIX)mimir:$(IMAGE_TAG) cmd/mimir +# We don't do separate building of single-platform and multiplatform images here (as we do for push-multiarch-build-image), as +# these Dockerfiles are not doing much, and are unlikely to fail. +push-multiarch-%/$(UPTODATE): + $(eval DIR := $(patsubst push-multiarch-%/$(UPTODATE),%,$@)) + + if [ -f $(DIR)/main.go ]; then \ + $(MAKE) GOOS=linux GOARCH=amd64 BINARY_SUFFIX=_linux_amd64 $(DIR)/$(shell basename $(DIR)); \ + $(MAKE) GOOS=linux GOARCH=arm64 BINARY_SUFFIX=_linux_arm64 $(DIR)/$(shell basename $(DIR)); \ + fi + $(SUDO) docker buildx build -o $(PUSH_MULTIARCH_TARGET) --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) --build-arg=USE_BINARY_SUFFIX=true -t $(IMAGE_PREFIX)$(shell basename $(DIR)):$(IMAGE_TAG) $(DIR)/ + +push-multiarch-mimir: push-multiarch-cmd/mimir/.uptodate # This target fetches current build image, and tags it with "latest" tag. It can be used instead of building the image locally. .PHONY: fetch-build-image @@ -117,8 +127,8 @@ fetch-build-image: push-multiarch-build-image: @echo # Build image for each platform separately... it tends to generate fewer errors. - $(SUDO) docker buildx build --platform linux/amd64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) mimir-build-image/ - $(SUDO) docker buildx build --platform linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) mimir-build-image/ + $(SUDO) docker buildx build --platform linux/amd64 --progress=plain --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) mimir-build-image/ + $(SUDO) docker buildx build --platform linux/arm64 --progress=plain --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) mimir-build-image/ # This command will run the same build as above, but it will reuse existing platform-specific images, # put them together and push to registry. $(SUDO) docker buildx build -o type=registry --platform linux/amd64,linux/arm64 --build-arg=revision=$(GIT_REVISION) --build-arg=goproxyValue=$(GOPROXY_VALUE) -t $(BUILD_IMAGE):$(IMAGE_TAG) mimir-build-image/ @@ -184,7 +194,7 @@ mimir-build-image/$(UPTODATE): mimir-build-image/* # All the boiler plate for building golang follows: SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E") BUILD_IN_CONTAINER ?= true -LATEST_BUILD_IMAGE_TAG ?= update-go-1.17.8-8a996bb57 +LATEST_BUILD_IMAGE_TAG ?= publish-multiarch-images-7a4b40a6d # TTY is parameterized to allow Google Cloud Builder to run builds, # as it currently disallows TTY devices. This value needs to be overridden @@ -281,6 +291,15 @@ lint: check-makefiles faillint -paths "github.com/thanos-io/thanos/pkg/block.{NewIgnoreDeletionMarkFilter}" \ ./pkg/compactor/... + faillint -paths "github.com/thanos-io/thanos/pkg/shipper.{New}" ./pkg/... + + faillint -paths "github.com/thanos-io/thanos/pkg/block/indexheader" ./pkg/... + + # We've copied github.com/NYTimes/gziphandler to pkg/util/gziphandler + # at least until https://github.com/nytimes/gziphandler/pull/112 is merged + faillint -paths "github.com/NYTimes/gziphandler" \ + ./pkg/... ./cmd/... ./tools/... ./integration/... + # Ensure packages we imported from Thanos are no longer used. GOFLAGS="-tags=requires_docker" faillint -paths \ "github.com/thanos/thanos-io/pkg/store,\ @@ -329,10 +348,12 @@ doc: ## Generates the config file documentation. doc: clean-doc $(DOC_TEMPLATES:.template=.md) $(DOC_EMBED:.md=.md.embedmd) # Make up markdown files prettier. When running with check-doc target, it will fail if this produces any change. prettier --write "**/*.md" + # Make operations/helm/charts/*/README.md + helm-docs # Add license header to files. license: - go run ./tools/add-license ./cmd ./integration ./pkg ./tools ./development ./mimir-build-image ./operations + go run ./tools/add-license ./cmd ./integration ./pkg ./tools ./development ./mimir-build-image ./operations ./.github check-license: license @git diff --exit-code || (echo "Please add the license header running 'make BUILD_IN_CONTAINER=false license'" && false) @@ -373,6 +394,7 @@ dist: ## Generates binaries for a Mimir release. build-mixin: check-mixin-jb @rm -rf $(MIXIN_OUT_PATH) && mkdir $(MIXIN_OUT_PATH) @mixtool generate all --output-alerts $(MIXIN_OUT_PATH)/alerts.yaml --output-rules $(MIXIN_OUT_PATH)/rules.yaml --directory $(MIXIN_OUT_PATH)/dashboards ${MIXIN_PATH}/mixin-compiled.libsonnet + @./tools/check-rules.sh $(MIXIN_OUT_PATH)/rules.yaml 20 # If any rule group has more than 20 rules, fail. 20 is our default per-tenant limit in the ruler. @cd $(MIXIN_OUT_PATH)/.. && zip -q -r mimir-mixin.zip $$(basename "$(MIXIN_OUT_PATH)") @echo "The mixin has been compiled to $(MIXIN_OUT_PATH) and archived to $$(realpath --relative-to=$$(pwd) $(MIXIN_OUT_PATH)/../mimir-mixin.zip)" @@ -396,6 +418,9 @@ format-makefiles: $(MAKE_FILES) clean: $(SUDO) docker rmi $(IMAGE_NAMES) >/dev/null 2>&1 || true rm -rf -- $(UPTODATE_FILES) $(EXES) .cache dist + # Remove executables built for multiarch images. + find . -type f -name '*_linux_arm64' -perm +u+x -exec rm {} \; + find . -type f -name '*_linux_amd64' -perm +u+x -exec rm {} \; go clean ./... clean-protos: @@ -405,34 +430,6 @@ clean-protos: list-image-targets: @echo $(UPTODATE_FILES) | tr " " "\n" -save-images: - @mkdir -p docker-images - for image_name in $(IMAGE_NAMES); do \ - if echo $$image_name | grep -q build; then \ - continue; \ - fi; \ - if [ "$$(docker images -q $$image_name:$(IMAGE_TAG) 2> /dev/null)" = "" ]; then \ - echo "Skipping $$image_name:$(IMAGE_TAG) because image does not exist"; \ - else \ - echo "Saving $$image_name:$(IMAGE_TAG)"; \ - docker save $$image_name:$(IMAGE_TAG) -o docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ - fi; \ - done - -load-images: - for image_name in $(IMAGE_NAMES); do \ - if echo $$image_name | grep -q build; then \ - continue; \ - fi; \ - image_path=docker-images/$$(echo $$image_name | tr "/" _):$(IMAGE_TAG); \ - if [ -e "$$image_path" ]; then \ - echo "Loading $$image_path"; \ - docker load -i "$$image_path"; \ - else \ - echo "Skipping $$image_path because image does not exist"; \ - fi; \ - done - clean-doc: rm -f $(DOC_TEMPLATES:.template=.md) @@ -456,7 +453,7 @@ clean-white-noise: check-white-noise: clean-white-noise @git diff --exit-code -- '*.md' || (echo "Please remove trailing whitespaces running 'make clean-white-noise'" && false) -check-mixin: build-mixin format-mixin check-mixin-jb check-mixin-mixtool check-mixin-playbook +check-mixin: build-mixin format-mixin check-mixin-jb check-mixin-mixtool check-mixin-runbooks @echo "Checking diff:" @git diff --exit-code -- $(MIXIN_PATH) $(MIXIN_OUT_PATH) || (echo "Please build and format mixin by running 'make build-mixin format-mixin'" && false) @@ -472,8 +469,8 @@ check-mixin-mixtool: check-mixin-jb @cd $(MIXIN_PATH) && \ mixtool lint mixin.libsonnet -check-mixin-playbook: build-mixin - @$(MIXIN_PATH)/scripts/lint-playbooks.sh +check-mixin-runbooks: build-mixin + @tools/lint-runbooks.sh mixin-serve: ## Runs Grafana (listening on port 3000) loading the mixin dashboards compiled at operations/mimir-mixin-compiled. @./operations/mimir-mixin-tools/serve/run.sh @@ -499,6 +496,12 @@ check-jsonnet-getting-started: | sed 's/\(jb install github.com\/grafana\/mimir\/operations\/mimir@main\)/\1 \&\& rm -fr .\/vendor\/mimir \&\& cp -r ..\/operations\/mimir .\/vendor\/mimir\//g' \ | bash +build-helm-tests: + @./operations/helm/tests/build.sh + +check-helm-tests: build-helm-tests + @git diff --exit-code -- ./operations/helm/tests || (echo "Please rebuild helm tests output 'make build-helm-tests'" && false) + build-jsonnet-tests: @./operations/mimir-tests/build.sh diff --git a/README.md b/README.md index e2424742137..eb95c06e850 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Grafana Mimir is an open source software project that provides a scalable long-term storage for [Prometheus](https://prometheus.io). Some of the core strengths of Grafana Mimir include: -- **Easy to install and maintain:** Grafana Mimir’s extensive documentation, tutorials, and deployment tooling make it quick to get started. Using its monolithic mode, you can get Grafana Mimir up and running with just one binary and no additional dependencies. Once deployed, the best-practice dashboards, alerts, and playbooks packaged with Grafana Mimir make it easy to monitor the health of the system. +- **Easy to install and maintain:** Grafana Mimir’s extensive documentation, tutorials, and deployment tooling make it quick to get started. Using its monolithic mode, you can get Grafana Mimir up and running with just one binary and no additional dependencies. Once deployed, the best-practice dashboards, alerts, and runbooks packaged with Grafana Mimir make it easy to monitor the health of the system. - **Massive scalability:** You can run Grafana Mimir's horizontally-scalable architecture across multiple machines, resulting in the ability to process orders of magnitude more time series than a single Prometheus instance. Internal testing shows that Grafana Mimir handles up to 1 billion active time series. - **Global view of metrics:** Grafana Mimir enables you to run queries that aggregate series from multiple Prometheus instances, giving you a global view of your systems. Its query engine extensively parallelizes query execution, so that even the highest-cardinality queries complete with blazing speed. - **Cheap, durable metric storage:** Grafana Mimir uses object storage for long-term data storage, allowing it to take advantage of this ubiquitous, cost-effective, high-durability technology. It is compatible with multiple object store implementations, including AWS S3, Google Cloud Storage, Azure Blob Storage, OpenStack Swift, as well as any S3-compatible object storage. diff --git a/RELEASE.md b/RELEASE.md index af240bd87fc..a1667cb1d1a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,7 +9,7 @@ A new Grafana Mimir release is cut approximately every 6 weeks. The following ta | Version | Date | Release shepherd | | ------- | ---------- | ----------------- | | 2.0.0 | 2022-03-20 | Marco Pracucci | -| 2.1.0 | 2022-05-16 | _To be announced_ | +| 2.1.0 | 2022-05-16 | Johanna Ratliff | | 2.2.0 | 2022-06-27 | _To be announced_ | | 2.3.0 | 2022-08-08 | _To be announced_ | @@ -80,7 +80,8 @@ To publish a release candidate: ### Publish a stable release -To publish a stable release: +> **Note:** Technical documentation is automatically published on release tags or release branches with a corresponding release tag. The workflow that publishes documentation is defined in [`publish-technical-documentation-release.yml`](.github/workflows/publish-technical-documentation-release.yml). +> To publish a stable release: 1. Do not change the release branch directly; make a PR to the release-X.Y branch with VERSION and any CHANGELOG changes. 1. Ensure the `VERSION` file has **no** `-rc.X` suffix @@ -104,10 +105,6 @@ To publish a stable release: - Temporarily enable "Allow merge commits" option in "Settings > Options" - Locally merge the `merge-release-X.Y-to-main` branch into `main`, and push the changes to `main` back to GitHub. This doesn't break `main` branch protection, since the PR has been approved already, and it also doesn't require removing the protection. 1. Open a PR to add the new version to the backward compatibility integration test (`integration/backward_compatibility_test.go`) -1. Publish release documentation using `./tools/release-docs`: - ```console - $ ./tools/release-docs /tmp/grafana/website mimir-2.0.0 - ``` 1. Publish dashboards (done by a Grafana Labs member) 1. Login to [https://grafana.com](https://grafana.com) with your Grafana Labs account 1. Open [https://grafana.com/orgs/grafana/dashboards](https://grafana.com/orgs/grafana/dashboards) diff --git a/VERSION b/VERSION index 359a5b952d4..50aea0e7aba 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.0.0 \ No newline at end of file +2.1.0 \ No newline at end of file diff --git a/cmd/metaconvert/Dockerfile b/cmd/metaconvert/Dockerfile index b02dbed0886..d88d58440e6 100644 --- a/cmd/metaconvert/Dockerfile +++ b/cmd/metaconvert/Dockerfile @@ -3,9 +3,15 @@ # Provenance-includes-license: Apache-2.0 # Provenance-includes-copyright: The Cortex Authors. -FROM alpine:3.15.0 +FROM alpine:3.16.0 RUN apk add --no-cache ca-certificates -COPY metaconvert / +# Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG. +ARG TARGETOS +ARG TARGETARCH +ARG BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}" +# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix. +ARG USE_BINARY_SUFFIX +COPY metaconvert${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /metaconvert ENTRYPOINT ["/metaconvert"] ARG revision diff --git a/cmd/metaconvert/main.go b/cmd/metaconvert/main.go index 97ac981ef3e..ba97eb43f78 100644 --- a/cmd/metaconvert/main.go +++ b/cmd/metaconvert/main.go @@ -14,6 +14,7 @@ import ( "os" "os/signal" "path" + "sort" "syscall" gklog "github.com/go-kit/log" @@ -106,20 +107,20 @@ func convertTenantBlocks(ctx context.Context, userBucketClient objstore.Bucket, updated := false - metaOrgID := meta.Thanos.Labels[mimir_tsdb.TenantIDExternalLabel] - if metaOrgID != tenant { - level.Warn(logger).Log("msg", "updating tenant label", "block", blockID.String(), "old_value", metaOrgID, "new_value", tenant) - updated = true - meta.Thanos.Labels[mimir_tsdb.TenantIDExternalLabel] = tenant + // Sort labels before processing to have stable output for testing. + var labels []string + for l := range meta.Thanos.Labels { + labels = append(labels, l) } + sort.Strings(labels) - for l, v := range meta.Thanos.Labels { + for _, l := range labels { switch l { - case mimir_tsdb.TenantIDExternalLabel, mimir_tsdb.IngesterIDExternalLabel, mimir_tsdb.CompactorShardIDExternalLabel: + case mimir_tsdb.CompactorShardIDExternalLabel: continue } - level.Warn(logger).Log("msg", "removing unknown label", "block", blockID.String(), "label", l, "value", v) + level.Warn(logger).Log("msg", "removing unknown label", "block", blockID.String(), "label", l, "value", meta.Thanos.Labels[l]) updated = true delete(meta.Thanos.Labels, l) } diff --git a/cmd/metaconvert/main_test.go b/cmd/metaconvert/main_test.go index 29b8b0babbc..58aa355a396 100644 --- a/cmd/metaconvert/main_test.go +++ b/cmd/metaconvert/main_test.go @@ -4,7 +4,6 @@ package main import ( "context" - "fmt" "path" "strings" "testing" @@ -67,8 +66,8 @@ func TestConvertTenantBlocks(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - "test": "label", - mimir_tsdb.TenantIDExternalLabel: "wrong tenant", + "test": "label", + "__org_id__": "wrong tenant", }, }, }, @@ -80,9 +79,9 @@ func TestConvertTenantBlocks(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: "fake", + "__org_id__": "fake", mimir_tsdb.CompactorShardIDExternalLabel: "1_of_10", - mimir_tsdb.IngesterIDExternalLabel: "ingester-1", + "__ingester_id__": "ingester-1", }, }, }, @@ -94,7 +93,7 @@ func TestConvertTenantBlocks(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: tenant, + "__org_id__": tenant, }, }, }, @@ -127,9 +126,6 @@ func TestConvertTenantBlocks(t *testing.T) { Thanos: metadata.Thanos{ Version: 10, - Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: tenant, - }, Downsample: metadata.ThanosDownsample{ Resolution: 15, }, @@ -141,12 +137,6 @@ func TestConvertTenantBlocks(t *testing.T) { BlockMeta: tsdb.BlockMeta{ ULID: blockWithWrongTenant, }, - - Thanos: metadata.Thanos{ - Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: tenant, - }, - }, }, blockWithManyMimirLabels: { @@ -156,9 +146,7 @@ func TestConvertTenantBlocks(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: tenant, mimir_tsdb.CompactorShardIDExternalLabel: "1_of_10", - mimir_tsdb.IngesterIDExternalLabel: "ingester-1", }, }, }, @@ -167,33 +155,33 @@ func TestConvertTenantBlocks(t *testing.T) { BlockMeta: tsdb.BlockMeta{ ULID: blockWithNoChangesRequired, }, - - Thanos: metadata.Thanos{ - Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: tenant, - }, - }, }, } for b, m := range expected { meta, err := block.DownloadMeta(ctx, logger, bkt, b) - require.NoError(t, err) - require.Equal(t, m, meta) + require.NoError(t, err, b.String()) + + // Normalize empty map to nil to simplify tests. + if len(meta.Thanos.Labels) == 0 { + meta.Thanos.Labels = nil + } + require.Equal(t, m, meta, b.String()) } assert.Equal(t, []string{ - `level=warn tenant=target_tenant msg="updating tenant label" block=00000000010000000000000000 old_value= new_value=target_tenant`, - `level=info tenant=target_tenant msg="changes required, uploading meta.json file" block=00000000010000000000000000`, - `level=info tenant=target_tenant msg="meta.json file uploaded successfully" block=00000000010000000000000000`, - `level=warn tenant=target_tenant msg="updating tenant label" block=00000000020000000000000000 old_value="wrong tenant" new_value=target_tenant`, + `level=info tenant=target_tenant msg="no changes required" block=00000000010000000000000000`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000020000000000000000 label=__org_id__ value="wrong tenant"`, `level=warn tenant=target_tenant msg="removing unknown label" block=00000000020000000000000000 label=test value=label`, `level=info tenant=target_tenant msg="changes required, uploading meta.json file" block=00000000020000000000000000`, `level=info tenant=target_tenant msg="meta.json file uploaded successfully" block=00000000020000000000000000`, - `level=warn tenant=target_tenant msg="updating tenant label" block=00000000030000000000000000 old_value=fake new_value=target_tenant`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000030000000000000000 label=__ingester_id__ value=ingester-1`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000030000000000000000 label=__org_id__ value=fake`, `level=info tenant=target_tenant msg="changes required, uploading meta.json file" block=00000000030000000000000000`, `level=info tenant=target_tenant msg="meta.json file uploaded successfully" block=00000000030000000000000000`, - `level=info tenant=target_tenant msg="no changes required" block=00000000040000000000000000`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000040000000000000000 label=__org_id__ value=target_tenant`, + `level=info tenant=target_tenant msg="changes required, uploading meta.json file" block=00000000040000000000000000`, + `level=info tenant=target_tenant msg="meta.json file uploaded successfully" block=00000000040000000000000000`, }, strings.Split(strings.TrimSpace(logs.String()), "\n")) } @@ -242,8 +230,8 @@ func TestConvertTenantBlocksDryMode(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - "test": "label", - mimir_tsdb.TenantIDExternalLabel: "wrong tenant", + "test": "label", + "__org_id__": "wrong tenant", }, }, }, @@ -255,9 +243,9 @@ func TestConvertTenantBlocksDryMode(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: "fake", + "__org_id__": "fake", mimir_tsdb.CompactorShardIDExternalLabel: "1_of_10", - mimir_tsdb.IngesterIDExternalLabel: "ingester-1", + "__ingester_id__": "ingester-1", }, }, }, @@ -269,7 +257,7 @@ func TestConvertTenantBlocksDryMode(t *testing.T) { Thanos: metadata.Thanos{ Labels: map[string]string{ - mimir_tsdb.TenantIDExternalLabel: tenant, + "__org_id__": tenant, }, }, }, @@ -291,16 +279,15 @@ func TestConvertTenantBlocksDryMode(t *testing.T) { require.Equal(t, m, meta) } - fmt.Println(logs.String()) - assert.Equal(t, []string{ - `level=warn tenant=target_tenant msg="updating tenant label" block=00000000010000000000000000 old_value= new_value=target_tenant`, - `level=warn tenant=target_tenant msg="changes required, not uploading back due to dry run" block=00000000010000000000000000`, - `level=warn tenant=target_tenant msg="updating tenant label" block=00000000020000000000000000 old_value="wrong tenant" new_value=target_tenant`, + `level=info tenant=target_tenant msg="no changes required" block=00000000010000000000000000`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000020000000000000000 label=__org_id__ value="wrong tenant"`, `level=warn tenant=target_tenant msg="removing unknown label" block=00000000020000000000000000 label=test value=label`, `level=warn tenant=target_tenant msg="changes required, not uploading back due to dry run" block=00000000020000000000000000`, - `level=warn tenant=target_tenant msg="updating tenant label" block=00000000030000000000000000 old_value=fake new_value=target_tenant`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000030000000000000000 label=__ingester_id__ value=ingester-1`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000030000000000000000 label=__org_id__ value=fake`, `level=warn tenant=target_tenant msg="changes required, not uploading back due to dry run" block=00000000030000000000000000`, - `level=info tenant=target_tenant msg="no changes required" block=00000000040000000000000000`, + `level=warn tenant=target_tenant msg="removing unknown label" block=00000000040000000000000000 label=__org_id__ value=target_tenant`, + `level=warn tenant=target_tenant msg="changes required, not uploading back due to dry run" block=00000000040000000000000000`, }, strings.Split(strings.TrimSpace(logs.String()), "\n")) } diff --git a/cmd/mimir-continuous-test/Dockerfile b/cmd/mimir-continuous-test/Dockerfile index f0b0b8b339a..42aaf8e5c2e 100644 --- a/cmd/mimir-continuous-test/Dockerfile +++ b/cmd/mimir-continuous-test/Dockerfile @@ -1,8 +1,14 @@ # SPDX-License-Identifier: AGPL-3.0-only -FROM alpine:3.15.0 +FROM alpine:3.16.0 RUN apk add --no-cache ca-certificates -COPY mimir-continuous-test / +# Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG. +ARG TARGETOS +ARG TARGETARCH +ARG BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}" +# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix. +ARG USE_BINARY_SUFFIX +COPY mimir-continuous-test${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /mimir-continuous-test ENTRYPOINT ["/mimir-continuous-test"] ARG revision diff --git a/cmd/mimir-continuous-test/main.go b/cmd/mimir-continuous-test/main.go index 73cef2bc510..e528bb47ec5 100644 --- a/cmd/mimir-continuous-test/main.go +++ b/cmd/mimir-continuous-test/main.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/client_golang/prometheus/collectors" "github.com/weaveworks/common/logging" "github.com/weaveworks/common/server" + "github.com/weaveworks/common/tracing" "github.com/grafana/mimir/pkg/continuoustest" "github.com/grafana/mimir/pkg/util/instrumentation" @@ -43,6 +44,14 @@ func main() { util_log.InitLogger(&server.Config{ LogLevel: cfg.LogLevel, }) + + // Setting the environment variable JAEGER_AGENT_HOST enables tracing. + if trace, err := tracing.NewFromEnv("mimir-continuous-test"); err != nil { + level.Error(util_log.Logger).Log("msg", "Failed to setup tracing", "err", err.Error()) + } else { + defer trace.Close() + } + logger := util_log.Logger // Run the instrumentation server. diff --git a/cmd/mimir/Dockerfile b/cmd/mimir/Dockerfile index d813a84617d..a7f195948ed 100644 --- a/cmd/mimir/Dockerfile +++ b/cmd/mimir/Dockerfile @@ -3,7 +3,7 @@ # Provenance-includes-license: Apache-2.0 # Provenance-includes-copyright: The Cortex Authors. -FROM alpine:3.15.0 +FROM alpine:3.16.0 RUN apk add --no-cache ca-certificates # Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG. ARG TARGETOS diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index e15c0e25bf8..63991730ee6 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -337,7 +337,7 @@ "required": false, "desc": "Limit on the size of a gRPC message this server can receive (bytes).", "fieldValue": null, - "fieldDefaultValue": 4194304, + "fieldDefaultValue": 104857600, "fieldFlag": "server.grpc-max-recv-msg-size-bytes", "fieldType": "int", "fieldCategory": "advanced" @@ -348,7 +348,7 @@ "required": false, "desc": "Limit on the size of a gRPC message this server can send (bytes).", "fieldValue": null, - "fieldDefaultValue": 4194304, + "fieldDefaultValue": 104857600, "fieldFlag": "server.grpc-max-send-msg-size-bytes", "fieldType": "int", "fieldCategory": "advanced" @@ -910,17 +910,6 @@ "fieldType": "duration", "fieldCategory": "advanced" }, - { - "kind": "field", - "name": "extend_writes", - "required": false, - "desc": "Try writing to an additional ingester in the presence of an ingester not in the ACTIVE state. It is useful to disable this along with -ingester.ring.unregister-on-shutdown=false in order to not spread samples to extra ingesters during rolling restarts with consistent naming.", - "fieldValue": null, - "fieldDefaultValue": true, - "fieldFlag": "distributor.extend-writes", - "fieldType": "boolean", - "fieldCategory": "advanced" - }, { "kind": "block", "name": "ring", @@ -1343,6 +1332,17 @@ "fieldFlag": "distributor.forwarding.request-timeout", "fieldType": "duration", "fieldCategory": "experimental" + }, + { + "kind": "field", + "name": "propagate_errors", + "required": false, + "desc": "If disabled then forwarding requests are always considered to be successful, errors are ignored.", + "fieldValue": null, + "fieldDefaultValue": true, + "fieldFlag": "distributor.forwarding.propagate-errors", + "fieldType": "boolean", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -1388,7 +1388,8 @@ "fieldValue": null, "fieldDefaultValue": 46800000000000, "fieldFlag": "querier.query-ingesters-within", - "fieldType": "duration" + "fieldType": "duration", + "fieldCategory": "advanced" }, { "kind": "field", @@ -1396,9 +1397,10 @@ "required": false, "desc": "The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'.", "fieldValue": null, - "fieldDefaultValue": 0, + "fieldDefaultValue": 43200000000000, "fieldFlag": "querier.query-store-after", - "fieldType": "duration" + "fieldType": "duration", + "fieldCategory": "advanced" }, { "kind": "field", @@ -1491,9 +1493,9 @@ "kind": "field", "name": "shuffle_sharding_ingesters_lookback_period", "required": false, - "desc": "When distributor's sharding strategy is shuffle-sharding and this setting is \u003e 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).", + "desc": "When this setting is \u003e 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled).", "fieldValue": null, - "fieldDefaultValue": 0, + "fieldDefaultValue": 46800000000000, "fieldFlag": "querier.shuffle-sharding-ingesters-lookback-period", "fieldType": "duration", "fieldCategory": "advanced" @@ -2182,7 +2184,7 @@ "kind": "field", "name": "unregister_on_shutdown", "required": false, - "desc": "Unregister from the ring upon clean shutdown. It can be useful to disable for rolling restarts with consistent naming in conjunction with -distributor.extend-writes=false.", + "desc": "Unregister from the ring upon clean shutdown. It can be useful to disable for rolling restarts with consistent naming.", "fieldValue": null, "fieldDefaultValue": true, "fieldFlag": "ingester.ring.unregister-on-shutdown", @@ -2200,17 +2202,6 @@ "fieldType": "duration", "fieldCategory": "advanced" }, - { - "kind": "field", - "name": "join_after", - "required": false, - "desc": "Period to wait for a claim from another member; will join automatically after this.", - "fieldValue": null, - "fieldDefaultValue": 0, - "fieldFlag": "ingester.ring.join-after", - "fieldType": "duration", - "fieldCategory": "advanced" - }, { "kind": "field", "name": "min_ready_duration", @@ -2420,6 +2411,28 @@ "required": false, "desc": "", "blockEntries": [ + { + "kind": "field", + "name": "request_rate", + "required": false, + "desc": "Per-tenant request rate limit in requests per second. 0 to disable.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "distributor.request-rate-limit", + "fieldType": "float", + "fieldCategory": "experimental" + }, + { + "kind": "field", + "name": "request_burst_size", + "required": false, + "desc": "Per-tenant allowed request burst size. 0 to disable.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "distributor.request-burst-size", + "fieldType": "int", + "fieldCategory": "experimental" + }, { "kind": "field", "name": "ingestion_rate", @@ -3773,6 +3786,17 @@ "fieldFlag": "blocks-storage.backend", "fieldType": "string" }, + { + "kind": "field", + "name": "storage_prefix", + "required": false, + "desc": "Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "blocks-storage.storage-prefix", + "fieldType": "string", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "s3", @@ -5015,7 +5039,7 @@ "required": false, "desc": "Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter.", "fieldValue": null, - "fieldDefaultValue": 0, + "fieldDefaultValue": 36000000000000, "fieldFlag": "blocks-storage.bucket-store.ignore-blocks-within", "fieldType": "duration", "fieldCategory": "advanced" @@ -5109,15 +5133,25 @@ "fieldCategory": "advanced" }, { - "kind": "field", - "name": "index_header_thread_pool_size", + "kind": "block", + "name": "index_header", "required": false, - "desc": "Number of threads that are dedicated for use reading index headers. Set to 0 to disable use of dedicated threads for reading index headers.", + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "map_populate_enabled", + "required": false, + "desc": "If enabled, the store-gateway will attempt to pre-populate the file system cache when memory-mapping index-header files.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "blocks-storage.bucket-store.index-header.map-populate-enabled", + "fieldType": "boolean", + "fieldCategory": "experimental" + } + ], "fieldValue": null, - "fieldDefaultValue": 0, - "fieldFlag": "blocks-storage.bucket-store.index-header-thread-pool-size", - "fieldType": "int", - "fieldCategory": "experimental" + "fieldDefaultValue": null } ], "fieldValue": null, @@ -5307,7 +5341,7 @@ "kind": "field", "name": "head_chunks_write_queue_size", "required": false, - "desc": "The size of the write queue used by the head chunks mapper. Lower values reduce memory utilisation at the cost of potentially higher ingest latency. Value of 0 switches chunks mapper to implementation without a queue.", + "desc": "The size of the write queue used by the head chunks mapper. Lower values reduce memory utilisation at the cost of potentially higher ingest latency. Value of 0 switches chunks mapper to implementation without a queue. This flag is only used if the new chunk disk mapper is enabled with -blocks-storage.tsdb.new-chunk-disk-mapper.", "fieldValue": null, "fieldDefaultValue": 0, "fieldFlag": "blocks-storage.tsdb.head-chunks-write-queue-size", @@ -6454,6 +6488,17 @@ ], "fieldValue": null, "fieldDefaultValue": null + }, + { + "kind": "field", + "name": "thread_pool_size", + "required": false, + "desc": "Number of OS threads that are dedicated for handling requests. Set to 0 to disable use of dedicated OS threads for handling requests.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "store-gateway.thread-pool-size", + "fieldType": "int", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -7360,70 +7405,189 @@ "fieldType": "string" }, { - "kind": "field", - "name": "tls_enabled", - "required": false, - "desc": "Set to true if query-frontend connection requires TLS.", - "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-enabled", - "fieldType": "boolean", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_cert_path", - "required": false, - "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-cert-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_key_path", - "required": false, - "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-key-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_ca_path", - "required": false, - "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-ca-path", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_server_name", - "required": false, - "desc": "Override the expected name on the server certificate.", - "fieldValue": null, - "fieldDefaultValue": "", - "fieldFlag": "ruler.query-frontend.tls-server-name", - "fieldType": "string", - "fieldCategory": "advanced" - }, - { - "kind": "field", - "name": "tls_insecure_skip_verify", + "kind": "block", + "name": "grpc_client_config", "required": false, - "desc": "Skip validating server certificate.", + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "max_recv_msg_size", + "required": false, + "desc": "gRPC client max receive message size (bytes).", + "fieldValue": null, + "fieldDefaultValue": 104857600, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_send_msg_size", + "required": false, + "desc": "gRPC client max send message size (bytes).", + "fieldValue": null, + "fieldDefaultValue": 104857600, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "grpc_compression", + "required": false, + "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression)", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-compression", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "rate_limit", + "required": false, + "desc": "Rate limit for gRPC client; 0 means disabled.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-client-rate-limit", + "fieldType": "float", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "rate_limit_burst", + "required": false, + "desc": "Rate limit burst for gRPC client.", + "fieldValue": null, + "fieldDefaultValue": 0, + "fieldFlag": "ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst", + "fieldType": "int", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "backoff_on_ratelimits", + "required": false, + "desc": "Enable backoff and retry when we hit ratelimits.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-on-ratelimits", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "block", + "name": "backoff_config", + "required": false, + "desc": "", + "blockEntries": [ + { + "kind": "field", + "name": "min_period", + "required": false, + "desc": "Minimum delay when backing off.", + "fieldValue": null, + "fieldDefaultValue": 100000000, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-min-period", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_period", + "required": false, + "desc": "Maximum delay when backing off.", + "fieldValue": null, + "fieldDefaultValue": 10000000000, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-max-period", + "fieldType": "duration", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "max_retries", + "required": false, + "desc": "Number of times to backoff and retry before failing.", + "fieldValue": null, + "fieldDefaultValue": 10, + "fieldFlag": "ruler.query-frontend.grpc-client-config.backoff-retries", + "fieldType": "int", + "fieldCategory": "advanced" + } + ], + "fieldValue": null, + "fieldDefaultValue": null + }, + { + "kind": "field", + "name": "tls_enabled", + "required": false, + "desc": "Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-enabled", + "fieldType": "boolean", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_cert_path", + "required": false, + "desc": "Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-cert-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_key_path", + "required": false, + "desc": "Path to the key file for the client certificate. Also requires the client certificate to be configured.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-key-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_ca_path", + "required": false, + "desc": "Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-ca-path", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_server_name", + "required": false, + "desc": "Override the expected name on the server certificate.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-server-name", + "fieldType": "string", + "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "tls_insecure_skip_verify", + "required": false, + "desc": "Skip validating server certificate.", + "fieldValue": null, + "fieldDefaultValue": false, + "fieldFlag": "ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify", + "fieldType": "boolean", + "fieldCategory": "advanced" + } + ], "fieldValue": null, - "fieldDefaultValue": false, - "fieldFlag": "ruler.query-frontend.tls-insecure-skip-verify", - "fieldType": "boolean", - "fieldCategory": "advanced" + "fieldDefaultValue": null } ], "fieldValue": null, @@ -7469,6 +7633,17 @@ "fieldFlag": "ruler-storage.backend", "fieldType": "string" }, + { + "kind": "field", + "name": "storage_prefix", + "required": false, + "desc": "Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler-storage.storage-prefix", + "fieldType": "string", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "s3", @@ -8776,6 +8951,17 @@ "fieldFlag": "alertmanager-storage.backend", "fieldType": "string" }, + { + "kind": "field", + "name": "storage_prefix", + "required": false, + "desc": "Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "alertmanager-storage.storage-prefix", + "fieldType": "string", + "fieldCategory": "experimental" + }, { "kind": "block", "name": "s3", diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index e0ecc66ccc9..c340611d852 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -66,6 +66,8 @@ Usage of ./cmd/mimir/mimir: Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. -alertmanager-storage.s3.tls-handshake-timeout duration Maximum time to wait for a TLS handshake. 0 means no limit. (default 10s) + -alertmanager-storage.storage-prefix string + [experimental] Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. -alertmanager-storage.swift.auth-url string OpenStack Swift authentication URL -alertmanager-storage.swift.auth-version int @@ -305,7 +307,7 @@ Usage of ./cmd/mimir/mimir: -blocks-storage.bucket-store.consistency-delay duration Minimum age of a block before it's being read. Set it to safe value (e.g 30m) if your object storage is eventually consistent. GCS and S3 are (roughly) strongly consistent. -blocks-storage.bucket-store.ignore-blocks-within duration - Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter. + Blocks with minimum time within this duration are ignored, and not loaded by store-gateway. Useful when used together with -querier.query-store-after to prevent loading young blocks, because there are usually many of them (depending on number of ingesters) and they are not yet compacted. Negative values or 0 disable the filter. (default 10h0m0s) -blocks-storage.bucket-store.ignore-deletion-marks-delay duration Duration after which the blocks marked for deletion will be filtered out while fetching blocks. The idea of ignore-deletion-marks-delay is to ignore blocks that are marked for deletion with some delay. This ensures store can still serve blocks that are meant to be deleted but do not have a replacement yet. (default 1h0m0s) -blocks-storage.bucket-store.index-cache.backend string @@ -332,8 +334,8 @@ Usage of ./cmd/mimir/mimir: If enabled, store-gateway will lazy load an index-header only once required by a query. (default true) -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout duration If index-header lazy loading is enabled and this setting is > 0, the store-gateway will offload unused index-headers after 'idle timeout' inactivity. (default 1h0m0s) - -blocks-storage.bucket-store.index-header-thread-pool-size int - [experimental] Number of threads that are dedicated for use reading index headers. Set to 0 to disable use of dedicated threads for reading index headers. + -blocks-storage.bucket-store.index-header.map-populate-enabled + [experimental] If enabled, the store-gateway will attempt to pre-populate the file system cache when memory-mapping index-header files. -blocks-storage.bucket-store.max-chunk-pool-bytes uint Max size - in bytes - of a chunks pool, used to reduce memory allocations. The pool is shared across all tenants. 0 to disable the limit. (default 2147483648) -blocks-storage.bucket-store.max-concurrent int @@ -437,6 +439,8 @@ Usage of ./cmd/mimir/mimir: Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. -blocks-storage.s3.tls-handshake-timeout duration Maximum time to wait for a TLS handshake. 0 means no limit. (default 10s) + -blocks-storage.storage-prefix string + [experimental] Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. -blocks-storage.swift.auth-url string OpenStack Swift authentication URL -blocks-storage.swift.auth-version int @@ -486,7 +490,7 @@ Usage of ./cmd/mimir/mimir: -blocks-storage.tsdb.head-chunks-write-buffer-size-bytes int The write buffer size used by the head chunks mapper. Lower values reduce memory utilisation on clusters with a large number of tenants at the cost of increased disk I/O operations. (default 4194304) -blocks-storage.tsdb.head-chunks-write-queue-size int - [experimental] The size of the write queue used by the head chunks mapper. Lower values reduce memory utilisation at the cost of potentially higher ingest latency. Value of 0 switches chunks mapper to implementation without a queue. + [experimental] The size of the write queue used by the head chunks mapper. Lower values reduce memory utilisation at the cost of potentially higher ingest latency. Value of 0 switches chunks mapper to implementation without a queue. This flag is only used if the new chunk disk mapper is enabled with -blocks-storage.tsdb.new-chunk-disk-mapper. -blocks-storage.tsdb.head-compaction-concurrency int Maximum number of tenants concurrently compacting TSDB head into a new block (default 5) -blocks-storage.tsdb.head-compaction-idle-timeout duration @@ -643,10 +647,10 @@ Usage of ./cmd/mimir/mimir: How frequently to clean up clients for ingesters that have gone away. (default 15s) -distributor.drop-label value This flag can be used to specify label names that to drop during sample ingestion within the distributor and can be repeated in order to drop multiple labels. - -distributor.extend-writes - Try writing to an additional ingester in the presence of an ingester not in the ACTIVE state. It is useful to disable this along with -ingester.ring.unregister-on-shutdown=false in order to not spread samples to extra ingesters during rolling restarts with consistent naming. (default true) -distributor.forwarding.enabled [experimental] Enables the feature to forward certain metrics in remote_write requests, depending on defined rules. + -distributor.forwarding.propagate-errors + [experimental] If disabled then forwarding requests are always considered to be successful, errors are ignored. (default true) -distributor.forwarding.request-timeout duration [experimental] Timeout for requests to ingestion endpoints to which we forward metrics. (default 10s) -distributor.ha-tracker.cluster string @@ -727,6 +731,10 @@ Usage of ./cmd/mimir/mimir: remote_write API max receive message size (bytes). (default 104857600) -distributor.remote-timeout duration Timeout for downstream ingesters. (default 20s) + -distributor.request-burst-size int + [experimental] Per-tenant allowed request burst size. 0 to disable. + -distributor.request-rate-limit float + [experimental] Per-tenant request rate limit in requests per second. 0 to disable. -distributor.ring.consul.acl-token string ACL Token used to interact with Consul. -distributor.ring.consul.client-timeout duration @@ -913,8 +921,6 @@ Usage of ./cmd/mimir/mimir: List of network interface names to look up when finding the instance IP address. (default []) -ingester.ring.instance-port int Port to advertise in the ring (defaults to -server.grpc-listen-port). - -ingester.ring.join-after duration - Period to wait for a claim from another member; will join automatically after this. -ingester.ring.min-ready-duration duration Minimum duration to wait after the internal readiness checks have passed but before succeeding the readiness endpoint. This is used to slowdown deployment controllers (eg. Kubernetes) after an instance is ready and before they proceed with a rolling update, to give the rest of the cluster instances enough time to receive ring updates. (default 15s) -ingester.ring.multi.mirror-enabled @@ -940,7 +946,7 @@ Usage of ./cmd/mimir/mimir: -ingester.ring.tokens-file-path string File path where tokens are stored. If empty, tokens are not stored at shutdown and restored at startup. -ingester.ring.unregister-on-shutdown - Unregister from the ring upon clean shutdown. It can be useful to disable for rolling restarts with consistent naming in conjunction with -distributor.extend-writes=false. (default true) + Unregister from the ring upon clean shutdown. It can be useful to disable for rolling restarts with consistent naming. (default true) -ingester.ring.zone-awareness-enabled True to enable the zone-awareness and replicate ingested samples across different availability zones. This option needs be set on ingesters, distributors, queriers and rulers when running in microservices mode. -ingester.stream-chunks-when-using-blocks @@ -1090,11 +1096,11 @@ Usage of ./cmd/mimir/mimir: -querier.query-ingesters-within duration Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester. (default 13h0m0s) -querier.query-store-after duration - The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'. + The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'. (default 12h0m0s) -querier.scheduler-address string Address of the query-scheduler component, in host:port format. Only one of -querier.frontend-address or -querier.scheduler-address can be set. If neither is set, queries are only received via HTTP endpoint. -querier.shuffle-sharding-ingesters-lookback-period duration - When distributor's sharding strategy is shuffle-sharding and this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled). + When this setting is > 0, queriers fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. The lookback period should be greater or equal than the configured -querier.query-store-after and -querier.query-ingesters-within. If this setting is 0, queriers always query all ingesters (ingesters shuffle sharding on read path is disabled). (default 13h0m0s) -querier.store-gateway-client.tls-ca-path string Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used. -querier.store-gateway-client.tls-cert-path string @@ -1298,6 +1304,8 @@ Usage of ./cmd/mimir/mimir: Enable AWS Server Side Encryption. Supported values: SSE-KMS, SSE-S3. -ruler-storage.s3.tls-handshake-timeout duration Maximum time to wait for a TLS handshake. 0 means no limit. (default 10s) + -ruler-storage.storage-prefix string + [experimental] Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters. -ruler-storage.swift.auth-url string OpenStack Swift authentication URL -ruler-storage.swift.auth-version int @@ -1412,17 +1420,35 @@ Usage of ./cmd/mimir/mimir: How frequently to poll for rule changes (default 1m0s) -ruler.query-frontend.address string GRPC listen address of the query-frontend(s). Must be a DNS address (prefixed with dns:///) to enable client side load balancing. - -ruler.query-frontend.tls-ca-path string + -ruler.query-frontend.grpc-client-config.backoff-max-period duration + Maximum delay when backing off. (default 10s) + -ruler.query-frontend.grpc-client-config.backoff-min-period duration + Minimum delay when backing off. (default 100ms) + -ruler.query-frontend.grpc-client-config.backoff-on-ratelimits + Enable backoff and retry when we hit ratelimits. + -ruler.query-frontend.grpc-client-config.backoff-retries int + Number of times to backoff and retry before failing. (default 10) + -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit float + Rate limit for gRPC client; 0 means disabled. + -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst int + Rate limit burst for gRPC client. + -ruler.query-frontend.grpc-client-config.grpc-compression string + Use compression when sending messages. Supported values are: 'gzip', 'snappy' and '' (disable compression) + -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size int + gRPC client max receive message size (bytes). (default 104857600) + -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size int + gRPC client max send message size (bytes). (default 104857600) + -ruler.query-frontend.grpc-client-config.tls-ca-path string Path to the CA certificates file to validate server certificate against. If not set, the host's root CA certificates are used. - -ruler.query-frontend.tls-cert-path string + -ruler.query-frontend.grpc-client-config.tls-cert-path string Path to the client certificate file, which will be used for authenticating with the server. Also requires the key path to be configured. - -ruler.query-frontend.tls-enabled - Set to true if query-frontend connection requires TLS. - -ruler.query-frontend.tls-insecure-skip-verify + -ruler.query-frontend.grpc-client-config.tls-enabled + Enable TLS in the GRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used. + -ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify Skip validating server certificate. - -ruler.query-frontend.tls-key-path string + -ruler.query-frontend.grpc-client-config.tls-key-path string Path to the key file for the client certificate. Also requires the client certificate to be configured. - -ruler.query-frontend.tls-server-name string + -ruler.query-frontend.grpc-client-config.tls-server-name string Override the expected name on the server certificate. -ruler.query-stats-enabled Report the wall time for ruler queries to complete as a per-tenant metric and as an info level log message. @@ -1513,9 +1539,9 @@ Usage of ./cmd/mimir/mimir: -server.grpc-max-concurrent-streams uint Limit on the number of concurrent streams for gRPC calls (0 = unlimited) (default 100) -server.grpc-max-recv-msg-size-bytes int - Limit on the size of a gRPC message this server can receive (bytes). (default 4194304) + Limit on the size of a gRPC message this server can receive (bytes). (default 104857600) -server.grpc-max-send-msg-size-bytes int - Limit on the size of a gRPC message this server can send (bytes). (default 4194304) + Limit on the size of a gRPC message this server can send (bytes). (default 104857600) -server.grpc-tls-ca-path string GRPC TLS Client CA path. -server.grpc-tls-cert-path string @@ -1644,6 +1670,8 @@ Usage of ./cmd/mimir/mimir: True to enable zone-awareness and replicate blocks across different availability zones. This option needs be set both on the store-gateway, querier and ruler when running in microservices mode. -store-gateway.tenant-shard-size int The tenant's shard size, used when store-gateway sharding is enabled. Value of 0 disables shuffle sharding for the tenant, that is all tenant blocks are sharded across all store-gateway replicas. + -store-gateway.thread-pool-size uint + [experimental] Number of OS threads that are dedicated for handling requests. Set to 0 to disable use of dedicated OS threads for handling requests. -store.max-labels-query-length value Limit the time range (end - start time) of series, label names and values queries. This limit is enforced in the querier. If the requested time range is outside the allowed range, the request will not fail but will be manipulated to only query data within the allowed time range. 0 to disable. -store.max-query-length value diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index 9cf462171af..467b52695f6 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -335,10 +335,6 @@ Usage of ./cmd/mimir/mimir: Maximum number of split (by time) or partial (by shard) queries that will be scheduled in parallel by the query-frontend for a single input query. This limit is introduced to have a fairer query scheduling and avoid a single query over a large time range saturating all available queriers. (default 14) -querier.max-samples int Maximum number of samples a single query can load into memory. This config option should be set on query-frontend too when query sharding is enabled. (default 50000000) - -querier.query-ingesters-within duration - Maximum lookback beyond which queries are not sent to ingester. 0 means all queries are sent to ingester. (default 13h0m0s) - -querier.query-store-after duration - The time after which a metric should be queried from storage and not just ingesters. 0 means all queries are sent to store. If this option is enabled, the time range of the query sent to the store-gateway will be manipulated to ensure the query end is not more recent than 'now - query-store-after'. -querier.scheduler-address string Address of the query-scheduler component, in host:port format. Only one of -querier.frontend-address or -querier.scheduler-address can be set. If neither is set, queries are only received via HTTP endpoint. -querier.timeout duration diff --git a/cmd/mimir/main.go b/cmd/mimir/main.go index 4777ee2c73b..72bddb289d4 100644 --- a/cmd/mimir/main.go +++ b/cmd/mimir/main.go @@ -26,6 +26,7 @@ import ( "github.com/grafana/mimir/pkg/mimir" util_log "github.com/grafana/mimir/pkg/util/log" + "github.com/grafana/mimir/pkg/util/usage" "github.com/grafana/mimir/pkg/util/version" ) @@ -113,7 +114,7 @@ func main() { if mainFlags.printHelp || mainFlags.printHelpAll { // Print available parameters to stdout, so that users can grep/less them easily. flag.CommandLine.SetOutput(os.Stdout) - if err := usage(&mainFlags, &cfg); err != nil { + if err := usage.Usage(mainFlags.printHelpAll, &mainFlags, &cfg); err != nil { fmt.Fprintf(os.Stderr, "error printing usage: %s\n", err) os.Exit(1) } diff --git a/cmd/mimirtool/Dockerfile b/cmd/mimirtool/Dockerfile index 3d029782786..0f797e6ce8b 100644 --- a/cmd/mimirtool/Dockerfile +++ b/cmd/mimirtool/Dockerfile @@ -1,12 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 -FROM alpine:3.15.0 +FROM alpine:3.16.0 RUN apk add --no-cache ca-certificates # Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG. ARG TARGETOS ARG TARGETARCH ARG BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}" -# Set to non-empty value to use ${TARGET_SUFFIX} when copying mimir binary, leave unset to use no suffix. +# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix. ARG USE_BINARY_SUFFIX COPY mimirtool${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /bin/mimirtool ENTRYPOINT [ "/bin/mimirtool" ] diff --git a/cmd/query-tee/Dockerfile b/cmd/query-tee/Dockerfile index 8571fb2bc95..008e65e6637 100644 --- a/cmd/query-tee/Dockerfile +++ b/cmd/query-tee/Dockerfile @@ -3,9 +3,15 @@ # Provenance-includes-license: Apache-2.0 # Provenance-includes-copyright: The Cortex Authors. -FROM alpine:3.15.0 +FROM alpine:3.16.0 RUN apk add --no-cache ca-certificates -COPY query-tee / +# Expose TARGETOS and TARGETARCH variables. These are supported by Docker when using BuildKit, but must be "enabled" using ARG. +ARG TARGETOS +ARG TARGETARCH +ARG BINARY_SUFFIX="_${TARGETOS}_${TARGETARCH}" +# Set to non-empty value to use ${TARGET_SUFFIX} when copying binary, leave unset to use no suffix. +ARG USE_BINARY_SUFFIX +COPY query-tee${USE_BINARY_SUFFIX:+${BINARY_SUFFIX}} /query-tee ENTRYPOINT ["/query-tee"] ARG revision diff --git a/development/tsdb-blocks-storage-s3-single-binary/config/mimir.yaml b/development/tsdb-blocks-storage-s3-single-binary/config/mimir.yaml index 0e8231f1e27..34211b4dde3 100644 --- a/development/tsdb-blocks-storage-s3-single-binary/config/mimir.yaml +++ b/development/tsdb-blocks-storage-s3-single-binary/config/mimir.yaml @@ -14,7 +14,6 @@ ingester_client: ingester: ring: # We want to start immediately. - join_after: 0 final_sleep: 0s num_tokens: 512 kvstore: @@ -23,9 +22,6 @@ ingester: host: consul:8500 replication_factor: 1 -querier: - query_ingesters_within: 3h - store_gateway: sharding_ring: replication_factor: 1 diff --git a/development/tsdb-blocks-storage-s3-single-binary/dev.dockerfile b/development/tsdb-blocks-storage-s3-single-binary/dev.dockerfile index 865216008c3..7844498ee10 100644 --- a/development/tsdb-blocks-storage-s3-single-binary/dev.dockerfile +++ b/development/tsdb-blocks-storage-s3-single-binary/dev.dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15.4 +FROM alpine:3.16 RUN mkdir /mimir WORKDIR /mimir diff --git a/development/tsdb-blocks-storage-s3/config/mimir.yaml b/development/tsdb-blocks-storage-s3/config/mimir.yaml index d26ab0ad3da..defad27077f 100644 --- a/development/tsdb-blocks-storage-s3/config/mimir.yaml +++ b/development/tsdb-blocks-storage-s3/config/mimir.yaml @@ -20,7 +20,6 @@ ingester_client: ingester: ring: # We want to start immediately. - join_after: 0 final_sleep: 0s num_tokens: 512 kvstore: @@ -36,9 +35,6 @@ memberlist: abort_if_cluster_join_fails: false rejoin_interval: 10s -querier: - query_ingesters_within: 3h - blocks_storage: backend: s3 diff --git a/development/tsdb-blocks-storage-s3/dev.dockerfile b/development/tsdb-blocks-storage-s3/dev.dockerfile index facd2333a0c..efa0fbd145a 100644 --- a/development/tsdb-blocks-storage-s3/dev.dockerfile +++ b/development/tsdb-blocks-storage-s3/dev.dockerfile @@ -2,7 +2,7 @@ FROM golang:1.17.8 ENV CGO_ENABLED=0 RUN go install github.com/go-delve/delve/cmd/dlv@v1.7.3 -FROM alpine:3.15.4 +FROM alpine:3.16 RUN mkdir /mimir WORKDIR /mimir diff --git a/development/tsdb-blocks-storage-swift-single-binary/config/mimir.yaml b/development/tsdb-blocks-storage-swift-single-binary/config/mimir.yaml index 21ced14b220..3dbffde8311 100644 --- a/development/tsdb-blocks-storage-swift-single-binary/config/mimir.yaml +++ b/development/tsdb-blocks-storage-swift-single-binary/config/mimir.yaml @@ -14,7 +14,6 @@ ingester_client: ingester: ring: # We want to start immediately. - join_after: 0 final_sleep: 0s num_tokens: 512 kvstore: @@ -23,9 +22,6 @@ ingester: host: consul:8500 replication_factor: 1 -querier: - query_ingesters_within: 3h - store_gateway: sharding_ring: replication_factor: 1 diff --git a/development/tsdb-blocks-storage-swift-single-binary/dev.dockerfile b/development/tsdb-blocks-storage-swift-single-binary/dev.dockerfile index 1a7db8a45d8..dd3d067f8cc 100644 --- a/development/tsdb-blocks-storage-swift-single-binary/dev.dockerfile +++ b/development/tsdb-blocks-storage-swift-single-binary/dev.dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.15.0 +FROM alpine:3.16.0 RUN mkdir /mimir WORKDIR /mimir diff --git a/docs/configurations/demo.yaml b/docs/configurations/demo.yaml index e6064d46601..2c83f967067 100644 --- a/docs/configurations/demo.yaml +++ b/docs/configurations/demo.yaml @@ -31,9 +31,9 @@ ingester: replication_factor: 1 ruler_storage: - backend: local - local: - directory: /tmp/mimir/rules + backend: filesystem + filesystem: + dir: /tmp/mimir/rules server: http_listen_port: 9009 diff --git a/docs/configurations/grafanacon-2022/agent-setup.yaml b/docs/configurations/grafanacon-2022/agent-setup.yaml new file mode 100644 index 00000000000..c796031aded --- /dev/null +++ b/docs/configurations/grafanacon-2022/agent-setup.yaml @@ -0,0 +1,74 @@ +apiVersion: monitoring.grafana.com/v1alpha1 +kind: GrafanaAgent +metadata: + name: grafana-agent + namespace: default + labels: + app: grafana-agent +spec: + image: grafana/agent:v0.24.1 + logLevel: info + serviceAccountName: grafana-agent + metrics: + instanceSelector: + matchLabels: + agent: grafana-agent-metrics + externalLabels: + k8s_cluster: docker-desktop + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: grafana-agent + namespace: default + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: grafana-agent +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - events + verbs: + - get + - list + - watch + - apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch + - nonResourceURLs: + - /metrics + - /metrics/cadvisor + verbs: + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: grafana-agent +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: grafana-agent +subjects: + - kind: ServiceAccount + name: grafana-agent + namespace: default diff --git a/docs/configurations/grafanacon-2022/alerts.yaml b/docs/configurations/grafanacon-2022/alerts.yaml new file mode 120000 index 00000000000..ab56b372c63 --- /dev/null +++ b/docs/configurations/grafanacon-2022/alerts.yaml @@ -0,0 +1 @@ +../../../operations/mimir-mixin-compiled/alerts.yaml \ No newline at end of file diff --git a/docs/configurations/grafanacon-2022/index.md b/docs/configurations/grafanacon-2022/index.md new file mode 100644 index 00000000000..c4dccf005de --- /dev/null +++ b/docs/configurations/grafanacon-2022/index.md @@ -0,0 +1,81 @@ +--- +title: "GrafanaCon 2022 Mimir session" +menuTitle: "GrafanaCon 2022 Mimir session" +description: "Configuration files used during GrafanaCon 20202 Mimir session" +weight: 1 +keywords: + - grafanacon 2022 + - mimir +author: + - peter +associated_technologies: + - mimir +--- + +# Before you start + +**Warning:** Following commands will not specify explicit context for `kubectl` commands. Make sure to select correct +context and namespace. For example Docker for Desktop comes with `docker-desktop` context preinstalled. We will +use `default` namespace in this demo. + +``` +kubectl config use-context docker-desktop +kubectl config set-context docker-desktop --namespace=default +``` + +# Adding Grafana Helm Charts repository + +``` +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +# Install Ingress + +``` +kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.1.2/deploy/static/provider/cloud/deploy.yaml +``` + +# Grafana Agent Operator + +Links: + +- [Installing Grafana Agent Operator with Helm](https://grafana.com/docs/agent/latest/operator/helm-getting-started/) +- [Grafana Agent Operator Custom Resource Quickstart](https://grafana.com/docs/agent/latest/operator/custom-resource-quickstart/) + +``` +helm install agent grafana/grafana-agent-operator + +kubectl apply -f agent-setup.yaml +``` + +# Deploy Mimir + +`mimir-values.yaml` file uses `mimir-helm` as ingress hostname. Make sure it resolves to IP address of your Ingress. + +``` +helm install gcon grafana/mimir-distributed -f mimir-values.yaml +``` + +# Configure Grafana Agent to scrape Mimir metrics and send them to Mimir + +``` +kubectl apply -f metrics-instance.yaml +``` + +# Deploy Rules and Alerts + +``` +mimirtool rules load --address=http://mimir-helm/ --id=anonymous ./rules.yaml + +mimirtool rules load --address=http://mimir-helm/ --id=anonymous ./alerts.yaml +``` + +# Scrape Kubernetes metrics + +This step adds more ServiceMonitors, which will be used by Grafana Agent to discover metrics to scrape, and forward to Mimir. +ServiceMonitors in this case are used to scrape metrics from Kubernetes itself. + +``` +kubectl apply -f kubernetes-service-monitors.yaml +``` diff --git a/docs/configurations/grafanacon-2022/kubernetes-service-monitors.yaml b/docs/configurations/grafanacon-2022/kubernetes-service-monitors.yaml new file mode 100644 index 00000000000..d18945c78cb --- /dev/null +++ b/docs/configurations/grafanacon-2022/kubernetes-service-monitors.yaml @@ -0,0 +1,71 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + instance: primary + name: kubelet-monitor + namespace: default +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + interval: 60s + metricRelabelings: + - action: keep + regex: kubelet_cgroup_manager_duration_seconds_count|go_goroutines|kubelet_pod_start_duration_seconds_count|kubelet_runtime_operations_total|kubelet_pleg_relist_duration_seconds_bucket|volume_manager_total_volumes|kubelet_volume_stats_capacity_bytes|container_cpu_usage_seconds_total|container_network_transmit_bytes_total|kubelet_runtime_operations_errors_total|container_network_receive_bytes_total|container_memory_swap|container_network_receive_packets_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|kubelet_running_pod_count|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate|container_memory_working_set_bytes|storage_operation_errors_total|kubelet_pleg_relist_duration_seconds_count|kubelet_running_pods|rest_client_request_duration_seconds_bucket|process_resident_memory_bytes|storage_operation_duration_seconds_count|kubelet_running_containers|kubelet_runtime_operations_duration_seconds_bucket|kubelet_node_config_error|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_running_container_count|kubelet_volume_stats_available_bytes|kubelet_volume_stats_inodes|container_memory_rss|kubelet_pod_worker_duration_seconds_count|kubelet_node_name|kubelet_pleg_relist_interval_seconds_bucket|container_network_receive_packets_dropped_total|kubelet_pod_worker_duration_seconds_bucket|container_start_time_seconds|container_network_transmit_packets_dropped_total|process_cpu_seconds_total|storage_operation_duration_seconds_bucket|container_memory_cache|container_network_transmit_packets_total|kubelet_volume_stats_inodes_used|up|rest_client_requests_total + sourceLabels: + - __name__ + - action: replace + targetLabel: job + replacement: integrations/kubernetes/kubelet + port: https-metrics + relabelings: + - sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https + tlsConfig: + insecureSkipVerify: true + namespaceSelector: + matchNames: + - default + selector: + matchLabels: + app.kubernetes.io/name: kubelet +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + instance: primary + name: cadvisor-monitor + namespace: default +spec: + endpoints: + - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + honorLabels: true + honorTimestamps: false + interval: 60s + metricRelabelings: + - action: keep + regex: kubelet_cgroup_manager_duration_seconds_count|go_goroutines|kubelet_pod_start_duration_seconds_count|kubelet_runtime_operations_total|kubelet_pleg_relist_duration_seconds_bucket|volume_manager_total_volumes|kubelet_volume_stats_capacity_bytes|container_cpu_usage_seconds_total|container_network_transmit_bytes_total|kubelet_runtime_operations_errors_total|container_network_receive_bytes_total|container_memory_swap|container_network_receive_packets_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|kubelet_running_pod_count|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate|container_memory_working_set_bytes|storage_operation_errors_total|kubelet_pleg_relist_duration_seconds_count|kubelet_running_pods|rest_client_request_duration_seconds_bucket|process_resident_memory_bytes|storage_operation_duration_seconds_count|kubelet_running_containers|kubelet_runtime_operations_duration_seconds_bucket|kubelet_node_config_error|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_running_container_count|kubelet_volume_stats_available_bytes|kubelet_volume_stats_inodes|container_memory_rss|kubelet_pod_worker_duration_seconds_count|kubelet_node_name|kubelet_pleg_relist_interval_seconds_bucket|container_network_receive_packets_dropped_total|kubelet_pod_worker_duration_seconds_bucket|container_start_time_seconds|container_network_transmit_packets_dropped_total|process_cpu_seconds_total|storage_operation_duration_seconds_bucket|container_memory_cache|container_network_transmit_packets_total|kubelet_volume_stats_inodes_used|up|rest_client_requests_total + sourceLabels: + - __name__ + - action: replace + targetLabel: job + replacement: integrations/kubernetes/cadvisor + path: /metrics/cadvisor + port: https-metrics + relabelings: + - sourceLabels: + - __metrics_path__ + targetLabel: metrics_path + scheme: https + tlsConfig: + insecureSkipVerify: true + namespaceSelector: + matchNames: + - default + selector: + matchLabels: + app.kubernetes.io/name: kubelet diff --git a/docs/configurations/grafanacon-2022/metrics-instance.yaml b/docs/configurations/grafanacon-2022/metrics-instance.yaml new file mode 100644 index 00000000000..80a74171a5a --- /dev/null +++ b/docs/configurations/grafanacon-2022/metrics-instance.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.grafana.com/v1alpha1 +kind: MetricsInstance + +metadata: + name: metrics + namespace: default + # Label used by GrafanaAgent resource to find + # MetricsInstance resources. + labels: + agent: grafana-agent-metrics + +spec: + remoteWrite: + - url: http://gcon-mimir-distributor:8080/api/v1/push + + # Use all ServiceMonitor resources in this namespace + # (namespace is configured by serviceMonitorNamespaceSelector, defaults to + # same namespace as MetricsInstance) + serviceMonitorSelector: {} diff --git a/docs/configurations/grafanacon-2022/mimir-values.yaml b/docs/configurations/grafanacon-2022/mimir-values.yaml new file mode 100644 index 00000000000..0bd83bedaab --- /dev/null +++ b/docs/configurations/grafanacon-2022/mimir-values.yaml @@ -0,0 +1,15 @@ +nginx: + ingress: + enabled: true + ingressClassName: nginx + hosts: + - host: mimir-helm + paths: + - path: / + pathType: Prefix + tls: + # empty, disabled. + +serviceMonitor: + enabled: true + interval: 10s diff --git a/docs/configurations/grafanacon-2022/rules.yaml b/docs/configurations/grafanacon-2022/rules.yaml new file mode 120000 index 00000000000..d942b75aa09 --- /dev/null +++ b/docs/configurations/grafanacon-2022/rules.yaml @@ -0,0 +1 @@ +../../../operations/mimir-mixin-compiled/rules.yaml \ No newline at end of file diff --git a/docs/configurations/single-process-config-blocks-gossip-1.yaml b/docs/configurations/single-process-config-blocks-gossip-1.yaml index 272efc3c278..7cd5d2ff095 100644 --- a/docs/configurations/single-process-config-blocks-gossip-1.yaml +++ b/docs/configurations/single-process-config-blocks-gossip-1.yaml @@ -28,10 +28,6 @@ ingester: # Defaults to hostname, but we run both ingesters in this demonstration on the same machine. instance_id: "Ingester 1" - # We don't want to join immediately, but wait a bit to see other ingesters and their tokens first. - # It can take a while to have the full picture when using gossip - join_after: 10s - # To avoid generating same tokens by multiple ingesters, they can "observe" the ring for a while, # after putting their own tokens into it. This is only useful when using gossip, since multiple # ingesters joining at the same time can have conflicting tokens if they don't see each other yet. diff --git a/docs/configurations/single-process-config-blocks-gossip-2.yaml b/docs/configurations/single-process-config-blocks-gossip-2.yaml index 965dd97113a..fc0b05afc36 100644 --- a/docs/configurations/single-process-config-blocks-gossip-2.yaml +++ b/docs/configurations/single-process-config-blocks-gossip-2.yaml @@ -28,10 +28,6 @@ ingester: # Defaults to hostname, but we run both ingesters in this demonstration on the same machine. instance_id: "Ingester 2" - # We don't want to join immediately, but wait a bit to see other ingesters and their tokens first. - # It can take a while to have the full picture when using gossip - join_after: 10s - # To avoid generating same tokens by multiple ingesters, they can "observe" the ring for a while, # after putting their own tokens into it. This is only useful when using gossip, since multiple # ingesters joining at the same time can have conflicting tokens if they don't see each other yet. diff --git a/docs/configurations/single-process-config-blocks-tls.yaml b/docs/configurations/single-process-config-blocks-tls.yaml index ae9ec0f11a1..80298c4e143 100644 --- a/docs/configurations/single-process-config-blocks-tls.yaml +++ b/docs/configurations/single-process-config-blocks-tls.yaml @@ -38,7 +38,6 @@ ingester: # address: 127.0.0.1 # We want to start immediately and flush on shutdown. - join_after: 0 min_ready_duration: 0s final_sleep: 0s num_tokens: 512 diff --git a/docs/configurations/single-process-config-blocks.yaml b/docs/configurations/single-process-config-blocks.yaml index 5ce7a6e056e..30f0ed1fc7f 100644 --- a/docs/configurations/single-process-config-blocks.yaml +++ b/docs/configurations/single-process-config-blocks.yaml @@ -29,7 +29,6 @@ ingester: # address: 127.0.0.1 # We want to start immediately and flush on shutdown. - join_after: 0 min_ready_duration: 0s final_sleep: 0s num_tokens: 512 diff --git a/docs/internal/contributing/README.md b/docs/internal/contributing/README.md index 38c86ee8165..4965a262e76 100644 --- a/docs/internal/contributing/README.md +++ b/docs/internal/contributing/README.md @@ -11,9 +11,13 @@ a piece of work is finished it should: - Be organised into one or more commits, each of which has a commit message that describes all changes made in that commit ('why' more than 'what' - we can read the diffs to see the code that changed). - Each commit should build towards the whole - don't leave in back-tracks and mistakes that you later corrected. - Have unit and/or [integration](./how-integration-tests-work.md) tests for new functionality or tests that would have caught the bug being fixed. -- Include a CHANGELOG message if users of Grafana Mimir need to hear about what you did. +- Include a [CHANGELOG](#changelog) message if users of Grafana Mimir need to hear about what you did. - If you have made any changes to flags or config, run `make doc` and commit the changed files to update the config file documentation. +## Grafana Mimir Helm chart + +Please see the dedicated "[Contributing to Grafana Mimir helm chart](contributing-to-helm-chart.md)" page. + ## Formatting Grafana Mimir uses `goimports` tool (`go get golang.org/x/tools/cmd/goimports` to install) to format the Go files, and sort imports. We use goimports with `-local github.com/grafana/mimir` parameter, to put Grafana Mimir internal imports into a separate group. We try to keep imports sorted into three groups: imports from standard library, imports of 3rd party packages and internal Grafana Mimir imports. Goimports will fix the order, but will keep existing newlines between imports in the groups. We try to avoid extra newlines like that. @@ -22,16 +26,6 @@ You're using an IDE you may find useful the following settings for the Grafana M - [VSCode](vscode-goimports-settings.json) -## Developer Certificates of Origin (DCOs) - -Before submitting your work in a pull request, make sure that _all_ commits are signed off with a **Developer Certificate of Origin** (DCO). Here's an example: - -```shell -git commit -s -m "Here is my signed commit" -``` - -You can find further instructions [here](https://github.com/probot/dco#how-it-works). - ## Building Grafana Mimir To build: @@ -87,3 +81,28 @@ Please see the dedicated "[Design patterns and Code conventions](design-patterns The Grafana Mimir documentation is compiled into a website published at [grafana.com](https://grafana.com/). Please see "[How to run the website locally](./how-to-run-website-locally.md)" for instructions. Note: if you attempt to view pages on Github, it's likely that you might find broken links or pages. That is expected and should not be addressed unless it is causing issues with the site that occur as part of the build. + +## Changelog + +When appending to the changelog, the changes must be listed with a corresponding scope. A scope denotes the type of change that has occurred. + +The ordering of entries in the changelog should be `[CHANGE]`, `[FEATURE]`, `[ENHANCEMENT]`, `[BUGFIX]`. + +#### [CHANGE] + +The CHANGE scope denotes a change that changes the expected behavior of the project while not adding new functionality or fixing an underling issue. This commonly occurs when renaming things to make them more consistent or to accommodate updated versions of vendored dependencies. + +#### [FEATURE] + +The FEATURE scope denotes a change that adds new functionality to the project/service. + +#### [ENHANCEMENT] + +The ENHANCEMENT scope denotes a change that improves upon the current functionality of the project/service. Generally, an enhancement is something that improves upon something that is already present. Either by making it simpler, more powerful, or more performant. For Example: + +- An optimization on a particular process in a service that makes it more performant +- Simpler syntax for setting a configuration value, like allowing `1m` instead of 60 for a duration setting. + +#### [BUGFIX] + +The BUGFIX scope denotes a change that fixes an issue with the project in question. A BUGFIX should align the behaviour of the service with the current expected behaviour of the service. If a BUGFIX introduces new unexpected behaviour to ameliorate the issue, a corresponding FEATURE or ENHANCEMENT scope should also be added to the changelog. diff --git a/docs/internal/contributing/contributing-to-helm-chart.md b/docs/internal/contributing/contributing-to-helm-chart.md new file mode 100644 index 00000000000..119e31bb4e3 --- /dev/null +++ b/docs/internal/contributing/contributing-to-helm-chart.md @@ -0,0 +1,27 @@ +# Contributing to the Grafana Mimir Helm Chart + +## Differences to general workflow + +Please see the [general workflow](README.md#workflow) for reference. + +- Changelog is in the chart itself [operations/helm/charts/mimir-distributed/CHANGELOG.md](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/CHANGELOG.md). +- If you made any changes to the [operations/helm/charts/mimir-distributed/Chart.yaml](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/Chart.yaml), run `make doc` and commit the changed files to update the [operations/helm/charts/mimir-distributed/README.md](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/README.md). +- If your changes impact the test configurations in the [operations/helm/charts/mimir-distributed/ci](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/ci) directory, see [Updating compiled manifests](#updating-compiled-manifests). + +## Updating compiled manifests + +We keep a compiled version of the helm chart for each values file in the `ci` directory. +This makes it easy to see how a given PR impacts the final output. +A PR check will fail if you forget to update the compiled manifests, and you can use `make build-helm-tests` to update them. + +## Versioning + +Normally contributors need _not_ bump the version. The chart will be released with a beta version weekly by maintainers (unless no changes were made) and also regular stable releases will be released by cherry picking commits from the `main` branch to a release branch (e.g. `release-2.1`). + +If version increase is need, the version is set in the chart itself [operations/helm/charts/mimir-distributed/Chart.yaml](https://github.com/grafana/mimir/blob/main/operations/helm/charts/mimir-distributed/Chart.yaml). On the `main` branch, versions should be _pre-release_ only, meaning that the version should be suffixed by `-beta.`, for example `2.1.0-beta.1` for a pre-release of the next stable `2.1.0` chart version. Versioning should follow the [Helm3 standard](https://helm.sh/docs/topics/charts/#charts-and-versioning) which is [SemVer 2](https://semver.org/spec/v2.0.0.html). + +## Using beta version + +Once a PR that updates the chart version is merged to `main`, it takes a couple of minutes for it to be published in [https://grafana.github.io/helm-charts](https://grafana.github.io/helm-charts) Helm repository. + +In order to search, template, install, upgrade, etc beta versions of charts, Helm commands require the user to specify the `--devel` flag. This means that checking for whether the beta version is published should be done with `helm search repo --devel`. diff --git a/docs/internal/how-to-update-the-build-image.md b/docs/internal/how-to-update-the-build-image.md index 8a6610735af..0119bd6ffdc 100644 --- a/docs/internal/how-to-update-the-build-image.md +++ b/docs/internal/how-to-update-the-build-image.md @@ -1,6 +1,8 @@ The build image currently can only be updated by a Grafana Mimir maintainer. If you're not a maintainer you can still open a PR with the changes, asking a maintainer to assist you in publishing the updated image. The procedure is: -1. Update `mimir-build-image/Dockerfile`. -2. Build and publish the image by using `make push-multiarch-build-image`. This will build and push multiplatform docker image (for Linux/amd64 and Linux/arm64). Pushing to the `grafana/mimir-build-image` repository can only be done by a maintainer. Running this step successfully requires [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/), but does not require a specific platform. -3. Replace the image tag in `.github/workflows/*` (_there may be multiple references_) and Makefile (variable `LATEST_BUILD_IMAGE_TAG`). -4. Open a PR and make sure the CI with the new build image passes +1. Update `mimir-build-image/Dockerfile` on a new branch. Note: the resulting images have the tag name derived from the branch name. +2. Make sure to have [Docker Buildx](https://docs.docker.com/buildx/working-with-buildx/). Docker Desktop and major distributions have it in the docker package. +3. On Linux you'll need some extra qemu packages as well: `sudo apt-get install qemu qemu-user-static binfmt-support debootstrap`. And set up docker buildx: `docker buildx create --name armBuilder ; docker buildx use armBuilder`. +4. Build and publish the image by using `make push-multiarch-build-image`. This will build and push multiplatform docker image (for Linux/amd64 and Linux/arm64). +5. Replace the image tag in `.github/workflows/*` (_there may be multiple references_) and Makefile (variable `LATEST_BUILD_IMAGE_TAG`). +6. Open a PR and make sure the CI with the new build image passes diff --git a/docs/sources/_index.md b/docs/sources/_index.md index b3995a789c7..82b4bba8e62 100644 --- a/docs/sources/_index.md +++ b/docs/sources/_index.md @@ -14,13 +14,13 @@ keywords: - observability --- -# Grafana Mimir Documentation +# Grafana Mimir documentation ![Grafana Mimir](mimir-logo.png) Grafana Mimir is an open source software project that provides a scalable long-term storage for [Prometheus](https://prometheus.io). Some of the core strengths of Grafana Mimir include: -- **Easy to install and maintain:** Grafana Mimir’s extensive documentation, tutorials, and deployment tooling make it quick to get started. Using its monolithic mode, you can get Grafana Mimir up and running with just one binary and no additional dependencies. Once deployed, the best-practice dashboards, alerts, and playbooks packaged with Grafana Mimir make it easy to monitor the health of the system. +- **Easy to install and maintain:** Grafana Mimir’s extensive documentation, tutorials, and deployment tooling make it quick to get started. Using its monolithic mode, you can get Grafana Mimir up and running with just one binary and no additional dependencies. Once deployed, the best-practice dashboards, alerts, and runbooks packaged with Grafana Mimir make it easy to monitor the health of the system. - **Massive scalability:** You can run Grafana Mimir's horizontally-scalable architecture across multiple machines, resulting in the ability to process orders of magnitude more time series than a single Prometheus instance. Internal testing shows that Grafana Mimir handles up to 1 billion active time series. - **Global view of metrics:** Grafana Mimir enables you to run queries that aggregate series from multiple Prometheus instances, giving you a global view of your systems. Its query engine extensively parallelizes query execution, so that even the highest-cardinality queries complete with blazing speed. - **Cheap, durable metric storage:** Grafana Mimir uses object storage for long-term data storage, allowing it to take advantage of this ubiquitous, cost-effective, high-durability technology. It is compatible with multiple object store implementations, including AWS S3, Google Cloud Storage, Azure Blob Storage, OpenStack Swift, as well as any S3-compatible object storage. diff --git a/docs/sources/operators-guide/architecture/bucket-index/index.md b/docs/sources/operators-guide/architecture/bucket-index/index.md index a8e4d1f9dee..6e30985da6c 100644 --- a/docs/sources/operators-guide/architecture/bucket-index/index.md +++ b/docs/sources/operators-guide/architecture/bucket-index/index.md @@ -7,7 +7,7 @@ weight: 50 # Grafana Mimir bucket index -The bucket index is a per-tenant file that contains the list of blocks and block deletion marks in the storage. The bucket index is stored in the backend object storage, is periodically updated by the compactor, and used by queriers, store-gateways, and rulers to discover blocks in the storage. +The bucket index is a per-tenant file that contains the list of blocks and block deletion marks in the storage. The bucket index is stored in the backend object storage, is periodically updated by the compactor, and used by queriers, store-gateways, and rulers (in [internal]({{< relref "../components/ruler/index.md#internal" >}}) operational mode) to discover blocks in the storage. The bucket index is enabled by default, but is optional. It can be disabled via `-blocks-storage.bucket-store.bucket-index.enabled=false` (or its respective YAML configuration option). Disabling the bucket index is not recommended. diff --git a/docs/sources/operators-guide/architecture/components/distributor.md b/docs/sources/operators-guide/architecture/components/distributor.md index 51ced7dea37..357bbdcbe22 100644 --- a/docs/sources/operators-guide/architecture/components/distributor.md +++ b/docs/sources/operators-guide/architecture/components/distributor.md @@ -33,21 +33,29 @@ The distributor validation includes the following checks: ## Rate limiting -The distributor includes a built-in rate limiter that it applies to each tenant. -The rate limit is the maximum ingestion rate for each tenant across the Grafana Mimir cluster. -If the rate exceeds the maximum number of samples per second, the distributor drops the request and returns an HTTP 429 response code. +The distributor includes two different types of rate limiters that apply to each tenant. -Internally, the limit is implemented using a per-distributor local rate limiter. -The local rate limiter for each distributor is configured with a limit of `ingestion rate limit / N`, where `N` is the number of healthy distributor replicas. -The distributor automatically adjusts the ingestion rate limit if the number of distributor replicas change. -Because the rate limit is implemented using a per-distributor local rate limiter, the ingestion rate limit requires that write requests are [evenly distributed across the pool of distributors]({{< relref "#load-balancing-across-distributors" >}}). +- **Request rate**
+ The maximum number of requests per second that can be served across Grafana Mimir cluster for each tenant. -Use the following flags to configure the rate limit: +- **Ingestion rate**
+ The maximum samples per second that can be ingested across Grafana Mimir cluster for each tenant. +If any of these rates is exceeded, the distributor drops the request and returns an HTTP 429 response code. + +Internally, these limits are implemented using a per-distributor local rate limiter. +The local rate limiter for each distributor is configured with a limit of `limit / N`, where `N` is the number of healthy distributor replicas. +The distributor automatically adjusts the request and ingestion rate limits if the number of distributor replicas change. +Because these rate limits are implemented using a per-distributor local rate limiter, they require that write requests are [evenly distributed across the pool of distributors]({{< relref "#load-balancing-across-distributors" >}}). + +Use the following flags to configure the rate limits: + +- `-distributor.request-rate-limit`: Request rate limit, which is per tenant, and which is in requests per second +- `-distributor.request-burst-size`: Request burst size (in number of requests) allowed, which is per tenant - `-distributor.ingestion-rate-limit`: Ingestion rate limit, which is per tenant, and which is in samples per second - `-distributor.ingestion-burst-size`: Ingestion burst size (in number of samples) allowed, which is per tenant -> **Note:** You can override rate limiting on a per-tenant basis by setting `ingestion_rate` and `ingestion_burst_size` in the overrides section of the runtime configuration. +> **Note:** You can override rate limiting on a per-tenant basis by setting `request_rate`, `ingestion_rate`, `request_burst_size` and `ingestion_burst_size` in the overrides section of the runtime configuration. > **Note:** By default, Prometheus remote write doesn't retry requests on 429 HTTP response status code. To modify this behavior, use `retry_on_http_429: true` in the Prometheus [`remote_write` configuration](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write). diff --git a/docs/sources/operators-guide/architecture/components/ruler/index.md b/docs/sources/operators-guide/architecture/components/ruler/index.md index 16322a3aeb2..d4d517c5577 100644 --- a/docs/sources/operators-guide/architecture/components/ruler/index.md +++ b/docs/sources/operators-guide/architecture/components/ruler/index.md @@ -10,22 +10,41 @@ weight: 130 The ruler is an optional component that evaluates PromQL expressions defined in recording and alerting rules. Each tenant has a set of recording and alerting rules and can group those rules into namespaces. -[//]: # "Diagram source of ruler interactions https://docs.google.com/presentation/d/1LemaTVqa4Lf_tpql060vVoDGXrthp-Pie_SQL7qwHjc/edit#slide=id.g11658e7e4c6_0_938" +## Operational modes -![Architecture of Grafana Mimir's ruler component](ruler.svg) +The ruler supports two different rule evaluation modes: -## Recording rules +### Internal + +This is the default mode. The ruler internally runs a querier and distributor, and evaluates recording and alerting rules in the ruler process itself. +To evaluate rules, the ruler connects directly to ingesters and store-gateways, and writes any resulting series to the ingesters. -The ruler evaluates the expressions in the recording rules at regular intervals and writes the results back to the ingesters. -The ruler has a built-in querier that evaluates the PromQL expressions and a built-in distributor, so that it can write directly to the ingesters. Configuration of the built-in querier and distributor uses their respective configuration parameters: - [Querier]({{< relref "../../../configuring/reference-configuration-parameters/index.md#querier" >}}) - [Distributor]({{< relref "../../../configuring/reference-configuration-parameters/index.md#distributor" >}}) +> **Note**: When this mode is used, no query acceleration techniques are used and the evaluation of very high cardinality queries could take longer than the evaluation interval, eventually leading to missing data points in the evaluated recording rules. + +[//]: # "Diagram source of ruler interactions https://docs.google.com/presentation/d/1LemaTVqa4Lf_tpql060vVoDGXrthp-Pie_SQL7qwHjc/edit#slide=id.g11658e7e4c6_0_938" + +![Architecture of Grafana Mimir's ruler component in internal mode](ruler-internal.svg) + +### Remote + +In this mode the ruler delegates rules evaluation to the query-frontend. When enabled, the ruler leverages all the query acceleration techniques employed by the query-frontend, such as [query sharding]({{< relref "../../query-sharding/index.md" >}}). +To enable the remote operational mode, set the `-ruler.query-frontend.address` CLI flag or its respective YAML configuration parameter for the ruler. +Communication between ruler and query-frontend is established over gRPC, so you can make use of client-side load balancing by prefixing the query-frontend address URL with `dns://`. + +![Architecture of Grafana Mimir's ruler component in remote mode](ruler-remote.svg) + +## Recording rules + +The ruler evaluates the expressions in the [recording rules](https://prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) at regular intervals and writes the results back to the ingesters. + ## Alerting rules -The ruler evaluates the expressions in alerting rules at regular intervals and if the result includes any series, the alert becomes active. +The ruler evaluates the expressions in [alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/#alerting-rules) at regular intervals and if the result includes any series, the alert becomes active. If an alerting rule has a defined `for` duration, it enters the **PENDING** (`pending`) state. After the alert has been active for the entire `for` duration, it enters the **FIRING** (`firing`) state. The ruler then notifies Alertmanagers of any **FIRING** (`firing`) alerts. @@ -33,6 +52,47 @@ The ruler then notifies Alertmanagers of any **FIRING** (`firing`) alerts. Configure the addresses of Alertmanagers with the `-ruler.alertmanager-url` flag, which supports the DNS service discovery format. For more information about DNS service discovery, refer to [Supported discovery modes]({{< relref "../../../configuring/about-dns-service-discovery.md" >}}). +## Federated rule groups + +A federated rule group is a rule group with a non-empty `source_tenants`. + +The `source_tenants` field allows aggregating data from multiple tenants while evaluating a rule group. The expressions +of each rule in the group will be evaluated against the data of all tenants in `source_tenants`. If `source_tenants` is +empty or omitted, then the tenant under which the group is created will be treated as the `source_tenant`. + +Below is an example of how a federated rule group would look like: + +```yaml +name: MyGroupName +source_tenants: ["tenant-a", "tenant-b"] +rules: + - record: sum:metric + expr: sum(metric) +``` + +_In this example `MyGroupName` rules will be evaluated against `tenant-a` and `tenant-b` tenants._ + +Federated rule groups are skipped during evaluation by default. This feature depends on +the cross-tenant query federation feature. To enable federated rules +set `-ruler.tenant-federation.enabled=true` and `-tenant-federation.enabled=true` CLI flags (or their respective YAML +config options). + +During evaluation query limits applied to single tenants are also applied to each query in the rule group. For example, +if `tenant-a` has a federated rule group with `source_tenants: [tenant-b, tenant-c]`, then query limits for `tenant-b` +and `tenant-c` will be applied. If any of these limits is exceeded, the whole evaluation will fail. No partial results +will be saved. The same "no partial results" guarantee applies to queries failing for other reasons (e.g. ingester +unavailability). + +The time series used during evaluation of federated rules will have the `__tenant_id__` label, similar to how it is +present on series returned with cross-tenant query federation. + +> **Note**: Federated rule groups allow data from multiple source tenants to be written into a single +> destination tenant. This makes the existing separation of tenants' data less clear. For example, `tenant-a` has a +> federated rule group that aggregates over `tenant-b`'s data (e.g. `sum(metric_b)`) and writes the result back +> into `tenant-a`'s storage (e.g. as metric `sum:metric_b`). Now part of `tenant-b`'s data is copied to `tenant-a` (albeit +> aggregated). Have this in mind when configuring the access control layer in front of mimir and when enabling federated +> rules via `-ruler.tenant-federation.enabled`. + ## Sharding The ruler supports multi-tenancy and horizontal scalability. diff --git a/docs/sources/operators-guide/architecture/components/ruler/ruler-internal.svg b/docs/sources/operators-guide/architecture/components/ruler/ruler-internal.svg new file mode 100644 index 00000000000..bf966a7d395 --- /dev/null +++ b/docs/sources/operators-guide/architecture/components/ruler/ruler-internal.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/sources/operators-guide/architecture/components/ruler/ruler-remote.svg b/docs/sources/operators-guide/architecture/components/ruler/ruler-remote.svg new file mode 100644 index 00000000000..0c19015dacc --- /dev/null +++ b/docs/sources/operators-guide/architecture/components/ruler/ruler-remote.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/sources/operators-guide/architecture/components/ruler/ruler.svg b/docs/sources/operators-guide/architecture/components/ruler/ruler.svg deleted file mode 100644 index cfbdea25b64..00000000000 --- a/docs/sources/operators-guide/architecture/components/ruler/ruler.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/docs/sources/operators-guide/configuring/about-configurations.md b/docs/sources/operators-guide/configuring/about-configurations.md index 5904417d8d0..561d9bb3b18 100644 --- a/docs/sources/operators-guide/configuring/about-configurations.md +++ b/docs/sources/operators-guide/configuring/about-configurations.md @@ -19,7 +19,7 @@ To see the current configuration state of any component, use the [`/config`]({{< Use a single configuration file, and either pass it to all replicas of Grafana Mimir (when running multiple single-process Mimir replicas) or to all components of Grafana Mimir (when running Grafana Mimir as microservices). When running Grafana Mimir on Kubernetes, you can achieve this by storing the configuration file in a [ConfigMap](https://kubernetes.io/docs/concepts/configuration/configmap/) and mounting it in each Grafana Mimir container. -This recommendation helps to avoid a common misconfiguration pitfall: while certain configuration parameters might look like they’re only needed by one type of component, they might in fact be used by multiple components. For example, the `-ingester.ring.replication-factor` CLI flag is not only required by ingesters, but also by distributors, queriers, and rulers. +This recommendation helps to avoid a common misconfiguration pitfall: while certain configuration parameters might look like they’re only needed by one type of component, they might in fact be used by multiple components. For example, the `-ingester.ring.replication-factor` CLI flag is not only required by ingesters, but also by distributors, queriers, and rulers (in [internal]({{< relref "../architecture/components/ruler/index.md#internal" >}}) operational mode). By using a single configuration file, you ensure that each component gets all of the configuration that it needs without needing to track which parameter belongs to which component. There is no harm in passing a configuration that is specific to one component (such as an ingester) to another component (such as a querier). In such case, the configuration is simply ignored. diff --git a/docs/sources/operators-guide/configuring/about-versioning.md b/docs/sources/operators-guide/configuring/about-versioning.md index 032347e6cde..6458c1706c6 100644 --- a/docs/sources/operators-guide/configuring/about-versioning.md +++ b/docs/sources/operators-guide/configuring/about-versioning.md @@ -43,8 +43,14 @@ Experimental configuration and flags are subject to change. The following features are currently experimental: -- Ruler: Tenant federation -- Distributor: Metrics relabeling +- Ruler + - Tenant federation + - Use query-frontend for rule evaluation +- Distributor + - Metrics relabeling + - Request rate limit + - `-distributor.request-rate-limit` + - `-distributor.request-burst-limit` - Purger: Tenant deletion API - Exemplar storage - `-ingester.max-global-exemplars-per-user` @@ -76,6 +82,10 @@ The following features are currently experimental: - `-query-scheduler.querier-forget-delay` - Store-gateway - `-blocks-storage.bucket-store.index-header-thread-pool-size` +- Blocks Storage, Alertmanager, and Ruler support for partitioning access to the same storage bucket + - `-alertmanager-storage.storage-prefix` + - `-blocks-storage.storage-prefix` + - `-ruler-storage.storage-prefix` ## Deprecated features diff --git a/docs/sources/operators-guide/configuring/configuring-custom-trackers.md b/docs/sources/operators-guide/configuring/configuring-custom-trackers.md index 2709a62bb0d..7a8ae8717fe 100644 --- a/docs/sources/operators-guide/configuring/configuring-custom-trackers.md +++ b/docs/sources/operators-guide/configuring/configuring-custom-trackers.md @@ -25,14 +25,37 @@ Each custom tracker counts the active series matching its label pattern on a per Series with metric name `cortex_ingester_active_series_custom_tracker` have two labels applied: `name` and `user`. The value of the `name` label is the name of the custom tracker specified in the configuration. The value of the `user` label is the tenant-id for which the series count applies. -Assume two custom trackers are configured as in the example above, and that your Grafana Mimir cluster has three tenants: `tenant_1`, `tenant_2`, and `tenant_with_only_prod_metrics`. Assume all series within `tenant_with_only_prod_metrics` have labels that match the pattern `{namespace=~"prod-.*"}` and none that match `{namespace=~"dev-.*"}`. +To illustrate this, assume that two custom trackers are configured as in the preceding YAML snippet, and that your Grafana Mimir cluster has two tenants: `tenant_1` and `tenant_with_only_prod_metrics`. Assume that `tenant_with_only_prod_metrics` has three series with labels that match the pattern `{namespace=~"prod-.*"}` and none that match the patten `{namespace=~"dev-.*"}`. Also assume that `tenant_1` has five series that match the pattern `{namespace=~"dev-.*"}` and 10 series that match the pattern `{namespace=~"prod-.*"}`. In this example, the following output appears when the `/metrics` endpoint for the ingester component is scraped: ``` -cortex_ingester_active_series_custom_tracker{name="dev", user="tenant_1"} -cortex_ingester_active_series_custom_tracker{name="prod", user="tenant_2"} -cortex_ingester_active_series_custom_tracker{name="prod", user="tenant_with_only_prod_metrics"} +cortex_ingester_active_series_custom_tracker{name="dev", user="tenant_1"} 5 +cortex_ingester_active_series_custom_tracker{name="prod", user="tenant_1"} 10 +cortex_ingester_active_series_custom_tracker{name="prod", user="tenant_with_only_prod_metrics"} 3 ``` +For specific tenants, you can override the default configuration as previously described. To do so, edit the [runtime configuration]({{< relref "./about-runtime-configuration.md" >}}). + +You can override the active series custom trackers’ configuration for the tenant `tenant_with_only_prod_metrics` to track two services instead of the default matchers. See the following example: + +``` +overrides: + tenant_with_only_prod_metrics: + active_series_custom_trackers: + service1: '{service="service1"}' + service2: '{service="service2"}' +``` + +After adding this override, and assuming that there is one matching series for `service1` and two matching series for `service2`, the output at `/metrics` changes: + +``` +cortex_ingester_active_series_custom_tracker{name="dev", user="tenant_1"} 5 +cortex_ingester_active_series_custom_tracker{name="prod", user="tenant_1"} 10 +cortex_ingester_active_series_custom_tracker{name="service1", user="tenant_with_only_prod_metrics"} 1 +cortex_ingester_active_series_custom_tracker{name="service2", user="tenant_with_only_prod_metrics"} 2 +``` + +To set up runtime overrides, refer to [runtime configuration]({{< relref "./about-runtime-configuration.md" >}}). + > **Note:** The custom active series trackers are exposed on each ingester. To understand the count of active series matching a particular label pattern in your Grafana Mimir cluster at a global level, you must collect and sum this metric across all ingesters. If you're running Grafana Mimir with a `replication_factor` > 1, you must also adjust for the fact that the same series will be replicated `RF` times across your ingesters. diff --git a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md index 1c4de7e62fa..8b4aca215ea 100644 --- a/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configuring/reference-configuration-parameters/index.md @@ -310,11 +310,11 @@ grpc_tls_config: # (advanced) Limit on the size of a gRPC message this server can receive # (bytes). # CLI flag: -server.grpc-max-recv-msg-size-bytes -[grpc_server_max_recv_msg_size: | default = 4194304] +[grpc_server_max_recv_msg_size: | default = 104857600] # (advanced) Limit on the size of a gRPC message this server can send (bytes). # CLI flag: -server.grpc-max-send-msg-size-bytes -[grpc_server_max_send_msg_size: | default = 4194304] +[grpc_server_max_send_msg_size: | default = 104857600] # (advanced) Limit on the number of concurrent streams for gRPC calls (0 = # unlimited) @@ -475,13 +475,6 @@ ha_tracker: # CLI flag: -distributor.remote-timeout [remote_timeout: | default = 20s] -# (advanced) Try writing to an additional ingester in the presence of an -# ingester not in the ACTIVE state. It is useful to disable this along with -# -ingester.ring.unregister-on-shutdown=false in order to not spread samples to -# extra ingesters during rolling restarts with consistent naming. -# CLI flag: -distributor.extend-writes -[extend_writes: | default = true] - ring: kvstore: # Backend storage to use for the ring. Supported values are: consul, etcd, @@ -569,6 +562,11 @@ forwarding: # forward metrics. # CLI flag: -distributor.forwarding.request-timeout [request_timeout: | default = 10s] + + # (experimental) If disabled then forwarding requests are always considered to + # be successful, errors are ignored. + # CLI flag: -distributor.forwarding.propagate-errors + [propagate_errors: | default = true] ``` ### ingester @@ -677,8 +675,7 @@ ring: [instance_availability_zone: | default = ""] # (advanced) Unregister from the ring upon clean shutdown. It can be useful to - # disable for rolling restarts with consistent naming in conjunction with - # -distributor.extend-writes=false. + # disable for rolling restarts with consistent naming. # CLI flag: -ingester.ring.unregister-on-shutdown [unregister_on_shutdown: | default = true] @@ -687,11 +684,6 @@ ring: # CLI flag: -ingester.ring.observe-period [observe_period: | default = 0s] - # (advanced) Period to wait for a claim from another member; will join - # automatically after this. - # CLI flag: -ingester.ring.join-after - [join_after: | default = 0s] - # (advanced) Minimum duration to wait after the internal readiness checks have # passed but before succeeding the readiness endpoint. This is used to # slowdown deployment controllers (eg. Kubernetes) after an instance is ready @@ -799,17 +791,18 @@ The `querier` block configures the querier. # CLI flag: -querier.batch-iterators [batch_iterators: | default = true] -# Maximum lookback beyond which queries are not sent to ingester. 0 means all -# queries are sent to ingester. +# (advanced) Maximum lookback beyond which queries are not sent to ingester. 0 +# means all queries are sent to ingester. # CLI flag: -querier.query-ingesters-within [query_ingesters_within: | default = 13h] -# The time after which a metric should be queried from storage and not just -# ingesters. 0 means all queries are sent to store. If this option is enabled, -# the time range of the query sent to the store-gateway will be manipulated to -# ensure the query end is not more recent than 'now - query-store-after'. +# (advanced) The time after which a metric should be queried from storage and +# not just ingesters. 0 means all queries are sent to store. If this option is +# enabled, the time range of the query sent to the store-gateway will be +# manipulated to ensure the query end is not more recent than 'now - +# query-store-after'. # CLI flag: -querier.query-store-after -[query_store_after: | default = 0s] +[query_store_after: | default = 12h] # (advanced) Maximum duration into the future you can query. 0 to disable. # CLI flag: -querier.max-query-into-future @@ -843,15 +836,14 @@ store_gateway_client: # CLI flag: -querier.store-gateway-client.tls-insecure-skip-verify [tls_insecure_skip_verify: | default = false] -# (advanced) When distributor's sharding strategy is shuffle-sharding and this -# setting is > 0, queriers fetch in-memory series from the minimum set of -# required ingesters, selecting only ingesters which may have received series -# since 'now - lookback period'. The lookback period should be greater or equal -# than the configured -querier.query-store-after and +# (advanced) When this setting is > 0, queriers fetch in-memory series from the +# minimum set of required ingesters, selecting only ingesters which may have +# received series since 'now - lookback period'. The lookback period should be +# greater or equal than the configured -querier.query-store-after and # -querier.query-ingesters-within. If this setting is 0, queriers always query # all ingesters (ingesters shuffle sharding on read path is disabled). # CLI flag: -querier.shuffle-sharding-ingesters-lookback-period -[shuffle_sharding_ingesters_lookback_period: | default = 0s] +[shuffle_sharding_ingesters_lookback_period: | default = 13h] # The maximum number of concurrent queries. This config option should be set on # query-frontend too when query sharding is enabled. @@ -1408,32 +1400,74 @@ query_frontend: # CLI flag: -ruler.query-frontend.address [address: | default = ""] - # (advanced) Set to true if query-frontend connection requires TLS. - # CLI flag: -ruler.query-frontend.tls-enabled - [tls_enabled: | default = false] - - # (advanced) Path to the client certificate file, which will be used for - # authenticating with the server. Also requires the key path to be configured. - # CLI flag: -ruler.query-frontend.tls-cert-path - [tls_cert_path: | default = ""] - - # (advanced) Path to the key file for the client certificate. Also requires - # the client certificate to be configured. - # CLI flag: -ruler.query-frontend.tls-key-path - [tls_key_path: | default = ""] - - # (advanced) Path to the CA certificates file to validate server certificate - # against. If not set, the host's root CA certificates are used. - # CLI flag: -ruler.query-frontend.tls-ca-path - [tls_ca_path: | default = ""] - - # (advanced) Override the expected name on the server certificate. - # CLI flag: -ruler.query-frontend.tls-server-name - [tls_server_name: | default = ""] - - # (advanced) Skip validating server certificate. - # CLI flag: -ruler.query-frontend.tls-insecure-skip-verify - [tls_insecure_skip_verify: | default = false] + grpc_client_config: + # (advanced) gRPC client max receive message size (bytes). + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-recv-msg-size + [max_recv_msg_size: | default = 104857600] + + # (advanced) gRPC client max send message size (bytes). + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-max-send-msg-size + [max_send_msg_size: | default = 104857600] + + # (advanced) Use compression when sending messages. Supported values are: + # 'gzip', 'snappy' and '' (disable compression) + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-compression + [grpc_compression: | default = ""] + + # (advanced) Rate limit for gRPC client; 0 means disabled. + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit + [rate_limit: | default = 0] + + # (advanced) Rate limit burst for gRPC client. + # CLI flag: -ruler.query-frontend.grpc-client-config.grpc-client-rate-limit-burst + [rate_limit_burst: | default = 0] + + # (advanced) Enable backoff and retry when we hit ratelimits. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-on-ratelimits + [backoff_on_ratelimits: | default = false] + + backoff_config: + # (advanced) Minimum delay when backing off. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-min-period + [min_period: | default = 100ms] + + # (advanced) Maximum delay when backing off. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-max-period + [max_period: | default = 10s] + + # (advanced) Number of times to backoff and retry before failing. + # CLI flag: -ruler.query-frontend.grpc-client-config.backoff-retries + [max_retries: | default = 10] + + # (advanced) Enable TLS in the GRPC client. This flag needs to be enabled + # when any other TLS flag is set. If set to false, insecure connection to + # gRPC server will be used. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-enabled + [tls_enabled: | default = false] + + # (advanced) Path to the client certificate file, which will be used for + # authenticating with the server. Also requires the key path to be + # configured. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-cert-path + [tls_cert_path: | default = ""] + + # (advanced) Path to the key file for the client certificate. Also requires + # the client certificate to be configured. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-key-path + [tls_key_path: | default = ""] + + # (advanced) Path to the CA certificates file to validate server certificate + # against. If not set, the host's root CA certificates are used. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-ca-path + [tls_ca_path: | default = ""] + + # (advanced) Override the expected name on the server certificate. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-server-name + [tls_server_name: | default = ""] + + # (advanced) Skip validating server certificate. + # CLI flag: -ruler.query-frontend.grpc-client-config.tls-insecure-skip-verify + [tls_insecure_skip_verify: | default = false] tenant_federation: # Enable running rule groups against multiple tenants. The tenant IDs involved @@ -1454,6 +1488,11 @@ The `ruler_storage` block configures the ruler storage backend. # CLI flag: -ruler-storage.backend [backend: | default = "filesystem"] +# (experimental) Prefix for all objects stored in the backend storage. For +# simplicity, it may only contain digits and English alphabet letters. +# CLI flag: -ruler-storage.storage-prefix +[storage_prefix: | default = ""] + s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an @@ -1899,6 +1938,11 @@ The `alertmanager_storage` block configures the alertmanager storage backend. # CLI flag: -alertmanager-storage.backend [backend: | default = "filesystem"] +# (experimental) Prefix for all objects stored in the backend storage. For +# simplicity, it may only contain digits and English alphabet letters. +# CLI flag: -alertmanager-storage.storage-prefix +[storage_prefix: | default = ""] + s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an @@ -2556,6 +2600,15 @@ The `memberlist` block configures the Gossip memberlist. The `limits` block configures default and per-tenant limits imposed by components. ```yaml +# (experimental) Per-tenant request rate limit in requests per second. 0 to +# disable. +# CLI flag: -distributor.request-rate-limit +[request_rate: | default = 0] + +# (experimental) Per-tenant allowed request burst size. 0 to disable. +# CLI flag: -distributor.request-burst-size +[request_burst_size: | default = 0] + # Per-tenant ingestion rate limit in samples per second. # CLI flag: -distributor.ingestion-rate-limit [ingestion_rate: | default = 10000] @@ -2889,6 +2942,11 @@ The `blocks_storage` block configures the blocks storage. # CLI flag: -blocks-storage.backend [backend: | default = "filesystem"] +# (experimental) Prefix for all objects stored in the backend storage. For +# simplicity, it may only contain digits and English alphabet letters. +# CLI flag: -blocks-storage.storage-prefix +[storage_prefix: | default = ""] + s3: # The S3 bucket endpoint. It could be an AWS S3 endpoint listed at # https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an @@ -3297,7 +3355,7 @@ bucket_store: # are usually many of them (depending on number of ingesters) and they are not # yet compacted. Negative values or 0 disable the filter. # CLI flag: -blocks-storage.bucket-store.ignore-blocks-within - [ignore_blocks_within: | default = 0s] + [ignore_blocks_within: | default = 10h] # (advanced) Max size - in bytes - of a chunks pool, used to reduce memory # allocations. The pool is shared across all tenants. 0 to disable the limit. @@ -3339,11 +3397,11 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.posting-offsets-in-mem-sampling [postings_offsets_in_mem_sampling: | default = 32] - # (experimental) Number of threads that are dedicated for use reading index - # headers. Set to 0 to disable use of dedicated threads for reading index - # headers. - # CLI flag: -blocks-storage.bucket-store.index-header-thread-pool-size - [index_header_thread_pool_size: | default = 0] + index_header: + # (experimental) If enabled, the store-gateway will attempt to pre-populate + # the file system cache when memory-mapping index-header files. + # CLI flag: -blocks-storage.bucket-store.index-header.map-populate-enabled + [map_populate_enabled: | default = false] tsdb: # Directory to store TSDBs (including WAL) in the ingesters. This directory is @@ -3438,7 +3496,8 @@ tsdb: # (experimental) The size of the write queue used by the head chunks mapper. # Lower values reduce memory utilisation at the cost of potentially higher # ingest latency. Value of 0 switches chunks mapper to implementation without - # a queue. + # a queue. This flag is only used if the new chunk disk mapper is enabled with + # -blocks-storage.tsdb.new-chunk-disk-mapper. # CLI flag: -blocks-storage.tsdb.head-chunks-write-queue-size [head_chunks_write_queue_size: | default = 0] @@ -3776,6 +3835,11 @@ sharding_ring: # Unregister from the ring upon clean shutdown. # CLI flag: -store-gateway.sharding-ring.unregister-on-shutdown [unregister_on_shutdown: | default = true] + +# (experimental) Number of OS threads that are dedicated for handling requests. +# Set to 0 to disable use of dedicated OS threads for handling requests. +# CLI flag: -store-gateway.thread-pool-size +[thread_pool_size: | default = 0] ``` ### sse diff --git a/docs/sources/operators-guide/deploying-grafana-mimir/_index.md b/docs/sources/operators-guide/deploying-grafana-mimir/_index.md index 93bc327f038..dd21156e88f 100644 --- a/docs/sources/operators-guide/deploying-grafana-mimir/_index.md +++ b/docs/sources/operators-guide/deploying-grafana-mimir/_index.md @@ -18,6 +18,6 @@ A [mimir-distributed](https://github.com/grafana/helm-charts/tree/main/charts/mi ## Tanka -Grafana Labs also publishes [jsonnet](https://jsonnet.org/) files that you can use to deploy Grafana Mimir in [microservices mode]({{< relref "../architecture/deployment-modes/index.md#microservices-mode" >}}). To locate the Jsonnet files and a README file, refer to [Jsonnet for Mimir on Kubernetes](https://github.com/grafana/mimir/tree/main/operations/mimir). +Grafana Labs also publishes [Jsonnet](https://jsonnet.org/) files that you can use to deploy Grafana Mimir in [microservices mode]({{< relref "../architecture/deployment-modes/index.md#microservices-mode" >}}). To locate the Jsonnet files and a README file, refer to [Jsonnet for Mimir on Kubernetes](https://github.com/grafana/mimir/tree/main/operations/mimir). -The README explains how to use [tanka](https://tanka.dev/) and [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler) to generate Kubernetes YAML manifests from the jsonnet files. Alternatively, if you are familiar with tanka, you can use it directly to deploy Grafana Mimir. +The README explains how to use [Tanka](https://tanka.dev/) and [jsonnet-bundler](https://github.com/jsonnet-bundler/jsonnet-bundler) to generate Kubernetes YAML manifests from the jsonnet files. Alternatively, if you are familiar with Tanka, you can use it directly to deploy Grafana Mimir. diff --git a/docs/sources/operators-guide/getting-started/_index.md b/docs/sources/operators-guide/getting-started/_index.md index 1116f404ff0..bd330f20772 100644 --- a/docs/sources/operators-guide/getting-started/_index.md +++ b/docs/sources/operators-guide/getting-started/_index.md @@ -82,9 +82,9 @@ ingester: replication_factor: 1 ruler_storage: - backend: local - local: - directory: /tmp/mimir/rules + backend: filesystem + filesystem: + dir: /tmp/mimir/rules server: http_listen_port: 9009 @@ -103,7 +103,7 @@ In a terminal, run one of the following commands: - Using Docker: ```bash - docker run --rm --name mimir --publish 9009:9009 --volume "$(pwd)"/demo.yaml:/etc/mimir/demo.yaml grafana/mimir:${MIMIR_LATEST} --config.file=/etc/mimir/demo.yaml + docker run --rm --name mimir --publish 9009:9009 --volume "$(pwd)"/demo.yaml:/etc/mimir/demo.yaml grafana/mimir:latest --config.file=/etc/mimir/demo.yaml ``` - Using a local binary: diff --git a/operations/mimir-mixin/docs/playbooks.md b/docs/sources/operators-guide/mimir-runbooks/_index.md similarity index 70% rename from operations/mimir-mixin/docs/playbooks.md rename to docs/sources/operators-guide/mimir-runbooks/_index.md index c048f69e210..426a51b5d0c 100644 --- a/operations/mimir-mixin/docs/playbooks.md +++ b/docs/sources/operators-guide/mimir-runbooks/_index.md @@ -1,6 +1,15 @@ -# Playbooks +--- +title: "Grafana Mimir runbooks" +menuTitle: "Runbooks" +description: "Grafana Mimir runbooks." +weight: 110 +keywords: + - Mimir runbooks +--- + +# Grafana Mimir runbooks -This document contains playbooks, or at least a checklist of what to look for, for alerts in the mimir-mixin and logs from Mimir. This document assumes that you are running a Mimir cluster: +This document contains runbooks, or at least a checklist of what to look for, for alerts in the mimir-mixin and logs from Mimir. This document assumes that you are running a Mimir cluster: 1. Using this mixin config 2. Using GCS as object store (but similar procedures apply to other backends) @@ -54,7 +63,7 @@ How the limit is **configured**: - When configured in the runtime config, changes are applied live without requiring an ingester restart - The configured limit can be queried via `cortex_ingester_instance_limits{limit="max_series"}` -How to **fix**: +How to **fix** it: 1. **Temporarily increase the limit**
If the actual number of series is very close to or already hit the limit, or if you foresee the ingester will hit the limit before dropping the stale series as an effect of the scale up, you should also temporarily increase the limit. @@ -125,19 +134,19 @@ How the limit is **configured**: - When configured in the runtime config, changes are applied live without requiring an ingester restart - The configured limit can be queried via `cortex_ingester_instance_limits{limit="max_tenants"}` -How to **fix**: +How to **fix** it: 1. Ensure shuffle-sharding is enabled in the Mimir cluster 1. Assuming shuffle-sharding is enabled, scaling up ingesters will lower the number of tenants per ingester. However, the effect of this change will be visible only after `-blocks-storage.tsdb.close-idle-tsdb-timeout` period so you may have to temporarily increase the limit ### MimirDistributorReachingInflightPushRequestLimit -This alert fires when the `cortex_distributor_inflight_push_requests` per distributor instance limit is enabled and the actual number of inflight push requests is approaching the set limit. Once the limit is reached, push requests to the distributor will fail (5xx) for new requests, while existing inflight push requests will continue to succeed. +This alert fires when the `cortex_distributor_inflight_push_requests` per distributor instance limit is enabled and the actual number of in-flight push requests is approaching the set limit. Once the limit is reached, push requests to the distributor will fail (5xx) for new requests, while existing in-flight push requests will continue to succeed. In case of **emergency**: -- If the actual number of inflight push requests is very close to or already at the set limit, then you can increase the limit via CLI flag or config to gain some time -- Increasing the limit will increase the number of inflight push requests which will increase distributors' memory utilization. Please monitor the distributors' memory utilization via the `Mimir / Writes Resources` dashboard +- If the actual number of in-flight push requests is very close to or already at the set limit, then you can increase the limit via CLI flag or config to gain some time +- Increasing the limit will increase the number of in-flight push requests which will increase distributors' memory utilization. Please monitor the distributors' memory utilization via the `Mimir / Writes Resources` dashboard How the limit is **configured**: @@ -150,12 +159,12 @@ How the limit is **configured**: - These changes are applied with a distributor restart. - The configured limit can be queried via `cortex_distributor_instance_limits{limit="max_inflight_push_requests"})` -How to **fix**: +How to **fix** it: 1. **Temporarily increase the limit**
- If the actual number of inflight push requests is very close to or already hit the limit. + If the actual number of in-flight push requests is very close to or already hit the limit. 2. **Scale up distributors**
- Scaling up distributors will lower the number of inflight push requests per distributor. + Scaling up distributors will lower the number of in-flight push requests per distributor. ### MimirRequestLatency @@ -213,6 +222,15 @@ How to **investigate**: - Check `Memcached Overview` dashboard - If memcached eviction rate is high, then you should scale up memcached replicas. Check the recommendations by `Mimir / Scaling` dashboard and make reasonable adjustments as necessary. - If memcached eviction rate is zero or very low, then it may be caused by "first time" queries + - Cache query timeouts + - Check store-gateway logs and look for warnings about timed out Memcached queries (example query: `{namespace="example-mimir-cluster", name=~"store-gateway.*"} |= "level=warn" |= "memcached" |= "timeout"`) + - If there are indeed a lot of timed out Memcached queries, consider whether the store-gateway Memcached timeout setting (`-blocks-storage.bucket-store.chunks-cache.memcached.timeout`) is sufficient + - By consulting the "Queue length" panel of the `Mimir / Queries` dashboard, determine if queries are waiting in queue due to busy queriers (an indication of this would be queue length > 0 for some time) + - If queries are waiting in queue + - Consider scaling up number of queriers if they're not auto-scaled; if auto-scaled, check auto-scaling parameters + - If queries are not waiting in queue + - Consider [enabling query sharding]({{< relref "../architecture/query-sharding/index.md#how-to-enable-query-sharding" >}}) if not already enabled, to increase query parallelism + - If query sharding already enabled, consider increasing total number of query shards (`query_sharding_total_shards`) for tenants submitting slow queries, so their queries can be further parallelized #### Alertmanager @@ -264,7 +282,7 @@ This alert goes off when an ingester is marked as unhealthy. Check the ring web This alert fires when a Mimir process has a number of memory map areas close to the limit. The limit is a per-process limit imposed by the kernel and this issue is typically caused by a large number of mmap-ed failes. -How to **fix**: +How to **fix** it: - Increase the limit on your system: `sysctl -w vm.max_map_count=` - If it's caused by a store-gateway, consider enabling `-blocks-storage.bucket-store.index-header-lazy-loading-enabled=true` to lazy mmap index-headers at query time @@ -285,7 +303,7 @@ This alert fires when rulers cannot push new samples (result of rule evaluation) In general, pushing samples can fail due to problems with Mimir operations (eg. too many ingesters have crashed, and ruler cannot write samples to them), or due to problems with resulting data (eg. user hitting limit for number of series, out of order samples, etc.). This alert fires only for first kind of problems, and not for problems caused by limits or invalid rules. -How to **fix**: +How to **fix** it: - Investigate the ruler logs to find out the reason why ruler cannot write samples. Note that ruler logs all push errors, including "user errors", but those are not causing the alert to fire. Focus on problems with ingesters. @@ -297,13 +315,24 @@ Each rule evaluation may fail due to many reasons, eg. due to invalid PromQL exp There is a category of errors that is more important: errors due to failure to read data from store-gateways or ingesters. These errors would result in 500 when run from querier. This alert fires if there is too many of such failures. -How to **fix**: +How to **fix** it: - Investigate the ruler logs to find out the reason why ruler cannot evaluate queries. Note that ruler logs rule evaluation errors even for "user errors", but those are not causing the alert to fire. Focus on problems with ingesters or store-gateways. +- In case remote operational mode is enabled the problem could be at any of the ruler query path components (ruler-query-frontend, ruler-query-scheduler and ruler-querier). Check the `Mimir / Remote ruler reads` and `Mimir / Remote ruler reads resources` dashboards to find out in which Mimir service the error is being originated. ### MimirRulerMissedEvaluations -_TODO: this playbook has not been written yet._ +This alert fires when there is a rule group that is taking longer to evaluate than its evaluation interval. + +How it **works**: + +- The Mimir ruler will evaluate a rule group according to the evaluation interval on the rule group. +- If an evaluation is not finished by the time the next evaluation should happen, the next evaluation is missed. + +How to **fix** it: + +- Increase the evaluation interval of the rule group. You can use the rate of missed evaluation to estimate how long the rule group evaluation actually takes. +- Try splitting up the rule group into multiple rule groups. Rule groups are evaluated in parallel, so the same rules may still fit in the same resolution. ### MimirIngesterHasNotShippedBlocks @@ -446,6 +475,25 @@ How to **investigate**: - Look for any scan error in the store-gateway logs (ie. networking or rate limiting issues) +### MimirStoreGatewayNoSyncedTenants + +This alert fires when a store-gateway doesn't own any tenant. Effectively it is sitting idle because no blocks are sharded to it. + +How it **works**: + +- Store-gateways join a hash ring to shard tenants and blocks across all store-gateway replicas. +- A tenant can be sharded across multiple store-gateways. How many exactly is determined by `-store-gateway.tenant-shard-size` or the `store_gateway_tenant_shard_size` limit. +- When the tenant shard size is less than the replicas of store-gateways, some store-gateways may not get any tenants' blocks sharded to them. +- This is more likely to happen in Mimir clusters with fewer number of tenants. + +How to **fix** it: + +There are three options: + +- Reduce the replicas of store-gateways so that they match the highest number of shards per tenant or +- Increase the shard size of one or more tenants to match the number of replicas or +- Set the shard size of one or more tenant to `0`; this will shard this tenant's blocks across all store-gateways. + ### MimirCompactorHasNotSuccessfullyCleanedUpBlocks This alert fires when a Mimir compactor is not successfully deleting blocks marked for deletion for a long time. @@ -522,7 +570,7 @@ gsutil mv gs://BUCKET/TENANT/BLOCK gs://BUCKET/TENANT/corrupted-BLOCK Where: -- `BUCKET` is the gcs bucket name the compactor is using. The cell's bucket name is specified as the `blocks_storage_bucket_name` in the cell configuration +- `BUCKET` is the gcs bucket name the compactor is using. The cluster's bucket name is specified as the `blocks_storage_bucket_name` in the cluster configuration - `TENANT` is the tenant id reported in the example error message above as `REDACTED-TENANT` - `BLOCK` is the last part of the file path reported as `REDACTED-BLOCK` in the example error message above @@ -546,7 +594,7 @@ How to **investigate**: 1. Look for partial blocks in the logs. Example Loki query: `{cluster="",namespace="",container="compactor"} |= "skipped partial block"` 1. Pick a block and note its ID (`block` field in log entry) and tenant ID (`org_id` in log entry) -1. Find the bucket used by the Mimir cell, such as checking the configured `blocks_storage_bucket_name` if you are using Jsonnet. +1. Find the bucket used by the Mimir cluster, such as checking the configured `blocks_storage_bucket_name` if you are using Jsonnet. 1. Find out which Mimir component operated on the block last (e.g. uploaded by ingester/compactor, or deleted by compactor) 1. Determine when the partial block was uploaded: `gsutil ls -l gs://${BUCKET}/${TENANT_ID}/${BLOCK_ID}`. Alternatively you can use `ulidtime` command from Mimir tools directory `ulidtime ${BLOCK_ID}` to find block creation time. 1. Search in the logs around that time to find the log entry from when the compactor created the block ("compacted blocks" for log message) @@ -558,7 +606,7 @@ How to **investigate**: ### MimirQueriesIncorrect -_TODO: this playbook has not been written yet._ +_TODO: this runbook has not been written yet._ ### MimirInconsistentRuntimeConfig @@ -594,7 +642,7 @@ How to **investigate**: This alert fires if Mimir is running without query-scheduler and queries are piling up in the query-frontend queue. -The procedure to investigate it is the same as the one for [`MimirSchedulerQueriesStuck`](#MimirSchedulerQueriesStuck): please see the other playbook for more details. +The procedure to investigate it is the same as the one for [`MimirSchedulerQueriesStuck`](#MimirSchedulerQueriesStuck): please see the other runbook for more details. ### MimirSchedulerQueriesStuck @@ -661,7 +709,7 @@ How to **investigate**: This alert fires if the average number of in-memory series per ingester is above our target (1.5M). -How to **fix**: +How to **fix** it: - Scale up ingesters - To find out the Mimir clusters where ingesters should be scaled up and how many minimum replicas are expected: @@ -675,7 +723,7 @@ How to **fix**: This alert fires if the average number of samples ingested / sec in ingesters is above our target. -How to **fix**: +How to **fix** it: - Scale up ingesters - To compute the desired number of ingesters to satisfy the average samples rate you can run the following query, replacing `` with the namespace to analyse and `` with the target number of samples/sec per ingester (check out the alert threshold to see the current target): @@ -695,7 +743,7 @@ How it **works**: - Ingester memory short spikes are primarily influenced by queries and TSDB head compaction into new blocks (occurring every 2h) - A pod gets `OOMKilled` once its working set memory reaches the configured limit, so it's important to prevent ingesters' memory utilization (working set memory) from getting close to the limit (we need to keep at least 30% room for spikes due to queries) -How to **fix**: +How to **fix** it: - Check if the issue occurs only for few ingesters. If so: - Restart affected ingesters 1 by 1 (proceed with the next one once the previous pod has restarted and it's Ready) @@ -854,7 +902,7 @@ How it **works**: - Alertmanager memory baseline usage is primarily influenced by memory allocated by the process (mostly Go heap) for alerts and silences. - A pod gets `OOMKilled` once its working set memory reaches the configured limit, so it's important to prevent alertmanager's memory utilization (working set memory) from going over to the limit. The memory usage is typically sustained and does not suffer from spikes, hence thresholds are set very close to the limit. -How to **fix**: +How to **fix** it: - Scale up alertmanager replicas; you can use e.g. the `Mimir / Scaling` dashboard for reference, in order to determine the needed amount of alertmanagers. @@ -982,6 +1030,362 @@ How to **investigate**: 1. The alert fired because of a bug in Mimir: fix it. 1. The alert fired because of a bug or edge case in the continuous test tool, causing a false positive: fix it. +## Codified errors + +Mimir has some codified error IDs that you might see in HTTP responses or logs. +These error IDs allow you to read related details in the documentation that follows. + +### err-mimir-missing-metric-name + +This non-critical error occurs when Mimir receives a write request that contains a series without a metric name. +Each series must have a metric name. Rarely it does not, in which case there might be a bug in the sender client. + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-metric-name-invalid + +This non-critical error occurs when Mimir receives a write request that contains a series with an invalid metric name. +A metric name can only contain characters as defined by Prometheus’ [Metric names and labels](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-max-label-names-per-series + +This non-critical error occurs when Mimir receives a write request that contains a series with a number of labels that exceed the configured limit. +The limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-validation.max-label-names-per-series` option. + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-label-invalid + +This non-critical error occurs when Mimir receives a write request that contains a series with an invalid label name. +A label name name can only contain characters as defined by Prometheus’ [Metric names and labels](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels). + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-label-name-too-long + +This non-critical error occurs when Mimir receives a write request that contains a series with a label name whose length exceeds the configured limit. +The limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-validation.max-length-label-name` option. + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-label-value-too-long + +This non-critical error occurs when Mimir receives a write request that contains a series with a label value whose length exceeds the configured limit. +The limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-validation.max-length-label-value` option. + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-duplicate-label-names + +This non-critical error occurs when Mimir receives a write request that contains a series with the same label name two or more times. +A series that contains a duplicated label name is invalid and gets skipped during the ingestion. + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-labels-not-sorted + +This error occurs when Mimir receives a write request that contains a series whose label names are not sorted alphabetically. +However, Mimir internally sorts labels for series that it receives, so this error should not occur in practice. +If you experience this error, [open an issue in the Mimir repository](https://github.com/grafana/mimir/issues). + +> **Note**: Invalid series are skipped during the ingestion, and valid series within the same request are ingested. + +### err-mimir-too-far-in-future + +This non-critical error occurs when Mimir receives a write request that contains a sample whose timestamp is in the future compared to the current "real world" time. +Mimir accepts timestamps that are slightly in the future, due to skewed clocks for example. It rejects timestamps that are too far in the future, based on the definition that you can set via the `-validation.create-grace-period` option. +On a per-tenant basis, you can fine tune the tolerance by configuring the `-validation.max-length-label-value` option. + +> **Note**: Series with invalid samples are skipped during the ingestion, and series within the same request are ingested. + +### err-mimir-exemplar-labels-missing + +This non-critical error occurs when Mimir receives a write request that contains an exemplar without a label that identifies the related metric. +An exemplar must have at least one valid label pair, otherwise it cannot be associated with any metric. + +> **Note**: Invalid exemplars are skipped during the ingestion, and valid exemplars within the same request are ingested. + +### err-mimir-exemplar-labels-too-long + +This non-critical error occurs when Mimir receives a write request that contains an exemplar where the combined set size of its labels exceeds the limit. +The limit is used to protect the system’s stability from potential abuse or mistakes, and it cannot be configured. + +> **Note**: Invalid exemplars are skipped during the ingestion, and valid exemplars within the same request are ingested. + +### err-mimir-exemplar-timestamp-invalid + +This non-critical error occurs when Mimir receives a write request that contains an exemplar without a timestamp. +An exemplar must have a valid timestamp, otherwise it cannot be correlated to any point in time. + +> **Note**: Invalid exemplars are skipped during the ingestion, and valid exemplars within the same request are ingested. + +### err-mimir-metadata-missing-metric-name + +This non-critical error occurs when Mimir receives a write request that contains a metric metadata without a metric name. +Each metric metadata must have a metric name. Rarely it does not, in which case there might be a bug in the sender client. + +> **Note**: Invalid metrics metadata are skipped during the ingestion, and valid metadata within the same request are ingested. + +### err-mimir-metric-name-too-long + +This non-critical error occurs when Mimir receives a write request that contains a metric metadata with a metric name whose length exceeds the configured limit. +The limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-validation.max-metadata-length` option. + +> **Note**: Invalid metrics metadata are skipped during the ingestion, and valid metadata within the same request are ingested. + +### err-mimir-help-too-long + +This non-critical error occurs when Mimir receives a write request that contains a metric metadata with an help description whose length exceeds the configured limit. +The limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-validation.max-metadata-length` option. + +> **Note**: Invalid metrics metadata are skipped during the ingestion, and valid metadata within the same request are ingested. + +### err-mimir-unit-too-long + +This non-critical error occurs when Mimir receives a write request that contains a metric metadata with unit name whose length exceeds the configured limit. +The limit protects the system’s stability from potential abuse or mistakes. To configure the limit on a per-tenant basis, use the `-validation.max-metadata-length` option. + +> **Note**: Invalid metrics metadata are skipped during the ingestion, and valid metadata within the same request are ingested. + +### err-mimir-distributor-max-ingestion-rate + +This critical error occurs when the rate of received samples, exemplars and metadata per second is exceeded in a distributor. + +The distributor implements a rate limit on the samples per second that can be ingested, and it's used to protect a distributor from overloading in case of high traffic. +This per-instance limit is applied to all samples, exemplars, and all of the metadata that it receives. +Also, the limit spans all of the tenants within each distributor. + +How to **fix** it: + +- Scale up the distributors. +- Increase the limit by using the `-distributor.instance-limits.max-ingestion-rate` option. + +### err-mimir-distributor-max-inflight-push-requests + +This error occurs when a distributor rejects a write request because the maximum in-flight requests limit has been reached. + +How it **works**: + +- The distributor has a per-instance limit on the number of in-flight write (push) requests. +- The limit applies to all in-flight write requests, across all tenants, and it protects the distributor from becoming overloaded in case of high traffic. +- To configure the limit, set the `-distributor.instance-limits.max-inflight-push-requests` option. + +How to **fix** it: + +- Increase the limit by setting the `-distributor.instance-limits.max-inflight-push-requests` option. +- Check the write requests latency through the `Mimir / Writes` dashboard and come back to investigate the root cause of high latency (the higher the latency, the higher the number of in-flight write requests). +- Consider scaling out the distributors. + +### err-mimir-ingester-max-ingestion-rate + +This critical error occurs when the rate of received samples per second is exceeded in an ingester. + +The ingester implements a rate limit on the samples per second that can be ingested, and it's used to protect an ingester from overloading in case of high traffic. +This per-instance limit is applied to all samples that it receives. +Also, the limit spans all of the tenants within each ingester. + +How to **fix** it: + +- Scale up the ingesters. +- Increase the limit by using the `-ingester.instance-limits.max-ingestion-rate` option (or `max_ingestion_rate` in the runtime config). + +### err-mimir-ingester-max-tenants + +This critical error occurs when the ingester receives a write request for a new tenant (a tenant for which no series have been stored yet) but the ingester cannot accept it because the maximum number of allowed tenants per ingester has been reached. + +How to **fix** it: + +- Increase the limit by using the `-ingester.instance-limits.max-tenants` option (or `max_tenants` in the runtime config). +- Consider configuring ingesters shuffle sharding to reduce the number of tenants per ingester. + +### err-mimir-ingester-max-series + +This critical error occurs when an ingester rejects a write request because it reached the maximum number of in-memory series. + +How it **works**: + +- The ingester keeps most recent series data in-memory. +- The ingester has a per-instance limit on the number of in-memory series, used to protect the ingester from overloading in case of high traffic. +- When the limit on the number of in-memory series is reached, new series are rejected, while samples can still be appended to existing ones. +- To configure the limit, set the `-ingester.instance-limits.max-series` option (or `max_series` in the runtime config). + +How to **fix** it: + +- See [`MimirIngesterReachingSeriesLimit`](#MimirIngesterReachingSeriesLimit) runbook. + +### err-mimir-ingester-max-inflight-push-requests + +This error occurs when an ingester rejects a write request because the maximum in-flight requests limit has been reached. + +How it **works**: + +- The ingester has a per-instance limit on the number of in-flight write (push) requests. +- The limit applies to all in-flight write requests, across all tenants, and it protects the ingester from becoming overloaded in case of high traffic. +- To configure the limit, set the `-ingester.instance-limits.max-inflight-push-requests` option (or `max_inflight_push_requests` in the runtime config). + +How to **fix** it: + +- Increase the limit by setting the `-ingester.instance-limits.max-inflight-push-requests` option (or `max_inflight_push_requests` in the runtime config). +- Check the write requests latency through the `Mimir / Writes` dashboard and come back to investigate the root cause of high latency (the higher the latency, the higher the number of in-flight write requests). +- Consider scaling out the ingesters. + +### err-mimir-max-series-per-user + +This error occurs when the number of in-memory series for a given tenant exceeds the configured limit. + +The limit is used to protect ingesters from overloading in case a tenant writes a high number of series, as well as to protect the whole system’s stability from potential abuse or mistakes. +To configure the limit on a per-tenant basis, use the `-ingester.max-global-series-per-user` option (or `max_global_series_per_user` in the runtime configuration). + +How to **fix** it: + +- Ensure the actual number of series written by the affected tenant is legit. +- Consider increasing the per-tenant limit by using the `-ingester.max-global-series-per-user` option (or `max_global_series_per_user` in the runtime configuration). + +### err-mimir-max-series-per-metric + +This error occurs when the number of in-memory series for a given tenant and metric name exceeds the configured limit. + +The limit is primarily used to protect a tenant from potential mistakes on their metrics instrumentation. +For example, if an instrumented application exposes a metric with a label value including very dynamic data (e.g. a timestamp) the ingestion of that metric would quickly lead to hit the per-tenant series limit, causing other metrics to be rejected too. +This limit introduces a cap on the maximum number of series each metric name can have, rejecting exceeding series only for that metric name, before the per-tenant series limit is reached. +To configure the limit on a per-tenant basis, use the `-ingester.max-global-series-per-metric` option (or `max_global_series_per_metric` in the runtime configuration). + +How to **fix** it: + +- Check the details in the error message to find out which is the affected metric name. +- Investigate if the high number of series exposed for the affected metric name is legit. +- Consider reducing the cardinality of the affected metric, by tuning or removing some of its labels. +- Consider increasing the per-tenant limit by using the `-ingester.max-global-series-per-metric` option. +- Consider excluding specific metric names from this limit's check by using the `-ingester.ignore-series-limit-for-metric-names` option (or `max_global_series_per_metric` in the runtime configuration). + +### err-mimir-max-metadata-per-user + +This non-critical error occurs when the number of in-memory metrics with metadata for a given tenant exceeds the configured limit. + +Metric metadata is a set of information attached to a metric name, like its unit (e.g. counter) and description. +Metric metadata can be included by the sender in the write request, and it's returned when querying the `/api/v1/metadata` API endpoint. +Metric metadata is stored in the ingesters memory, so the higher the number of metrics metadata stored, the higher the memory utilization. + +Mimir has a per-tenant limit of the number of metric names that have metadata attached. +This limit is used to protect the whole system’s stability from potential abuse or mistakes. +To configure the limit on a per-tenant basis, use the `-ingester.max-global-series-per-user` option (or `max_global_metadata_per_user` in the runtime configuration). + +How to **fix** it: + +- Check the current number of metric names for the affected tenant, running the instant query `count(count by(__name__) ({__name__=~".+"}))`. Alternatively, you can get the cardinality of `__name__` label calling the API endpoint `/api/v1/cardinality/label_names`. +- Consider increasing the per-tenant limit setting to a value greater than the number of unique metric names returned by the previous query. + +### err-mimir-max-metadata-per-metric + +This non-critical error occurs when the number of different metadata for a given metric name exceeds the configured limit. + +Metric metadata is a set of information attached to a metric name, like its unit (e.g. counter) and description. +Typically, for a given metric name there's only one set of metadata (e.g. the same metric name exposed by different application has the same counter and description). +However, there could be some edge cases where the same metric name has a different meaning between applications or the same meaning but a slightly different description. +In these edge cases, different applications would expose different metadata for the same metric name. + +This limit is used to protect the whole system’s stability from potential abuse or mistakes, in case the number of metadata variants for a given metric name grows indefinitely. +To configure the limit on a per-tenant basis, use the `-ingester.max-global-series-per-metric` option (or `max_global_metadata_per_metric` in the runtime configuration). + +How to **fix** it: + +- Check the metadata for the affected metric name, querying the `/api/v1/metadata?metric=` API endpoint (replace `` with the metric name). +- If the different metadata is unexpected, consider fixing the discrepancy in the instrumented applications. +- If the different metadata is expected, consider increasing the per-tenant limit by using the `-ingester.max-global-series-per-metric` option (or `max_global_metadata_per_metric` in the runtime configuration). + +### err-mimir-max-chunks-per-query + +This error occurs when a query execution exceeds the limit on the number of series chunks fetched. + +This limit is used to protect the system’s stability from potential abuse or mistakes, when running a query fetching a huge amount of data. +To configure the limit on a per-tenant basis, use the `-querier.max-fetched-chunks-per-query` option (or `max_fetched_chunks_per_query` in the runtime configuration). + +How to **fix** it: + +- Consider reducing the time range and/or cardinality of the query. To reduce the cardinality of the query, you can add more label matchers to the query, restricting the set of matching series. +- Consider increasing the per-tenant limit by using the `-querier.max-fetched-chunks-per-query` option (or `max_fetched_chunks_per_query` in the runtime configuration). + +### err-mimir-max-series-per-query + +This error occurs when a query execution exceeds the limit on the maximum number of series. + +This limit is used to protect the system’s stability from potential abuse or mistakes, when running a query fetching a huge amount of data. +To configure the limit on a per-tenant basis, use the `-querier.max-fetched-series-per-query` option (or `max_fetched_series_per_query` in the runtime configuration). + +How to **fix** it: + +- Consider reducing the time range and/or cardinality of the query. To reduce the cardinality of the query, you can add more label matchers to the query, restricting the set of matching series. +- Consider increasing the per-tenant limit by using the `-querier.max-fetched-series-per-query` option (or `max_fetched_series_per_query` in the runtime configuration). + +### err-mimir-max-chunks-bytes-per-query + +This error occurs when a query execution exceeds the limit on aggregated size (in bytes) of fetched chunks. + +This limit is used to protect the system’s stability from potential abuse or mistakes, when running a query fetching a huge amount of data. +To configure the limit on a per-tenant basis, use the `-querier.max-fetched-chunk-bytes-per-query` option (or `max_fetched_chunk_bytes_per_query` in the runtime configuration). + +How to **fix** it: + +- Consider reducing the time range and/or cardinality of the query. To reduce the cardinality of the query, you can add more label matchers to the query, restricting the set of matching series. +- Consider increasing the per-tenant limit by using the `-querier.max-fetched-chunk-bytes-per-query` option (or `max_fetched_chunk_bytes_per_query` in the runtime configuration). + +### err-mimir-max-query-length + +This error occurs when the time range of a query exceeds the configured maximum length. + +Both PromQL instant and range queries can fetch metrics data over a period of time. +A [range query](https://prometheus.io/docs/prometheus/latest/querying/api/#range-queries) requires a `start` and `end` timestamp, so the difference of `end` minus `start` is the time range length of the query. +An [instant query](https://prometheus.io/docs/prometheus/latest/querying/api/#instant-queries) requires a `time` parameter and the query is executed fetching samples at that point in time. +However, even an instant query can fetch metrics data over a period of time by using the [range vector selectors](https://prometheus.io/docs/prometheus/latest/querying/basics/#range-vector-selectors). +For example, the instant query `sum(rate(http_requests_total{job="prometheus"}[1h]))` fetches metrics over a 1 hour period. +This time period is what Grafana Mimir calls the _query time range length_ (or _query length_). + +Mimir has a limit on the query length. +This limit is applied to partial queries, after they've split (according to time) by the query-frontend. This limit protects the system’s stability from potential abuse or mistakes. +To configure the limit on a per-tenant basis, use the `-store.max-query-length` option (or `max_query_length` in the runtime configuration). + +### err-mimir-tenant-max-request-rate + +This error occurs when the rate of write requests per second is exceeded for this tenant. + +How it **works**: + +- There is a per-tenant rate limit on the write requests per second, and it's applied across all distributors for this tenant. +- The limit is implemented using [token buckets](https://en.wikipedia.org/wiki/Token_bucket). + +How to **fix** it: + +- Increase the per-tenant limit by using the `-distributor.request-rate-limit` (requests per second) and `-distributor.request-burst-size` (number of requests) options (or `request_rate` and `request_burst_size` in the runtime configuration). The configurable burst represents how many requests can temporarily exceed the limit, in case of short traffic peaks. The configured burst size must be greater or equal than the configured limit. + +### err-mimir-tenant-max-ingestion-rate + +This error occurs when the rate of received samples, exemplars and metadata per second is exceeded for this tenant. + +How it **works**: + +- There is a per-tenant rate limit on the samples, exemplars and metadata that can be ingested per second, and it's applied across all distributors for this tenant. +- The limit is implemented using [token buckets](https://en.wikipedia.org/wiki/Token_bucket). + +How to **fix** it: + +- Increase the per-tenant limit by using the `-distributor.ingestion-rate-limit` (samples per second) and `-distributor.ingestion-burst-size` (number of samples) options (or `ingestion_rate` and `ingestion_burst_size` in the runtime configuration). The configurable burst represents how many samples, exemplars and metadata can temporarily exceed the limit, in case of short traffic peaks. The configured burst size must be greater or equal than the configured limit. + +### err-mimir-tenant-too-many-ha-clusters + +This error occurs when a distributor rejects a write request because the number of [high-availability (HA) clusters]({{< relref "../configuring/configuring-high-availability-deduplication.md" >}}) has hit the configured limit for this tenant. + +How it **works**: + +- The distributor implements an upper limit on the number of clusters that the HA tracker will keep track of for a single tenant. +- It is triggered when the write request would add a new cluster while the number the tenant currently has is already equal to the limit. +- To configure the limit, set the `-distributor.ha-tracker.max-clusters` option (or `ha_max_clusters` in the runtime configuration). + +How to **fix** it: + +- Increase the per-tenant limit by using the `-distributor.ha-tracker.max-clusters` option (or `ha_max_clusters` in the runtime configuration). + ## Mimir routes by path **Write path**: @@ -1099,7 +1503,7 @@ This is the short version of an extensive documentation on [how to resize Kubern In some scenarios, it may be useful to preserve current volume status for inspection, but keep using the volume. [Google Persistent Disk supports "Clone"](https://cloud.google.com/compute/docs/disks/add-persistent-disk#source-disk) operation that can be used to do that. -Newly cloned disk is independant from its original, and can be used for further investigation by attaching it to a new Machine / Pod. +Newly cloned disk is independent from its original, and can be used for further investigation by attaching it to a new Machine / Pod. When using Kubernetes, here is YAML file that creates PV (`clone-ingester-7-pv`) pointing to the new disk clone (`clone-pvc-80cc0efa-4996-11ea-ba79-42010a96008c` in this example), PVC (`clone-ingester-7-pvc`) pointing to PV, and finally Pod (`clone-ingester-7-dataaccess`) using the PVC to access the disk. @@ -1193,7 +1597,7 @@ A PVC can be manually deleted by an operator. When a PVC claim is deleted, what ### Recover accidentally deleted blocks (Google Cloud specific) -_This playbook assumes you've enabled versioning in your GCS bucket and the retention of deleted blocks didn't expire yet._ +_This runbook assumes you've enabled versioning in your GCS bucket and the retention of deleted blocks didn't expire yet._ These are just example actions but should give you a fair idea on how you could go about doing this. Read the [GCS doc](https://cloud.google.com/storage/docs/using-versioned-objects#gsutil_1) before you proceed. diff --git a/docs/sources/operators-guide/reference-glossary.md b/docs/sources/operators-guide/reference-glossary.md index 13ba1ad523b..266a03f38d0 100644 --- a/docs/sources/operators-guide/reference-glossary.md +++ b/docs/sources/operators-guide/reference-glossary.md @@ -2,7 +2,7 @@ title: "Reference: Grafana Mimir glossary" menuTitle: "Reference: Glossary" description: "Grafana Mimir glossary terms." -weight: 100 +weight: 130 --- # Reference: Grafana Mimir glossary diff --git a/docs/sources/operators-guide/reference-http-api/index.md b/docs/sources/operators-guide/reference-http-api/index.md index 2e78afaf4af..f02bdc9b377 100644 --- a/docs/sources/operators-guide/reference-http-api/index.md +++ b/docs/sources/operators-guide/reference-http-api/index.md @@ -2,7 +2,7 @@ title: "Reference: Grafana Mimir HTTP API" menuTitle: "Reference: HTTP API" description: "Use the HTTP API to write and query time-series data and operate a Grafana Mimir cluster." -weight: 100 +weight: 120 keywords: - Mimir API - Mimir endpoints @@ -37,7 +37,7 @@ This document groups API endpoints by service. Note that the API endpoints are e | [HA tracker status](#ha-tracker-status) | Distributor | `GET /distributor/ha_tracker` | | [Flush chunks / blocks](#flush-chunks--blocks) | Ingester | `GET,POST /ingester/flush` | | [Shutdown](#shutdown) | Ingester | `GET,POST /ingester/shutdown` | -| [Ingesters ring status](#ingesters-ring-status) | Ingester | `GET /ingester/ring` | +| [Ingesters ring status](#ingesters-ring-status) | Distributor,Ingester | `GET /ingester/ring` | | [Instant query](#instant-query) | Querier, Query-frontend | `GET,POST /api/v1/query` | | [Range query](#range-query) | Querier, Query-frontend | `GET,POST /api/v1/query_range` | | [Exemplar query](#exemplar-query) | Querier, Query-frontend | `GET,POST /api/v1/query_exemplars` | @@ -683,65 +683,26 @@ POST /api/v1/rules/{namespace} POST /rules/{namespace} ``` -Creates or updates a rule group. This endpoint expects a request with `Content-Type: application/yaml` header and the -rules **YAML** definition in the request body, and returns `202` on success. +Creates or updates a rule group. +This endpoint expects a request with `Content-Type: application/yaml` header and the rules group **YAML** definition in the request body, and returns `202` on success. +The request body must contain the definition of one and only one rule group. This endpoint can be disabled via the `-ruler.enable-api` CLI flag (or its respective YAML config option). Requires [authentication](#authentication). -#### Federated rule groups - -A federated rule groups is a rule group with a non-empty `source_tenants`. - -The `source_tenants` field allows aggregating data from multiple tenants while evaluating a rule group. The expressions -of each rule in the group will be evaluated against the data of all tenants in `source_tenants`. If `source_tenants` is -empty or omitted, then the tenant under which the group is created will be treated as the `source_tenant`. - -Federated rule groups are skipped during evaluation by default. This feature depends on -the cross-tenant query federation feature. To enable federated rules -set `-ruler.tenant-federation.enabled=true` and `-tenant-federation.enabled=true` CLI flags (or their respective YAML -config options). - -During evaluation query limits applied to single tenants are also applied to each query in the rule group. For example, -if `tenant-a` has a federated rule group with `source_tenants: [tenant-b, tenant-c]`, then query limits for `tenant-b` -and `tenant-c` will be applied. If any of these limits is exceeded, the whole evaluation will fail. No partial results -will be saved. The same "no partial results" guarantee applies to queries failing for other reasons (e.g. ingester -unavailability). - -The time series used during evaluation of federated rules will have the `__tenant_id__` label, similar to how it is -present on series returned with cross-tenant query federation. - -**Considerations:** Federated rule groups allow data from multiple source tenants to be written into a single -destination tenant. This makes the existing separation of tenants' data less clear. For example, `tenant-a` has a -federated rule group that aggregates over `tenant-b`'s data (e.g. `sum(metric_b)`) and writes the result back -into `tenant-a`'s storage (e.g. as metric `sum:metric_b`). Now part of `tenant-b`'s data is copied to `tenant-a` (albeit -aggregated). Have this in mind when configuring the access control layer in front of mimir and when enabling federated -rules via `-ruler.tenant-federation.enabled`. - -**Example request** - -Request headers: - -- `Content-Type: application/yaml` +> **Note:** When using `curl` send the request body from a file, ensure that you use the `--data-binary` flag instead of `-d`, `--data`, or `--data-ascii`. +> The latter options do not preserve carriage returns and newlines. -Request body: +#### Example request body ```yaml -name: -interval: -source_tenants: - - +name: MyGroupName rules: - - record: - expr: - - alert: - expr: - for: - annotations: - : + - alert: MyAlertName + expr: up == 0 labels: - : + severity: warning ``` ### Delete rule group diff --git a/docs/sources/operators-guide/running-production-environment/planning-capacity.md b/docs/sources/operators-guide/running-production-environment/planning-capacity.md index a459e19fc59..ca4407fcd68 100644 --- a/docs/sources/operators-guide/running-production-environment/planning-capacity.md +++ b/docs/sources/operators-guide/running-production-environment/planning-capacity.md @@ -126,7 +126,10 @@ Estimated required CPU, memory, and disk space: ### (Optional) Ruler The [ruler]({{< relref "../architecture/components/ruler/index.md" >}}) component resources utilization is determined by the number of rules evaluated per second. -The rules evaluation is computationally equal to queries execution, so the querier resources recommendations apply to ruler too. + +When [internal]({{< relref "../architecture/components/ruler/index.md#internal" >}}) mode is used (default), rules evaluation is computationally equal to queries execution, so the querier resources recommendations apply to ruler too. + +When [remote]({{< relref "../architecture/components/ruler/index.md#internal" >}}) operational mode is used, most of the computational load is shifted to query-frontend and querier components. So those should be scaled accordingly to deal both with queries and rules evaluation workload. ### Compactor diff --git a/docs/sources/operators-guide/running-production-environment/production-tips/index.md b/docs/sources/operators-guide/running-production-environment/production-tips/index.md index 11a84e8c735..4a1103d94ff 100644 --- a/docs/sources/operators-guide/running-production-environment/production-tips/index.md +++ b/docs/sources/operators-guide/running-production-environment/production-tips/index.md @@ -30,6 +30,14 @@ The required disk space depends on the number of time series stored in the inges For more information about estimating the required ingester disk space, refer to [Planning capacity]({{< relref "../planning-capacity.md#ingester" >}}). +### Ingester disk IOPS + +The IOPS (input/output operations per second) and latency performances of the ingester disks can affect both write and read requests. +On the write path, the ingester writes to the write-ahead log (WAL) on disk. +On the read path, the ingester reads from the series whose chunks have already been written to disk. + +For these reasons, run the ingesters on disks such as SSDs that have fast disk speed. + ## Querier ### Ensure caching is enabled @@ -46,14 +54,15 @@ When running Grafana Mimir at scale, querying non-compacted blocks might be inef - Non compacted blocks contain duplicated samples, as a result of the ingesters replication. - Querying many small TSDB indexes is slower than querying a few compacted TSDB indexes. -Configure Grafana Mimir to ensure only compacted blocks are queried: +The default values for `-querier.query-store-after`, `-querier.query-ingesters-within`, and `-blocks-storage.bucket-store.ignore-blocks-within` are set such that only compacted blocks are queried. In most cases, no additional configuration is required. + +Configure Grafana Mimir so large tenants are parallelized by the compactor: 1. Configure compactor's `-compactor.split-and-merge-shards` and `-compactor.split-groups` for every tenant with more than 20 million active series. For more information about configuring the compactor's split and merge shards, refer to [compactor]({{< relref "../../architecture/components/compactor/index.md" >}}). -1. Configure querier's `-querier.query-store-after` equal to `-querier.query-ingesters-within` minus five minutes. The five-minute delta is recommended to ensure the time range on the boundary is queried both from ingesters and queriers. #### How to estimate `-querier.query-store-after` -Set the `-querier.query-store-after` to a duration that is large enough to give compactor enough time to compact newly uploaded blocks, and queriers and store-gateways to discover and synchronize newly compacted blocks. +If you are not using the defaults, set the `-querier.query-store-after` to a duration that is large enough to give compactor enough time to compact newly uploaded blocks, and queriers and store-gateways to discover and synchronize newly compacted blocks. The following diagram shows all of the timings involved in the estimation. This diagram should be used only as a template and you can modify the assumptions based on real measurements in your Mimir cluster. The example makes the following assumptions: @@ -84,6 +93,13 @@ The total number of file descriptors used to load index-headers linearly increas We recommend configuring the system's `file-max` ulimit at least to `65536` to avoid reaching the maximum number of open file descriptors. +### Store-gateway disk IOPS + +The IOPS and latency performances of the store-gateway disk can affect queries. +The store-gateway downloads the block’s [index-headers]({{< relref "../../architecture/binary-index-header.md" >}}) onto local disk, and reads them for each query that needs to fetch data from the long-term storage. + +For these reasons, run the store-gateways on disks such as SSDs that have fast disk speed. + ## Compactor ### Ensure the compactor has enough disk space diff --git a/docs/sources/operators-guide/securing/securing-communications-with-tls.md b/docs/sources/operators-guide/securing/securing-communications-with-tls.md index 57bea8f2e9f..c1b69eaba8b 100644 --- a/docs/sources/operators-guide/securing/securing-communications-with-tls.md +++ b/docs/sources/operators-guide/securing/securing-communications-with-tls.md @@ -100,6 +100,7 @@ The following Grafana Mimir components support TLS for inter-communication, whic - Query-frontend gRPC client used to connect to query-schedulers: `-query-frontend.grpc-client-config.*` - Querier gRPC client used to connect to query-frontends and query-schedulers: `-querier.frontend-client.*` - Ruler gRPC client used to connect to other ruler instances: `-ruler.client.*` +- Ruler gRPC client used to connect to query-frontend: `-ruler.query-frontend.grpc-client-config.*` - Alertmanager gRPC client used to connect to other Alertmanager instances: `-alertmanager.alertmanager-client.*` - gRPC client used by distributors, queriers, and rulers to connect to ingesters: `-ingester.client.*` - etcd client used by all Mimir components to connect to etcd, which is required only if you're running the hash ring or HA tracker on the etcd backend: `-.etcd.*` diff --git a/docs/sources/operators-guide/tools/mimir-continuous-test.md b/docs/sources/operators-guide/tools/mimir-continuous-test.md index 41f9e920d64..0e50e87f12e 100644 --- a/docs/sources/operators-guide/tools/mimir-continuous-test.md +++ b/docs/sources/operators-guide/tools/mimir-continuous-test.md @@ -7,8 +7,12 @@ weight: 30 # Grafana mimir-continuous-test -As a developer, you can use the standalone mimir-continuous-test tool to continuously run smoke tests on live Grafana Mimir clusters. +As a developer, you can use the standalone mimir-continuous-test tool to run smoke tests on live Grafana Mimir clusters. This tool identifies a class of bugs that could be difficult to spot during development. +Two operating modes are supported: + +- As a continuously running deployment in your environment, mimir-continuous-test can be used to detect issues on a live Grafana Mimir cluster over time. +- As an ad-hoc smoke test tool, mimir-continuous-test can be used to validate basic functionality after configuration changes are made to a Grafana Mimir cluster. ## Download mimir-continuous-test @@ -36,6 +40,7 @@ Mimir-continuous-test requires the endpoints of the backend Grafana Mimir cluste - Set `-tests.write-endpoint` to the base endpoint on the write path. Remove any trailing slash from the URL. The tool appends the specific API path to the URL, for example `/api/v1/push` for the remote-write API. - Set `-tests.read-endpoint` to the base endpoint on the read path. Remove any trailing slash from the URL. The tool appends the specific API path to the URL, for example `/api/v1/query_range` for the range-query API. - Set `-tests.tenant-id` to the tenant ID to use to write and read metrics in tests. +- Set `-tests.smoke-test` to run the test once and immediately exit. In this mode, the process exit code is non-zero when the test fails. > **Note:** You can run `mimir-continuous-test -help` to list all available configuration options. diff --git a/docs/sources/operators-guide/tools/mimirtool.md b/docs/sources/operators-guide/tools/mimirtool.md index 2be407ffe86..df6a22c030b 100644 --- a/docs/sources/operators-guide/tools/mimirtool.md +++ b/docs/sources/operators-guide/tools/mimirtool.md @@ -660,11 +660,20 @@ It supports converting both CLI flags and [YAML configuration files]({{< relref `mimirtool config convert` helps you migrate from Cortex to Grafana Mimir. There are changes to the default values of some configuration parameters in Mimir v2.0.0 that you might not want to use as part of this migration: -- `blocks_storage.backend` - Cortex default is `s3`, Mimir default is `filesystem` -- `ruler_storage.backend` - Cortex default is `s3`, Mimir default is `filesystem` -- `alertmanager_storage.backend` - Cortex default is `s3`, Mimir default is `filesystem` -- `server.http_listen_port` - Cortex default is `80`, Mimir default is `8080` -- (GEM only) `auth.type` - GEM 1.x default is `trust`, 2.x default is `enterprise` +| Parameter | Cortex/GEM 1.7 default | Mimir/GEM 2.0 default | +| --------------------------------------------- | ------------------------ | ------------------------ | +| `blocks_storage.backend` | `s3` | `filesystem` | +| `ruler_storage.backend` | `s3` | `filesystem` | +| `alertmanager_storage.backend` | `s3` | `filesystem` | +| `server.http_listen_port` | `80` | `8080` | +| `activity_tracker.filepath` | `./active-query-tracker` | `./metrics-activity.log` | +| `alertmanager.data_dir` | `data/` | `./data-alertmanager/` | +| `blocks_storage.filesystem.dir` | `` | `blocks` | +| `compactor.data_dir` | `./data` | `./data-compactor/` | +| `ruler.rule_path` | `/rules` | `./data-ruler/` | +| `ruler_storage.filesystem.dir` | `` | `ruler` | +| (GEM only) `auth.type` | `trust` | `enterprise` | +| (GEM only) `graphite.querier.schemas.backend` | `s3` | `filesystem` | For these parameters `mimirtool config convert` will output the Cortex default value even when the configuration parameter is not explicitly set in the input configuration. If in the input configuration you explicitly set the Cortex default value, and you have provided @@ -803,7 +812,6 @@ The script outputs results that are similar to the following: ```console -consul.hostname=consul.cortex-to-mimir.svc.cluster.local:8500 --distributor.extend-writes=true -distributor.ha-tracker.enable=false -distributor.ha-tracker.enable-for-all-users=true -distributor.ha-tracker.etcd.endpoints=etcd-client.cortex-to-mimir.svc.cluster.local.:2379 diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager-resources/mimir-alertmanager-resources.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager-resources/mimir-alertmanager-resources.png index dd340fea473..7ed9076092d 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager-resources/mimir-alertmanager-resources.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager-resources/mimir-alertmanager-resources.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager/mimir-alertmanager.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager/mimir-alertmanager.png index 901fe6d2508..3997aeb968a 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager/mimir-alertmanager.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/alertmanager/mimir-alertmanager.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor-resources/mimir-compactor-resources.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor-resources/mimir-compactor-resources.png index 08daa74e01e..b2941e7eb3b 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor-resources/mimir-compactor-resources.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor-resources/mimir-compactor-resources.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor/mimir-compactor.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor/mimir-compactor.png index 0c7ad4ff942..5f231ccda5c 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor/mimir-compactor.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/compactor/mimir-compactor.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/config/mimir-config.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/config/mimir-config.png index f759d4ab2da..22bd3347032 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/config/mimir-config.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/config/mimir-config.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/object-store/mimir-object-store.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/object-store/mimir-object-store.png index d5579c7a369..548ff97e60a 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/object-store/mimir-object-store.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/object-store/mimir-object-store.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/overrides/mimir-overrides.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/overrides/mimir-overrides.png index 4ff2afa5ade..e18766b05d6 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/overrides/mimir-overrides.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/overrides/mimir-overrides.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/queries/mimir-queries.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/queries/mimir-queries.png index 3a9dfbc613a..ed4d913410d 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/queries/mimir-queries.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/queries/mimir-queries.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-networking/mimir-reads-networking.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-networking/mimir-reads-networking.png index 6d3815f8f15..544b235b216 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-networking/mimir-reads-networking.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-networking/mimir-reads-networking.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-resources/mimir-reads-resources.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-resources/mimir-reads-resources.png index f91efd79a22..d72553aaea6 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-resources/mimir-reads-resources.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/reads-resources/mimir-reads-resources.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/reads/mimir-reads.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/reads/mimir-reads.png index 62f86db86e9..e4f2e9ebe4a 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/reads/mimir-reads.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/reads/mimir-reads.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads-resources/index.md b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads-resources/index.md new file mode 100644 index 00000000000..facf573b9c7 --- /dev/null +++ b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads-resources/index.md @@ -0,0 +1,20 @@ +--- +title: "Grafana Mimir Remote ruler reads resources dashboard" +menuTitle: "Remote ruler reads resources" +description: "View an example Remote ruler reads resources dashboard." +weight: 110 +--- + +# Grafana Mimir Remote ruler reads resources dashboard + +The Remote ruler reads resources dashboard shows CPU, memory, disk, and other resources utilization metrics for ruler query path components when remote operational mode is enabled. + +The dashboard isolates each service on the ruler read path into its own section and displays the order in which a read request flows. + +This dashboard requires [additional resources metrics]({{< relref "../../requirements.md#additional-resources-metrics" >}}). + +## Example + +The following example shows a Remote ruler reads resources dashboard from a demo cluster. + +![Grafana Mimir Remote ruler reads resources dashboard](mimir-remote-ruler-reads-resources.png) diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads-resources/mimir-remote-ruler-reads-resources.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads-resources/mimir-remote-ruler-reads-resources.png new file mode 100644 index 00000000000..c9af1aa97ca Binary files /dev/null and b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads-resources/mimir-remote-ruler-reads-resources.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads/index.md b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads/index.md new file mode 100644 index 00000000000..dd2bed89e15 --- /dev/null +++ b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads/index.md @@ -0,0 +1,18 @@ +--- +title: "Grafana Mimir Remote ruler reads dashboard" +menuTitle: "Remote ruler reads" +description: "View an example Remote ruler reads dashboard." +weight: 90 +--- + +# Grafana Mimir Remote ruler reads dashboard + +The Remote ruler reads dashboard shows health metrics for the ruler read path when remote operational mode is enabled. + +The dashboard isolates each service on the ruler read path into its own section and displays the order in which a read request flows. + +## Example + +The following example shows a Remote ruler reads dashboard from a demo cluster. + +![Grafana Mimir Remote ruler reads dashboard](mimir-remote-ruler-reads.png) diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads/mimir-remote-ruler-reads.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads/mimir-remote-ruler-reads.png new file mode 100644 index 00000000000..97c8e07fee8 Binary files /dev/null and b/docs/sources/operators-guide/visualizing-metrics/dashboards/remote-ruler-reads/mimir-remote-ruler-reads.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/rollout-progress/mimir-rollout-progress.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/rollout-progress/mimir-rollout-progress.png index 9f2ae344508..fd43d040786 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/rollout-progress/mimir-rollout-progress.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/rollout-progress/mimir-rollout-progress.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/ruler/mimir-ruler.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/ruler/mimir-ruler.png index 45fff9ab2c7..04c67b75eed 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/ruler/mimir-ruler.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/ruler/mimir-ruler.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/scaling/mimir-scaling.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/scaling/mimir-scaling.png index dfdcbb0fd92..20e9b5272cc 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/scaling/mimir-scaling.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/scaling/mimir-scaling.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/tenants/mimir-tenants.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/tenants/mimir-tenants.png index e75ef9ae9ff..1d08bbde6e5 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/tenants/mimir-tenants.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/tenants/mimir-tenants.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-networking/mimir-writes-networking.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-networking/mimir-writes-networking.png index 14d59adc794..38737bab1f5 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-networking/mimir-writes-networking.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-networking/mimir-writes-networking.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-resources/mimir-writes-resources.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-resources/mimir-writes-resources.png index 39e91d4eb7c..c73082f99e7 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-resources/mimir-writes-resources.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/writes-resources/mimir-writes-resources.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/dashboards/writes/mimir-writes.png b/docs/sources/operators-guide/visualizing-metrics/dashboards/writes/mimir-writes.png index 43b22759591..f76bb76f101 100644 Binary files a/docs/sources/operators-guide/visualizing-metrics/dashboards/writes/mimir-writes.png and b/docs/sources/operators-guide/visualizing-metrics/dashboards/writes/mimir-writes.png differ diff --git a/docs/sources/operators-guide/visualizing-metrics/deploying-monitor-mixin.md b/docs/sources/operators-guide/visualizing-metrics/deploying-monitor-mixin.md index e360d7cab77..d1e1909d8e3 100644 --- a/docs/sources/operators-guide/visualizing-metrics/deploying-monitor-mixin.md +++ b/docs/sources/operators-guide/visualizing-metrics/deploying-monitor-mixin.md @@ -17,7 +17,7 @@ Grafana Mimir is shipped with a comprehensive set of production-ready Grafana da Dashboards provide both a high-level and in-depth view of every aspect of a Grafana Mimir cluster. You can take a look at all the available dashboards in [this overview]({{< relref "dashboards/_index.md" >}}). -Alerts allow you to monitor the health of a Mimir cluster. For each alert, we provide detailed [playbooks](https://github.com/grafana/mimir/blob/main/operations/mimir-mixin/docs/playbooks.md) to further investigate and fix the issue. +Alerts allow you to monitor the health of a Mimir cluster. For each alert, we provide detailed [runbooks]({{< relref "../mimir-runbooks/_index.md" >}}) to further investigate and fix the issue. The [requirements documentation]({{< relref "requirements.md" >}}) lists prerequisites for using the Grafana Mimir dashboards and alerts. diff --git a/docs/sources/operators-guide/visualizing-metrics/installing-dashboards-and-alerts.md b/docs/sources/operators-guide/visualizing-metrics/installing-dashboards-and-alerts.md index 3bb4daa15c9..3b8e894790d 100644 --- a/docs/sources/operators-guide/visualizing-metrics/installing-dashboards-and-alerts.md +++ b/docs/sources/operators-guide/visualizing-metrics/installing-dashboards-and-alerts.md @@ -59,11 +59,15 @@ make BUILD_IN_CONTAINER=true build-mixin In case you're already using Jsonnet to define your infrastructure as a code, you can vendor the Grafana Mimir mixin directly into your infrastructure repository and configure it overriding the `_config` fields. Given the exact setup really depends on a case-by-case basis, the following instructions are not meant to be prescriptive but just show the main steps required to vendor the mixin. -1. Install Grafana Mimir mixin +1. Initialise Jsonnet ```bash - jb install github.com/grafana/mimir/operations/mimir + jb init ``` -2. Import and configure it +2. Install Grafana Mimir mixin + ```bash + jb install github.com/grafana/mimir/operations/mimir-mixin@main + ``` +3. Import and configure it ```jsonnet (import 'github.com/grafana/mimir/operations/mimir-mixin/mixin.libsonnet') + { _config+:: { diff --git a/docs/sources/release-notes/v2.0.md b/docs/sources/release-notes/v2.0.md index 198ff86da94..973dcbc1b77 100644 --- a/docs/sources/release-notes/v2.0.md +++ b/docs/sources/release-notes/v2.0.md @@ -37,7 +37,7 @@ These features and enhancements distinguish Grafana Mimir from Cortex 1.10.0: - We switched to `memberlist` as the default store for Grafana Mimir’s [hash rings]({{< relref "../operators-guide/architecture/hash-ring/index.md" >}}). With this, users no longer have to run Consul or etcd as external dependencies. We made performance optimizations to `memberlist` to reduce its CPU utilization, which ensures that `memberlist` runs smoothly on Grafana Mimir clusters with lots of active series. - - We’ve included our own internal best practice dashboards, mixins, and alerts for [monitoring Grafana Mimir]({{< relref "../operators-guide/visualizing-metrics" >}}). While installing monitoring best practices such as these has typically required use of Jsonnet, we’ve eliminated this requirement. We include dashboards as JSON and alerting and recording rules as YAML which can be directly imported into your Grafana and Prometheus deployments. The alerts are accompanied by [playbooks](https://github.com/grafana/mimir/blob/main/operations/mimir-mixin/docs/playbooks.md) distilled from our own internal operations. + - We’ve included our own internal best practice dashboards, mixins, and alerts for [monitoring Grafana Mimir]({{< relref "../operators-guide/visualizing-metrics" >}}). While installing monitoring best practices such as these has typically required use of Jsonnet, we’ve eliminated this requirement. We include dashboards as JSON and alerting and recording rules as YAML which can be directly imported into your Grafana and Prometheus deployments. The alerts are accompanied by [runbooks]({{< relref "../operators-guide/mimir-runbooks/_index.md" >}}) distilled from our own internal operations. - **Configuration parameter reduction and classification**: We removed 36% of the configuration parameters in Grafana Mimir. All remaining configuration parameters have been classified as basic, advanced, or experimental. This is meant to make Grafana Mimir’s configuration more approachable for new users. In a default installation, you can focus exclusively on basic configuration. As you become more familiar with Grafana Mimir and want to push your clusters further, you can choose to tune advanced parameters or use experimental parameters. Refer to [parameter categories]({{< relref "../operators-guide/configuring/reference-configuration-parameters/index.md#parameter-categories" >}}) to learn more. @@ -45,7 +45,7 @@ These features and enhancements distinguish Grafana Mimir from Cortex 1.10.0: - **Query sharding for improved query speeds**: Grafana Mimir introduces query sharding to accelerate the execution of high-cardinality or CPU-intensive queries. Query sharding distributes the execution of a single query across multiple machines, to significantly reduce query execution time. We have seen speedups of 10 to 30x in our Grafana Cloud Metrics clusters. Refer to [query sharding]({{< relref "../operators-guide/architecture/query-sharding/index.md" >}}) to learn more. -- **Federated rule groups**: Grafana Mimir makes it possible to write alerting and recording rules that use metrics data from multiple tenants. For example, a user can now create a recording rule that adds metricA from tenantA to metricB from tenantB and writes the result to tenantC. This feature is experimental. For more information on how to use it, refer to [federated rule groups]({{< relref "../operators-guide/reference-http-api/index.md#federated-rule-groups" >}}). +- **Federated rule groups**: Grafana Mimir makes it possible to write alerting and recording rules that use metrics data from multiple tenants. For example, a user can now create a recording rule that adds metricA from tenantA to metricB from tenantB and writes the result to tenantC. This feature is experimental. For more information on how to use it, refer to [federated rule groups]({{< relref "../operators-guide/architecture/components/ruler/index.md#federated-rule-groups" >}}). - **Understand your metrics cardinality**: Grafana Mimir adds two API endpoints to help users identify high-cardinality metrics. The [`label_names` endpoint]({{< relref "../operators-guide/reference-http-api/index.md#label-names-cardinality" >}}) takes a metric name and returns all label names applied to that metric, as well as the count of values for each label name. When run without a metric name, it returns the highest-cardinality label names. The [`label_values` endpoint]({{< relref "../operators-guide/reference-http-api/index.md#label-values-cardinality" >}}) returns the highest-cardinality metrics, and can be used to get a count of how many series have a given label-value pair applied. The new [custom tracker feature]({{< relref "../operators-guide/configuring/configuring-custom-trackers.md" >}}) takes this one step further by allowing you to track the count of active series over time that match a specific label matcher. diff --git a/docs/sources/release-notes/v2.1.md b/docs/sources/release-notes/v2.1.md new file mode 100644 index 00000000000..952b16175c6 --- /dev/null +++ b/docs/sources/release-notes/v2.1.md @@ -0,0 +1,41 @@ +--- +title: "Grafana Mimir version 2.1 release notes" +menuTitle: "V2.1 release notes" +description: "Release notes for Grafana Mimir version 2.1" +weight: 200 +--- + +# Grafana Mimir version 2.1 release notes + +Grafana Labs is excited to announce version 2.1 of Grafana Mimir, the most scalable, most performant open source time series database in the world. + +Below we highlight the top features, enhancements and bugfixes in this release, as well as relevant callouts for those upgrading from Grafana Mimir 2.0. The complete list of changes is recorded in the [Changelog](https://github.com/grafana/mimir/blob/main/CHANGELOG.md). + +## Features and enhancements + +- **Mimir on ARM**: We now publish Docker images for both `amd64` and `arm64`, making it easier for those on arm-based machines to develop and run Mimir. Multiplaform images are available from the the [Mimir docker registry](https://hub.docker.com/r/grafana/mimir). Note that our existing integration test suite only uses the `amd64` images, which means we cannot make any functional or performance guarantees about the `arm64` images. + +- **`Remote` ruler mode for improved rule evaluation performance**: We've added a `remote` mode for the Grafana Mimir ruler, in which the ruler delegates rule evaluation to the [query-frontend]({{< relref "../operators-guide/architecture/components/query-frontend/index.md" >}}) rather than evaluating rules directly within the ruler process itself. This allows recording and alerting rules to benefit from the query parallelization techniques implemented in the query-frontend (like query sharding). `Remote` mode is considered experimental and is off by default. To enable, see [remote ruler]({{< relref "../operators-guide/architecture/components/ruler/index.md#remote" >}}). + +- **Per-tenant custom trackers for monitoring cardinality**: In Grafana Mimir 2.0, we introduced a [custom tracker feature]({{< relref "../operators-guide/configuring/configuring-custom-trackers.md" >}}) that allows you to track the count of active series over time that match a specific label matcher. In Grafana Mimir 2.1, we've made it possible to configure custom trackers via the [runtime configuration file]({{< relref "../operators-guide/configuring/about-runtime-configuration.md" >}}). This means you can now define different trackers for each tenant in your cluster and modify those trackers without an ingester restart. + +- **Reduce cardinality of Grafana Mimir's `/metrics` endpoint**: While Grafana Mimir does a good job of exposing a relatively small number of series about its own state, this number can tick up when running Grafana Mimir clusters with high tenant counts or high active series counts. To reduce this number (and the accompanying cost of scraping and storing these time series), we made [several optimizations](https://github.com/grafana/mimir/issues/1750) which decreased series count on the `/metrics` endpoint by more than 10%. + +## Upgrade considerations + +We've updated the default values for 2 parameters in Grafana Mimir to give users better out-of-the-box performance: + +- We've changed the default for `-blocks-storage.tsdb.isolation-enabled` from `true` to `false`. We've marked this flag as deprecated and will remove it completely in 2 releases. [TSDB isolation](https://grafana.com/blog/2020/05/05/how-isolation-improves-queries-in-prometheus-2.17/) is a feature inherited from Prometheus that didn't provide any benefit given Grafana Mimir's distributed architecture and in our [1 billion series load test](https://grafana.com/blog/2022/04/08/how-we-scaled-our-new-prometheus-tsdb-grafana-mimir-to-1-billion-active-series/#prometheus-tsdb-enhancements) we found it actually hurt performance. Disabling it reduced our ingester 99th percentile latency by 90%. + +- The store-gateway attributes cache is now enabled by default (achieved by updating the default for `-blocks-storage.bucket-store.chunks-cache.attributes-in-memory-max-items` from `0` to `50000`). This in-memory cache makes it faster to look up object attributes for chunk data. We've been running this optional cache internally for a while and upon a recent configuration audit, realized it made sense to do the same for all users. The increase in store-gateway memory utilization from enabling this cache is negligible and easily justified given the performance gains. + +- As part of the enhancement to make active series custom trackers configurable on a per-tenant basis, we've moved where they get configured. Users of this feature should migrate from defining `active_series_custom_trackers_config` in the `ingester` section of Grafana Mimir's [YAML configuration]({{< relref "../operators-guide/configuring/reference-configuration-parameters/index.md#configuration-parameters" >}}) to defining it in the `limits` section. Both options are supported for now to give users time to make the change, but configuring `active_series_custom_trackers_config` via the `ingester` section has been marked deprecated and will be removed in Mimir 2.3. Note: Configuring `active_series_custom_trackers_config` in both the `limits` and `ingester` sections will cause Grafana Mimir to fail on startup. + +## Bug fixes + +### 2.1.0 bug fixes + +- [PR 1704](https://github.com/grafana/mimir/pull/1704): Fixed a bug that previously caused Grafana Mimir to crash on startup when trying to run in monolithic mode with the results cache enabled due to duplicate metric names. +- [PR 1835](https://github.com/grafana/mimir/pull/1835): Fixed a bug that caused Grafana Mimir to crash when an invalid Alertmanager configuration was set even though the Alertmanager component was disabled. After this fix, the Alertmanager configuration is only validated if the Alertmanager component is loaded. +- [PR 1836](https://github.com/grafana/mimir/pull/1836): The ability to run Alertmanager with `local` storage broke in Grafana Mimir 2.0 when we removed the ability to run the Alertmanager without sharding. With this bugfix, we've made it possible to again run Alertmanager with `local` storage. However, for production use, we still recommend using external store since this is needed to persist Alertmanager state (e.g. silences) between replicas. +- [PR 1715](https://github.com/grafana/mimir/pull/1715): Restored Grafana Mimir's ability to use CNAME DNS records to reach memcached servers. The bug was inherited from an upstream change to Thanos; we contributed a fix to Thanos and subsequently updated our Thanos version. diff --git a/docs/sources/tutorials/play-with-grafana-mimir/index.md b/docs/sources/tutorials/play-with-grafana-mimir/index.md index 5fe6a43cd32..d2ef9bd0905 100644 --- a/docs/sources/tutorials/play-with-grafana-mimir/index.md +++ b/docs/sources/tutorials/play-with-grafana-mimir/index.md @@ -100,7 +100,7 @@ To start, we recommend looking at these dashboards: A couple of caveats: - It typically takes a few minutes after Grafana Mimir starts to display meaningful metrics in the dashboards. -- Because this tutorial runs Grafana Mimir without any ingress gateway, query-scheduler, or memcached, the related panels are expected to be empty. +- Because this tutorial runs Grafana Mimir without any query-scheduler, or memcached, the related panels are expected to be empty. The dashboards installed in the Grafana are taken from the Grafana Mimir mixin which packages up Grafana Labs' best practice dashboards, recording rules, and alerts for monitoring Grafana Mimir. To learn more about the mixin, check out the Grafana Mimir mixin documentation. To learn more about how Grafana is connecting to Grafana Mimir, review the [Mimir datasource](http://localhost:9000/datasources). diff --git a/go.mod b/go.mod index ceddfd5ae9f..3c0a49eaaab 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/grafana/mimir go 1.17 require ( - github.com/NYTimes/gziphandler v1.1.1 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 github.com/cespare/xxhash/v2 v2.1.2 // indirect github.com/dustin/go-humanize v1.0.0 @@ -18,8 +17,8 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/gopacket v1.1.19 github.com/gorilla/mux v1.8.0 - github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca - github.com/grafana/e2e v0.1.0 + github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b + github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe github.com/hashicorp/golang-lru v0.5.4 github.com/json-iterator/go v1.1.12 github.com/leanovate/gopter v0.2.4 @@ -39,7 +38,7 @@ require ( github.com/sirupsen/logrus v1.8.1 github.com/spf13/afero v1.6.0 github.com/stretchr/testify v1.7.1 - github.com/thanos-io/thanos v0.24.1-0.20220416232747-81218afa5b01 + github.com/thanos-io/thanos v0.26.1-0.20220602051129-a6f6ce060ed4 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/weaveworks/common v0.0.0-20211109170639-0684aab3d884 go.uber.org/atomic v1.9.0 @@ -50,7 +49,7 @@ require ( golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 google.golang.org/grpc v1.45.0 gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -58,8 +57,8 @@ require ( github.com/google/go-github/v32 v32.1.0 github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db + golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 gopkg.in/alecthomas/kingpin.v2 v2.2.6 ) @@ -139,6 +138,7 @@ require ( github.com/googleapis/gax-go/v2 v2.2.0 // indirect github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de // indirect github.com/gosimple/slug v1.1.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7 // indirect github.com/hashicorp/consul/api v1.12.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -208,7 +208,6 @@ require ( go.uber.org/zap v1.19.1 // indirect golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect - golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886 // indirect golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.10 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect @@ -230,7 +229,7 @@ replace github.com/bradfitz/gomemcache => github.com/themihai/gomemcache v0.0.0- replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus v0.0.0-20220603131538-5f73bf911b89 // OOO Support forces us to fork thanos because we've changed the ChunkReader interface -replace github.com/thanos-io/thanos => github.com/jesusvazquez/thanos v0.19.1-0.20220427121402-46735047853b +replace github.com/thanos-io/thanos => github.com/jesusvazquez/thanos v0.19.1-0.20220610094531-ab07eb568317 // Pin hashicorp depencencies since the Prometheus fork, go mod tries to update them. replace github.com/hashicorp/go-immutable-radix => github.com/hashicorp/go-immutable-radix v1.2.0 @@ -239,6 +238,9 @@ replace github.com/hashicorp/go-hclog => github.com/hashicorp/go-hclog v0.12.2 // Replace memberlist with our fork which includes some fixes that haven't been // merged upstream yet: https://github.com/hashicorp/memberlist/pull/260 -replace github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b +replace github.com/hashicorp/memberlist => github.com/grafana/memberlist v0.3.1-0.20220425183535-6b97a09b7167 replace github.com/vimeo/galaxycache => github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e + +// Use fork of grafana go sdk, which includes fixes that haven't been merged into upstream +replace github.com/grafana-tools/sdk => github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f diff --git a/go.sum b/go.sum index 945e9a09e3d..14a76668c58 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,7 @@ github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.17/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.23/go.mod h1:BAWYUWGPEtKPzjVkp0Q6an0MJcJDsoh5Z1BFAEFs4Xs= github.com/Azure/go-autorest/autorest v0.11.25 h1:yp+V8DGur2aIUE87ebP8twPLz6k68jtJTlg61mEoByA= github.com/Azure/go-autorest/autorest v0.11.25/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= @@ -85,6 +86,7 @@ github.com/Azure/go-autorest/autorest/adal v0.9.2/go.mod h1:/3SMAM86bP6wC9Ev35pe github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.11/go.mod h1:nBKAnTomx8gDtl+3ZCJv2v0KACFHWTB2drffI1B68Pk= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.14/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 h1:TzPg6B6fTZ0G1zBf3T54aI7p3cAT6u//TOXGPmFMOXg= @@ -211,6 +213,7 @@ github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2z github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.40.45/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go v1.42.8/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.42.31/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.43.31 h1:yJZIr8nMV1hXjAvvOLUFqZRJcHV7udPQBfhJqawDzI0= github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= @@ -327,6 +330,8 @@ github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f h1:Mc/WpMhT0pzDD5zGjhge7PiO7nkrMME4GuGS1y4HGwk= +github.com/colega/grafana-tools-sdk v0.0.0-20220323154849-711bca56d13f/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -531,8 +536,8 @@ github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= -github.com/efficientgo/e2e v0.11.2-0.20220224081107-b67f7b039363 h1:wlimY9L7RuHjNugLWsYH1wFTpkonEc3rktYSLx8aXu0= -github.com/efficientgo/e2e v0.11.2-0.20220224081107-b67f7b039363/go.mod h1:vDnF4AAEZmO0mvyFIATeDJPFaSRM7ywaOnKd61zaSoE= +github.com/efficientgo/e2e v0.12.1 h1:ZYNTf09ptlba0I3ZStYaF7gCbevWdalriiX7usOSiFM= +github.com/efficientgo/e2e v0.12.1/go.mod h1:xDHUyIqAWyVWU29Lf+BaZoavW7xAbDEvTwHWWI/3bhk= github.com/efficientgo/tools/core v0.0.0-20210129205121-421d0828c9a6/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= github.com/efficientgo/tools/core v0.0.0-20210829154005-c7bad8450208 h1:jIALuFymwBqVsF32JhgzVsbCB6QsWvXqhetn8QgyrZ4= github.com/efficientgo/tools/core v0.0.0-20210829154005-c7bad8450208/go.mod h1:OmVcnJopJL8d3X3sSXTiypGoUSgFq1aDGmlrdi9dn/M= @@ -915,6 +920,7 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210715191844-86eeefc3e471/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/pprof v0.0.0-20220318212150-b2ab0324ddda h1:KdHPvlgeNEDs8rae032MqFG8LVwcSEivcCjNdVOXRmg= github.com/google/pprof v0.0.0-20220318212150-b2ab0324ddda/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -954,16 +960,14 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosimple/slug v1.1.1 h1:fRu/digW+NMwBIP+RmviTK97Ho/bEj/C9swrCspN3D4= github.com/gosimple/slug v1.1.1/go.mod h1:ER78kgg1Mv0NQGlXiDe57DpCyfbNywXXZ9mIorhxAf0= -github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9 h1:LQAhgcUPnzdjU/OjCJaLlPQI7NmQCRlfjMPSA1VegvA= -github.com/grafana-tools/sdk v0.0.0-20211220201350-966b3088eec9/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/grafana/dskit v0.0.0-20211021180445-3bd016e9d7f1/go.mod h1:uPG2nyK4CtgNDmWv7qyzYcdI+S90kHHRWvHnBtEMBXM= github.com/grafana/dskit v0.0.0-20220112093026-95274ccc858d/go.mod h1:M0/dlftwBvH7+hdNNpjMa/CUXD7gsew67mbkCuDlFXE= -github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca h1:0qHzm6VS0bCsSWKHuyfpt+pdpyScdZbzY/IFIyKSYOk= -github.com/grafana/dskit v0.0.0-20220331160727-49faf69f72ca/go.mod h1:q51XdMLLHNZJSG6KOGujC20ed2OoLFdx0hBmOEVfRs0= -github.com/grafana/e2e v0.1.0 h1:nThd0U0TjUqyOOupSb+qDd4BOdhqwhR/oYbjoqiMlZk= -github.com/grafana/e2e v0.1.0/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo= -github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b h1:UlCBLaqvS4wVYNrMKSfqTBVsed/EOY9dnenhYZMUnrA= -github.com/grafana/memberlist v0.2.5-0.20211201083710-c7bc8e9df94b/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b h1:9h79WowXGj6wErMzoi35pBECxnh7ucKJlQTs4Gs0yOI= +github.com/grafana/dskit v0.0.0-20220526081034-789ec0ca4a3b/go.mod h1:9It/K30QPyj/FuTqBb/SYnaS4/BJCP5YL4SRfXB7dG0= +github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe h1:mxrRWDjKtob43xF9nEhJthdtCzX35/800Sk7nE//YHQ= +github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe/go.mod h1:+26VJWpczg2OU3D0537acnHSHzhJORpxOs6F+M27tZo= +github.com/grafana/memberlist v0.3.1-0.20220425183535-6b97a09b7167 h1:PgEQkGHR4YimSCEGT5IoswN9gJKZDVskf+he6UClCLw= +github.com/grafana/memberlist v0.3.1-0.20220425183535-6b97a09b7167/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/grafana/mimir-prometheus v0.0.0-20220603131538-5f73bf911b89 h1:k6c1K5kkkdruuW+lvX0g4qFnnLTG+ZzUp+tc76r7WT4= github.com/grafana/mimir-prometheus v0.0.0-20220603131538-5f73bf911b89/go.mod h1:W59JUgfj423JtdkiZLvblAJD4IQeE04y26z0CL7DVKc= github.com/grafana/regexp v0.0.0-20220304095617-2e8d9baf4ac2 h1:uirlL/j72L93RhV4+mkWhjv0cov2I0MIgPOG9rMDr1k= @@ -1068,8 +1072,8 @@ github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGk github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jesusvazquez/thanos v0.19.1-0.20220427121402-46735047853b h1:5DyOpfQlkHOv9FE9ZsnfCFUJyH4JGcKdc5cPbSn7wEE= -github.com/jesusvazquez/thanos v0.19.1-0.20220427121402-46735047853b/go.mod h1:/WiTX1fRzebFlnvMkPNojcB3QfKsaz/o89JnDMomcsQ= +github.com/jesusvazquez/thanos v0.19.1-0.20220610094531-ab07eb568317 h1:hvM2NqbuhELZHcEti7+yCdY1DPXgneBf8mcmHyPoxyA= +github.com/jesusvazquez/thanos v0.19.1-0.20220610094531-ab07eb568317/go.mod h1:9e/ytDfVepSKxihUWXyy1irj+ipM/DAlOBqsyXs+Y10= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -1215,6 +1219,7 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= +github.com/miekg/dns v1.1.45/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -1450,7 +1455,6 @@ github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+ github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.15.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.30.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= @@ -1753,6 +1757,7 @@ go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs= +go.opentelemetry.io/otel/sdk v1.5.0/go.mod h1:CU4J1v+7iEljnm1G14QjdFWOXUyYLHVh0Lh+/BTYyFg= go.opentelemetry.io/otel/sdk v1.6.1 h1:ZmcNyMhcuAYIb/Nr6QhBPTMopMTbov/47wHt1gibkoY= go.opentelemetry.io/otel/sdk v1.6.1/go.mod h1:IVYrddmFZ+eJqu2k38qD3WezFR2pymCzm8tdxyh3R4E= go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE= @@ -1777,6 +1782,7 @@ go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= go.uber.org/automaxprocs v1.5.1/go.mod h1:BF4eumQw0P9GtnuxxovUd06vwm1o18oMzFtK66vU6XU= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -1952,6 +1958,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211123203042-d83791d6bcd9/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de h1:pZB1TWnKi+o4bENlbzAgLrEbY4RMYmUIRobMcSmfeYc= @@ -2156,6 +2163,7 @@ golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 h1:M73Iuj3xbbb9Uk1DYhzydthsj6oOd6l9bpuFcNoUvTs= golang.org/x/time v0.0.0-20220224211638-0e9765cccd65/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2242,6 +2250,7 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= @@ -2289,6 +2298,7 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.60.0/go.mod h1:d7rl65NZAkEQ90JFzqBjcRq1TVeG5ZoGV3sSpEnnVb4= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= @@ -2381,6 +2391,7 @@ google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ6 google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= @@ -2491,8 +2502,9 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/integration/alertmanager_test.go b/integration/alertmanager_test.go index c83083b7fec..d262aa27fef 100644 --- a/integration/alertmanager_test.go +++ b/integration/alertmanager_test.go @@ -125,6 +125,43 @@ func TestAlertmanager(t *testing.T) { require.Equal(t, "Accept-Encoding", res.Header.Get("Vary")) } +func TestAlertmanagerLocalStore(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + consul := e2edb.NewConsul() + require.NoError(t, s.StartAndWaitReady(consul)) + + require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(mimirAlertmanagerUserConfigYaml))) + + alertmanager := e2emimir.NewAlertmanager( + "alertmanager", + mergeFlags( + AlertmanagerFlags(), + AlertmanagerLocalFlags(), + AlertmanagerShardingFlags(consul.NetworkHTTPEndpoint(), 1), + ), + ) + require.NoError(t, s.StartAndWaitReady(alertmanager)) + require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_config_last_reload_successful")) + require.NoError(t, alertmanager.WaitSumMetrics(e2e.Greater(0), "cortex_alertmanager_config_hash")) + + c, err := e2emimir.NewClient("", "", alertmanager.HTTPEndpoint(), "", "user-1") + require.NoError(t, err) + + cfg, err := c.GetAlertmanagerConfig(context.Background()) + require.NoError(t, err) + + // Ensure the returned status config matches alertmanager_test_fixtures/user-1.yaml + require.NotNil(t, cfg) + require.Equal(t, "example_receiver", cfg.Route.Receiver) + require.Len(t, cfg.Route.GroupByStr, 1) + require.Equal(t, "example_groupby", cfg.Route.GroupByStr[0]) + require.Len(t, cfg.Receivers, 1) + require.Equal(t, "example_receiver", cfg.Receivers[0].Name) +} + func TestAlertmanagerStoreAPI(t *testing.T) { s, err := e2e.NewScenario(networkName) require.NoError(t, err) diff --git a/integration/backward_compatibility.go b/integration/backward_compatibility.go index fe57ffafef5..389283fa6a7 100644 --- a/integration/backward_compatibility.go +++ b/integration/backward_compatibility.go @@ -7,7 +7,7 @@ import "github.com/grafana/mimir/integration/e2emimir" // DefaultPreviousVersionImages is used by `tools/pre-pull-images` so it needs // to be in a non `_test.go` file. var DefaultPreviousVersionImages = map[string]e2emimir.FlagMapper{ - "grafana/mimir:2.0.0": e2emimir.NoopFlagMapper, + "grafana/mimir:2.1.0": e2emimir.NoopFlagMapper, } var ( diff --git a/integration/configs.go b/integration/configs.go index 2859c075278..15bc1a3b41a 100644 --- a/integration/configs.go +++ b/integration/configs.go @@ -10,10 +10,12 @@ package integration import ( "fmt" "os" + "path/filepath" "strconv" "strings" "text/template" + e2e "github.com/grafana/e2e" e2edb "github.com/grafana/e2e/db" ) @@ -95,6 +97,13 @@ var ( } } + AlertmanagerLocalFlags = func() map[string]string { + return map[string]string{ + "-alertmanager-storage.backend": "local", + "-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"), + } + } + AlertmanagerS3Flags = func() map[string]string { return map[string]string{ "-alertmanager-storage.backend": "s3", diff --git a/integration/getting_started_with_gossiped_ring_test.go b/integration/getting_started_with_gossiped_ring_test.go index bff60d7126b..7e7ae28799d 100644 --- a/integration/getting_started_with_gossiped_ring_test.go +++ b/integration/getting_started_with_gossiped_ring_test.go @@ -41,6 +41,7 @@ func TestGettingStartedWithGossipedRing(t *testing.T) { "-ingester.ring.observe-period": "5s", // to avoid conflicts in tokens "-blocks-storage.bucket-store.bucket-index.enabled": "false", "-blocks-storage.bucket-store.sync-interval": "1s", // sync continuously + "-blocks-storage.bucket-store.ignore-blocks-within": "0", "-blocks-storage.backend": "s3", "-blocks-storage.s3.bucket-name": blocksBucketName, "-blocks-storage.s3.access-key-id": e2edb.MinioAccessKey, diff --git a/integration/ingester_sharding_test.go b/integration/ingester_sharding_test.go index d1f3ddfd477..a6be8ba4c69 100644 --- a/integration/ingester_sharding_test.go +++ b/integration/ingester_sharding_test.go @@ -51,6 +51,7 @@ func TestIngesterSharding(t *testing.T) { // Enable shuffle sharding on read path but not lookback, otherwise all ingesters would be // queried being just registered. + flags["-querier.query-store-after"] = "0" flags["-querier.shuffle-sharding-ingesters-lookback-period"] = "1ns" // Start dependencies. diff --git a/integration/querier_remote_read_test.go b/integration/querier_remote_read_test.go index 4486d50230b..467e1f13567 100644 --- a/integration/querier_remote_read_test.go +++ b/integration/querier_remote_read_test.go @@ -10,7 +10,9 @@ package integration import ( "bytes" "context" + "io" "io/ioutil" + "math/rand" "net/http" "testing" "time" @@ -82,7 +84,7 @@ func TestQuerierRemoteRead(t *testing.T) { req := &prompb.ReadRequest{ Queries: []*prompb.Query{q}, - AcceptedResponseTypes: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS}, + AcceptedResponseTypes: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_SAMPLES}, } data, err := proto.Marshal(req) @@ -125,3 +127,119 @@ func TestQuerierRemoteRead(t *testing.T) { require.Equal(t, int64(expectedVectors[0].Timestamp), resp.Results[0].Timeseries[0].Samples[0].Timestamp) require.Equal(t, float64(expectedVectors[0].Value), resp.Results[0].Timeseries[0].Samples[0].Value) } + +func TestQuerierStreamingRemoteRead(t *testing.T) { + s, err := e2e.NewScenario(networkName) + require.NoError(t, err) + defer s.Close() + + flags := mergeFlags(BlocksStorageFlags(), map[string]string{ + "-distributor.ingestion-rate-limit": "1048576", + "-distributor.ingestion-burst-size": "1048576", + }) + + // Start dependencies. + minio := e2edb.NewMinio(9000, blocksBucketName) + + consul := e2edb.NewConsul() + require.NoError(t, s.StartAndWaitReady(minio, consul)) + + // Start Mimir components for the write path. + distributor := e2emimir.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), flags) + ingester := e2emimir.NewIngester("ingester", consul.NetworkHTTPEndpoint(), flags) + require.NoError(t, s.StartAndWaitReady(distributor, ingester)) + + // Wait until the distributor has updated the ring. + // The distributor should have 512 tokens for the ingester ring and 1 for the distributor ring + require.NoError(t, distributor.WaitSumMetrics(e2e.Equals(512+1), "cortex_ring_tokens_total")) + + querier := e2emimir.NewQuerier("querier", consul.NetworkHTTPEndpoint(), BlocksStorageFlags()) + require.NoError(t, s.StartAndWaitReady(querier)) + + // Wait until the querier has updated the ring. + require.NoError(t, querier.WaitSumMetrics(e2e.Equals(512), "cortex_ring_tokens_total")) + + // Push a series to Mimir. + now := time.Now() + + c, err := e2emimir.NewClient(distributor.HTTPEndpoint(), "", "", "", "user-1") + require.NoError(t, err) + + // Generate the series + startMs := now.Add(-time.Minute).Unix() * 1000 + endMs := now.Add(time.Minute).Unix() * 1000 + + var samples []prompb.Sample + for i := startMs; i < endMs; i++ { + samples = append(samples, prompb.Sample{ + Value: rand.Float64(), + Timestamp: i, + }) + } + + var series []prompb.TimeSeries + series = append(series, prompb.TimeSeries{ + Labels: []prompb.Label{ + {Name: labels.MetricName, Value: "series_1"}, + }, + Samples: samples, + }) + + res, err := c.Push(series) + require.NoError(t, err) + require.Equal(t, 200, res.StatusCode) + + matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "series_1") + require.NoError(t, err) + + q, err := remote.ToQuery(startMs, endMs, []*labels.Matcher{matcher}, &storage.SelectHints{ + Step: 1, + Start: startMs, + End: endMs, + }) + require.NoError(t, err) + + req := &prompb.ReadRequest{ + Queries: []*prompb.Query{q}, + AcceptedResponseTypes: []prompb.ReadRequest_ResponseType{prompb.ReadRequest_STREAMED_XOR_CHUNKS}, + } + + data, err := proto.Marshal(req) + require.NoError(t, err) + compressed := snappy.Encode(nil, data) + + // Call the remote read API endpoint with a timeout. + httpReqCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + httpReq, err := http.NewRequestWithContext(httpReqCtx, "POST", "http://"+querier.HTTPEndpoint()+"/prometheus/api/v1/read", bytes.NewReader(compressed)) + require.NoError(t, err) + httpReq.Header.Set("X-Scope-OrgID", "user-1") + httpReq.Header.Set("User-Agent", "Prometheus/1.8.2") + httpReq.Header.Set("X-Prometheus-Remote-Read-Version", "0.1.0") + + httpResp, err := http.DefaultClient.Do(httpReq) + require.NoError(t, err) + require.Equal(t, http.StatusOK, httpResp.StatusCode) + + // Fetch streaming response + stream := remote.NewChunkedReader(httpResp.Body, remote.DefaultChunkedReadLimit, nil) + + results := []prompb.ChunkedReadResponse{} + for { + var res prompb.ChunkedReadResponse + err := stream.NextProto(&res) + if err == io.EOF { + break + } + require.NoError(t, err) + results = append(results, res) + } + + // Validate the returned remote read data + require.Len(t, results, 1) + require.Len(t, results[0].ChunkedSeries, 1) + require.Len(t, results[0].ChunkedSeries[0].Labels, 1) + require.Equal(t, "series_1", results[0].ChunkedSeries[0].Labels[0].GetValue()) + require.True(t, len(results[0].ChunkedSeries[0].Chunks) > 0) +} diff --git a/integration/querier_sharding_test.go b/integration/querier_sharding_test.go index 951fc8a1880..bd892be0236 100644 --- a/integration/querier_sharding_test.go +++ b/integration/querier_sharding_test.go @@ -71,7 +71,6 @@ func runQuerierShardingTest(t *testing.T, cfg querierShardingTestConfig) { flags := mergeFlags(BlocksStorageFlags(), map[string]string{ "-query-frontend.cache-results": "true", - "-querier.query-ingesters-within": "12h", // Required by the test on query /series out of ingesters time range "-query-frontend.results-cache.backend": "memcached", "-query-frontend.results-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort), "-query-frontend.results-cache.compression": "snappy", diff --git a/integration/querier_tenant_federation_test.go b/integration/querier_tenant_federation_test.go index 4a4f2dfc4c6..fe76200600c 100644 --- a/integration/querier_tenant_federation_test.go +++ b/integration/querier_tenant_federation_test.go @@ -66,7 +66,6 @@ func runQuerierTenantFederationTest(t *testing.T, cfg querierTenantFederationCon flags := mergeFlags(BlocksStorageFlags(), map[string]string{ "-query-frontend.cache-results": "true", - "-querier.query-ingesters-within": "12h", // Required by the test on query /series out of ingesters time range "-query-frontend.results-cache.backend": "memcached", "-query-frontend.results-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort), "-tenant-federation.enabled": "true", diff --git a/integration/querier_test.go b/integration/querier_test.go index d4322ed9bd9..dc736b1db58 100644 --- a/integration/querier_test.go +++ b/integration/querier_test.go @@ -148,8 +148,10 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) { "-blocks-storage.bucket-store.index-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort), "-blocks-storage.bucket-store.sync-interval": "1s", "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, + "-blocks-storage.bucket-store.ignore-blocks-within": "0", "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), "-store-gateway.tenant-shard-size": fmt.Sprintf("%d", testCfg.tenantShardSize), + "-querier.query-store-after": "0", "-query-frontend.query-stats-enabled": "true", "-query-frontend.parallelize-shardable-queries": strconv.FormatBool(testCfg.queryShardingEnabled), }) @@ -308,8 +310,9 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) { func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { tests := map[string]struct { - indexCacheBackend string - bucketIndexEnabled bool + indexCacheBackend string + bucketIndexEnabled bool + queryShardingEnabled bool }{ "inmemory index cache": { indexCacheBackend: tsdb.IndexCacheBackendInMemory, @@ -321,6 +324,10 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { indexCacheBackend: tsdb.IndexCacheBackendMemcached, bucketIndexEnabled: true, }, + "inmemory index cache, query sharding enabled": { + indexCacheBackend: tsdb.IndexCacheBackendInMemory, + queryShardingEnabled: true, + }, } for testName, testCfg := range tests { @@ -348,10 +355,12 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { "-blocks-storage.tsdb.block-ranges-period": blockRangePeriod.String(), "-blocks-storage.tsdb.ship-interval": "1s", "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.bucket-store.ignore-blocks-within": "0", "-blocks-storage.tsdb.retention-period": ((blockRangePeriod * 2) - 1).String(), "-blocks-storage.bucket-store.index-cache.backend": testCfg.indexCacheBackend, "-blocks-storage.bucket-store.index-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort), "-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled), + // Ingester. "-ingester.ring.store": "consul", "-ingester.ring.consul.hostname": consul.NetworkHTTPEndpoint(), @@ -366,6 +375,10 @@ func TestQuerierWithBlocksStorageRunningInSingleBinaryMode(t *testing.T) { "-compactor.ring.store": "consul", "-compactor.ring.consul.hostname": consul.NetworkHTTPEndpoint(), "-compactor.cleanup-interval": "2s", // Update bucket index often. + // Querier. + "-querier.query-store-after": "0", + // Query-frontend. + "-query-frontend.parallelize-shardable-queries": strconv.FormatBool(testCfg.queryShardingEnabled), }) // Start Mimir replicas. @@ -765,10 +778,12 @@ func TestQuerierWithBlocksStorageOnMissingBlocksFromStorage(t *testing.T) { // blocks (less than 3*sync-interval age) as they could be unnoticed by the store-gateway and ingesters // have them anyway. We turn down the sync-interval to speed up the test. storeGateway := e2emimir.NewStoreGateway("store-gateway", consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{ - "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.bucket-store.sync-interval": "1s", + "-blocks-storage.bucket-store.ignore-blocks-within": "0", })) querier := e2emimir.NewQuerier("querier", consul.NetworkHTTPEndpoint(), mergeFlags(flags, map[string]string{ "-blocks-storage.bucket-store.sync-interval": "1s", + "-querier.query-store-after": "0", })) require.NoError(t, s.StartAndWaitReady(querier, storeGateway)) @@ -871,7 +886,7 @@ func TestQueryLimitsWithBlocksStorageRunningInMicroServices(t *testing.T) { _, err = c.QueryRange("{__name__=~\"series_.+\"}", series1Timestamp, series4Timestamp.Add(1*time.Hour), blockRangePeriod) require.Error(t, err) - assert.Contains(t, err.Error(), "max number of series limit") + assert.ErrorContains(t, err, "the query exceeded the maximum number of series") } func TestHashCollisionHandling(t *testing.T) { diff --git a/integration/query_frontend_cache_test.go b/integration/query_frontend_cache_test.go index 01ad608a567..62cf9ddb204 100644 --- a/integration/query_frontend_cache_test.go +++ b/integration/query_frontend_cache_test.go @@ -35,7 +35,6 @@ func TestQueryFrontendUnalignedQuery(t *testing.T) { flags = mergeFlags(flags, map[string]string{ "-query-frontend.cache-results": "true", "-query-frontend.split-queries-by-interval": "2m", - "-querier.query-ingesters-within": "12h", // Required by the test on query /series out of ingesters time range "-query-frontend.align-querier-with-step": "true", "-query-frontend.max-cache-freshness": "0", // Cache everything. "-query-frontend.results-cache.backend": "memcached", diff --git a/integration/query_frontend_test.go b/integration/query_frontend_test.go index dee2cc1b021..f910d73266f 100644 --- a/integration/query_frontend_test.go +++ b/integration/query_frontend_test.go @@ -31,6 +31,7 @@ import ( "github.com/grafana/mimir/integration/ca" "github.com/grafana/mimir/integration/e2emimir" + "github.com/grafana/mimir/pkg/util/validation" ) type queryFrontendTestConfig struct { @@ -169,7 +170,6 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) { flags = mergeFlags(flags, map[string]string{ "-query-frontend.cache-results": "true", - "-querier.query-ingesters-within": "12h", // Required by the test on query /series out of ingesters time range "-query-frontend.results-cache.backend": "memcached", "-query-frontend.results-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort), "-query-frontend.query-stats-enabled": strconv.FormatBool(cfg.queryStatsEnabled), @@ -493,7 +493,7 @@ overrides: return c.QueryRangeRaw(`sum_over_time(metric[31d:1s])`, now.Add(-time.Minute), now, time.Minute) }, expStatusCode: http.StatusUnprocessableEntity, - expBody: `{"error":"expanding series: the query time range exceeds the limit (query length: 744h6m0s, limit: 720h0m0s)", "errorType":"execution", "status":"error"}`, + expBody: fmt.Sprintf(`{"error":"expanding series: %s", "errorType":"execution", "status":"error"}`, validation.NewMaxQueryLengthError((744*time.Hour)+(6*time.Minute), 720*time.Hour)), }, { name: "execution error", diff --git a/integration/ruler_test.go b/integration/ruler_test.go index cf687240831..61a8ca2c5cf 100644 --- a/integration/ruler_test.go +++ b/integration/ruler_test.go @@ -287,7 +287,7 @@ func TestRulerEvaluationDelay(t *testing.T) { // Wait until all the pushed samples have been evaluated by the rule. This // ensures that rule results are successfully written even after a // staleness period. - require.NoError(t, mimir.WaitSumMetrics(e2e.GreaterOrEqual(ruleEvaluationsAfterPush[0]+float64(samplesToSend)), "cortex_prometheus_rule_evaluations_total")) + require.NoError(t, mimir.WaitSumMetrics(e2e.Greater(ruleEvaluationsAfterPush[0]+float64(samplesToSend)), "cortex_prometheus_rule_evaluations_total")) // query all results to verify rules have been evaluated correctly result, err = c.QueryRange("stale_nan_eval", now.Add(-evaluationDelay), now, time.Second) diff --git a/mimir-build-image/Dockerfile b/mimir-build-image/Dockerfile index 88aec268cc3..8716256c831 100644 --- a/mimir-build-image/Dockerfile +++ b/mimir-build-image/Dockerfile @@ -3,10 +3,10 @@ # Provenance-includes-license: Apache-2.0 # Provenance-includes-copyright: The Cortex Authors. -FROM golang:1.17.8-buster +FROM golang:1.17.8-bullseye ARG goproxyValue ENV GOPROXY=${goproxyValue} -RUN apt-get update && apt-get install -y curl python-requests python-yaml file jq zip unzip protobuf-compiler libprotobuf-dev shellcheck libpcap-dev && \ +RUN apt-get update && apt-get install -y curl python3-requests python3-yaml file jq zip unzip protobuf-compiler libprotobuf-dev shellcheck libpcap-dev skopeo && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN go install golang.org/x/tools/cmd/goimports@3fce476f0a782aeb5034d592c189e63be4ba6c9e RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - @@ -45,6 +45,7 @@ RUN GO111MODULE=on \ go install github.com/monitoring-mixins/mixtool/cmd/mixtool@bca3066 && \ go install github.com/mikefarah/yq/v4@v4.13.4 && \ go install github.com/google/go-jsonnet/cmd/jsonnetfmt@v0.17.0 && \ + go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.8.1 && \ rm -rf /go/pkg /go/src /root/.cache ENV NODE_PATH=/usr/lib/node_modules diff --git a/operations/helm/charts/mimir-distributed/.gitignore b/operations/helm/charts/mimir-distributed/.gitignore new file mode 100644 index 00000000000..17c7db916f7 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/.gitignore @@ -0,0 +1,2 @@ +# Ignore dependencies downloaded by helm +charts/ diff --git a/operations/helm/charts/mimir-distributed/.helmignore b/operations/helm/charts/mimir-distributed/.helmignore new file mode 100644 index 00000000000..e3c5915b672 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/.helmignore @@ -0,0 +1,7 @@ +.envrc +.gitignore +CHANGELOG.md +ci/ +INTERNAL* +README.md.gotmpl +scripts/ diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md new file mode 100644 index 00000000000..4ca319b4bee --- /dev/null +++ b/operations/helm/charts/mimir-distributed/CHANGELOG.md @@ -0,0 +1,353 @@ +# Changelog + +This changelog is continued from `enterprise-metrics` after Grafana Enterprise Metrics was added to `mimir-distributed` in PR #1203. +All notable changes to this chart will be documented in this file. + +Entries should be ordered as follows: +- [CHANGE] +- [FEATURE] +- [ENHANCEMENT] +- [BUGFIX] + +Entries should include a reference to the Pull Request that introduced the change. + +## main / unreleased + +* [CHANGE] Change default value for `blocks_storage.bucket_store.chunks_cache.memcached.timeout` to `450ms` to increase use of cached data. #2035 +* [ENHANCEMENT] Add `global.extraEnv` and `global.extraEnvFrom` to values. This enables setting common environment variables and common injection of secrets to the POD environment of Mimir/GEM services and Nginx. Memcached and minio are out of scope for now. #2031 +* [ENHANCEMENT] Add `extraEnvFrom` capability to all Mimir services to enable injecting secrets via environment variables. #2017 +* [ENHANCEMENT] Enable `-config.expand-env=true` option in all Mimir services to be able to take secrets/settings from the environment and inject them into the Mimir configuration file. #2017 +* [ENHANCEMENT] Add a simple test for enterprise installation #2027 + +## 2.1.0-beta.7 + +* [ENHANCEMENT] Bump image version to 2.1 #2001 + - For Grafana Mimir, see the release notes here: [Grafana Mimir 2.1](https://grafana.com/docs/mimir/latest/release-notes/v2.1/) + - For Grafana Enterprise Metrics, see the release notes here: [Grafana Enterprise Metrics 2.1](https://grafana.com/docs/enterprise-metrics/v2.1.x/release-notes/v2-1/) + +## 2.1.0-beta.6 + +* [ENHANCEMENT] Disable `ingester.ring.unregister-on-shutdown` and `distributor.extend-writes` #1994 + - This will prevent resharding every series during a rolling ingester restart + - Under some circumstances the previous values (both enabled) could cause write path degredation during rolling restarts + +## 2.1.0-beta.5 + +* [ENHANCEMENT] Add support for the results cache used by the query frontend #1993 + - This will result in additional resource usage due to the addition of one or + more memcached replicas. This applies when using small.yaml, large.yaml, + capped-large.yaml, capped-small.yaml, or when setting + `memcached-results.enabled=true` + +## 2.1.0-beta.4 + +* [BUGFIX] Set up using older bitnami chart repository for memcached as old charts were deleted from the current one. #1998 + +## 2.1.0-beta.3 + +* [BUGFIX] Use grpc round-robin for distributor clients in GEM gateway and self-monitoring + - This utilizes an additional headless service for the distributor pods + +## 2.1.0-beta.2 + +* [ENHANCEMENT] Version bump only for release tests. + +## 2.1.0-beta.1 + +* [ENHANCEMENT] Version bump only for release tests. + +## 2.0.14 + +* [BUGFIX] exclude headless services from ServiceMonitors to prevent duplication of prometheus scrape targets #1308 + +## 2.0.13 + +* [ENHANCEMENT] Removed `rbac.create` option. #1317 + +## 2.0.12 + +* [ENHANCEMENT] Add memberlist named port to container spec. #1311 + +## 2.0.11 + +* [ENHANCEMENT] Turn `ruler` and `override-exporter` into optional components. #1304 + +## 2.0.10 + +* [ENHANCEMENT] Reorder some values for consistency. #1302 +* [BUGFIX] Add missing `admin_api.env`, `gateway.env` and `overrides_exporter.env` values. #1302 +* [BUGFIX] Remove `.extraPorts` from values as it has no effect. #1302 + +## 2.0.9 + +* [ENHANCEMENT] Disable gateway ingress by default. #1303 +* [BUGFIX] Fix null port at gateway ingress definition. #1303 + +## 2.0.8 + +* [ENHANCEMENT] Add validation if `activity_tracker.filepath` is missing in `mimir.config`. #1290 +* [ENHANCEMENT] Add validation if `server.http_listen_port` or `server.grpc_listen_port` is set in `mimir.config`. #1290 +* [BUGFIX] Add missing empty array definition for `extraVolumeMounts` in admin_api, gateway and override-exporter. #1290 +* [BUGFIX] Fix wrong template called in nginx helper. #1290 + +## 2.0.7 + +* [ENHANCEMENT] Add option to modify the port for the GEM gateway service. #1270 + +## 2.0.6 + +* [ENHANCEMENT] Add option for an ingress on GEM gateway. #1266 + +## 2.0.5 + +* [BUGFIX] Use new component name system for gateway ingress. This regression has been introduced with #1203. #1260 + +## 2.0.4 + +* [ENHANCEMENT] Determine PodDisruptionBudget APIVersion based on running version of k8s #1229 + +## 2.0.3 + +* [ENHANCEMENT] Update README.md with helm-docs version 1.8.1 instead of old 1.4.0. #1230 + +## 2.0.2 + +* [ENHANCEMENT] Update Grafana Enterprise Metrics docker image tag to v2.0.1 #1241 + +## 2.0.1 + +* [BUGFIX] Honor `global.clusterDomain` when referencing internal services, e.g. alertmanager or nginx gateway. #1227 + +## 2.0.0 + +* [CHANGE] **Breaking** for existing users of `mimir-distributed`: the naming convention is changed to have shorter resource names, as in `-mimir-distributed-store-gateway` is now just `-mimir-store-gateway`. To have the previous names, please specify `nameOverride: mimir-distributed` in the values. #1203 +* [CHANGE] The chart `enterprise-metrics` is renamed to `mimir-distributed`. #1203 +* [CHANGE] **Breaking** Configuration for Grafana Enterprise Metrics is now in the value `mimir.config` as a helm template **string**. + Please consult the [Grafana Enterprise Migration Guide](https://grafana.com/docs/enterprise-metrics/latest/migrating-from-gem-1.7/) to learn more about how to upgrade the configuration. + Except for the following parameters specified as command line parameters in the Pod templates, + everything is now set in this string-typed value, giving a definitive source of configuration. + Exceptions: + > The `-target=` must be provided individually.\ + The `-config.file=` obviously.\ + User defined arguments from `..extraArgs`. +* [CHANGE] **Breaking** Kubernetes object labels now follow the [kubernetes standard](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/) (e.g. `app.kubernetes.io/component=ingester`). To enable smooth upgrade and compatibility with previous Grafana Enterprise Metrics Helm chart, the value `enterprise.legacyLabels` should be set to `true`. +* [CHANGE] **Breaking** Ingesters only support `StatefulSet` from now on as chunks storage was removed in favour of blocks storage. +* [CHANGE] **Breaking** Compactor is a required component, the value `compactor.enabled` is removed. +* [CHANGE] **Breaking** The configuration parameter `server.http_listen_port` and `server.grpc_listen_port` cannot be changed from their defaults. +* [CHANGE] The default for `ingester.ring.replication_factor` is now 3 and there will be 3 ingesters started even with the default `values.yaml`. + On the other hand, Pod anti affinity is turned off by default to allow single node deployment. +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v2.0.0](https://grafana.com/docs/enterprise-metrics/v2.0.x/) +* [FEATURE] Reworked chart to enable installing Grafana Mimir open source software version without licensed features. +* [FEATURE] Added the value `nameOverride` to enable migration from Cortex helm chart. +* [FEATURE] The alertmanager can be disabled with `alertmanager.enabled: false`, to support the use case of external alertmanager. +* [FEATURE] Added definitions of `ServiceMonitor` objects for Prometheus monitoring. Configuration is done via the `serviceMonitor` values. This enables partial functionality of Grafana Mimir dashboards out of the box - without alerts and recording rules pre-loaded. +* [ENHANCEMENT] Minio bucket creation is not tied to `admin-api` anymore, moved to its own job `templates/minio/create-bucket-job.yaml`. +* [BUGFIX] `..PodDisruptionBudget` was not working. Added template definition for all services. Pod disruption budget is enabled for the ingesters and store-gateways by default. +* [BUGFIX] Fix typo in value `.alertmanager.statefulset` to `.alertmanager.statefulSet`. +* [BUGFIX] Remove unused value `.useExternalLicense`. + +## Entries from enterprise-metrics chart + +## 1.8.1 + +* [ENHANCEMENT] Support Grafana Mimir monitoring mixin labels by setting container names to the component names. + This will make it easier to select different components in cadvisor metrics. + Previously, all containers used "enterprise-metrics" as the container name. + Now, for example, the ingester Pod will have a container name "ingester" rather than "enterprise-metrics". + +## 1.8.0 + +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.7.0](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v170----january-6th-2022). + +## 1.7.3 + +* [BUGFIX] Alertmanager does not fail anymore to load configuration via the API. #945 + +## 1.7.2 + +* [CHANGE] The Ingester statefulset now uses podManagementPolicy Parallel, upgrading requires recreating the statefulset #920 + +## 1.7.1 + +* [BUGFIX] Remove chunks related default limits. #867 + +## 1.7.0 + +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.6.1](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v161----november-18th-2021). #839 + +## 1.6.0 + +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.5.1](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v151----september-21st-2021). #729 +* [CHANGE] Production values set the ingester replication factor to three to avoid data loss. + The resource calculations of these values already factored in this replication factor but did not apply it in the configuration. + If you have not reduced the compute resources in these values then this change should have no impact besides increased resilience to ingester failure. + If you have reduced the compute resources, consider increasing them back to the recommended values before installing this version. #729 + +## 1.5.6 + +* [BUGFIX] YAML exports are no longer included as part of the Helm chart. #726 + +## 1.5.5 + +* [BUGFIX] Ensure all PodSpecs have configurable initContainers. #708 + +## 1.5.4 + +* [BUGFIX] Adds a `Service` resource for the Compactor Pods and adds Compactor to the default set of gateway proxy URLs. In previous chart versions the Compactor would not show up in the GEM plugin "Ring Health" tab because the gateway did not know how to reach Compactor. #714 + +## 1.5.3 + +* [BUGFIX] This change does not affect single replica deployments of the + admin-api but does fix the potential for an inconsistent state when + running with multiple replicas of the admin-api and experiencing + parallel writes for the same objects. #675 + +## 1.5.2 + +* [CHANGE] Removed all references to Consul in the yaml files since GEM will be focused on deploying with memberlist. Deleted the multi-kv-consul-primary-values.yaml and multi-kv-memberlist-primary-values.yaml files since they assume you're running Consul as your primary or second kvstore. #674 + +## 1.5.1 + +* [BUGFIX] Unused `ingress` configuration section removed from `values.yaml`. #658 + +## 1.5.0 + +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.5.0](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v150----august-24th-2021). #641 + +## 1.4.7 + +* [CHANGE] Enabled enterprise authentication by default. + > **Breaking:** This change can cause losing access to the GEM cluster in case `auth.type` has not + > been set explicitly. + > This is a security related change and therefore released in a patch release. + +## 1.4.6 + +* [FEATURE] Run an instance of the GEM overrides-exporter by default. #590 + +## 1.4.5 + +* [BUGFIX] Add `memberlist.join` configuration to the ruler. #618 + +## 1.4.4 + +* [CHANGE] Removed livenessProbe configuration as it can often be more detrimental than having none. Users can still configure livenessProbes with the per App configuration hooks. #594 + +## 1.4.3 + +* [ENHANCEMENT] Added values files for installations that require setting resource limits. #583 + +## 1.4.2 + +* [CHANGE] The compactor data directory configuration has been corrected to `/data`. #562 + > **Note:** The compactor is stateless and no data stored in the existing data directory needs to be moved in order to facilitate this upgrade. + > For more information, refer to the [Cortex Compactor documentation](https://cortexmetrics.io/docs/blocks-storage/compactor/). +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.4.2](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v142----jul-21st-2021) #562 + +## 1.4.1 + +* [BUGFIX] Fixed DNS address of distributor client for self-monitoring. #569 + +## 1.4.0 + +* [CHANGE] Use updated querier response compression configuration, changed in 1.4.0. #524 +* [CHANGE] Use updated alertmanager storage configuration, changed in 1.4.0. #524 +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.4.1](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v141----june-29th-2021). #524 +* [FEATURE] Enable [GEM self-monitoring](https://grafana.com/docs/metrics-enterprise/latest/self-monitoring/). #524 + +## 1.3.5 + +* [CHANGE] The GRPC port on the query-frontend and store-gateway Kubernetes Services have been changed to match the naming of all other services. #523 +* [FEATURE] Expose GRPC port on all GEM services. #523 + +## 1.3.4 + +* [BUGFIX] Removed symlinks from chart to fix Rancher repository imports. #504 + +## 1.3.3 + +* [FEATURE] The GEM config now uses the `{{ .Release.Name }}` variable as the default value for `cluster_name` which removes the need to additionally override this setting during an initial install. #500 + +## 1.3.2 + +* [FEATURE] Chart memcached dependencies are now at the latest release. This includes the memcached and the related exporter. #467 + +## 1.3.1 + +* [BUGFIX] Use non-deprecated alertmanager flags for cluster peers. #441 +* [BUGFIX] Make store-gateway Service not headless. #441 + +## 1.3.0 + +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.3.0](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v130----april-26th-2021). #415 + +## 1.2.0 + +* [CHANGE] The chart now uses memberlist for the ring key-value store removing the need to run Consul. #340 + > **Warning:** Existing clusters will need to follow an upgrade procedure. + > **Warning:** Existing clusters should first be upgraded to `v1.1.1` and use that version for migration before upgrading to `v1.2.0`. + To upgrade to using memberlist: + 1. Ensure you are running the `v1.1.1` version of the chart. + 2. Deploy runtime `multi_kv_config` to use Consul as a primary and memberlist as the secondary key-value store. + The values for such a change can be found in the [`multi-kv-consul-primary-values.yaml`](./multi-kv-consul-primary-values.yaml). + 3. Verify the configuration is in use by querying the [Configuration](https://cortexmetrics.io/docs/api/#configuration) HTTP API endpoint. + 4. Deploy runtime `multi_kv_config` to use memberlist as the primary and Consul as the secondary key-value store. + The values for such a change can be found in [`multi-kv-memberlist-primary-values.yaml`](./multi-kv-memberlist-primary-values.yaml) + 5. Verify the configuration is in use by querying the [Configuration](https://cortexmetrics.io/docs/api/#configuration) HTTP API endpoint. + 6. Deploy `v1.2.0` helm chart which configures memberlist as the sole key-value store and removes the Consul resources. + +## 1.1.1 + +* [FEATURE] Facilitate some runtime configuration of microservices. #342 +* [FEATURE] Upgrade to [Grafana Enterprise Metrics v1.2.0](https://grafana.com/docs/metrics-enterprise/latest/downloads/#v120----march-10-2021). #342 + +## 1.1.0 + +* [CHANGE] The memcached chart from the deprecated Helm stable repository has been removed and replaced with a Bitnami chart. #333 + > **Warning:** This change will result in the cycling of your memcached Pods and will invalidate the existing cache. +* [CHANGE] Memcached Pod resource limits have been lowered to match requests. #333 +* [FEATURE] YAML exports have been created for all chart values files. #333 +* [BUGFIX] The values for the querier/ruler/store-gateway `-.memcached.max-item-size` have been corrected to match the limit configured on the memcached server. #333 + +## 1.0.0 + +* [FEATURE] Initial versioned release. ##168 + +## Entries from mimir-distributed chart + +## 0.1.8 + +* [BUGFIX] Fix nginx routing for rules and expose buildinfo. #1233 + +## 0.1.7 + +* [BUGFIX] Remove misplaced config value and add affinity rules in `capped-small.yaml` and `capped-large.yaml`. #1225 + +## 0.1.6 + +* [CHANGE] **Breaking** Compactor is a required component, the value `compactor.enabled` is removed. #1193 +* [FEATURE] The alertmanager can be disabled with `alertmanager.enabled: false`, to support the use case of external alertmanager. #1193 + +## 0.1.5 + +* [BUGFIX] Fix labels for Mimir dashboards. #1190 + +## 0.1.4 + +* [BUGFIX] Fix documentation link missing slash. #1177 + +## 0.1.3 + +* [FEATURE] Add ServiceMonitor definitions. #1156 + +## 0.1.2 + +* [BUGFIX] Fix the naming of minio configmap and secret in the parent chart. #1152 + +## 0.1.1 + +* [BUGFIX] CI fixes. #1144 + +## 0.1.0 + +* [FEATURE] Initial commit, Mimir only, derived from `enterprise-metrics` chart. #1141 diff --git a/operations/helm/charts/mimir-distributed/Chart.lock b/operations/helm/charts/mimir-distributed/Chart.lock new file mode 100644 index 00000000000..9c5179eb26f --- /dev/null +++ b/operations/helm/charts/mimir-distributed/Chart.lock @@ -0,0 +1,18 @@ +dependencies: +- name: memcached + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + version: 5.5.2 +- name: memcached + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + version: 5.5.2 +- name: memcached + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + version: 5.5.2 +- name: memcached + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + version: 5.5.2 +- name: minio + repository: https://helm.min.io/ + version: 8.0.10 +digest: sha256:0b58716cf86880510e4ce9dacb312db80918d14704e68127b833b81b5fd7d7f3 +generated: "2022-06-02T09:31:09.709064-05:00" diff --git a/operations/helm/charts/mimir-distributed/Chart.yaml b/operations/helm/charts/mimir-distributed/Chart.yaml new file mode 100644 index 00000000000..2c262d75e3c --- /dev/null +++ b/operations/helm/charts/mimir-distributed/Chart.yaml @@ -0,0 +1,35 @@ +apiVersion: v2 +version: 2.2.0-weekly.189 +appVersion: 2.1.0 +description: "Grafana Mimir" +engine: gotpl +home: https://grafana.com/docs/mimir/v2.1.x/ +icon: https://grafana.com/static/img/logos/logo-mimir.svg +kubeVersion: ^1.10.0-0 +name: mimir-distributed +dependencies: + - name: memcached + alias: memcached + version: 5.5.2 + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + condition: memcached.enabled + - name: memcached + alias: memcached-queries + version: 5.5.2 + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + condition: memcached-queries.enabled + - name: memcached + alias: memcached-metadata + version: 5.5.2 + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + condition: memcached-metadata.enabled + - name: memcached + alias: memcached-results + version: 5.5.2 + repository: https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + condition: memcached-results.enabled + - name: minio + alias: minio + version: 8.0.10 + repository: https://helm.min.io/ + condition: minio.enabled diff --git a/operations/helm/charts/mimir-distributed/README.md b/operations/helm/charts/mimir-distributed/README.md new file mode 100644 index 00000000000..9b6ae1c1aea --- /dev/null +++ b/operations/helm/charts/mimir-distributed/README.md @@ -0,0 +1,188 @@ +# Grafana Mimir Helm Chart + +Helm chart for deploying [Grafana Mimir](https://grafana.com/docs/mimir/v2.1.x/) or optionally [Grafana Enterprise Metrics](https://grafana.com/docs/enterprise-metrics/v2.1.x/) to Kubernetes. Derived from [Grafana Enterprise Metrics Helm Chart](https://github.com/grafana/helm-charts/blob/main/charts/enterprise-metrics/README.md) + +# mimir-distributed + +![Version: 2.2.0-weekly.189](https://img.shields.io/badge/Version-2.2.0--weekly.189-informational?style=flat-square) ![AppVersion: 2.1.0](https://img.shields.io/badge/AppVersion-2.1.0-informational?style=flat-square) + +Grafana Mimir + +## Requirements + +Kubernetes: `^1.10.0-0` + +| Repository | Name | Version | +|------------|------|---------| +| https://helm.min.io/ | minio(minio) | 8.0.10 | +| https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami | memcached(memcached) | 5.5.2 | +| https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami | memcached-queries(memcached) | 5.5.2 | +| https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami | memcached-metadata(memcached) | 5.5.2 | +| https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami | memcached-results(memcached) | 5.5.2 | + +## Dependencies + +### Storage + +Grafana Mimir and Grafana Enterprise Metrics require an object storage backend to store metrics and indexes. + +The default chart values will deploy [Minio](https://min.io) for initial set up. Production deployments should use a separately deployed object store. +See [Grafana Mimir documentation](https://grafana.com/docs/mimir/v2.1.x/) for details on storage types and documentation. + +### Grafana Enterprise Metrics license + +In order to use the enterprise features of this chart, you need to provide the contents of a Grafana Enterprise Metrics license file as the value for the `license.contents` variable. +To obtain a Grafana Enterprise Metrics license, refer to [Get a license](https://grafana.com/docs/enterprise-metrics/v2.1.x/setup/#get-a-gem-license). + +### Helm3 + +The chart requires at least Helm version 3 to work. + +## Installation + +This section describes various use cases for installation, upgrade and migration from different systems and versions. + +### Preparation + +These are the common tasks to perform before any of the use cases. + +```bash +# Add the repository +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +### Installation of Grafana Mimir + +```bash +helm install grafana/mimir-distributed +``` + +As part of this chart many different pods and services are installed which all +have varying resource requirements. Please make sure that you have sufficient +resources (CPU/memory) available in your cluster before installing Grafana Mimir Helm Chart. + +### Migration from Cortex to Grafana Mimir + +Please consult the [Migration from Cortex to Grafana](https://grafana.com/docs/mimir/v2.1.x/migration-guide/migrating-from-cortex/) guide on how to update the configuration. +Prepare a custom values file with the contents: + +```yaml +nameOverride: cortex + +mimir: + config: | + +``` + +Perform the upgrade: + +```bash +helm upgrade grafana/mimir-distributed -f +``` + +## Installation of Grafana Enterprise Metrics + +To install the chart with licensed features enabled, using a local Grafana Enterprise Metrics license file called `license.jwt`, provide the license as a value and set the `enterprise.enabled` value to `true`. + +```bash +helm install grafana/mimir-distributed --set 'enterprise.enabled=true' --set-file 'license.contents=./license.jwt' +``` + +### Upgrade from a previous version of Grafana Enterprise Metrics + +Please consult the [migration guide](https://grafana.com/docs/enterprise-metrics/v2.1.x/migrating-from-gem-1.7/) for details on how to prepare the configuration. Prepare a custom values file, with the contents: + +```yaml +enterprise: + enabled: true + legacyLabels: true + +mimir: + config: | + +``` + +The value (`enterprise.legacyLabels`) is needed because this chart installs objects with kubernetes de-facto standard labels by default which are different from older Grafana Enterprise Metrics labels. + +```bash +helm upgrade grafana/mimir-distributed -f --set-file 'license.contents=./license.jwt' +``` + +### Upgrade from Grafana Mimir to Grafana Enterprise Metrics + +Use the name override to align labels and selectors and enable licensed features. + +```yaml +nameOverride: mimir-distributed + +enterprise: + enabled: true +``` + +## Scale values + +The default Helm chart values in the `values.yaml` file are configured to allow you to quickly test out Grafana Mimir. +Alternative values files are included to provide a more realistic configuration that should facilitate a certain level of ingest load. + +### Small + +The `small.yaml` values file configures the Grafana Mimir cluster to +handle production ingestion of ~1M active series using the blocks storage engine. +Query requirements can vary dramatically depending on query rate and query +ranges. The values here satisfy a "usual" query load as seen from our +production clusters at this scale. +It is important to ensure that you run no more than one ingester replica +per node so that a single node failure does not cause data loss. Zone aware +replication can be configured to ensure data replication spans availability +zones. Refer to [Zone Aware Replication](https://grafana.com/docs/mimir/v2.1.x/operators-guide/configuring/configuring-zone-aware-replication/) +for more information. +Minio is no longer enabled and you are encouraged to use your cloud providers +object storage service for production deployments. + +To deploy a cluster using `small.yaml` values file: + +```bash +helm install grafana/mimir-distributed -f small.yaml +``` + +### Large + +The `large.yaml` values file configures the Grafana Mimir cluster to +handle production ingestion of ~10M active series using the blocks +storage engine. +Query requirements can vary dramatically depending on query rate and query +ranges. The values here satisfy a "usual" query load as seen from our +production clusters at this scale. +It is important to ensure that you run no more than one ingester replica +per node so that a single node failure does not cause data loss. Zone aware +replication can be configured to ensure data replication spans availability +zones. Refer to [Zone Aware Replication](https://grafana.com/docs/mimir/v2.1.x/operators-guide/configuring/configuring-zone-aware-replication/) +for more information. +Minio is no longer enabled and you are encouraged to use your cloud providers +object storage service for production deployments. + +To deploy a cluster using the `large.yaml` values file: + +```bash +helm install grafana/mimir-distributed -f large.yaml +``` + +# Development + +To configure a local default storage class for k3d: + +```bash +kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml +kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +``` + +To install the chart with the values used in CI tests: + +```bash +helm install test ./ --values ./ci/test-values.yaml +``` + +# Contributing/Releasing + +Please see the dedicated "[Contributing to Grafana Mimir helm chart](https://github.com/grafana/mimir/tree/main/docs/internal/contributing/contributing-to-helm-chart.md)" page. diff --git a/operations/helm/charts/mimir-distributed/README.md.gotmpl b/operations/helm/charts/mimir-distributed/README.md.gotmpl new file mode 100644 index 00000000000..4ecd2149666 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/README.md.gotmpl @@ -0,0 +1,181 @@ +# Grafana Mimir Helm Chart + +Helm chart for deploying [Grafana Mimir]({{ template "chart.homepage" . }}) or optionally [Grafana Enterprise Metrics](https://grafana.com/docs/enterprise-metrics/v2.1.x/) to Kubernetes. Derived from [Grafana Enterprise Metrics Helm Chart](https://github.com/grafana/helm-charts/blob/main/charts/enterprise-metrics/README.md) + +{{ template "chart.header" . }} + +{{ template "chart.versionBadge" . }}{{ template "chart.typeBadge" . }}{{ template "chart.appVersionBadge" . }} + +{{ template "chart.description" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +## Dependencies + +### Storage + +Grafana Mimir and Grafana Enterprise Metrics require an object storage backend to store metrics and indexes. + +The default chart values will deploy [Minio](https://min.io) for initial set up. Production deployments should use a separately deployed object store. +See [Grafana Mimir documentation]({{ template "chart.homepage" . }}) for details on storage types and documentation. + +### Grafana Enterprise Metrics license + +In order to use the enterprise features of this chart, you need to provide the contents of a Grafana Enterprise Metrics license file as the value for the `license.contents` variable. +To obtain a Grafana Enterprise Metrics license, refer to [Get a license](https://grafana.com/docs/enterprise-metrics/v2.1.x/setup/#get-a-gem-license). + +### Helm3 + +The chart requires at least Helm version 3 to work. + +## Installation + +This section describes various use cases for installation, upgrade and migration from different systems and versions. + +### Preparation + +These are the common tasks to perform before any of the use cases. + +```bash +# Add the repository +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update +``` + +### Installation of Grafana Mimir + +```bash +helm install grafana/mimir-distributed +``` + +As part of this chart many different pods and services are installed which all +have varying resource requirements. Please make sure that you have sufficient +resources (CPU/memory) available in your cluster before installing Grafana Mimir Helm Chart. + +### Migration from Cortex to Grafana Mimir + +Please consult the [Migration from Cortex to Grafana]({{ template "chart.homepage" . }}migration-guide/migrating-from-cortex/) guide on how to update the configuration. +Prepare a custom values file with the contents: + +```yaml +nameOverride: cortex + +mimir: + config: | + +``` + +Perform the upgrade: + +```bash +helm upgrade grafana/mimir-distributed -f +``` + + +## Installation of Grafana Enterprise Metrics + +To install the chart with licensed features enabled, using a local Grafana Enterprise Metrics license file called `license.jwt`, provide the license as a value and set the `enterprise.enabled` value to `true`. + +```bash +helm install grafana/mimir-distributed --set 'enterprise.enabled=true' --set-file 'license.contents=./license.jwt' +``` + +### Upgrade from a previous version of Grafana Enterprise Metrics + +Please consult the [migration guide](https://grafana.com/docs/enterprise-metrics/v2.1.x/migrating-from-gem-1.7/) for details on how to prepare the configuration. Prepare a custom values file, with the contents: + +```yaml +enterprise: + enabled: true + legacyLabels: true + +mimir: + config: | + +``` + +The value (`enterprise.legacyLabels`) is needed because this chart installs objects with kubernetes de-facto standard labels by default which are different from older Grafana Enterprise Metrics labels. + +```bash +helm upgrade grafana/mimir-distributed -f --set-file 'license.contents=./license.jwt' +``` + +### Upgrade from Grafana Mimir to Grafana Enterprise Metrics + +Use the name override to align labels and selectors and enable licensed features. + +```yaml +nameOverride: mimir-distributed + +enterprise: + enabled: true +``` + +## Scale values + +The default Helm chart values in the `values.yaml` file are configured to allow you to quickly test out Grafana Mimir. +Alternative values files are included to provide a more realistic configuration that should facilitate a certain level of ingest load. + +### Small + +The `small.yaml` values file configures the Grafana Mimir cluster to +handle production ingestion of ~1M active series using the blocks storage engine. +Query requirements can vary dramatically depending on query rate and query +ranges. The values here satisfy a "usual" query load as seen from our +production clusters at this scale. +It is important to ensure that you run no more than one ingester replica +per node so that a single node failure does not cause data loss. Zone aware +replication can be configured to ensure data replication spans availability +zones. Refer to [Zone Aware Replication]({{ template "chart.homepage" . }}operators-guide/configuring/configuring-zone-aware-replication/) +for more information. +Minio is no longer enabled and you are encouraged to use your cloud providers +object storage service for production deployments. + +To deploy a cluster using `small.yaml` values file: + +```bash +helm install grafana/mimir-distributed -f small.yaml +``` + +### Large + +The `large.yaml` values file configures the Grafana Mimir cluster to +handle production ingestion of ~10M active series using the blocks +storage engine. +Query requirements can vary dramatically depending on query rate and query +ranges. The values here satisfy a "usual" query load as seen from our +production clusters at this scale. +It is important to ensure that you run no more than one ingester replica +per node so that a single node failure does not cause data loss. Zone aware +replication can be configured to ensure data replication spans availability +zones. Refer to [Zone Aware Replication]({{ template "chart.homepage" . }}operators-guide/configuring/configuring-zone-aware-replication/) +for more information. +Minio is no longer enabled and you are encouraged to use your cloud providers +object storage service for production deployments. + +To deploy a cluster using the `large.yaml` values file: + +```bash +helm install grafana/mimir-distributed -f large.yaml +``` + +# Development + +To configure a local default storage class for k3d: + +```bash +kubectl apply -f https://raw.githubusercontent.com/rancher/local-path-provisioner/master/deploy/local-path-storage.yaml +kubectl patch storageclass local-path -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}' +``` + +To install the chart with the values used in CI tests: + +```bash +helm install test ./ --values ./ci/test-values.yaml +``` + +# Contributing/Releasing + +Please see the dedicated "[Contributing to Grafana Mimir helm chart](https://github.com/grafana/mimir/tree/main/docs/internal/contributing/contributing-to-helm-chart.md)" page. diff --git a/operations/helm/charts/mimir-distributed/capped-large.yaml b/operations/helm/charts/mimir-distributed/capped-large.yaml new file mode 100644 index 00000000000..6a196584d7a --- /dev/null +++ b/operations/helm/charts/mimir-distributed/capped-large.yaml @@ -0,0 +1,159 @@ +# These values configure the Grafana Mimir or Grafana Enterprise Metrics cluster to +# handle production ingestion of ~10M active series using the blocks +# storage engine scraped with a 15s interval. +# Query requirements can vary dramatically depending on query rate and query +# ranges. The values here satisfy a "usual" query load as seen from our +# production clusters at this scale. +# It is important to ensure that you run no more than one ingester replica +# per node so that a single node failure does not cause data loss. Zone aware +# replication can be configured to ensure data replication spans availability +# zones. Refer to [Zone Aware Replication](https://grafana.com/docs/mimir/v2.0.x/operators-guide/configuring/configuring-zone-aware-replication/) +# for more information. +# Minio is no longer enabled and you are encouraged to use your cloud providers +# object storage service for production deployments. +# Ingesters are configured with a replication factor of 3 to ensure that a single ingester failur does not +# result in data loss. + +alertmanager: + persistentVolume: + enabled: true + replicas: 3 + resources: + requests: + cpu: 1 + memory: 8Gi + limits: + cpu: 1 + memory: 8Gi + statefulSet: + enabled: true + +compactor: + persistentVolume: + size: 50Gi + resources: + requests: + cpu: 2 + memory: 2Gi + limits: + cpu: 2 + memory: 2Gi + +distributor: + replicas: 15 + resources: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi + +ingester: + persistentVolume: + size: 50Gi + replicas: 25 + resources: + requests: + cpu: 4 + memory: 15Gi + limits: + cpu: 4 + memory: 15Gi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - ingester + topologyKey: 'kubernetes.io/hostname' + +memcached: + enabled: true + replicaCount: 32 + +memcached-queries: + enabled: true + replicaCount: 10 + +memcached-metadata: + enabled: true + +minio: + enabled: false + +overrides_exporter: + replicas: 1 + resources: + limits: + cpu: 1 + memory: 256Mi + requests: + cpu: 1 + memory: 256Mi + +querier: + replicas: 6 + resources: + requests: + cpu: 1 + memory: 12Gi + limits: + cpu: 1 + memory: 12Gi + +query_frontend: + replicas: 2 + resources: + requests: + cpu: 2 + memory: 6Gi + limits: + cpu: 2 + memory: 6Gi + +ruler: + replicas: 2 + resources: + requests: + cpu: 1 + memory: 6Gi + limits: + cpu: 1 + memory: 6Gi + +store_gateway: + persistentVolume: + size: 50Gi + replicas: 1 + resources: + requests: + cpu: 1 + memory: 6Gi + limits: + cpu: 1 + memory: 6Gi + +# Grafana Enterprise Metrics feature related +admin_api: + replicas: 3 + resources: + requests: + cpu: 1 + memory: 256Mi + limits: + cpu: 1 + memory: 256Mi + +gateway: + replicas: 8 + resources: + requests: + cpu: 1 + memory: 384Mi + limits: + cpu: 1 + memory: 384Mi diff --git a/operations/helm/charts/mimir-distributed/capped-small.yaml b/operations/helm/charts/mimir-distributed/capped-small.yaml new file mode 100644 index 00000000000..e864a30edb2 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/capped-small.yaml @@ -0,0 +1,162 @@ +# These values configure the Grafana Mimir or Grafana Enterprise Metrics cluster to +# handle production ingestion of ~1M active series using the blocks +# storage engine scraped with a 15s interval. +# Query requirements can vary dramatically depending on query rate and query +# ranges. The values here satisfy a "usual" query load as seen from our +# production clusters at this scale. +# It is important to ensure that you run no more than one ingester replica +# per node so that a single node failure does not cause data loss. Zone aware +# replication can be configured to ensure data replication spans availability +# zones. Refer to [Zone Aware Replication](https://grafana.com/docs/mimir/v2.0.x/operators-guide/configuring/configuring-zone-aware-replication/) +# for more information. +# Minio is no longer enabled and you are encouraged to use your cloud providers +# object storage service for production deployments. +# Ingesters are configured with a replication factor of 3 to ensure that a single ingester failur does not +# result in data loss. + +alertmanager: + persistentVolume: + enabled: true + replicas: 3 + resources: + requests: + cpu: 1 + memory: 8Gi + limits: + cpu: 1 + memory: 8Gi + statefulSet: + enabled: true + +compactor: + persistentVolume: + size: 50Gi + resources: + requests: + cpu: 2 + memory: 2Gi + limits: + cpu: 2 + memory: 2Gi + +distributor: + replicas: 3 + resources: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 2 + memory: 4Gi + +ingester: + persistentVolume: + size: 50Gi + replicas: 4 + resources: + requests: + cpu: 4 + memory: 15Gi + limits: + cpu: 4 + memory: 15Gi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - ingester + topologyKey: 'kubernetes.io/hostname' + +memcached: + enabled: true + replicaCount: 2 + +memcached-queries: + enabled: true + replicaCount: 3 + +memcached-metadata: + enabled: true + +memcached-results: + enabled: true + +minio: + enabled: false + +overrides_exporter: + replicas: 1 + resources: + limits: + cpu: 1 + memory: 256Mi + requests: + cpu: 1 + memory: 256Mi + +querier: + replicas: 2 + resources: + requests: + cpu: 1 + memory: 12Gi + limits: + cpu: 1 + memory: 12Gi + +query_frontend: + replicas: 1 + resources: + requests: + cpu: 2 + memory: 6Gi + limits: + cpu: 2 + memory: 6Gi + +ruler: + replicas: 2 + resources: + requests: + cpu: 1 + memory: 6Gi + limits: + cpu: 1 + memory: 6Gi + +store_gateway: + persistentVolume: + size: 50Gi + replicas: 1 + resources: + requests: + cpu: 1 + memory: 6Gi + limits: + cpu: 1 + memory: 6Gi + +# Grafana Enterprise Metrics feature related +admin_api: + replicas: 3 + resources: + requests: + cpu: 1 + memory: 256Mi + limits: + cpu: 1 + memory: 256Mi + +gateway: + replicas: 3 + resources: + requests: + cpu: 1 + memory: 384Mi + limits: + cpu: 1 + memory: 384Mi diff --git a/operations/helm/charts/mimir-distributed/ci/test-enterprise-values.yaml b/operations/helm/charts/mimir-distributed/ci/test-enterprise-values.yaml new file mode 100644 index 00000000000..05cd858de7b --- /dev/null +++ b/operations/helm/charts/mimir-distributed/ci/test-enterprise-values.yaml @@ -0,0 +1,24 @@ +# Test values to limit the load during CI +kubeVersionOverride: "1.20" + +enterprise: + enabled: true + +alertmanager: + persistentVolume: + enabled: false + +compactor: + persistentVolume: + enabled: false + +ingester: + persistentVolume: + enabled: false + resources: + requests: + cpu: 10m + +store_gateway: + persistentVolume: + enabled: false diff --git a/operations/helm/charts/mimir-distributed/ci/test-oss-values.yaml b/operations/helm/charts/mimir-distributed/ci/test-oss-values.yaml new file mode 100644 index 00000000000..b713a3b1599 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/ci/test-oss-values.yaml @@ -0,0 +1,21 @@ +# Test values to limit the load during CI +kubeVersionOverride: "1.20" + +alertmanager: + persistentVolume: + enabled: false + +compactor: + persistentVolume: + enabled: false + +ingester: + persistentVolume: + enabled: false + resources: + requests: + cpu: 10m + +store_gateway: + persistentVolume: + enabled: false diff --git a/operations/helm/charts/mimir-distributed/large.yaml b/operations/helm/charts/mimir-distributed/large.yaml new file mode 100644 index 00000000000..47583be960a --- /dev/null +++ b/operations/helm/charts/mimir-distributed/large.yaml @@ -0,0 +1,151 @@ +# These values configure the Grafana Mimir or Grafana Enterprise Metrics cluster to +# handle production ingestion of ~10M active series using the blocks +# storage engine scraped with a 15s interval. +# Query requirements can vary dramatically depending on query rate and query +# ranges. The values here satisfy a "usual" query load as seen from our +# production clusters at this scale. +# It is important to ensure that you run no more than one ingester replica +# per node so that a single node failure does not cause data loss. Zone aware +# replication can be configured to ensure data replication spans availability +# zones. Refer to [Zone Aware Replication](https://grafana.com/docs/mimir/v2.0.x/operators-guide/configuring/configuring-zone-aware-replication/) +# for more information. +# Minio is no longer enabled and you are encouraged to use your cloud providers +# object storage service for production deployments. +# Ingesters are configured with a replication factor of 3 to ensure that a single ingester failur does not +# result in data loss. + +alertmanager: + persistentVolume: + enabled: true + replicas: 3 + resources: + limits: + memory: 8Gi + requests: + cpu: 300m + memory: 6Gi + statefulSet: + enabled: true + +compactor: + persistentVolume: + size: 50Gi + resources: + limits: + cpu: 1200m + memory: 2Gi + requests: + cpu: 1 + memory: 1Gi + +distributor: + replicas: 15 + resources: + limits: + memory: 4Gi + requests: + cpu: 2 + memory: 2Gi + +ingester: + persistentVolume: + size: 50Gi + replicas: 25 + resources: + limits: + memory: 25Gi + requests: + cpu: 4 + memory: 15Gi + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - ingester + topologyKey: 'kubernetes.io/hostname' + + +memcached: + enabled: true + replicaCount: 32 + +memcached-queries: + enabled: true + replicaCount: 10 + +memcached-metadata: + enabled: true + +memcached-results: + enabled: true + replicaCount: 4 + +minio: + enabled: false + +overrides_exporter: + replicas: 1 + resources: + limits: + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi + +querier: + replicas: 6 + resources: + limits: + memory: 24Gi + requests: + cpu: 1 + memory: 12Gi + +query_frontend: + replicas: 2 + resources: + limits: + memory: 6Gi + requests: + cpu: 2 + memory: 2Gi + +ruler: + replicas: 2 + resources: + limits: + memory: 16Gi + requests: + cpu: 1 + memory: 6Gi + +store_gateway: + persistentVolume: + size: 50Gi + replicas: 1 + resources: + requests: + cpu: 1 + memory: 6Gi + +# Grafana Enterprise Metrics feature related +admin_api: + replicas: 3 + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 10m + memory: 64Mi + +gateway: + replicas: 8 + resources: + requests: + cpu: 1 + memory: 384Mi diff --git a/operations/helm/charts/mimir-distributed/scripts/create-pdb b/operations/helm/charts/mimir-distributed/scripts/create-pdb new file mode 100755 index 00000000000..0a19f2d9f9e --- /dev/null +++ b/operations/helm/charts/mimir-distributed/scripts/create-pdb @@ -0,0 +1,100 @@ +#!/usr/bin/env bash + +set -euf -o pipefail + +function usage { + cat < + +Options: + -g component is member of memberlist + -e is enterprise feature + -t toggleable with "enabled" + -ne opensource only feature + +Examples: + $0 overrides-exporter +EOF +} + +if [[ $# -eq 0 ]]; then + usage + exit 1 +fi + +component="$1" +enterprise=false +non_enterprise=false +toggle=false +memberlist='' + +while [[ $# -gt 0 ]] ; do +case "$1" in + -g) + memberlist=' "memberlist" true' + ;; + -e) + enterprise=true + ;; + -ne) + non_enterprise=true + ;; + -t) + toggle=true + ;; + -h) usage && exit 0 ;; + *) + component="$1" ;; +esac +shift +done + +# Convert kebab-case to snake_case. +function snake_case { + sed -E -e 's/-/_/' <<<"$1" +} + +snake_cased="$(snake_case "${component}")" + +if [ "${enterprise}" = "true" ] ; then +echo "{{- if .Values.enterprise.enabled -}}" +fi + +if [ "${non_enterprise}" = "true" ] ; then +echo "{{- if not .Values.enterprise.enabled -}}" +fi + +if [ "${toggle}" = "true" ] ; then +echo "{{- if .Values.${snake_cased}.enabled -}}" +fi + +cat < + +Options: + -g component is member of memberlist + -e is enterprise feature + -t toggleable with "enabled" + -ne opensource only feature + +Examples: + $0 overrides-exporter +EOF +} + +if [[ $# -eq 0 ]]; then + usage + exit 1 +fi + +component="$1" +enterprise=false +non_enterprise=false +toggle=false +memberlist='' + +while [[ $# -gt 0 ]] ; do +case "$1" in + -g) + memberlist=' "memberlist" true' + ;; + -e) + enterprise=true + ;; + -ne) + non_enterprise=true + ;; + -t) + toggle=true + ;; + -h) usage && exit 0 ;; + *) + component="$1" ;; +esac +shift +done + +# Convert kebab-case to snake_case. +function snake_case { + sed -E -e 's/-/_/' <<<"$1" +} + +snake_cased="$(snake_case "${component}")" + +if [ "${enterprise}" = "true" ] ; then +echo "{{- if .Values.enterprise.enabled -}}" +fi + +if [ "${non_enterprise}" = "true" ] ; then +echo "{{- if not .Values.enterprise.enabled -}}" +fi + +if [ "${toggle}" = "true" ] ; then +echo "{{- if .Values.${snake_cased}.enabled -}}" +fi + +cat < "templates/${component}/${component}-pdb.yaml" +done + +for component in "admin-api" "alertmanager" "compactor" "distributor" "gateway" "ingester" "querier" "query-frontend" "ruler" "store-gateway" ; do + ./scripts/create-servmon ${COMPONENTARGS["${component}"]} "${component}" > "templates/${component}/${component}-servmon.yaml" +done + +for component in "admin-api" "alertmanager" "compactor" "distributor" "gateway" "ingester" "overrides-exporter" "querier" "query-frontend" "ruler" "store-gateway" ; do + for filename in $(find "templates/${component}" -name "*.yaml") ; do + cp "${filename}" "${filename}.ori" + ./scripts/replace ${COMPONENTARGS["${component}"]} "${component}" "${filename}.ori" > "${filename}" + rm "${filename}.ori" + done +done diff --git a/operations/helm/charts/mimir-distributed/scripts/replace b/operations/helm/charts/mimir-distributed/scripts/replace new file mode 100755 index 00000000000..d8733adcc48 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/scripts/replace @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +set -euf -o pipefail + +function usage { + cat < + +Options: + -g component is member of memberlist + -e is enterprise feature + -t toggleable with "enabled" + -ne opensource only feature + +Examples: + $0 overrides-exporter +EOF +} + +if [[ $# -eq 0 ]]; then + usage + exit 1 +fi + +component="$1" +memberlist='' +SED=('sed' '-z' '-E' '-f/dev/stdin') + +while [[ $# -gt 0 ]] ; do +case "$1" in + -g) + memberlist=' "memberlist" true' + ;; + -e) + ;; + -ne) + ;; + -t) + ;; + -h) usage && exit 0 ;; + *) + break + ;; +esac +shift +done + +# Convert kebab-case to snake_case. +function snake_case { + sed -E -e 's/-/_/' <<<"$1" +} + +component="$1" +snake_cased="$(snake_case "${component}")" +headless="" +if grep -iqE '^ clusterip:[[:space:]]none$' "$2" ; then + headless="-headless" +fi + +# Example of updating labels +# update_app updates the .metadata.name, .metadata.labels, .spec.selector.matchLabels, and .spec.template.metadata.labels for an App like resource to use component specific helper templates. +# It is naïve and relies on indentation to for the correct field replacement. +update_app=$( + cat <= 1.21-0" (include "mimir.kubeVersion" .)) -}} + {{- print "policy/v1" -}} + {{- else -}} + {{- print "policy/v1beta1" -}} + {{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-dep.yaml b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-dep.yaml new file mode 100644 index 00000000000..8901db56a08 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-dep.yaml @@ -0,0 +1,131 @@ +{{- if .Values.enterprise.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {{- toYaml .Values.admin_api.annotations | nindent 4 }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }} + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "admin-api") }} +spec: + replicas: {{ .Values.admin_api.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 6 }} + strategy: + {{- toYaml .Values.admin_api.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 8 }} + {{- with .Values.admin_api.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end}} + {{- with .Values.admin_api.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.admin_api.priorityClassName }} + priorityClassName: {{ .Values.admin_api.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.admin_api.securityContext | nindent 8 }} + initContainers: + {{- with .Values.admin_api.initContainers }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: admin-api + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=admin-api" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.admin_api.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.admin_api.extraVolumeMounts }} + {{ toYaml .Values.admin_api.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: license + mountPath: /license + - name: storage + mountPath: "/data" + subPath: {{ .Values.admin_api.persistence.subPath }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.admin_api.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.admin_api.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.admin_api.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.admin_api.env }} + {{ toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.admin_api.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.admin_api.extraContainers }} + {{ toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml .Values.admin_api.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.admin_api.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.admin_api.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.admin_api.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if .Values.admin_api.extraVolumes }} + {{ toYaml .Values.admin_api.extraVolumes | nindent 8}} + {{- end }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + - name: storage + emptyDir: {} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-pdb.yaml new file mode 100644 index 00000000000..527f992b1aa --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enterprise.enabled -}} +{{- if .Values.admin_api.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "admin-api") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.admin_api.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-servmon.yaml new file mode 100644 index 00000000000..e42f7765984 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-servmon.yaml @@ -0,0 +1,54 @@ +{{- if .Values.enterprise.enabled -}} +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "admin-api") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "admin-api" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "admin-api" "memberlist" true) | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/admin-api" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-svc.yaml b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-svc.yaml new file mode 100644 index 00000000000..61904bfcb4e --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/admin-api/admin-api-svc.yaml @@ -0,0 +1,26 @@ +{{- if .Values.enterprise.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "admin-api") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }} + {{- with .Values.admin_api.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.admin_api.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "admin-api" "memberlist" true) | nindent 4 }} +{{- end -}} \ No newline at end of file diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml new file mode 100644 index 00000000000..cdaf83cb7fe --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-dep.yaml @@ -0,0 +1,139 @@ +{{- if .Values.alertmanager.enabled -}} +{{- if not .Values.alertmanager.statefulSet.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.alertmanager.annotations | nindent 4 }} +spec: + replicas: {{ .Values.alertmanager.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 6 }} + strategy: + {{- toYaml .Values.alertmanager.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 8 }} + {{- with .Values.alertmanager.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: +{{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} +{{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end}} + {{- with .Values.alertmanager.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.alertmanager.priorityClassName }} + priorityClassName: {{ .Values.query_frontend.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.alertmanager.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.alertmanager.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + containers: + - name: alertmanager + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=alertmanager" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.alertmanager.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.alertmanager.extraVolumeMounts }} + {{ toYaml .Values.alertmanager.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + subPath: {{ .Values.alertmanager.persistence.subPath }} + - name: tmp + mountPath: /tmp + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.alertmanager.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.alertmanager.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.alertmanager.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.alertmanager.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.alertmanager.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- if .Values.alertmanager.extraContainers }} +{{ toYaml .Values.alertmanager.extraContainers | indent 8}} +{{- end }} + nodeSelector: + {{- toYaml .Values.alertmanager.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.alertmanager.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.alertmanager.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.alertmanager.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime +{{- if .Values.alertmanager.extraVolumes }} +{{ toYaml .Values.alertmanager.extraVolumes | indent 8}} +{{- end }} + - name: storage + emptyDir: {} + - name: tmp + emptyDir: {} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-pdb.yaml new file mode 100644 index 00000000000..51de66d84d7 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.alertmanager.enabled -}} +{{- if .Values.alertmanager.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.alertmanager.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-servmon.yaml new file mode 100644 index 00000000000..439f0eac5d9 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-servmon.yaml @@ -0,0 +1,59 @@ +{{- if .Values.alertmanager.enabled -}} +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "alertmanager") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "alertmanager" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "alertmanager" "memberlist" true) | nindent 6 }} + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/alertmanager" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml new file mode 100644 index 00000000000..81774f9f932 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml @@ -0,0 +1,167 @@ +{{- if .Values.alertmanager.enabled -}} +{{- if .Values.alertmanager.statefulSet.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.alertmanager.annotations | nindent 4 }} +spec: + replicas: {{ .Values.alertmanager.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 6 }} + updateStrategy: + {{- toYaml .Values.alertmanager.statefulStrategy | nindent 4 }} + serviceName: {{ template "mimir.fullname" . }}-alertmanager + {{- if .Values.alertmanager.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: storage + {{- if .Values.alertmanager.persistentVolume.annotations }} + annotations: + {{ toYaml .Values.alertmanager.persistentVolume.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.alertmanager.persistentVolume.storageClass }} + {{- if (eq "-" .Values.alertmanager.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.alertmanager.persistentVolume.storageClass }}" + {{- end }} + {{- end }} + accessModes: + {{ toYaml .Values.alertmanager.persistentVolume.accessModes | nindent 10 }} + resources: + requests: + storage: "{{ .Values.alertmanager.persistentVolume.size }}" + {{- end }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 8 }} + {{- with .Values.alertmanager.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.alertmanager.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.alertmanager.priorityClassName }} + priorityClassName: {{ .Values.alertmanager.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.alertmanager.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.alertmanager.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + {{- toYaml .Values.alertmanager.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.alertmanager.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.alertmanager.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.alertmanager.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if not .Values.alertmanager.persistentVolume.enabled }} + - name: storage + emptyDir: {} + {{- end }} + - name: tmp + emptyDir: {} + {{- if .Values.alertmanager.extraVolumes }} + {{ toYaml .Values.alertmanager.extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- if .Values.alertmanager.extraContainers }} + {{ toYaml .Values.alertmanager.extraContainers | nindent 8 }} + {{- end }} + - name: alertmanager + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=alertmanager" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.alertmanager.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.alertmanager.extraVolumeMounts }} + {{ toYaml .Values.alertmanager.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + {{- if .Values.alertmanager.persistentVolume.subPath }} + subPath: {{ .Values.alertmanager.persistentVolume.subPath }} + {{- else }} + {{- end }} + - name: tmp + mountPath: /tmp + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.alertmanager.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.alertmanager.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.alertmanager.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.alertmanager.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.alertmanager.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml new file mode 100644 index 00000000000..852c4cc6880 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml @@ -0,0 +1,33 @@ +{{- if .Values.alertmanager.enabled -}} +{{- $clusterPort := regexReplaceAll ".+[:]" (default "0.0.0.0:9094" (include "mimir.alertmanagerClusterBindAddress" .) ) "" -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }}-headless + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} + prometheus.io/service-monitor: "false" + {{- with .Values.alertmanager.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.alertmanager.service.annotations | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + - port: {{ $clusterPort }} + protocol: TCP + name: cluster + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml new file mode 100644 index 00000000000..1b816c14b09 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml @@ -0,0 +1,26 @@ +{{- if .Values.alertmanager.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "alertmanager") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} + {{- with .Values.alertmanager.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.alertmanager.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "alertmanager" "memberlist" true) | nindent 4 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/compactor/compactor-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-pdb.yaml new file mode 100644 index 00000000000..c2835a59706 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.compactor.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "compactor") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.compactor.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/compactor/compactor-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-servmon.yaml new file mode 100644 index 00000000000..a64a3c7e8a4 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-servmon.yaml @@ -0,0 +1,52 @@ +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "compactor") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "compactor" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "compactor" "memberlist" true) | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/compactor" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/compactor/compactor-statefulset.yaml b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-statefulset.yaml new file mode 100644 index 00000000000..46cd3a3c49c --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-statefulset.yaml @@ -0,0 +1,158 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "compactor") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.compactor.annotations | nindent 4 }} +spec: + replicas: {{ .Values.compactor.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 6 }} + updateStrategy: + {{- toYaml .Values.compactor.strategy | nindent 4 }} + serviceName: {{ template "mimir.fullname" . }}-compactor + {{- if .Values.compactor.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: storage + {{- if .Values.compactor.persistentVolume.annotations }} + annotations: + {{ toYaml .Values.compactor.persistentVolume.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.compactor.persistentVolume.storageClass }} + {{- if (eq "-" .Values.compactor.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.compactor.persistentVolume.storageClass }}" + {{- end }} + {{- end }} + accessModes: + {{ toYaml .Values.compactor.persistentVolume.accessModes | nindent 10 }} + resources: + requests: + storage: "{{ .Values.compactor.persistentVolume.size }}" + {{- end }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 8 }} + {{- with .Values.compactor.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.compactor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.compactor.priorityClassName }} + priorityClassName: {{ .Values.compactor.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.compactor.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.compactor.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + {{- toYaml .Values.compactor.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.compactor.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.compactor.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.compactor.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if not .Values.compactor.persistentVolume.enabled }} + - name: storage + emptyDir: {} + {{- end }} + {{- if .Values.compactor.extraVolumes }} + {{ toYaml .Values.compactor.extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- if .Values.compactor.extraContainers }} + {{ toYaml .Values.compactor.extraContainers | nindent 8 }} + {{- end }} + - name: compactor + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=compactor" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.compactor.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.compactor.extraVolumeMounts }} + {{ toYaml .Values.compactor.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + {{- if .Values.compactor.persistentVolume.subPath }} + subPath: {{ .Values.compactor.persistentVolume.subPath }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.compactor.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.compactor.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.compactor.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.compactor.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.compactor.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/compactor/compactor-svc.yaml b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-svc.yaml new file mode 100644 index 00000000000..855e341e6c6 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/compactor/compactor-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "compactor") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }} + {{- with .Values.compactor.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.compactor.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "compactor" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/distributor/distributor-dep.yaml b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-dep.yaml new file mode 100644 index 00000000000..593a179c9d3 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-dep.yaml @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.distributor.annotations | nindent 4 }} +spec: + replicas: {{ .Values.distributor.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 6 }} + strategy: + {{- toYaml .Values.distributor.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 8 }} + {{- with .Values.distributor.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: +{{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} +{{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end}} + {{- with .Values.distributor.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.distributor.priorityClassName }} + priorityClassName: {{ .Values.distributor.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.distributor.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.distributor.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + containers: + - name: distributor + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=distributor" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.distributor.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.distributor.extraVolumeMounts }} + {{ toYaml .Values.distributor.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + subPath: {{ .Values.distributor.persistence.subPath }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.distributor.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.distributor.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.distributor.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.distributor.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.distributor.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- if .Values.distributor.extraContainers }} +{{ toYaml .Values.distributor.extraContainers | indent 8}} +{{- end }} + nodeSelector: + {{- toYaml .Values.distributor.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.distributor.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.distributor.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.distributor.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime +{{- if .Values.distributor.extraVolumes }} +{{ toYaml .Values.distributor.extraVolumes | indent 8}} +{{- end }} + - name: storage + emptyDir: {} + diff --git a/operations/helm/charts/mimir-distributed/templates/distributor/distributor-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-pdb.yaml new file mode 100644 index 00000000000..c9fca6342a3 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.distributor.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.distributor.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/distributor/distributor-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-servmon.yaml new file mode 100644 index 00000000000..3d7309b0930 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-servmon.yaml @@ -0,0 +1,52 @@ +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "distributor") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "distributor" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "distributor" "memberlist" true) | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/distributor" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/distributor/distributor-svc-headless.yaml b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-svc-headless.yaml new file mode 100644 index 00000000000..049f4406a79 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-svc-headless.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }}-headless + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }} + prometheus.io/service-monitor: "false" + {{- with .Values.distributor.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.distributor.service.annotations | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/distributor/distributor-svc.yaml b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-svc.yaml new file mode 100644 index 00000000000..c3a911a66df --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/distributor/distributor-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "distributor") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }} + {{- with .Values.distributor.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.distributor.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "distributor" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/gateway/_helpers.tpl b/operations/helm/charts/mimir-distributed/templates/gateway/_helpers.tpl new file mode 100644 index 00000000000..27a61abff7e --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gateway/_helpers.tpl @@ -0,0 +1,33 @@ +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "mimir.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return if ingress is stable. +*/}} +{{- define "mimir.ingress.isStable" -}} + {{- eq (include "mimir.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "mimir.ingress.supportsIngressClassName" -}} + {{- or (eq (include "mimir.ingress.isStable" .) "true") (and (eq (include "mimir.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "mimir.ingress.supportsPathType" -}} + {{- or (eq (include "mimir.ingress.isStable" .) "true") (and (eq (include "mimir.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gateway/gateway-dep.yaml b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-dep.yaml new file mode 100644 index 00000000000..e7f726a02d4 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-dep.yaml @@ -0,0 +1,123 @@ +{{- if .Values.enterprise.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {{- toYaml .Values.gateway.annotations | nindent 4 }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }} + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }} +spec: + replicas: {{ .Values.gateway.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "gateway") | nindent 6 }} + strategy: + {{- toYaml .Values.gateway.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "gateway") | nindent 8 }} + {{- with .Values.gateway.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end}} + {{- with .Values.gateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.gateway.priorityClassName }} + priorityClassName: {{ .Values.gateway.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.gateway.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.gateway.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: gateway + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=gateway" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.gateway.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.gateway.extraVolumeMounts }} + {{ toYaml .Values.gateway.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: license + mountPath: /license + - name: storage + mountPath: "/data" + subPath: {{ .Values.gateway.persistence.subPath }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.gateway.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.gateway.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.gateway.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.env }} + {{ toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.gateway.extraContainers }} + {{ toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml .Values.gateway.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.gateway.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.gateway.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.gateway.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if .Values.gateway.extraVolumes }} + {{ toYaml .Values.gateway.extraVolumes | nindent 8}} + {{- end }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + - name: storage + emptyDir: {} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gateway/gateway-ingress.yaml b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-ingress.yaml new file mode 100644 index 00000000000..6fc634fb6cb --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.enterprise.enabled -}} +{{- if .Values.gateway.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "mimir.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "mimir.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "mimir.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "mimir.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }} + {{- with .Values.gateway.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.gateway.ingress.ingressClassName }} + ingressClassName: {{ .Values.gateway.ingress.ingressClassName }} + {{- end -}} + {{- if .Values.gateway.ingress.tls }} + tls: + {{- range .Values.gateway.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + {{- with .secretName }} + secretName: {{ . }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.gateway.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if $ingressSupportsPathType }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "gateway") }} + port: + number: {{ $.Values.gateway.service.port | default (include "mimir.serverHttpListenPort" . ) }} + {{- else }} + serviceName: {{ include "mimir.resourceName" (dict "ctx" $ "component" "gateway") }} + servicePort: {{ $.Values.gateway.service.port | default (include "mimir.serverHttpListenPort" . ) }} + {{- end }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gateway/gateway-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-pdb.yaml new file mode 100644 index 00000000000..7ae256b410f --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.enterprise.enabled -}} +{{- if .Values.gateway.podDisruptionBudget -}} +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "gateway") | nindent 6 }} +{{ toYaml .Values.gateway.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gateway/gateway-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-servmon.yaml new file mode 100644 index 00000000000..161028c8a24 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-servmon.yaml @@ -0,0 +1,54 @@ +{{- if .Values.enterprise.enabled -}} +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "gateway") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "gateway") | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "gateway") | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/gateway" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gateway/gateway-svc.yaml b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-svc.yaml new file mode 100644 index 00000000000..4912ab28941 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gateway/gateway-svc.yaml @@ -0,0 +1,22 @@ +{{- if .Values.enterprise.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gateway") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "gateway") | nindent 4 }} + {{- with .Values.gateway.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.gateway.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ .Values.gateway.service.port | default (include "mimir.serverHttpListenPort" . ) }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "gateway") | nindent 4 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gossip-ring/_helpers.gossip-ring.tpl b/operations/helm/charts/mimir-distributed/templates/gossip-ring/_helpers.gossip-ring.tpl new file mode 100644 index 00000000000..b4a932eba0e --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gossip-ring/_helpers.gossip-ring.tpl @@ -0,0 +1,11 @@ +{{/* +gossip-ring selector labels +*/}} +{{- define "mimir.gossipRingSelectorLabels" -}} +{{ include "mimir.selectorLabels" . }} +{{- if .ctx.Values.enterprise.legacyLabels }} +gossip_ring_member: "true" +{{- else }} +app.kubernetes.io/part-of: memberlist +{{- end }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml b/operations/helm/charts/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml new file mode 100644 index 00000000000..ff81e2ba773 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "gossip-ring") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "gossip-ring") | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: gossip-ring + port: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + targetPort: {{ include "mimir.memberlistBindPort" . }} + publishNotReadyAddresses: true + selector: + {{- include "mimir.gossipRingSelectorLabels" (dict "ctx" .) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-dep.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-dep.yaml new file mode 100644 index 00000000000..006b98745e9 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-dep.yaml @@ -0,0 +1,3 @@ +{{- if not .Values.ingester.statefulSet.enabled -}} +{{- fail "Ingesters are stateful, kubernetes Deployment workloads are no longer supported. Please refer to the documentation for more details." }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-pdb.yaml new file mode 100644 index 00000000000..e4358f086af --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.ingester.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ingester") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.ingester.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-servmon.yaml new file mode 100644 index 00000000000..10f1d85c26d --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-servmon.yaml @@ -0,0 +1,57 @@ +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "ingester") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "ingester" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "ingester" "memberlist" true) | nindent 6 }} + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/ingester" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml new file mode 100644 index 00000000000..71c45d7995c --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-statefulset.yaml @@ -0,0 +1,162 @@ +{{- if .Values.ingester.statefulSet.enabled -}} +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ingester") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.ingester.annotations | nindent 4 }} +spec: + podManagementPolicy: {{ .Values.ingester.podManagementPolicy }} + replicas: {{ .Values.ingester.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 6 }} + updateStrategy: + {{- toYaml .Values.ingester.statefulStrategy | nindent 4 }} + serviceName: {{ template "mimir.fullname" . }}-ingester{{- if not .Values.enterprise.legacyLabels -}}-headless{{- end -}} + {{- if .Values.ingester.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: storage + {{- if .Values.ingester.persistentVolume.annotations }} + annotations: + {{ toYaml .Values.ingester.persistentVolume.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.ingester.persistentVolume.storageClass }} + {{- if (eq "-" .Values.ingester.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.ingester.persistentVolume.storageClass }}" + {{- end }} + {{- end }} + accessModes: + {{ toYaml .Values.ingester.persistentVolume.accessModes | nindent 10 }} + resources: + requests: + storage: "{{ .Values.ingester.persistentVolume.size }}" + {{- end }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 8 }} + {{- with .Values.ingester.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.ingester.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.ingester.priorityClassName }} + priorityClassName: {{ .Values.ingester.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.ingester.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.ingester.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + {{- toYaml .Values.ingester.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.ingester.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.ingester.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ingester.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if not .Values.ingester.persistentVolume.enabled }} + - name: storage + emptyDir: {} + {{- end }} + {{- if .Values.ingester.extraVolumes }} + {{ toYaml .Values.ingester.extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- if .Values.ingester.extraContainers }} + {{ toYaml .Values.ingester.extraContainers | nindent 8 }} + {{- end }} + - name: ingester + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=ingester" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.ingester.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.ingester.extraVolumeMounts }} + {{ toYaml .Values.ingester.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + {{- if .Values.ingester.persistentVolume.subPath }} + subPath: {{ .Values.ingester.persistentVolume.subPath }} + {{- else }} + {{- end }} + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.ingester.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.ingester.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.ingester.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ingester.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-svc-headless.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-svc-headless.yaml new file mode 100644 index 00000000000..fc67ba377b5 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-svc-headless.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ingester") }}-headless + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }} + prometheus.io/service-monitor: "false" + {{- with .Values.ingester.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.ingester.service.annotations | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/ingester/ingester-svc.yaml b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-svc.yaml new file mode 100644 index 00000000000..a72aaa46009 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ingester/ingester-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ingester") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }} + {{- with .Values.ingester.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.ingester.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ingester" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/license-secret.yaml b/operations/helm/charts/mimir-distributed/templates/license-secret.yaml new file mode 100644 index 00000000000..da73d63fb1a --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/license-secret.yaml @@ -0,0 +1,12 @@ +{{- if .Values.enterprise.enabled -}} +{{- if not .Values.license.external }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ tpl .Values.license.secretName . }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} +data: + license.jwt: {{ .Values.license.contents | b64enc }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/minio/create-bucket-job.yaml b/operations/helm/charts/mimir-distributed/templates/minio/create-bucket-job.yaml new file mode 100644 index 00000000000..f36d4059caa --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/minio/create-bucket-job.yaml @@ -0,0 +1,86 @@ +{{- if .Values.minio.enabled }} +{{- if .Values.minio.buckets }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-make-bucket-job" (include "minio.fullname" .) | trunc 63 | trimSuffix "-" }} + namespace: {{ .Release.Namespace | quote }} + labels: + app: {{ template "minio.name" . }}-make-bucket-job + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- with .Values.minio.makeBucketJob.annotations }} +{{ toYaml . | indent 4 }} +{{- end }} +spec: + template: + metadata: + labels: + app: {{ template "minio.name" . }}-job + release: {{ .Release.Name }} +{{- if .Values.minio.podLabels }} +{{ toYaml .Values.minio.podLabels | indent 8 }} +{{- end }} +{{- if .Values.minio.makeBucketJob.podAnnotations }} + annotations: +{{ toYaml .Values.minio.makeBucketJob.podAnnotations | indent 8 }} +{{- end }} + spec: + restartPolicy: OnFailure +{{- include "minio.imagePullSecrets" . | indent 6 }} +{{- if .Values.minio.nodeSelector }} + nodeSelector: +{{ toYaml .Values.minio.makeBucketJob.nodeSelector | indent 8 }} +{{- end }} +{{- with .Values.minio.makeBucketJob.affinity }} + affinity: +{{ toYaml . | indent 8 }} +{{- end }} +{{- with .Values.minio.makeBucketJob.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} +{{- end }} +{{- if .Values.minio.makeBucketJob.securityContext.enabled }} + securityContext: + runAsUser: {{ .Values.minio.makeBucketJob.securityContext.runAsUser }} + runAsGroup: {{ .Values.minio.makeBucketJob.securityContext.runAsGroup }} + fsGroup: {{ .Values.minio.makeBucketJob.securityContext.fsGroup }} +{{- end }} + volumes: + - name: minio-configuration + projected: + sources: + - configMap: + name: {{ .Release.Name }}-minio + - secret: + name: {{ .Release.Name }}-minio + {{- if .Values.minio.tls.enabled }} + - name: cert-secret-volume-mc + secret: + secretName: {{ .Values.minio.tls.certSecret }} + items: + - key: {{ .Values.minio.tls.publicCrt }} + path: CAs/public.crt + {{ end }} + containers: + - name: minio-mc + image: "{{ .Values.minio.mcImage.repository }}:{{ .Values.minio.mcImage.tag }}" + imagePullPolicy: {{ .Values.minio.mcImage.pullPolicy }} + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: {{ .Release.Name }}-minio + - name: MINIO_PORT + value: {{ .Values.minio.service.port | quote }} + volumeMounts: + - name: minio-configuration + mountPath: /config + {{- if .Values.minio.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.minio.configPathmc }}certs + {{ end }} + resources: +{{ toYaml .Values.minio.makeBucketJob.resources | indent 10 }} +{{- end }} +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/_helpers.tpl b/operations/helm/charts/mimir-distributed/templates/nginx/_helpers.tpl new file mode 100644 index 00000000000..8ae04b3e767 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/_helpers.tpl @@ -0,0 +1,40 @@ +{{/* +nginx auth secret name +*/}} +{{- define "mimir.nginxAuthSecret" -}} +{{ .Values.nginx.basicAuth.existingSecret | default (include "mimir.resourceName" (dict "ctx" . "component" "nginx") ) }} +{{- end }} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "mimir.ingress.apiVersion" -}} + {{- if and (.Capabilities.APIVersions.Has "networking.k8s.io/v1") (semverCompare ">= 1.19-0" .Capabilities.KubeVersion.Version) -}} + {{- print "networking.k8s.io/v1" -}} + {{- else if .Capabilities.APIVersions.Has "networking.k8s.io/v1beta1" -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "extensions/v1beta1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return if ingress is stable. +*/}} +{{- define "mimir.ingress.isStable" -}} + {{- eq (include "mimir.ingress.apiVersion" .) "networking.k8s.io/v1" -}} +{{- end -}} + +{{/* +Return if ingress supports ingressClassName. +*/}} +{{- define "mimir.ingress.supportsIngressClassName" -}} + {{- or (eq (include "mimir.ingress.isStable" .) "true") (and (eq (include "mimir.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} + +{{/* +Return if ingress supports pathType. +*/}} +{{- define "mimir.ingress.supportsPathType" -}} + {{- or (eq (include "mimir.ingress.isStable" .) "true") (and (eq (include "mimir.ingress.apiVersion" .) "networking.k8s.io/v1beta1") (semverCompare ">= 1.18-0" .Capabilities.KubeVersion.Version)) -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/ingress.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/ingress.yaml new file mode 100644 index 00000000000..a203c15c673 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/ingress.yaml @@ -0,0 +1,57 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if .Values.nginx.enabled -}} +{{- if .Values.nginx.ingress.enabled -}} +{{- $ingressApiIsStable := eq (include "mimir.ingress.isStable" .) "true" -}} +{{- $ingressSupportsIngressClassName := eq (include "mimir.ingress.supportsIngressClassName" .) "true" -}} +{{- $ingressSupportsPathType := eq (include "mimir.ingress.supportsPathType" .) "true" -}} +apiVersion: {{ include "mimir.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} + {{- with .Values.nginx.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and $ingressSupportsIngressClassName .Values.nginx.ingress.ingressClassName }} + ingressClassName: {{ .Values.nginx.ingress.ingressClassName }} + {{- end -}} + {{- if .Values.nginx.ingress.tls }} + tls: + {{- range .Values.nginx.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + {{- with .secretName }} + secretName: {{ . }} + {{- end }} + {{- end }} + {{- end }} + rules: + {{- range .Values.nginx.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if $ingressSupportsPathType }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if $ingressApiIsStable }} + service: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "nginx") }} + port: + number: {{ $.Values.nginx.service.port }} + {{- else }} + serviceName: {{ include "mimir.resourceName" (dict "ctx" $ "component" "nginx") }} + servicePort: {{ $.Values.nginx.service.port }} + {{- end }} + {{- end }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/nginx-configmap.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-configmap.yaml new file mode 100644 index 00000000000..f78228b8a8b --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-configmap.yaml @@ -0,0 +1,13 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if .Values.nginx.enabled }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} +data: + nginx.conf: | + {{- tpl .Values.nginx.nginxConfig.file . | nindent 4 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/nginx-dep.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-dep.yaml new file mode 100644 index 00000000000..02661db6222 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-dep.yaml @@ -0,0 +1,117 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if .Values.nginx.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} + annotations: + {{- toYaml .Values.nginx.annotations | nindent 4 }} +spec: +{{- if not .Values.nginx.autoscaling.enabled }} + replicas: {{ .Values.nginx.replicas }} +{{- end }} +{{- with .Values.nginx.deploymentStrategy }} + strategy: +{{ toYaml . | trim | indent 4 }} +{{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "nginx") | nindent 6 }} + template: + metadata: + annotations: + checksum/config: {{ include (print .Template.BasePath "/nginx/nginx-configmap.yaml") . | sha256sum }} + {{- with .Values.nginx.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "nginx") | nindent 8 }} + {{- with .Values.nginx.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "mimir.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.nginx.priorityClassName }} + priorityClassName: {{ .Values.nginx.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.nginx.podSecurityContext | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.nginx.terminationGracePeriodSeconds }} + containers: + - name: nginx + image: {{ .Values.nginx.image.registry }}/{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }} + imagePullPolicy: {{ .Values.nginx.image.pullPolicy }} + ports: + - name: http-metric + containerPort: 8080 + protocol: TCP + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nginx.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.nginx.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + readinessProbe: + {{- toYaml .Values.nginx.readinessProbe | nindent 12 }} + securityContext: + {{- toYaml .Values.nginx.containerSecurityContext | nindent 12 }} + volumeMounts: + - name: config + mountPath: /etc/nginx + {{- if .Values.nginx.basicAuth.enabled }} + - name: auth + mountPath: /etc/nginx/secrets + {{- end }} + - name: tmp + mountPath: /tmp + - name: docker-entrypoint-d-override + mountPath: /docker-entrypoint.d + {{- if .Values.nginx.extraVolumeMounts }} + {{- toYaml .Values.nginx.extraVolumeMounts | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.nginx.resources | nindent 12 }} + {{- with .Values.nginx.affinity }} + affinity: + {{- tpl . $ | nindent 8 }} + {{- end }} + {{- with .Values.nginx.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nginx.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ include "mimir.fullname" . }}-nginx + {{- if .Values.nginx.basicAuth.enabled }} + - name: auth + secret: + secretName: {{ include "mimir.nginxAuthSecret" . }} + {{- end }} + - name: tmp + emptyDir: {} + - name: docker-entrypoint-d-override + emptyDir: {} + {{- if .Values.nginx.extraVolumes }} + {{- toYaml .Values.nginx.extraVolumes | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/nginx-hpa.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-hpa.yaml new file mode 100644 index 00000000000..b7061ca4d6d --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-hpa.yaml @@ -0,0 +1,30 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if .Values.nginx.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + minReplicas: {{ .Values.nginx.autoscaling.minReplicas }} + maxReplicas: {{ .Values.nginx.autoscaling.maxReplicas }} + metrics: + {{- with .Values.nginx.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ . }} + {{- end }} + {{- with .Values.nginx.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ . }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/nginx-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-pdb.yaml new file mode 100644 index 00000000000..09ac6f617a4 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-pdb.yaml @@ -0,0 +1,15 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if .Values.nginx.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "nginx") | nindent 6 }} +{{ toYaml .Values.nginx.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/nginx-secret.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-secret.yaml new file mode 100644 index 00000000000..1c3fe6a677a --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-secret.yaml @@ -0,0 +1,13 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if and .Values.nginx.enabled .Values.nginx.basicAuth.enabled (not .Values.nginx.basicAuth.existingSecret) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} +stringData: + .htpasswd: | + {{- tpl .Values.nginx.basicAuth.htpasswd $ | nindent 4 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/nginx/nginx-svc.yaml b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-svc.yaml new file mode 100644 index 00000000000..2df10354f83 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/nginx/nginx-svc.yaml @@ -0,0 +1,33 @@ +{{- if not .Values.enterprise.enabled -}} +{{- if .Values.nginx.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "nginx") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "nginx") | nindent 4 }} + {{- with .Values.nginx.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.nginx.service.annotations | nindent 4 }} +spec: + type: {{ .Values.nginx.service.type }} + {{- with .Values.nginx.service.clusterIP }} + clusterIP: {{ . }} + {{- end }} + {{- if and (eq "LoadBalancer" .Values.nginx.service.type) .Values.nginx.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.nginx.service.loadBalancerIP }} + {{- end }} + ports: + - name: http-metric + port: {{ .Values.nginx.service.port }} + targetPort: http-metric + {{- if and (eq "NodePort" .Values.nginx.service.type) .Values.nginx.service.nodePort }} + nodePort: {{ .Values.nginx.service.nodePort }} + {{- end }} + protocol: TCP + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "nginx") | nindent 4 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml b/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml new file mode 100644 index 00000000000..1164ad736ba --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml @@ -0,0 +1,130 @@ +{{- if .Values.overrides_exporter.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {{- toYaml .Values.overrides_exporter.annotations | nindent 4 }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }} + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "overrides-exporter") }} +spec: + replicas: {{ .Values.overrides_exporter.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 6 }} + strategy: + {{- toYaml .Values.overrides_exporter.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 8 }} + {{- with .Values.overrides_exporter.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end}} + {{- with .Values.overrides_exporter.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.overrides_exporter.priorityClassName }} + priorityClassName: {{ .Values.overrides_exporter.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.overrides_exporter.securityContext | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + initContainers: + {{- toYaml .Values.overrides_exporter.initContainers | nindent 8 }} + containers: + - name: overrides-exporter + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=overrides-exporter" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.overrides_exporter.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.overrides_exporter.extraVolumeMounts }} + {{ toYaml .Values.overrides_exporter.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + subPath: {{ .Values.overrides_exporter.persistence.subPath }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.overrides_exporter.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.overrides_exporter.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.overrides_exporter.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.overrides_exporter.env }} + {{ toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.overrides_exporter.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.overrides_exporter.extraContainers }} + {{ toYaml . | nindent 8 }} + {{- end }} + nodeSelector: + {{- toYaml .Values.overrides_exporter.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.overrides_exporter.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.overrides_exporter.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.overrides_exporter.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if .Values.overrides_exporter.extraVolumes }} + {{ toYaml .Values.overrides_exporter.extraVolumes | nindent 8}} + {{- end }} + - name: storage + emptyDir: {} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-pdb.yaml new file mode 100644 index 00000000000..11dfa23b6dd --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.overrides_exporter.enabled -}} +{{- if .Values.overrides_exporter.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "overrides-exporter") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 6 }} +{{ toYaml .Values.overrides_exporter.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml b/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml new file mode 100644 index 00000000000..24daeafd6dd --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml @@ -0,0 +1,26 @@ +{{- if .Values.overrides_exporter.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "overrides-exporter") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }} + {{- with .Values.overrides_exporter.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.overrides_exporter.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "overrides-exporter") | nindent 4 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/podsecuritypolicy.yaml b/operations/helm/charts/mimir-distributed/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000000..b5a615ae7ef --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/podsecuritypolicy.yaml @@ -0,0 +1,36 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" .) }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} +spec: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'configMap' + - 'emptyDir' + - 'persistentVolumeClaim' + - 'secret' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/querier/querier-dep.yaml b/operations/helm/charts/mimir-distributed/templates/querier/querier-dep.yaml new file mode 100644 index 00000000000..9f1ca631a98 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/querier/querier-dep.yaml @@ -0,0 +1,131 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.querier.annotations | nindent 4 }} +spec: + replicas: {{ .Values.querier.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 6 }} + strategy: + {{- toYaml .Values.querier.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 8 }} + {{- with .Values.querier.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: +{{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} +{{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end}} + {{- with .Values.querier.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.querier.priorityClassName }} + priorityClassName: {{ .Values.querier.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.querier.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.querier.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + containers: + - name: querier + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=querier" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.querier.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.querier.extraVolumeMounts }} + {{ toYaml .Values.querier.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + subPath: {{ .Values.querier.persistence.subPath }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.querier.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.querier.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.querier.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.querier.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.querier.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- if .Values.querier.extraContainers }} +{{ toYaml .Values.querier.extraContainers | indent 8}} +{{- end }} + nodeSelector: + {{- toYaml .Values.querier.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.querier.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.querier.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.querier.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime +{{- if .Values.querier.extraVolumes }} +{{ toYaml .Values.querier.extraVolumes | indent 8}} +{{- end }} + - name: storage + emptyDir: {} diff --git a/operations/helm/charts/mimir-distributed/templates/querier/querier-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/querier/querier-pdb.yaml new file mode 100644 index 00000000000..b5801e4c882 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/querier/querier-pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.querier.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.querier.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/querier/querier-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/querier/querier-servmon.yaml new file mode 100644 index 00000000000..64bc027d710 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/querier/querier-servmon.yaml @@ -0,0 +1,52 @@ +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "querier") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "querier" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "querier" "memberlist" true) | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/querier" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/querier/querier-svc.yaml b/operations/helm/charts/mimir-distributed/templates/querier/querier-svc.yaml new file mode 100644 index 00000000000..685da16a808 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/querier/querier-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "querier") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }} + {{- with .Values.querier.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.querier.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "querier" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml new file mode 100644 index 00000000000..5e27bf6a749 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml @@ -0,0 +1,130 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }} + annotations: + {{- toYaml .Values.query_frontend.annotations | nindent 4 }} +spec: + replicas: {{ .Values.query_frontend.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 6 }} + strategy: + {{- toYaml .Values.query_frontend.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "query-frontend") | nindent 8 }} + {{- with .Values.query_frontend.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: +{{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} +{{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end}} + {{- with .Values.query_frontend.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.query_frontend.priorityClassName }} + priorityClassName: {{ .Values.query_frontend.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.query_frontend.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.query_frontend.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + containers: + - name: query-frontend + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=query-frontend" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.query_frontend.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.query_frontend.extraVolumeMounts }} + {{ toYaml .Values.query_frontend.extraVolumeMounts | nindent 12}} + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: config + mountPath: /etc/mimir + - name: storage + mountPath: /data + {{- if .Values.query_frontend.persistence.subPath }} + subPath: {{ .Values.query_frontend.persistence.subPath }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.query_frontend.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.query_frontend.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.query_frontend.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.query_frontend.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.query_frontend.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- if .Values.query_frontend.extraContainers }} +{{ toYaml .Values.query_frontend.extraContainers | indent 8}} +{{- end }} + nodeSelector: + {{- toYaml .Values.query_frontend.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.query_frontend.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.query_frontend.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.query_frontend.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime +{{- if .Values.query_frontend.extraVolumes }} +{{ toYaml .Values.query_frontend.extraVolumes | indent 8}} +{{- end }} + - name: storage + emptyDir: {} diff --git a/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-pdb.yaml new file mode 100644 index 00000000000..df732771367 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.query_frontend.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 6 }} +{{ toYaml .Values.query_frontend.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-servmon.yaml new file mode 100644 index 00000000000..7807f8c5ac3 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-servmon.yaml @@ -0,0 +1,57 @@ +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "query-frontend") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "query-frontend") | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "query-frontend") | nindent 6 }} + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/query-frontend" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml new file mode 100644 index 00000000000..85630dd3f94 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }}-headless + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }} + prometheus.io/service-monitor: "false" + {{- with .Values.query_frontend.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.query_frontend.service.annotations | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml new file mode 100644 index 00000000000..7c97d6b7222 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "query-frontend") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "query-frontend") | nindent 4 }} + {{- with .Values.query_frontend.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.query_frontend.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" .}} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "query-frontend") | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/role.yaml b/operations/helm/charts/mimir-distributed/templates/role.yaml new file mode 100644 index 00000000000..770c4758a2f --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/role.yaml @@ -0,0 +1,13 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" .) }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [{{ include "mimir.resourceName" (dict "ctx" .) }}] +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/rolebinding.yaml b/operations/helm/charts/mimir-distributed/templates/rolebinding.yaml new file mode 100644 index 00000000000..8de7dbb9d81 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/rolebinding.yaml @@ -0,0 +1,15 @@ +{{- if .Values.rbac.pspEnabled }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" .) }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "mimir.resourceName" (dict "ctx" .) }} +subjects: +- kind: ServiceAccount + name: {{ template "mimir.serviceAccountName" . }} +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/ruler/ruler-dep.yaml b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-dep.yaml new file mode 100644 index 00000000000..beeccb1c1ee --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-dep.yaml @@ -0,0 +1,133 @@ +{{- if .Values.ruler.enabled -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.ruler.annotations | nindent 4 }} +spec: + replicas: {{ .Values.ruler.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 6 }} + strategy: + {{- toYaml .Values.ruler.strategy | nindent 4 }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 8 }} + {{- with .Values.ruler.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: +{{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} +{{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} +{{- end}} + {{- with .Values.ruler.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.ruler.priorityClassName }} + priorityClassName: {{ .Values.ruler.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.ruler.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.ruler.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end}} + {{- end }} + containers: + - name: ruler + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=ruler" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.ruler.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.ruler.extraVolumeMounts }} + {{ toYaml .Values.ruler.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + subPath: {{ .Values.ruler.persistence.subPath }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.ruler.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.ruler.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.ruler.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ruler.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.ruler.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} +{{- if .Values.ruler.extraContainers }} +{{ toYaml .Values.ruler.extraContainers | indent 8}} +{{- end }} + nodeSelector: + {{- toYaml .Values.ruler.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.ruler.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.ruler.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.ruler.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime +{{- if .Values.ruler.extraVolumes }} +{{ toYaml .Values.ruler.extraVolumes | indent 8}} +{{- end }} + - name: storage + emptyDir: {} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ruler/ruler-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-pdb.yaml new file mode 100644 index 00000000000..7308cd8b7fe --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-pdb.yaml @@ -0,0 +1,15 @@ +{{- if .Values.ruler.enabled -}} +{{- if .Values.ruler.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.ruler.podDisruptionBudget | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ruler/ruler-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-servmon.yaml new file mode 100644 index 00000000000..4212f1ae248 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-servmon.yaml @@ -0,0 +1,54 @@ +{{- if .Values.ruler.enabled -}} +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "ruler") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "ruler" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "ruler" "memberlist" true) | nindent 6 }} + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/ruler" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/ruler/ruler-svc.yaml b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-svc.yaml new file mode 100644 index 00000000000..b9abaeb4eb7 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-svc.yaml @@ -0,0 +1,22 @@ +{{- if .Values.ruler.enabled -}} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "ruler") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }} + {{- with .Values.ruler.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.ruler.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "ruler" "memberlist" true) | nindent 4 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/runtime-configmap.yaml b/operations/helm/charts/mimir-distributed/templates/runtime-configmap.yaml new file mode 100644 index 00000000000..6a65dcf8b0a --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/runtime-configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "runtime") }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} +data: + runtime.yaml: | + {{ tpl (toYaml .Values.runtimeConfig) . | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/secret.yaml b/operations/helm/charts/mimir-distributed/templates/secret.yaml new file mode 100644 index 00000000000..7af4286a189 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/secret.yaml @@ -0,0 +1,10 @@ +{{- if not .Values.useExternalConfig }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ tpl .Values.externalConfigSecretName . }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} +data: + mimir.yaml: {{ include "mimir.calculatedConfig" . | b64enc }} +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/serviceaccount.yaml b/operations/helm/charts/mimir-distributed/templates/serviceaccount.yaml new file mode 100644 index 00000000000..35f66871b32 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "mimir.serviceAccountName" . }} + labels: + {{- include "mimir.labels" (dict "ctx" .) | nindent 4 }} + annotations: + {{- toYaml .Values.serviceAccount.annotations | nindent 4 }} +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml new file mode 100644 index 00000000000..0ef24228ab9 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml @@ -0,0 +1,13 @@ +{{- if .Values.store_gateway.podDisruptionBudget -}} +apiVersion: {{ include "mimir.podDisruptionBudget.apiVersion" . }} +kind: PodDisruptionBudget +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "store-gateway") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 6 }} +{{ toYaml .Values.store_gateway.podDisruptionBudget | indent 2 }} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-servmon.yaml b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-servmon.yaml new file mode 100644 index 00000000000..2ed93c99755 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-servmon.yaml @@ -0,0 +1,57 @@ +{{- with .Values.serviceMonitor }} +{{- if .enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" $ "component" "store-gateway") }} + {{- with .namespace }} + namespace: {{ . }} + {{- end }} + labels: + {{- include "mimir.labels" (dict "ctx" $ "component" "store-gateway" "memberlist" true) | nindent 4 }} + {{- with .labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .namespaceSelector }} + namespaceSelector: + {{- toYaml . | nindent 4 }} + {{- end }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" $ "component" "store-gateway" "memberlist" true) | nindent 6 }} + matchExpressions: + - key: prometheus.io/service-monitor + operator: NotIn + values: + - "false" + endpoints: + - port: http-metrics + {{- with .interval }} + interval: {{ . }} + {{- end }} + {{- with .scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + relabelings: + - sourceLabels: [job] + replacement: "{{ $.Release.Namespace }}/store-gateway" + targetLabel: job + - replacement: "{{ include "mimir.clusterName" $ }}" + targetLabel: cluster + {{- with .relabelings }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .scheme }} + scheme: {{ . }} + {{- end }} + {{- with .tlsConfig }} + tlsConfig: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml new file mode 100644 index 00000000000..eae90c3c313 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml @@ -0,0 +1,158 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "store-gateway") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }} + annotations: + {{- toYaml .Values.store_gateway.annotations | nindent 4 }} +spec: + replicas: {{ .Values.store_gateway.replicas }} + selector: + matchLabels: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 6 }} + updateStrategy: + {{- toYaml .Values.store_gateway.strategy | nindent 4 }} + serviceName: {{ template "mimir.fullname" . }}-store-gateway{{- if not .Values.enterprise.legacyLabels -}}-headless{{- end -}} + {{- if .Values.store_gateway.persistentVolume.enabled }} + volumeClaimTemplates: + - metadata: + name: storage + {{- if .Values.store_gateway.persistentVolume.annotations }} + annotations: + {{ toYaml .Values.store_gateway.persistentVolume.annotations | nindent 10 }} + {{- end }} + spec: + {{- if .Values.store_gateway.persistentVolume.storageClass }} + {{- if (eq "-" .Values.store_gateway.persistentVolume.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.store_gateway.persistentVolume.storageClass }}" + {{- end }} + {{- end }} + accessModes: + {{ toYaml .Values.store_gateway.persistentVolume.accessModes | nindent 10 }} + resources: + requests: + storage: "{{ .Values.store_gateway.persistentVolume.size }}" + {{- end }} + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 8 }} + {{- with .Values.store_gateway.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + annotations: + {{- if .Values.useExternalConfig }} + checksum/config: {{ .Values.externalConfigVersion }} + {{- else }} + checksum/config: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.store_gateway.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.store_gateway.priorityClassName }} + priorityClassName: {{ .Values.store_gateway.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.store_gateway.securityContext | nindent 8 }} + initContainers: + {{- toYaml .Values.store_gateway.initContainers | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + nodeSelector: + {{- toYaml .Values.store_gateway.nodeSelector | nindent 8 }} + affinity: + {{- toYaml .Values.store_gateway.affinity | nindent 8 }} + tolerations: + {{- toYaml .Values.store_gateway.tolerations | nindent 8 }} + terminationGracePeriodSeconds: {{ .Values.store_gateway.terminationGracePeriodSeconds }} + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.enterprise.enabled }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + {{- end }} + - name: runtime-config + configMap: + name: {{ template "mimir.fullname" . }}-runtime + {{- if not .Values.store_gateway.persistentVolume.enabled }} + - name: storage + emptyDir: {} + {{- end }} + {{- if .Values.store_gateway.extraVolumes }} + {{ toYaml .Values.store_gateway.extraVolumes | nindent 8 }} + {{- end }} + containers: + {{- if .Values.store_gateway.extraContainers }} + {{ toYaml .Values.store_gateway.extraContainers | nindent 8 }} + {{- end }} + - name: store-gateway + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=store-gateway" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.store_gateway.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.store_gateway.extraVolumeMounts }} + {{ toYaml .Values.store_gateway.extraVolumeMounts | nindent 12}} + {{- end }} + - name: config + mountPath: /etc/mimir + {{- if .Values.enterprise.enabled }} + - name: license + mountPath: /license + {{- end }} + - name: runtime-config + mountPath: /var/{{ include "mimir.name" . }} + - name: storage + mountPath: "/data" + {{- if .Values.store_gateway.persistentVolume.subPath }} + subPath: {{ .Values.store_gateway.persistentVolume.subPath }} + {{- end }} + ports: + - name: http-metrics + containerPort: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + - name: grpc + containerPort: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + - name: memberlist + containerPort: {{ include "mimir.memberlistBindPort" . }} + protocol: TCP + livenessProbe: + {{- toYaml .Values.store_gateway.livenessProbe | nindent 12 }} + readinessProbe: + {{- toYaml .Values.store_gateway.readinessProbe | nindent 12 }} + resources: + {{- toYaml .Values.store_gateway.resources | nindent 12 }} + securityContext: + readOnlyRootFilesystem: true + env: + {{- with .Values.global.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.store_gateway.env }} + {{- toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.store_gateway.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml new file mode 100644 index 00000000000..d79c9f940e9 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "store-gateway") }}-headless + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }} + prometheus.io/service-monitor: "false" + {{- with .Values.store_gateway.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.store_gateway.service.annotations | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml new file mode 100644 index 00000000000..6d157316436 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "store-gateway") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }} + {{- with .Values.store_gateway.service.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + annotations: + {{- toYaml .Values.store_gateway.service.annotations | nindent 4 }} +spec: + type: ClusterIP + ports: + - port: {{ include "mimir.serverHttpListenPort" . }} + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: {{ include "mimir.serverGrpcListenPort" . }} + protocol: TCP + name: grpc + targetPort: grpc + selector: + {{- include "mimir.selectorLabels" (dict "ctx" . "component" "store-gateway" "memberlist" true) | nindent 4 }} diff --git a/operations/helm/charts/mimir-distributed/templates/tokengen/tokengen-job.yaml b/operations/helm/charts/mimir-distributed/templates/tokengen/tokengen-job.yaml new file mode 100644 index 00000000000..2b2b890ca1c --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/tokengen/tokengen-job.yaml @@ -0,0 +1,88 @@ +{{- if .Values.enterprise.enabled -}} +{{ if .Values.tokengenJob.enable }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "mimir.resourceName" (dict "ctx" . "component" "tokengen") }} + labels: + {{- include "mimir.labels" (dict "ctx" . "component" "tokengen") | nindent 4 }} + annotations: + {{- if .Values.tokengenJob.annotations }} + {{- toYaml .Values.tokengenJob.annotations | nindent 4 }} + {{- end }} + "helm.sh/hook": post-install +spec: + backoffLimit: 6 + completions: 1 + parallelism: 1 + selector: + template: + metadata: + labels: + {{- include "mimir.podLabels" (dict "ctx" . "component" "tokengen") | nindent 8 }} + {{- with .Values.tokengenJob.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ template "mimir.serviceAccountName" . }} + {{- if .Values.tokengenJob.priorityClassName }} + priorityClassName: {{ .Values.tokengenJob.priorityClassName }} + {{- end }} + securityContext: + {{- toYaml .Values.tokengenJob.securityContext | nindent 8 }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- range .Values.image.pullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + initContainers: + {{- toYaml .Values.tokengenJob.initContainers | nindent 8 }} + containers: + - name: tokengen + image: "{{ include "mimir.imageReference" . }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - "-target=tokengen" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + {{- range $key, $value := .Values.tokengenJob.extraArgs }} + - "-{{ $key }}={{ $value }}" + {{- end }} + volumeMounts: + {{- if .Values.tokengenJob.extraVolumeMounts }} + {{ toYaml .Values.tokengenJob.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + env: + {{- with .Values.global.extraEnv }} + {{ toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tokengenJob.env }} + {{ toYaml . | nindent 12 }} + {{- end }} + envFrom: + {{- with .Values.global.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tokengenJob.extraEnvFrom }} + {{- toYaml . | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + volumes: + - name: config + secret: + secretName: {{ tpl .Values.externalConfigSecretName . }} + {{- if .Values.tokengenJob.extraVolumes }} + {{ toYaml .Values.tokengenJob.extraVolumes | nindent 8 }} + {{- end }} + - name: license + secret: + secretName: {{ tpl .Values.license.secretName . }} + - name: storage + emptyDir: {} +{{- end -}} +{{- end -}} diff --git a/operations/helm/charts/mimir-distributed/templates/validate.yaml b/operations/helm/charts/mimir-distributed/templates/validate.yaml new file mode 100644 index 00000000000..0bbfe4975fe --- /dev/null +++ b/operations/helm/charts/mimir-distributed/templates/validate.yaml @@ -0,0 +1,21 @@ +{{- $calculatedConfig := include "mimir.calculatedConfig" . | fromYaml }} + +{{- if .Values.config }} +{{- fail "Top level 'config' is not allowed. In 'mimir.config', provide the configuration as a string that can contain template expressions. Alternatively, you can provide the configuration as an external secret." }} +{{- end }} + +{{- if .Values.podDisruptionBudget }} +{{- fail "Top level 'podDisruptionBudget' is removed in favour of per component podDisruptionBudget" }} +{{- end }} + +{{- if not ($calculatedConfig.activity_tracker).filepath }} +{{- fail "The 'activity_tracker.filepath' in 'mimir.config' should be set to a path that is on a writeable volume. For example: '/data/metrics-activity.log'." }} +{{- end }} + +{{- if ($calculatedConfig.server).http_listen_port }} +{{- fail "The setting 'server.http_listen_port' is not allowed in 'mimir.config' as it leads to circular dependency and thus is not supported. Use 'nginx.service.port' or 'gateway.service.port' for enterprise to expose services on a different port." }} +{{- end }} + +{{- if ($calculatedConfig.server).grpc_listen_port }} +{{- fail "The setting 'server.grpc_listen_port' is not allowed in 'mimir.config' as it leads to circular dependency and thus is not supported." }} +{{- end }} diff --git a/operations/helm/charts/mimir-distributed/values.yaml b/operations/helm/charts/mimir-distributed/values.yaml new file mode 100644 index 00000000000..b3bd7754e57 --- /dev/null +++ b/operations/helm/charts/mimir-distributed/values.yaml @@ -0,0 +1,1583 @@ +# Grafana Enterprise Metrics specific values and features are found after the `enterprise:` key +# +# The default values specified in this file are enough to deploy all of the +# Grafana Mimir or Grafana Enterprise Metrics microservices but are not suitable for +# production load. +# To configure the resources for production load, refer to the the small.yaml or +# large.yaml values files. + +# -- Overrides the chart's name +nameOverride: null + +# -- Overrides the chart's computed fullname +fullnameOverride: null + +# Container image settings. +# Since the image is unique for all microservices, so are image settings. +image: + repository: grafana/mimir + tag: 2.1.0 + pullPolicy: IfNotPresent + # Optionally specify an array of imagePullSecrets. + # Secrets must be manually created in the namespace. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + # pullSecrets: + # - myRegistryKeySecretName + +global: + # definitions to set up nginx resolver + dnsService: kube-dns + dnsNamespace: kube-system + clusterDomain: cluster.local + + # Common environment variables to add to all PODs directly managed by this chart (mimir services, nginx). Doesn't include memcached or minio. + extraEnv: [] + + # Common source of environment injections to add to all PODs directly managed by this chart (mimir services, nginx). Doesn't include memcached or minio. + extraEnvFrom: [] + +serviceAccount: + create: true + name: + annotations: {} + +# -- Configuration is loaded from the secret called 'externalConfigSecretName'. If 'useExternalConfig' is true, then the configuration is not generated, just consumed. +useExternalConfig: false + +# -- Name of the secret that contains the configuration (used for naming even if config is internal). +externalConfigSecretName: '{{ include "mimir.resourceName" (dict "ctx" . "component" "config") }}' + +# -- When 'useExternalConfig' is true, then changing 'externalConfigVersion' triggers restart of services - otherwise changes to the configuration cause a restart. +externalConfigVersion: '0' + +mimir: + # -- Config file for Grafana Mimir, enables templates. Needs to be copied in full for modifications. + config: | + {{- if not .Values.enterprise.enabled -}} + multitenancy_enabled: false + {{- end }} + + limits: {} + + activity_tracker: + filepath: /data/metrics-activity.log + + alertmanager: + data_dir: '/data' + enable_api: true + external_url: '/alertmanager' + + {{- if .Values.minio.enabled }} + alertmanager_storage: + backend: s3 + s3: + endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000 + bucket_name: {{ include "mimir.minioBucketPrefix" . }}-ruler + access_key_id: {{ .Values.minio.accessKey }} + secret_access_key: {{ .Values.minio.secretKey }} + insecure: true + {{- end }} + + frontend_worker: + frontend_address: {{ template "mimir.fullname" . }}-query-frontend-headless.{{ .Release.Namespace }}.svc:{{ include "mimir.serverGrpcListenPort" . }} + + ruler: + enable_api: true + rule_path: '/data' + alertmanager_url: dnssrvnoa+http://_http-metrics._tcp.{{ template "mimir.fullname" . }}-alertmanager-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}/alertmanager + + server: + grpc_server_max_recv_msg_size: 104857600 + grpc_server_max_send_msg_size: 104857600 + grpc_server_max_concurrent_streams: 1000 + + frontend: + log_queries_longer_than: 10s + align_queries_with_step: true + {{- if index .Values "memcached-results" "enabled" }} + results_cache: + backend: memcached + memcached: + addresses: dns+{{ .Release.Name }}-memcached-results.{{ .Release.Namespace }}.svc:11211 + max_item_size: {{ (index .Values "memcached-results").maxItemMemory }} + cache_results: true + {{- end }} + + compactor: + data_dir: "/data" + + ingester: + ring: + final_sleep: 0s + num_tokens: 512 + unregister_on_shutdown: false + + ingester_client: + grpc_client_config: + max_recv_msg_size: 104857600 + max_send_msg_size: 104857600 + + runtime_config: + file: /var/{{ include "mimir.name" . }}/runtime.yaml + + memberlist: + abort_if_cluster_join_fails: false + compression_enabled: false + join_members: + - {{ include "mimir.fullname" . }}-gossip-ring + + # This configures how the store-gateway synchronizes blocks stored in the bucket. It uses Minio by default for getting started (configured via flags) but this should be changed for production deployments. + blocks_storage: + backend: s3 + tsdb: + dir: /data/tsdb + bucket_store: + sync_dir: /data/tsdb-sync + {{- if .Values.memcached.enabled }} + chunks_cache: + backend: memcached + memcached: + addresses: dns+{{ .Release.Name }}-memcached.{{ .Release.Namespace }}.svc:11211 + max_item_size: {{ .Values.memcached.maxItemMemory }} + timeout: 450ms + {{- end }} + {{- if index .Values "memcached-metadata" "enabled" }} + metadata_cache: + backend: memcached + memcached: + addresses: dns+{{ .Release.Name }}-memcached-metadata.{{ .Release.Namespace }}.svc:11211 + max_item_size: {{ (index .Values "memcached-metadata").maxItemMemory }} + {{- end }} + {{- if index .Values "memcached-queries" "enabled" }} + index_cache: + backend: memcached + memcached: + addresses: dns+{{ .Release.Name }}-memcached-queries.{{ .Release.Namespace }}.svc:11211 + max_item_size: {{ (index .Values "memcached-queries").maxItemMemory }} + {{- end }} + {{- if .Values.minio.enabled }} + s3: + endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000 + bucket_name: {{ include "mimir.minioBucketPrefix" . }}-tsdb + access_key_id: {{ .Values.minio.accessKey }} + secret_access_key: {{ .Values.minio.secretKey }} + insecure: true + {{- end }} + + {{- if .Values.minio.enabled }} + ruler_storage: + backend: s3 + s3: + endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000 + bucket_name: {{ include "mimir.minioBucketPrefix" . }}-ruler + access_key_id: {{ .Values.minio.accessKey }} + secret_access_key: {{ .Values.minio.secretKey }} + insecure: true + {{- end }} + + {{- if .Values.enterprise.enabled }} + multitenancy_enabled: true + + admin_api: + leader_election: + enabled: true + ring: + kvstore: + store: "memberlist" + + {{- if .Values.minio.enabled }} + admin_client: + storage: + type: s3 + s3: + endpoint: {{ .Release.Name }}-minio.{{ .Release.Namespace }}.svc:9000 + bucket_name: enterprise-metrics-admin + access_key_id: {{ .Values.minio.accessKey }} + secret_access_key: {{ .Values.minio.secretKey }} + insecure: true + {{- end }} + + auth: + type: enterprise + + cluster_name: "{{ .Release.Name }}" + + license: + path: "/license/license.jwt" + + {{- if .Values.gateway.useDefaultProxyURLs }} + gateway: + proxy: + default: + url: http://{{ template "mimir.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + admin_api: + url: http://{{ template "mimir.fullname" . }}-admin-api.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + alertmanager: + url: http://{{ template "mimir.fullname" . }}-alertmanager.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + compactor: + url: http://{{ template "mimir.fullname" . }}-compactor.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + distributor: + url: dns:///{{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverGrpcListenPort" . }} + ingester: + url: http://{{ template "mimir.fullname" . }}-ingester.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + query_frontend: + url: http://{{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + ruler: + url: http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + store_gateway: + url: http://{{ template "mimir.fullname" . }}-store-gateway.{{ .Release.Namespace }}.svc:{{ include "mimir.serverHttpListenPort" . }} + {{- end }} + + instrumentation: + enabled: true + distributor_client: + address: 'dns:///{{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverGrpcListenPort" . }}' + + {{- end }} + + +# runtimeConfig provides a reloadable runtime configuration file for some specific configuration. +runtimeConfig: {} + +# RBAC configuration +rbac: + pspEnabled: true + +# ServiceMonitor configuration +serviceMonitor: + # -- If enabled, ServiceMonitor resources for Prometheus Operator are created + enabled: false + # -- Alternative namespace for ServiceMonitor resources + namespace: null + # -- Namespace selector for ServiceMonitor resources + namespaceSelector: {} + # -- ServiceMonitor annotations + annotations: {} + # -- Additional ServiceMonitor labels + labels: {} + # -- ServiceMonitor scrape interval + interval: null + # -- ServiceMonitor scrape timeout in Go duration format (e.g. 15s) + scrapeTimeout: null + # -- ServiceMonitor relabel configs to apply to samples before scraping + # https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig + relabelings: [] + # -- ServiceMonitor will use http by default, but you can pick https as well + scheme: http + # -- ServiceMonitor will use these tlsConfig settings to make the health check requests + tlsConfig: null + +alertmanager: + enabled: true + replicas: 1 + + statefulSet: + enabled: true + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 10m + memory: 32Mi + + extraArgs: {} + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: {} + annotations: {} + persistence: + # SubPath in emptyDir for persistence, only enabled if alertmanager.statefulSet.enabled is false + subPath: + + persistentVolume: + # If true and alertmanager.statefulSet.enabled is true, + # Alertmanager will create/use a Persistent Volume Claim + # If false, use emptyDir + enabled: true + + # Alertmanager data Persistent Volume Claim annotations + # + annotations: {} + + # Alertmanager data Persistent Volume access modes + # Must match those of existing PV or dynamic provisioner + # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + # + accessModes: + - ReadWriteOnce + + # Alertmanager data Persistent Volume size + # + size: 1Gi + + # Subdirectory of Alertmanager data Persistent Volume to mount + # Useful if the volume's root directory is not empty + # + subPath: '' + + # Alertmanager data Persistent Volume Storage Class + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + # + # storageClass: "-" + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + securityContext: {} + + # Tolerations for pod assignment + # ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + tolerations: [] + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + statefulStrategy: + type: RollingUpdate + + terminationGracePeriodSeconds: 60 + + initContainers: [] + # Init containers to be added to the alertmanager pod. + # - name: my-init-container + # image: busybox:latest + # command: ['sh', '-c', 'echo hello'] + + extraContainers: [] + # Additional containers to be added to the alertmanager pod. + # - name: reverse-proxy + # image: angelbarrera92/basic-auth-reverse-proxy:dev + # args: + # - "serve" + # - "--upstream=http://localhost:3100" + # - "--auth-config=/etc/reverse-proxy-conf/authn.yaml" + # ports: + # - name: http + # containerPort: 11811 + # protocol: TCP + # volumeMounts: + # - name: reverse-proxy-auth-config + # mountPath: /etc/reverse-proxy-conf + + extraVolumes: [] + # Additional volumes to the alertmanager pod. + # - name: reverse-proxy-auth-config + # secret: + # secretName: reverse-proxy-auth-config + + # Extra volume mounts that will be added to the alertmanager container + extraVolumeMounts: [] + + # Extra env variables to pass to the alertmanager container + env: [] + extraEnvFrom: [] + +distributor: + replicas: 1 + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 512Mi + + # Additional distributor container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - distributor + topologyKey: 'kubernetes.io/hostname' + + annotations: {} + persistence: + subPath: + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + securityContext: {} + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + terminationGracePeriodSeconds: 60 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +ingester: + replicas: 3 + + statefulSet: + enabled: true + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 512Mi + + # Additional ingester container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: + maxUnavailable: 1 + + podManagementPolicy: Parallel + + nodeSelector: {} + affinity: {} + annotations: {} + + persistentVolume: + # If true and ingester.statefulSet.enabled is true, + # Ingester will create/use a Persistent Volume Claim + # If false, use emptyDir + # + enabled: true + + # Ingester data Persistent Volume Claim annotations + # + annotations: {} + + # Ingester data Persistent Volume access modes + # Must match those of existing PV or dynamic provisioner + # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + accessModes: + - ReadWriteOnce + + # Ingester data Persistent Volume size + size: 2Gi + + # Subdirectory of Ingester data Persistent Volume to mount + # Useful if the volume's root directory is not empty + subPath: '' + + + # Ingester data Persistent Volume Storage Class + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + # + # storageClass: "-" + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + + securityContext: {} + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + statefulStrategy: + type: RollingUpdate + + terminationGracePeriodSeconds: 240 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +overrides_exporter: + enabled: true + replicas: 1 + + annotations: {} + + initContainers: [] + + service: + annotations: {} + labels: {} + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + podLabels: {} + podAnnotations: {} + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: {} + + securityContext: {} + + extraArgs: {} + + persistence: + subPath: + + livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + resources: + requests: + cpu: 100m + memory: 128Mi + + terminationGracePeriodSeconds: 60 + + tolerations: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +ruler: + enabled: true + replicas: 1 + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 128Mi + + # Additional ruler container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + # log.level: debug + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: {} + annotations: {} + persistence: + subPath: + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + securityContext: {} + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + terminationGracePeriodSeconds: 180 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +querier: + replicas: 2 + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 128Mi + + # Additional querier container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - querier + topologyKey: 'kubernetes.io/hostname' + + annotations: {} + persistence: + subPath: + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + securityContext: {} + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + terminationGracePeriodSeconds: 180 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +query_frontend: + replicas: 1 + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 128Mi + + # Additional query-frontend container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - query-frontend + topologyKey: 'kubernetes.io/hostname' + + annotations: {} + persistence: + subPath: + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + securityContext: {} + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + terminationGracePeriodSeconds: 180 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +store_gateway: + replicas: 1 + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 512Mi + + # Additional store-gateway container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: + maxUnavailable: 1 + + nodeSelector: {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - store-gateway + topologyKey: 'kubernetes.io/hostname' + + annotations: {} + + persistentVolume: + # If true Store-gateway will create/use a Persistent Volume Claim + # If false, use emptyDir + # + enabled: true + + # Store-gateway data Persistent Volume Claim annotations + # + annotations: {} + + # Store-gateway data Persistent Volume access modes + # Must match those of existing PV or dynamic provisioner + # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + # + accessModes: + - ReadWriteOnce + + # Store-gateway data Persistent Volume size + # + size: 2Gi + + # Subdirectory of Store-gateway data Persistent Volume to mount + # Useful if the volume's root directory is not empty + # + subPath: '' + + + # Store-gateway data Persistent Volume Storage Class + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + # + # storageClass: "-" + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + + securityContext: {} + + strategy: + type: RollingUpdate + + terminationGracePeriodSeconds: 240 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +compactor: + replicas: 1 + + service: + annotations: {} + labels: {} + + resources: + requests: + cpu: 100m + memory: 512Mi + + # Additional compactor container arguments, e.g. log level (debug, info, warn, error) + extraArgs: {} + + # Pod Labels + podLabels: {} + + # Pod Annotations + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - compactor + topologyKey: 'kubernetes.io/hostname' + + annotations: {} + + persistentVolume: + # If true compactor will create/use a Persistent Volume Claim + # If false, use emptyDir + # + enabled: true + + # compactor data Persistent Volume Claim annotations + # + annotations: {} + + # compactor data Persistent Volume access modes + # Must match those of existing PV or dynamic provisioner + # Ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ + # + accessModes: + - ReadWriteOnce + + # compactor data Persistent Volume size + # + size: 2Gi + + # Subdirectory of compactor data Persistent Volume to mount + # Useful if the volume's root directory is not empty + # + subPath: '' + + + # compactor data Persistent Volume Storage Class + # If defined, storageClassName: + # If set to "-", storageClassName: "", which disables dynamic provisioning + # If undefined (the default) or set to null, no storageClassName spec is + # set, choosing the default provisioner. (gp2 on AWS, standard on + # GKE, AWS & OpenStack) + # + # storageClass: "-" + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + + securityContext: {} + + strategy: + type: RollingUpdate + + terminationGracePeriodSeconds: 240 + + tolerations: [] + initContainers: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +memcached: + enabled: false + architecture: high-availability + arguments: + - -m 8192 + - -o + - modern + - -v + - -I 1m + - -c 4096 + image: + repository: memcached + tag: 1.6.9 + # maxItemMemory is in bytes. Should match memcached -I flag (which is in MB) + # It is a string to avoid https://github.com/helm/helm/issues/1707. + maxItemMemory: '1048576' # (* 1 (* 1024 1024)) + metrics: + enabled: true + image: + registry: quay.io + repository: prometheus/memcached-exporter + tag: v0.9.0 + replicaCount: 1 + resources: + limits: + # memory limits should match requests + memory: 9830Mi + requests: + cpu: 500m + # memory requests should be exceed memcached -m flag + memory: 9830Mi # (floor (* 1.2 8192)) + +memcached-queries: + enabled: false + architecture: high-availability + arguments: + - -m 2048 + - -o + - modern + - -v + - -I 15m + - -c 1024 + image: + repository: memcached + tag: 1.6.9 + # maxItemMemory is in bytes. Should match memcached -I flag (which is in MB) + # It is a string to avoid https://github.com/helm/helm/issues/1707. + maxItemMemory: '15728640' # (* 15 (* 1024 1024)) + metrics: + enabled: true + image: + registry: quay.io + repository: prometheus/memcached-exporter + tag: v0.9.0 + replicaCount: 1 + resources: + limits: + # memory limits should match requests + memory: 2457Mi + requests: + cpu: 500m + # memory requests should be exceed memcached -m flag + memory: 2457Mi # (floor (* 1.2 2048)) + +memcached-metadata: + enabled: false + architecture: high-availability + arguments: + - -m 512 + - -o + - modern + - -v + - -I 1m + - -c 1024 + image: + repository: memcached + tag: 1.6.9 + # maxItemMemory is in bytes. Should match memcached -I flag (which is in MB) + # It is a string to avoid https://github.com/helm/helm/issues/1707. + maxItemMemory: '1048576' # (* 1 (* 1024 1024)) + metrics: + enabled: true + image: + registry: quay.io + repository: prometheus/memcached-exporter + tag: v0.9.0 + replicaCount: 1 + resources: + limits: + # memory limits should match requests + memory: 614Mi + requests: + cpu: 500m + # memory requests should be exceed memcached -m flag + memory: 614Mi # (floor (* 1.2 512)) + +memcached-results: + enabled: false + architecture: high-availability + arguments: + - -m 512 + - -o + - modern + - -v + - -I 1m + - -c 1024 + image: + repository: memcached + tag: 1.6.9 + # maxItemMemory is in bytes. Should match memcached -I flag (which is in MB) + # It is a string to avoid https://github.com/helm/helm/issues/1707. + maxItemMemory: '1048576' # (* 1 (* 1024 1024)) + metrics: + enabled: true + image: + registry: quay.io + repository: prometheus/memcached-exporter + tag: v0.9.0 + replicaCount: 1 + resources: + limits: + # memory limits should match requests + memory: 614Mi + requests: + cpu: 500m + # memory requests should be exceed memcached -m flag + memory: 614Mi # (floor (* 1.2 512)) + +minio: + enabled: true + accessKey: grafana-mimir + buckets: + - name: mimir-tsdb + policy: none + purge: false + - name: mimir-ruler + policy: none + purge: false + - name: enterprise-metrics-tsdb + policy: none + purge: false + - name: enterprise-metrics-admin + policy: none + purge: false + - name: enterprise-metrics-ruler + policy: none + purge: false + persistence: + size: 5Gi + resources: + requests: + cpu: 100m + memory: 128Mi + secretKey: supersecret + +# Configuration for nginx gateway +nginx: + # -- Specifies whether nginx should be enabled + enabled: true + # -- Number of replicas for nginx + replicas: 1 + # -- Enable logging of 2xx and 3xx HTTP requests + verboseLogging: true + autoscaling: + # -- Enable autoscaling for nginx + enabled: false + # -- Minimum autoscaling replicas for nginx + minReplicas: 1 + # -- Maximum autoscaling replicas for nginx + maxReplicas: 3 + # -- Target CPU utilisation percentage for nginx + targetCPUUtilizationPercentage: 60 + # -- Target memory utilisation percentage for nginx + targetMemoryUtilizationPercentage: + # -- See `kubectl explain deployment.spec.strategy` for more, + # ref: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy + deploymentStrategy: + type: RollingUpdate + image: + # -- The Docker registry for nginx image + registry: docker.io + # -- The nginx image repository + repository: nginxinc/nginx-unprivileged + # -- The nginx image tag + tag: 1.19-alpine + # -- The nginx image pull policy + pullPolicy: IfNotPresent + # -- The name of the PriorityClass for nginx pods + priorityClassName: null + # -- Labels for nginx pods + podLabels: {} + # -- Annotations for nginx pods + podAnnotations: {} + # -- Pod Disruption Budget + podDisruptionBudget: {} + # -- Additional CLI args for nginx + extraArgs: [] + # -- Environment variables to add to the nginx pods + extraEnv: [] + # -- Environment variables from secrets or configmaps to add to the nginx pods + extraEnvFrom: [] + # -- Volumes to add to the nginx pods + extraVolumes: [] + # -- Volume mounts to add to the nginx pods + extraVolumeMounts: [] + # -- The SecurityContext for nginx containers + podSecurityContext: + fsGroup: 101 + runAsGroup: 101 + runAsNonRoot: true + runAsUser: 101 + # -- The SecurityContext for nginx containers + containerSecurityContext: + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + allowPrivilegeEscalation: false + # -- Resource requests and limits for the nginx + resources: {} + # -- Grace period to allow the nginx to shutdown before it is killed + terminationGracePeriodSeconds: 30 + # -- Affinity for nginx pods. Passed through `tpl` and, thus, to be configured as string + # @default -- Hard node and soft zone anti-affinity + affinity: | + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: component + operator: In + values: + - nginx + topologyKey: failure-domain.beta.kubernetes.io/zone + + annotations: {} + + # -- Node selector for nginx pods + nodeSelector: {} + # -- Tolerations for nginx pods + tolerations: [] + # Nginx service configuration + service: + # -- Port of the nginx service + port: 80 + # -- Type of the nginx service + type: ClusterIP + # -- ClusterIP of the nginx service + clusterIP: null + # -- Node port if service type is NodePort + nodePort: null + # -- Load balancer IPO address if service type is LoadBalancer + loadBalancerIP: null + # -- Annotations for the nginx service + annotations: {} + # -- Labels for nginx service + labels: {} + # Ingress configuration + ingress: + # -- Specifies whether an ingress for the nginx should be created + enabled: false + # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 + # ingressClassName: nginx + # -- Annotations for the nginx ingress + annotations: {} + # -- Hosts configuration for the nginx ingress + hosts: + - host: nginx.loki.example.com + paths: + - path: / + # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers + # pathType: Prefix + # -- TLS configuration for the nginx ingress + tls: + - secretName: loki-nginx-tls + hosts: + - nginx.loki.example.com + # Basic auth configuration + basicAuth: + # -- Enables basic authentication for nginx + enabled: false + # -- The basic auth username for nginx + username: null + # -- The basic auth password for nginx + password: null + # -- Uses the specified username and password to compute a htpasswd using Sprig's `htpasswd` function. + # The value is templated using `tpl`. Override this to use a custom htpasswd, e.g. in case the default causes + # high CPU load. + htpasswd: >- + {{ htpasswd (required "'nginx.basicAuth.username' is required" .Values.nginx.basicAuth.username) (required "'nginx.basicAuth.password' is required" .Values.nginx.basicAuth.password) }} + # -- Existing basic auth secret to use. Must contain '.htpasswd' + existingSecret: null + # Configures the readiness probe for nginx + readinessProbe: + httpGet: + path: / + port: http-metric + initialDelaySeconds: 15 + timeoutSeconds: 1 + nginxConfig: + # -- NGINX log format + logFormat: |- + main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + # -- Allows appending custom configuration to the server block + serverSnippet: "" + # -- Allows appending custom configuration to the http block + httpSnippet: "" + # -- Config file contents for Nginx. Passed through the `tpl` function to allow templating + # @default -- See values.yaml + file: | + worker_processes 5; ## Default: 1 + error_log /dev/stderr; + pid /tmp/nginx.pid; + worker_rlimit_nofile 8192; + + events { + worker_connections 4096; ## Default: 1024 + } + + http { + client_body_temp_path /tmp/client_temp; + proxy_temp_path /tmp/proxy_temp_path; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + default_type application/octet-stream; + log_format {{ .Values.nginx.nginxConfig.logFormat }} + + {{- if .Values.nginx.verboseLogging }} + access_log /dev/stderr main; + {{- else }} + + map $status $loggable { + ~^[23] 0; + default 1; + } + access_log /dev/stderr main if=$loggable; + {{- end }} + + sendfile on; + tcp_nopush on; + resolver {{ .Values.global.dnsService }}.{{ .Values.global.dnsNamespace }}.svc.{{ .Values.global.clusterDomain }}; + + {{- with .Values.nginx.nginxConfig.httpSnippet }} + {{ . | nindent 2 }} + {{- end }} + + server { + listen 8080; + + {{- if .Values.nginx.basicAuth.enabled }} + auth_basic "Mimir"; + auth_basic_user_file /etc/nginx/secrets/.htpasswd; + {{- end }} + + location = / { + return 200 'OK'; + auth_basic off; + } + + {{- if not (include "mimir.calculatedConfig" . | fromYaml).multitenancy_enabled }} + proxy_set_header X-Scope-OrgID 0; + {{- end }} + + # Distributor endpoints + location /distributor { + proxy_pass http://{{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location = /api/v1/push { + proxy_pass http://{{ template "mimir.fullname" . }}-distributor-headless.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + + # Alertmanager endpoints + location {{ template "mimir.alertmanagerHttpPrefix" . }} { + proxy_pass http://{{ template "mimir.fullname" . }}-alertmanager.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location = /multitenant_alertmanager/status { + proxy_pass http://{{ template "mimir.fullname" . }}-alertmanager.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location = /api/v1/alerts { + proxy_pass http://{{ template "mimir.fullname" . }}-alertmanager.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + + # Ruler endpoints + location {{ template "mimir.prometheusHttpPrefix" . }}/config/v1/rules { + proxy_pass http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location {{ template "mimir.prometheusHttpPrefix" . }}/api/v1/rules { + proxy_pass http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + + location /api/v1/rules { + proxy_pass http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location {{ template "mimir.prometheusHttpPrefix" . }}/api/v1/alerts { + proxy_pass http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location {{ template "mimir.prometheusHttpPrefix" . }}/rules { + proxy_pass http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + location = /ruler/ring { + proxy_pass http://{{ template "mimir.fullname" . }}-ruler.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + + # Rest of {{ template "mimir.prometheusHttpPrefix" . }} goes to the query frontend + location {{ template "mimir.prometheusHttpPrefix" . }} { + proxy_pass http://{{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + + # Buildinfo endpoint can go to any component + location = /api/v1/status/buildinfo { + proxy_pass http://{{ template "mimir.fullname" . }}-query-frontend.{{ .Release.Namespace }}.svc.{{ .Values.global.clusterDomain }}:{{ include "mimir.serverHttpListenPort" . }}$request_uri; + } + + {{- with .Values.nginx.nginxConfig.serverSnippet }} + {{ . | nindent 4 }} + {{- end }} + } + } + +############################################################################## +# The values in and after the `enterprise:` key configure the enterprise features +enterprise: + # Enable enterprise features. License must be provided, nginx gateway is not installed, instead + # the enterprise gateway is used. + enabled: false + + # Whether to generate pre-2.0 Grafana Enterprise Metrics resource labels and selectors, or generate new Kubernetes standard selectors. + # Rolling upgrade from version 1.7.x without downtime requires this setting to be true. Fresh installation or upgrade with downtime can set + # it to false. + legacyLabels: false + + # Container image settings for enterprise, note that pullPolicy and pullSecrets are set in top level .image + image: + repository: grafana/enterprise-metrics + tag: v2.1.0 + +# In order to use Grafana Enterprise Metrics features, you will need to provide the contents of your Grafana Enterprise Metrics +# license, either by providing the contents of the license.jwt, or the name Kubernetes Secret that contains your license.jwt. +# To set the license contents, use the flag `--set-file 'license.contents=./license.jwt'` +# To use your own Kubernetes Secret, `--set license.external=true`. +license: + contents: "NOTAVALIDLICENSE" + external: false + secretName: '{{ include "mimir.resourceName" (dict "ctx" . "component" "license") }}' + +# Settings for the initial admin(istrator) token generator job. Can only be enabled if +# enterprise.enabled is true - requires license. +tokengenJob: + enable: true + extraArgs: {} + env: [] + extraEnvFrom: [] + annotations: {} + initContainers: [] + +# Settings for the admin_api service providing authentication and authorization service. +# Can only be enabled if enterprise.enabled is true - requires license. +admin_api: + replicas: 1 + + annotations: {} + service: + annotations: {} + labels: {} + + initContainers: [] + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + podLabels: {} + podAnnotations: {} + + nodeSelector: {} + affinity: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + securityContext: {} + + extraArgs: {} + + persistence: + subPath: + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + resources: + requests: + cpu: 10m + memory: 32Mi + + terminationGracePeriodSeconds: 60 + + tolerations: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + +# Settings for the gateway service providing authentication and authorization via the admin_api. +# Can only be enabled if enterprise.enabled is true - requires license. +gateway: + # If you want to use your own proxy URLs, set this to false. + useDefaultProxyURLs: true + replicas: 1 + + annotations: {} + service: + annotations: {} + labels: {} + # -- If the port is left undefined, the service will listen on the same port as the pod + port: null + + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + podLabels: {} + podAnnotations: {} + + # Pod Disruption Budget + podDisruptionBudget: {} + + nodeSelector: {} + affinity: {} + + securityContext: {} + initContainers: [] + + extraArgs: {} + + persistence: + subPath: + + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + + resources: + requests: + cpu: 10m + memory: 32Mi + + terminationGracePeriodSeconds: 60 + + tolerations: [] + extraContainers: [] + extraVolumes: [] + extraVolumeMounts: [] + env: [] + extraEnvFrom: [] + + # Ingress configuration + ingress: + # -- Specifies whether an ingress for the gateway should be created + enabled: false + # -- Ingress Class Name. MAY be required for Kubernetes versions >= 1.18 + # ingressClassName: gateway + # -- Annotations for the gateway ingress + annotations: {} + # -- Hosts configuration for the gateway ingress + hosts: + - host: gateway.gem.example.com + paths: + - path: / + # -- pathType (e.g. ImplementationSpecific, Prefix, .. etc.) might also be required by some Ingress Controllers + # pathType: Prefix + # -- TLS configuration for the gateway ingress + tls: + - secretName: gem-gateway-tls + hosts: + - gateway.gem.example.com diff --git a/operations/helm/cr.yaml b/operations/helm/cr.yaml new file mode 100644 index 00000000000..45d502964bd --- /dev/null +++ b/operations/helm/cr.yaml @@ -0,0 +1,5 @@ +git-repo: helm-charts +key: Grafana Loki +owner: grafana +sign: true +skip-existing: true diff --git a/operations/helm/ct.yaml b/operations/helm/ct.yaml new file mode 100644 index 00000000000..ddbf04d5833 --- /dev/null +++ b/operations/helm/ct.yaml @@ -0,0 +1,11 @@ +# See https://github.com/helm/chart-testing#configuration +remote: origin +target-branch: main +chart-dirs: + - operations/helm/charts +chart-repos: + - bitnami=https://charts.bitnami.com/bitnami + - bitnami-pre-2022=https://raw.githubusercontent.com/bitnami/charts/eb5f9a9513d987b519f0ecd732e7031241c50328/bitnami + - minio=https://helm.min.io +helm-extra-args: --timeout 600s +validate-maintainers: false diff --git a/operations/helm/tests/build.sh b/operations/helm/tests/build.sh new file mode 100755 index 00000000000..b281e960299 --- /dev/null +++ b/operations/helm/tests/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: AGPL-3.0-only + +set -euo pipefail + +CHART_PATH=operations/helm/charts/mimir-distributed + +# Start from a clean slate +rm -rf operations/helm/tests/*-generated + +# Install chart dependencies for this branch +helm dependency update "$CHART_PATH" + +# Locally render the chart for every test file +TESTS=$(ls -1 ${CHART_PATH}/ci/*values.yaml) + +for FILEPATH in $TESTS; do + # Extract the filename (without extension). + TEST_NAME=$(basename -s '.yaml' "$FILEPATH") + + echo "Templating $TEST_NAME" + helm template "${TEST_NAME}" ${CHART_PATH} -f "${FILEPATH}" --output-dir "operations/helm/tests/${TEST_NAME}-generated" +done diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml new file mode 100644 index 00000000000..9ab8577d049 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml @@ -0,0 +1,111 @@ +--- +# Source: mimir-distributed/charts/minio/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-enterprise-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +data: + initialize: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for Minio service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/accesskey) ; SECRET=$(cat /config/secretkey) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to Minio server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} config host add myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkBucketExists ($bucket) + # Check if the bucket exists, by using the exit code of `mc ls` + checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} ls myminio/$BUCKET > /dev/null 2>&1) + return $? + } + + # createBucket ($bucket, $policy, $purge) + # Ensure bucket exists, purging if asked to + createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET ; then + echo "Purging bucket '$BUCKET'." + set +e ; # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e ; # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist + if ! checkBucketExists $BUCKET ; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + + + # set versioning for bucket + if [ ! -z $VERSIONING ] ; then + if [ $VERSIONING = true ] ; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ] ; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} policy set $POLICY myminio/$BUCKET + } + + # Try connecting to Minio instance + scheme=http + connectToMinio $scheme + # Create the buckets + createBucket mimir-tsdb none false + createBucket mimir-ruler none false + createBucket enterprise-metrics-tsdb none false + createBucket enterprise-metrics-admin none false + createBucket enterprise-metrics-ruler none false diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml new file mode 100644 index 00000000000..098d932170e --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml @@ -0,0 +1,72 @@ +--- +# Source: mimir-distributed/charts/minio/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-enterprise-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + selector: + matchLabels: + app: minio + release: test-enterprise-values + template: + metadata: + name: test-enterprise-values-minio + labels: + app: minio + release: test-enterprise-values + annotations: + checksum/secrets: aac2dea246043210c2649ffaf6c6ea57cd94d0aaf2b21759b8b38a093096478e + checksum/config: 8109517e3e9f729fb84cca8217b59099354497df0ca310f4e4b07d4951b02dc4 + spec: + serviceAccountName: "test-enterprise-values-minio" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: minio + image: "minio/minio:RELEASE.2021-02-14T04-01-33Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs/ server /export" + volumeMounts: + - name: export + mountPath: /export + ports: + - name: http + containerPort: 9000 + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: test-enterprise-values-minio + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: test-enterprise-values-minio + key: secretkey + resources: + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: export + persistentVolumeClaim: + claimName: test-enterprise-values-minio + - name: minio-user + secret: + secretName: test-enterprise-values-minio diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml new file mode 100644 index 00000000000..0f09bf58309 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml @@ -0,0 +1,47 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: test-enterprise-values-minio-make-bucket-job + labels: + app: minio-make-bucket-job + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + labels: + app: minio-job + release: test-enterprise-values + spec: + restartPolicy: OnFailure + volumes: + - name: minio-configuration + projected: + sources: + - configMap: + name: test-enterprise-values-minio + - secret: + name: test-enterprise-values-minio + serviceAccountName: "test-enterprise-values-minio" + containers: + - name: minio-mc + image: "minio/mc:RELEASE.2021-02-14T04-28-06Z" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: test-enterprise-values-minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: minio-configuration + mountPath: /config + resources: + requests: + memory: 128Mi diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml new file mode 100644 index 00000000000..060b5cf2bc2 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml @@ -0,0 +1,37 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: test-enterprise-values-minio-update-prometheus-secret + labels: + app: minio-update-prometheus-secret + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch + resourceNames: + - test-enterprise-values-minio-prometheus + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + resourceNames: + - test-enterprise-values-minio diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml new file mode 100644 index 00000000000..0d793749d0b --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml @@ -0,0 +1,19 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-enterprise-values-minio-update-prometheus-secret + labels: + app: minio-update-prometheus-secret + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-enterprise-values-minio-update-prometheus-secret +subjects: + - kind: ServiceAccount + name: test-enterprise-values-minio-update-prometheus-secret + namespace: "default" diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml new file mode 100644 index 00000000000..d41d8705aa8 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml @@ -0,0 +1,11 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-enterprise-values-minio-update-prometheus-secret + labels: + app: minio-update-prometheus-secret + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/pvc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/pvc.yaml new file mode 100644 index 00000000000..d2d46159fe6 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/pvc.yaml @@ -0,0 +1,17 @@ +--- +# Source: mimir-distributed/charts/minio/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-enterprise-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "5Gi" diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/secrets.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/secrets.yaml new file mode 100644 index 00000000000..94e52b472c6 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/secrets.yaml @@ -0,0 +1,15 @@ +--- +# Source: mimir-distributed/charts/minio/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-enterprise-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +type: Opaque +data: + accesskey: "Z3JhZmFuYS1taW1pcg==" + secretkey: "c3VwZXJzZWNyZXQ=" diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/service.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/service.yaml new file mode 100644 index 00000000000..21f8a4c4fb2 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/service.yaml @@ -0,0 +1,21 @@ +--- +# Source: mimir-distributed/charts/minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-enterprise-values + heritage: Helm +spec: + type: ClusterIP + ports: + - name: http + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: test-enterprise-values diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml new file mode 100644 index 00000000000..9c86f090671 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +--- +# Source: mimir-distributed/charts/minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "test-enterprise-values-minio" + namespace: "default" + labels: + app: minio + chart: minio-8.0.10 + release: "test-enterprise-values" diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-dep.yaml new file mode 100644 index 00000000000..6fc0d5ff1d7 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-dep.yaml @@ -0,0 +1,107 @@ +--- +# Source: mimir-distributed/templates/admin-api/admin-api-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {} + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: admin-api + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + name: test-enterprise-values-mimir-admin-api +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: admin-api + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: admin-api + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + containers: + - name: admin-api + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=admin-api" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: license + mountPath: /license + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-svc.yaml new file mode 100644 index 00000000000..dbd9cb9ba9a --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/admin-api/admin-api-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/admin-api/admin-api-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-admin-api + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: admin-api + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: admin-api diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml new file mode 100644 index 00000000000..3f593bc2bea --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml @@ -0,0 +1,109 @@ +--- +# Source: mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-enterprise-values-mimir-alertmanager + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: alertmanager + updateStrategy: + type: RollingUpdate + serviceName: test-enterprise-values-mimir-alertmanager + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} + - name: tmp + emptyDir: {} + containers: + - name: alertmanager + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=alertmanager" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + - name: tmp + mountPath: /tmp + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml new file mode 100644 index 00000000000..e5e72b636cb --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml @@ -0,0 +1,37 @@ +--- +# Source: mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-alertmanager-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + - port: 9094 + protocol: TCP + name: cluster + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: alertmanager diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml new file mode 100644 index 00000000000..37975bf32b0 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/alertmanager/alertmanager-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-alertmanager + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: alertmanager diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml new file mode 100644 index 00000000000..55f0cc868f8 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml @@ -0,0 +1,115 @@ +--- +# Source: mimir-distributed/templates/compactor/compactor-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-enterprise-values-mimir-compactor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: compactor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: compactor + updateStrategy: + type: RollingUpdate + serviceName: test-enterprise-values-mimir-compactor + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: compactor + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - compactor + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + [] + terminationGracePeriodSeconds: 240 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} + containers: + - name: compactor + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=compactor" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml new file mode 100644 index 00000000000..fe7b12606b3 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/compactor/compactor-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-compactor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: compactor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: compactor diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml new file mode 100644 index 00000000000..d81968bc00a --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml @@ -0,0 +1,116 @@ +--- +# Source: mimir-distributed/templates/distributor/distributor-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-enterprise-values-mimir-distributor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: distributor + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: distributor + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=distributor" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - distributor + topologyKey: kubernetes.io/hostname + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml new file mode 100644 index 00000000000..ad4a9f1ceff --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml @@ -0,0 +1,29 @@ +--- +# Source: mimir-distributed/templates/distributor/distributor-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-distributor-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: distributor diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml new file mode 100644 index 00000000000..af815ba312c --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/distributor/distributor-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-distributor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: distributor diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-dep.yaml new file mode 100644 index 00000000000..a946c6dbcca --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-dep.yaml @@ -0,0 +1,100 @@ +--- +# Source: mimir-distributed/templates/gateway/gateway-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {} + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: gateway + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + name: test-enterprise-values-mimir-gateway +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: gateway + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: gateway + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: gateway + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=gateway" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: license + mountPath: /license + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-svc.yaml new file mode 100644 index 00000000000..4d19d71a1a9 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gateway/gateway-svc.yaml @@ -0,0 +1,26 @@ +--- +# Source: mimir-distributed/templates/gateway/gateway-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: gateway + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: gateway diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml new file mode 100644 index 00000000000..51af4c76f2c --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml @@ -0,0 +1,26 @@ +--- +# Source: mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-gossip-ring + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: gossip-ring + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + clusterIP: None + ports: + - name: gossip-ring + port: 7946 + protocol: TCP + targetPort: 7946 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/part-of: memberlist diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml new file mode 100644 index 00000000000..1bb947a3e7e --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml @@ -0,0 +1,21 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-pdb.yaml +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: test-enterprise-values-mimir-ingester + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester + maxUnavailable: 1 diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml new file mode 100644 index 00000000000..5e1efc64f63 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml @@ -0,0 +1,106 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-enterprise-values-mimir-ingester + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester + updateStrategy: + type: RollingUpdate + serviceName: test-enterprise-values-mimir-ingester-headless + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 240 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} + containers: + - name: ingester + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=ingester" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + - name: license + mountPath: /license + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + resources: + requests: + cpu: 10m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-svc-headless.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-svc-headless.yaml new file mode 100644 index 00000000000..f91780d4ac9 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-svc-headless.yaml @@ -0,0 +1,29 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-ingester-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-svc.yaml new file mode 100644 index 00000000000..1301971836a --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ingester/ingester-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-ingester + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ingester diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/license-secret.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/license-secret.yaml new file mode 100644 index 00000000000..5846877f9bc --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/license-secret.yaml @@ -0,0 +1,14 @@ +--- +# Source: mimir-distributed/templates/license-secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-enterprise-values-mimir-license + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +data: + license.jwt: Tk9UQVZBTElETElDRU5TRQ== diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/minio/create-bucket-job.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/minio/create-bucket-job.yaml new file mode 100644 index 00000000000..11e3e42d177 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/minio/create-bucket-job.yaml @@ -0,0 +1,44 @@ +--- +# Source: mimir-distributed/templates/minio/create-bucket-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: test-enterprise-values-mimir-distributed-make-bucket-job + namespace: "default" + labels: + app: mimir-distributed-make-bucket-job + chart: mimir-distributed-2.2.0-weekly.189 + release: test-enterprise-values + heritage: Helm +spec: + template: + metadata: + labels: + app: mimir-distributed-job + release: test-enterprise-values + spec: + restartPolicy: OnFailure + volumes: + - name: minio-configuration + projected: + sources: + - configMap: + name: test-enterprise-values-minio + - secret: + name: test-enterprise-values-minio + containers: + - name: minio-mc + image: "minio/mc:RELEASE.2021-02-14T04-28-06Z" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: test-enterprise-values-minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: minio-configuration + mountPath: /config + resources: + requests: + memory: 128Mi diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml new file mode 100644 index 00000000000..f9c31aa7108 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml @@ -0,0 +1,106 @@ +--- +# Source: mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {} + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: overrides-exporter + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + name: test-enterprise-values-mimir-overrides-exporter +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: overrides-exporter + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: overrides-exporter + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: overrides-exporter + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=overrides-exporter" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml new file mode 100644 index 00000000000..c27fc7a1cf1 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml @@ -0,0 +1,30 @@ +--- +# Source: mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-overrides-exporter + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: overrides-exporter + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: overrides-exporter diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/podsecuritypolicy.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000000..3964fffda80 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/podsecuritypolicy.yaml @@ -0,0 +1,40 @@ +--- +# Source: mimir-distributed/templates/podsecuritypolicy.yaml +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: test-enterprise-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'configMap' + - 'emptyDir' + - 'persistentVolumeClaim' + - 'secret' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/querier/querier-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/querier/querier-dep.yaml new file mode 100644 index 00000000000..237f6aefd0d --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/querier/querier-dep.yaml @@ -0,0 +1,118 @@ +--- +# Source: mimir-distributed/templates/querier/querier-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-enterprise-values-mimir-querier + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: querier + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: querier + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: querier + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: querier + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=querier" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - querier + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + [] + terminationGracePeriodSeconds: 180 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/querier/querier-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/querier/querier-svc.yaml new file mode 100644 index 00000000000..ded06cef9e3 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/querier/querier-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/querier/querier-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-querier + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: querier + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: querier diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml new file mode 100644 index 00000000000..aa598680f71 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml @@ -0,0 +1,112 @@ +--- +# Source: mimir-distributed/templates/query-frontend/query-frontend-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-enterprise-values-mimir-query-frontend + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: query-frontend + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: query-frontend + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: query-frontend + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: query-frontend + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=query-frontend" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: runtime-config + mountPath: /var/mimir + - name: license + mountPath: /license + - name: config + mountPath: /etc/mimir + - name: storage + mountPath: /data + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - query-frontend + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + [] + terminationGracePeriodSeconds: 180 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml new file mode 100644 index 00000000000..55290abe0c7 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml @@ -0,0 +1,33 @@ +--- +# Source: mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-query-frontend-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: query-frontend + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: query-frontend diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml new file mode 100644 index 00000000000..134e5fa6655 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml @@ -0,0 +1,30 @@ +--- +# Source: mimir-distributed/templates/query-frontend/query-frontend-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-query-frontend + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: query-frontend + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: query-frontend diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/role.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/role.yaml new file mode 100644 index 00000000000..0bfe0d93990 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/role.yaml @@ -0,0 +1,17 @@ +--- +# Source: mimir-distributed/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: test-enterprise-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [test-enterprise-values-mimir] diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/rolebinding.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/rolebinding.yaml new file mode 100644 index 00000000000..daf73bf5a48 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +--- +# Source: mimir-distributed/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-enterprise-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-enterprise-values-mimir +subjects: +- kind: ServiceAccount + name: test-enterprise-values-mimir diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml new file mode 100644 index 00000000000..b40077a5b2a --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml @@ -0,0 +1,108 @@ +--- +# Source: mimir-distributed/templates/ruler/ruler-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-enterprise-values-mimir-ruler + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ruler + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ruler + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: ruler + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: ruler + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=ruler" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 180 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-svc.yaml new file mode 100644 index 00000000000..95bc0b22dd8 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-svc.yaml @@ -0,0 +1,27 @@ +--- +# Source: mimir-distributed/templates/ruler/ruler-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-ruler + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ruler + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: ruler diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/runtime-configmap.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/runtime-configmap.yaml new file mode 100644 index 00000000000..0974615fc37 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/runtime-configmap.yaml @@ -0,0 +1,16 @@ +--- +# Source: mimir-distributed/templates/runtime-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-enterprise-values-mimir-runtime + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +data: + runtime.yaml: | + + {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/secret.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/secret.yaml new file mode 100644 index 00000000000..a10a89ff534 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/secret.yaml @@ -0,0 +1,14 @@ +--- +# Source: mimir-distributed/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-enterprise-values-mimir-config + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +data: + mimir.yaml: CgpsaW1pdHM6IHt9CgphY3Rpdml0eV90cmFja2VyOgogIGZpbGVwYXRoOiAvZGF0YS9tZXRyaWNzLWFjdGl2aXR5LmxvZwoKYWxlcnRtYW5hZ2VyOgogIGRhdGFfZGlyOiAnL2RhdGEnCiAgZW5hYmxlX2FwaTogdHJ1ZQogIGV4dGVybmFsX3VybDogJy9hbGVydG1hbmFnZXInCmFsZXJ0bWFuYWdlcl9zdG9yYWdlOgogIGJhY2tlbmQ6IHMzCiAgczM6CiAgICBlbmRwb2ludDogdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW5pby5kZWZhdWx0LnN2Yzo5MDAwCiAgICBidWNrZXRfbmFtZTogbWltaXItcnVsZXIKICAgIGFjY2Vzc19rZXlfaWQ6IGdyYWZhbmEtbWltaXIKICAgIHNlY3JldF9hY2Nlc3Nfa2V5OiBzdXBlcnNlY3JldAogICAgaW5zZWN1cmU6IHRydWUKCmZyb250ZW5kX3dvcmtlcjoKICBmcm9udGVuZF9hZGRyZXNzOiB0ZXN0LWVudGVycHJpc2UtdmFsdWVzLW1pbWlyLXF1ZXJ5LWZyb250ZW5kLWhlYWRsZXNzLmRlZmF1bHQuc3ZjOjkwOTUKCnJ1bGVyOgogIGVuYWJsZV9hcGk6IHRydWUKICBydWxlX3BhdGg6ICcvZGF0YScKICBhbGVydG1hbmFnZXJfdXJsOiBkbnNzcnZub2EraHR0cDovL19odHRwLW1ldHJpY3MuX3RjcC50ZXN0LWVudGVycHJpc2UtdmFsdWVzLW1pbWlyLWFsZXJ0bWFuYWdlci1oZWFkbGVzcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsL2FsZXJ0bWFuYWdlcgoKc2VydmVyOgogIGdycGNfc2VydmVyX21heF9yZWN2X21zZ19zaXplOiAxMDQ4NTc2MDAKICBncnBjX3NlcnZlcl9tYXhfc2VuZF9tc2dfc2l6ZTogMTA0ODU3NjAwCiAgZ3JwY19zZXJ2ZXJfbWF4X2NvbmN1cnJlbnRfc3RyZWFtczogMTAwMAoKZnJvbnRlbmQ6CiAgbG9nX3F1ZXJpZXNfbG9uZ2VyX3RoYW46IDEwcwogIGFsaWduX3F1ZXJpZXNfd2l0aF9zdGVwOiB0cnVlCgpjb21wYWN0b3I6CiAgZGF0YV9kaXI6ICIvZGF0YSIKCmluZ2VzdGVyOgogIHJpbmc6CiAgICBmaW5hbF9zbGVlcDogMHMKICAgIG51bV90b2tlbnM6IDUxMgogICAgdW5yZWdpc3Rlcl9vbl9zaHV0ZG93bjogZmFsc2UKCmluZ2VzdGVyX2NsaWVudDoKICBncnBjX2NsaWVudF9jb25maWc6CiAgICBtYXhfcmVjdl9tc2dfc2l6ZTogMTA0ODU3NjAwCiAgICBtYXhfc2VuZF9tc2dfc2l6ZTogMTA0ODU3NjAwCgpydW50aW1lX2NvbmZpZzoKICBmaWxlOiAvdmFyL21pbWlyL3J1bnRpbWUueWFtbAoKbWVtYmVybGlzdDoKICBhYm9ydF9pZl9jbHVzdGVyX2pvaW5fZmFpbHM6IGZhbHNlCiAgY29tcHJlc3Npb25fZW5hYmxlZDogZmFsc2UKICBqb2luX21lbWJlcnM6CiAgLSB0ZXN0LWVudGVycHJpc2UtdmFsdWVzLW1pbWlyLWdvc3NpcC1yaW5nCgojIFRoaXMgY29uZmlndXJlcyBob3cgdGhlIHN0b3JlLWdhdGV3YXkgc3luY2hyb25pemVzIGJsb2NrcyBzdG9yZWQgaW4gdGhlIGJ1Y2tldC4gSXQgdXNlcyBNaW5pbyBieSBkZWZhdWx0IGZvciBnZXR0aW5nIHN0YXJ0ZWQgKGNvbmZpZ3VyZWQgdmlhIGZsYWdzKSBidXQgdGhpcyBzaG91bGQgYmUgY2hhbmdlZCBmb3IgcHJvZHVjdGlvbiBkZXBsb3ltZW50cy4KYmxvY2tzX3N0b3JhZ2U6CiAgYmFja2VuZDogczMKICB0c2RiOgogICAgZGlyOiAvZGF0YS90c2RiCiAgYnVja2V0X3N0b3JlOgogICAgc3luY19kaXI6IC9kYXRhL3RzZGItc3luYwogIHMzOgogICAgZW5kcG9pbnQ6IHRlc3QtZW50ZXJwcmlzZS12YWx1ZXMtbWluaW8uZGVmYXVsdC5zdmM6OTAwMAogICAgYnVja2V0X25hbWU6IG1pbWlyLXRzZGIKICAgIGFjY2Vzc19rZXlfaWQ6IGdyYWZhbmEtbWltaXIKICAgIHNlY3JldF9hY2Nlc3Nfa2V5OiBzdXBlcnNlY3JldAogICAgaW5zZWN1cmU6IHRydWUKcnVsZXJfc3RvcmFnZToKICBiYWNrZW5kOiBzMwogIHMzOgogICAgZW5kcG9pbnQ6IHRlc3QtZW50ZXJwcmlzZS12YWx1ZXMtbWluaW8uZGVmYXVsdC5zdmM6OTAwMAogICAgYnVja2V0X25hbWU6IG1pbWlyLXJ1bGVyCiAgICBhY2Nlc3Nfa2V5X2lkOiBncmFmYW5hLW1pbWlyCiAgICBzZWNyZXRfYWNjZXNzX2tleTogc3VwZXJzZWNyZXQKICAgIGluc2VjdXJlOiB0cnVlCm11bHRpdGVuYW5jeV9lbmFibGVkOiB0cnVlCgphZG1pbl9hcGk6CiAgbGVhZGVyX2VsZWN0aW9uOgogICAgZW5hYmxlZDogdHJ1ZQogICAgcmluZzoKICAgICAga3ZzdG9yZToKICAgICAgICBzdG9yZTogIm1lbWJlcmxpc3QiCmFkbWluX2NsaWVudDoKICBzdG9yYWdlOgogICAgdHlwZTogczMKICAgIHMzOgogICAgICBlbmRwb2ludDogdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW5pby5kZWZhdWx0LnN2Yzo5MDAwCiAgICAgIGJ1Y2tldF9uYW1lOiBlbnRlcnByaXNlLW1ldHJpY3MtYWRtaW4KICAgICAgYWNjZXNzX2tleV9pZDogZ3JhZmFuYS1taW1pcgogICAgICBzZWNyZXRfYWNjZXNzX2tleTogc3VwZXJzZWNyZXQKICAgICAgaW5zZWN1cmU6IHRydWUKCmF1dGg6CiAgdHlwZTogZW50ZXJwcmlzZQoKY2x1c3Rlcl9uYW1lOiAidGVzdC1lbnRlcnByaXNlLXZhbHVlcyIKCmxpY2Vuc2U6CiAgcGF0aDogIi9saWNlbnNlL2xpY2Vuc2Uuand0IgpnYXRld2F5OgogIHByb3h5OgogICAgZGVmYXVsdDoKICAgICAgdXJsOiBodHRwOi8vdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW1pci1hZG1pbi1hcGkuZGVmYXVsdC5zdmM6ODA4MAogICAgYWRtaW5fYXBpOgogICAgICB1cmw6IGh0dHA6Ly90ZXN0LWVudGVycHJpc2UtdmFsdWVzLW1pbWlyLWFkbWluLWFwaS5kZWZhdWx0LnN2Yzo4MDgwCiAgICBhbGVydG1hbmFnZXI6CiAgICAgIHVybDogaHR0cDovL3Rlc3QtZW50ZXJwcmlzZS12YWx1ZXMtbWltaXItYWxlcnRtYW5hZ2VyLmRlZmF1bHQuc3ZjOjgwODAKICAgIGNvbXBhY3RvcjoKICAgICAgdXJsOiBodHRwOi8vdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW1pci1jb21wYWN0b3IuZGVmYXVsdC5zdmM6ODA4MAogICAgZGlzdHJpYnV0b3I6CiAgICAgIHVybDogZG5zOi8vL3Rlc3QtZW50ZXJwcmlzZS12YWx1ZXMtbWltaXItZGlzdHJpYnV0b3ItaGVhZGxlc3MuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbDo5MDk1CiAgICBpbmdlc3RlcjoKICAgICAgdXJsOiBodHRwOi8vdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW1pci1pbmdlc3Rlci5kZWZhdWx0LnN2Yzo4MDgwCiAgICBxdWVyeV9mcm9udGVuZDoKICAgICAgdXJsOiBodHRwOi8vdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW1pci1xdWVyeS1mcm9udGVuZC5kZWZhdWx0LnN2Yzo4MDgwCiAgICBydWxlcjoKICAgICAgdXJsOiBodHRwOi8vdGVzdC1lbnRlcnByaXNlLXZhbHVlcy1taW1pci1ydWxlci5kZWZhdWx0LnN2Yzo4MDgwCiAgICBzdG9yZV9nYXRld2F5OgogICAgICB1cmw6IGh0dHA6Ly90ZXN0LWVudGVycHJpc2UtdmFsdWVzLW1pbWlyLXN0b3JlLWdhdGV3YXkuZGVmYXVsdC5zdmM6ODA4MAoKaW5zdHJ1bWVudGF0aW9uOgogIGVuYWJsZWQ6IHRydWUKICBkaXN0cmlidXRvcl9jbGllbnQ6CiAgICBhZGRyZXNzOiAnZG5zOi8vL3Rlc3QtZW50ZXJwcmlzZS12YWx1ZXMtbWltaXItZGlzdHJpYnV0b3ItaGVhZGxlc3MuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbDo5MDk1Jwo= diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/serviceaccount.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/serviceaccount.yaml new file mode 100644 index 00000000000..78a8ad657b4 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +--- +# Source: mimir-distributed/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-enterprise-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml new file mode 100644 index 00000000000..485f99bff0d --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml @@ -0,0 +1,21 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: test-enterprise-values-mimir-store-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway + maxUnavailable: 1 diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml new file mode 100644 index 00000000000..a6fd00b3770 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml @@ -0,0 +1,113 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-enterprise-values-mimir-store-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway + updateStrategy: + type: RollingUpdate + serviceName: test-enterprise-values-mimir-store-gateway-headless + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 20297fef45efa773cd872a3e65ada08c59029003e83d3b1eace4d87bbf72ce99 + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - store-gateway + topologyKey: kubernetes.io/hostname + tolerations: + [] + terminationGracePeriodSeconds: 240 + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: runtime-config + configMap: + name: test-enterprise-values-mimir-runtime + - name: storage + emptyDir: {} + containers: + - name: store-gateway + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=store-gateway" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml new file mode 100644 index 00000000000..33c3e07dd34 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml @@ -0,0 +1,29 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-store-gateway-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml new file mode 100644 index 00000000000..a94d269f9ae --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-enterprise-values-mimir-store-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: store-gateway diff --git a/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/tokengen/tokengen-job.yaml b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/tokengen/tokengen-job.yaml new file mode 100644 index 00000000000..0e2ea38b721 --- /dev/null +++ b/operations/helm/tests/test-enterprise-values-generated/mimir-distributed/templates/tokengen/tokengen-job.yaml @@ -0,0 +1,60 @@ +--- +# Source: mimir-distributed/templates/tokengen/tokengen-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: test-enterprise-values-mimir-tokengen + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/component: tokengen + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-install +spec: + backoffLimit: 6 + completions: 1 + parallelism: 1 + selector: + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-enterprise-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: tokengen + spec: + serviceAccountName: test-enterprise-values-mimir + securityContext: + null + initContainers: + [] + containers: + - name: tokengen + image: "grafana/enterprise-metrics:v2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=tokengen" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: license + mountPath: /license + env: + envFrom: + restartPolicy: OnFailure + volumes: + - name: config + secret: + secretName: test-enterprise-values-mimir-config + - name: license + secret: + secretName: test-enterprise-values-mimir-license + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml new file mode 100644 index 00000000000..b5dc96c16cd --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/configmap.yaml @@ -0,0 +1,111 @@ +--- +# Source: mimir-distributed/charts/minio/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-oss-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +data: + initialize: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for Minio service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/accesskey) ; SECRET=$(cat /config/secretkey) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to Minio server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} config host add myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkBucketExists ($bucket) + # Check if the bucket exists, by using the exit code of `mc ls` + checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} ls myminio/$BUCKET > /dev/null 2>&1) + return $? + } + + # createBucket ($bucket, $policy, $purge) + # Ensure bucket exists, purging if asked to + createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET ; then + echo "Purging bucket '$BUCKET'." + set +e ; # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e ; # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist + if ! checkBucketExists $BUCKET ; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + + + # set versioning for bucket + if [ ! -z $VERSIONING ] ; then + if [ $VERSIONING = true ] ; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ] ; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} policy set $POLICY myminio/$BUCKET + } + + # Try connecting to Minio instance + scheme=http + connectToMinio $scheme + # Create the buckets + createBucket mimir-tsdb none false + createBucket mimir-ruler none false + createBucket enterprise-metrics-tsdb none false + createBucket enterprise-metrics-admin none false + createBucket enterprise-metrics-ruler none false diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml new file mode 100644 index 00000000000..a9f44e76c20 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/deployment.yaml @@ -0,0 +1,72 @@ +--- +# Source: mimir-distributed/charts/minio/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-oss-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + selector: + matchLabels: + app: minio + release: test-oss-values + template: + metadata: + name: test-oss-values-minio + labels: + app: minio + release: test-oss-values + annotations: + checksum/secrets: f7d601b8641bf40718836a3781f3547aa41f4d5a36774f49fbe5353840847956 + checksum/config: 32ca71423a92c84343fe0ae8158915116831302093797101294d27b284a8c78b + spec: + serviceAccountName: "test-oss-values-minio" + securityContext: + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + containers: + - name: minio + image: "minio/minio:RELEASE.2021-02-14T04-01-33Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio -S /etc/minio/certs/ server /export" + volumeMounts: + - name: export + mountPath: /export + ports: + - name: http + containerPort: 9000 + env: + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + name: test-oss-values-minio + key: accesskey + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + name: test-oss-values-minio + key: secretkey + resources: + requests: + cpu: 100m + memory: 128Mi + volumes: + - name: export + persistentVolumeClaim: + claimName: test-oss-values-minio + - name: minio-user + secret: + secretName: test-oss-values-minio diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml new file mode 100644 index 00000000000..ab4cbae0e05 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml @@ -0,0 +1,47 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-create-bucket-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: test-oss-values-minio-make-bucket-job + labels: + app: minio-make-bucket-job + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + labels: + app: minio-job + release: test-oss-values + spec: + restartPolicy: OnFailure + volumes: + - name: minio-configuration + projected: + sources: + - configMap: + name: test-oss-values-minio + - secret: + name: test-oss-values-minio + serviceAccountName: "test-oss-values-minio" + containers: + - name: minio-mc + image: "minio/mc:RELEASE.2021-02-14T04-28-06Z" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: test-oss-values-minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: minio-configuration + mountPath: /config + resources: + requests: + memory: 128Mi diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml new file mode 100644 index 00000000000..7829ff55941 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml @@ -0,0 +1,37 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: test-oss-values-minio-update-prometheus-secret + labels: + app: minio-update-prometheus-secret + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch + resourceNames: + - test-oss-values-minio-prometheus + - apiGroups: + - "" + resources: + - secrets + verbs: + - create + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - get + resourceNames: + - test-oss-values-minio diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml new file mode 100644 index 00000000000..49d29f0d92b --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml @@ -0,0 +1,19 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-oss-values-minio-update-prometheus-secret + labels: + app: minio-update-prometheus-secret + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-oss-values-minio-update-prometheus-secret +subjects: + - kind: ServiceAccount + name: test-oss-values-minio-update-prometheus-secret + namespace: "default" diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml new file mode 100644 index 00000000000..d8dec25b606 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml @@ -0,0 +1,11 @@ +--- +# Source: mimir-distributed/charts/minio/templates/post-install-prometheus-metrics-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-oss-values-minio-update-prometheus-secret + labels: + app: minio-update-prometheus-secret + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/pvc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/pvc.yaml new file mode 100644 index 00000000000..a4915536b96 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/pvc.yaml @@ -0,0 +1,17 @@ +--- +# Source: mimir-distributed/charts/minio/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: test-oss-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "5Gi" diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/secrets.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/secrets.yaml new file mode 100644 index 00000000000..dbbcc64fd5b --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/secrets.yaml @@ -0,0 +1,15 @@ +--- +# Source: mimir-distributed/charts/minio/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-oss-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +type: Opaque +data: + accesskey: "Z3JhZmFuYS1taW1pcg==" + secretkey: "c3VwZXJzZWNyZXQ=" diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/service.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/service.yaml new file mode 100644 index 00000000000..09ff81bb01a --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/service.yaml @@ -0,0 +1,21 @@ +--- +# Source: mimir-distributed/charts/minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-minio + labels: + app: minio + chart: minio-8.0.10 + release: test-oss-values + heritage: Helm +spec: + type: ClusterIP + ports: + - name: http + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: test-oss-values diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml new file mode 100644 index 00000000000..117733a9d99 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/charts/minio/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +--- +# Source: mimir-distributed/charts/minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "test-oss-values-minio" + namespace: "default" + labels: + app: minio + chart: minio-8.0.10 + release: "test-oss-values" diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml new file mode 100644 index 00000000000..be6b9c174a5 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml @@ -0,0 +1,104 @@ +--- +# Source: mimir-distributed/templates/alertmanager/alertmanager-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-oss-values-mimir-alertmanager + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: alertmanager + updateStrategy: + type: RollingUpdate + serviceName: test-oss-values-mimir-alertmanager + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} + - name: tmp + emptyDir: {} + containers: + - name: alertmanager + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=alertmanager" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + - name: tmp + mountPath: /tmp + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 10m + memory: 32Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml new file mode 100644 index 00000000000..662fb5387b6 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml @@ -0,0 +1,37 @@ +--- +# Source: mimir-distributed/templates/alertmanager/alertmanager-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-alertmanager-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + - port: 9094 + protocol: TCP + name: cluster + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: alertmanager diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml new file mode 100644 index 00000000000..3c6dcb67190 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/alertmanager/alertmanager-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/alertmanager/alertmanager-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-alertmanager + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: alertmanager + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: alertmanager diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml new file mode 100644 index 00000000000..1205d24e97b --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/compactor/compactor-statefulset.yaml @@ -0,0 +1,110 @@ +--- +# Source: mimir-distributed/templates/compactor/compactor-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-oss-values-mimir-compactor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: compactor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: compactor + updateStrategy: + type: RollingUpdate + serviceName: test-oss-values-mimir-compactor + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: compactor + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - compactor + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + [] + terminationGracePeriodSeconds: 240 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} + containers: + - name: compactor + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=compactor" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml new file mode 100644 index 00000000000..2a5af3a6e46 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/compactor/compactor-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/compactor/compactor-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-compactor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: compactor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: compactor diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml new file mode 100644 index 00000000000..d27718199d8 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-dep.yaml @@ -0,0 +1,111 @@ +--- +# Source: mimir-distributed/templates/distributor/distributor-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-oss-values-mimir-distributor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: distributor + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: distributor + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=distributor" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - distributor + topologyKey: kubernetes.io/hostname + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml new file mode 100644 index 00000000000..2cecc4be2c9 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-svc-headless.yaml @@ -0,0 +1,29 @@ +--- +# Source: mimir-distributed/templates/distributor/distributor-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-distributor-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: distributor diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml new file mode 100644 index 00000000000..ef0dff05dc7 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/distributor/distributor-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/distributor/distributor-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-distributor + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: distributor + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: distributor diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml new file mode 100644 index 00000000000..ae66fd7c268 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml @@ -0,0 +1,26 @@ +--- +# Source: mimir-distributed/templates/gossip-ring/gossip-ring-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-gossip-ring + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: gossip-ring + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + clusterIP: None + ports: + - name: gossip-ring + port: 7946 + protocol: TCP + targetPort: 7946 + publishNotReadyAddresses: true + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/part-of: memberlist diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml new file mode 100644 index 00000000000..8772a9d15af --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-pdb.yaml @@ -0,0 +1,21 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-pdb.yaml +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: test-oss-values-mimir-ingester + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester + maxUnavailable: 1 diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml new file mode 100644 index 00000000000..8b9acade53d --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml @@ -0,0 +1,101 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-oss-values-mimir-ingester + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester + updateStrategy: + type: RollingUpdate + serviceName: test-oss-values-mimir-ingester-headless + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 240 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} + containers: + - name: ingester + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=ingester" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + resources: + requests: + cpu: 10m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-svc-headless.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-svc-headless.yaml new file mode 100644 index 00000000000..bdde7e36d02 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-svc-headless.yaml @@ -0,0 +1,29 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-ingester-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-svc.yaml new file mode 100644 index 00000000000..75966d41959 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ingester/ingester-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/ingester/ingester-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-ingester + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ingester diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/minio/create-bucket-job.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/minio/create-bucket-job.yaml new file mode 100644 index 00000000000..b714a38d655 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/minio/create-bucket-job.yaml @@ -0,0 +1,44 @@ +--- +# Source: mimir-distributed/templates/minio/create-bucket-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: test-oss-values-mimir-distributed-make-bucket-job + namespace: "default" + labels: + app: mimir-distributed-make-bucket-job + chart: mimir-distributed-2.2.0-weekly.189 + release: test-oss-values + heritage: Helm +spec: + template: + metadata: + labels: + app: mimir-distributed-job + release: test-oss-values + spec: + restartPolicy: OnFailure + volumes: + - name: minio-configuration + projected: + sources: + - configMap: + name: test-oss-values-minio + - secret: + name: test-oss-values-minio + containers: + - name: minio-mc + image: "minio/mc:RELEASE.2021-02-14T04-28-06Z" + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "/config/initialize"] + env: + - name: MINIO_ENDPOINT + value: test-oss-values-minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: minio-configuration + mountPath: /config + resources: + requests: + memory: 128Mi diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-configmap.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-configmap.yaml new file mode 100644 index 00000000000..e85527322ab --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-configmap.yaml @@ -0,0 +1,101 @@ +--- +# Source: mimir-distributed/templates/nginx/nginx-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-oss-values-mimir-nginx + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: nginx + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +data: + nginx.conf: | + worker_processes 5; ## Default: 1 + error_log /dev/stderr; + pid /tmp/nginx.pid; + worker_rlimit_nofile 8192; + + events { + worker_connections 4096; ## Default: 1024 + } + + http { + client_body_temp_path /tmp/client_temp; + proxy_temp_path /tmp/proxy_temp_path; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + + default_type application/octet-stream; + log_format main '$remote_addr - $remote_user [$time_local] $status ' + '"$request" $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /dev/stderr main; + + sendfile on; + tcp_nopush on; + resolver kube-dns.kube-system.svc.cluster.local; + + server { + listen 8080; + + location = / { + return 200 'OK'; + auth_basic off; + } + proxy_set_header X-Scope-OrgID 0; + + # Distributor endpoints + location /distributor { + proxy_pass http://test-oss-values-mimir-distributor-headless.default.svc.cluster.local:8080$request_uri; + } + location = /api/v1/push { + proxy_pass http://test-oss-values-mimir-distributor-headless.default.svc.cluster.local:8080$request_uri; + } + + # Alertmanager endpoints + location /alertmanager { + proxy_pass http://test-oss-values-mimir-alertmanager.default.svc.cluster.local:8080$request_uri; + } + location = /multitenant_alertmanager/status { + proxy_pass http://test-oss-values-mimir-alertmanager.default.svc.cluster.local:8080$request_uri; + } + location = /api/v1/alerts { + proxy_pass http://test-oss-values-mimir-alertmanager.default.svc.cluster.local:8080$request_uri; + } + + # Ruler endpoints + location /prometheus/config/v1/rules { + proxy_pass http://test-oss-values-mimir-ruler.default.svc.cluster.local:8080$request_uri; + } + location /prometheus/api/v1/rules { + proxy_pass http://test-oss-values-mimir-ruler.default.svc.cluster.local:8080$request_uri; + } + + location /api/v1/rules { + proxy_pass http://test-oss-values-mimir-ruler.default.svc.cluster.local:8080$request_uri; + } + location /prometheus/api/v1/alerts { + proxy_pass http://test-oss-values-mimir-ruler.default.svc.cluster.local:8080$request_uri; + } + location /prometheus/rules { + proxy_pass http://test-oss-values-mimir-ruler.default.svc.cluster.local:8080$request_uri; + } + location = /ruler/ring { + proxy_pass http://test-oss-values-mimir-ruler.default.svc.cluster.local:8080$request_uri; + } + + # Rest of /prometheus goes to the query frontend + location /prometheus { + proxy_pass http://test-oss-values-mimir-query-frontend.default.svc.cluster.local:8080$request_uri; + } + + # Buildinfo endpoint can go to any component + location = /api/v1/status/buildinfo { + proxy_pass http://test-oss-values-mimir-query-frontend.default.svc.cluster.local:8080$request_uri; + } + } + } diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-dep.yaml new file mode 100644 index 00000000000..7ed199b4cb0 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-dep.yaml @@ -0,0 +1,95 @@ +--- +# Source: mimir-distributed/templates/nginx/nginx-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-oss-values-mimir-nginx + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: nginx + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: nginx + template: + metadata: + annotations: + checksum/config: d7fa3f4c1e841a4634ee7561fa92990e6525d1bfdb176f416744e0b8f72ae99d + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: nginx + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + fsGroup: 101 + runAsGroup: 101 + runAsNonRoot: true + runAsUser: 101 + terminationGracePeriodSeconds: 30 + containers: + - name: nginx + image: docker.io/nginxinc/nginx-unprivileged:1.19-alpine + imagePullPolicy: IfNotPresent + ports: + - name: http-metric + containerPort: 8080 + protocol: TCP + env: + envFrom: + readinessProbe: + httpGet: + path: / + port: http-metric + initialDelaySeconds: 15 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - name: config + mountPath: /etc/nginx + - name: tmp + mountPath: /tmp + - name: docker-entrypoint-d-override + mountPath: /docker-entrypoint.d + resources: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: component + operator: In + values: + - nginx + topologyKey: failure-domain.beta.kubernetes.io/zone + + volumes: + - name: config + configMap: + name: test-oss-values-mimir-nginx + - name: tmp + emptyDir: {} + - name: docker-entrypoint-d-override + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-svc.yaml new file mode 100644 index 00000000000..e331454c698 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/nginx/nginx-svc.yaml @@ -0,0 +1,26 @@ +--- +# Source: mimir-distributed/templates/nginx/nginx-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-nginx + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: nginx + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - name: http-metric + port: 80 + targetPort: http-metric + protocol: TCP + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: nginx diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml new file mode 100644 index 00000000000..107d7216c8b --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml @@ -0,0 +1,101 @@ +--- +# Source: mimir-distributed/templates/overrides-exporter/overrides-exporter-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {} + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: overrides-exporter + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + name: test-oss-values-mimir-overrides-exporter +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: overrides-exporter + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: overrides-exporter + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: overrides-exporter + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=overrides-exporter" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + livenessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 60 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml new file mode 100644 index 00000000000..0ff1ee8ec9d --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml @@ -0,0 +1,30 @@ +--- +# Source: mimir-distributed/templates/overrides-exporter/overrides-exporter-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-overrides-exporter + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: overrides-exporter + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: overrides-exporter diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/podsecuritypolicy.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/podsecuritypolicy.yaml new file mode 100644 index 00000000000..395db399470 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/podsecuritypolicy.yaml @@ -0,0 +1,40 @@ +--- +# Source: mimir-distributed/templates/podsecuritypolicy.yaml +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: test-oss-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + privileged: false + allowPrivilegeEscalation: false + volumes: + - 'configMap' + - 'emptyDir' + - 'persistentVolumeClaim' + - 'secret' + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + rule: 'MustRunAsNonRoot' + seLinux: + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + - min: 1 + max: 65535 + readOnlyRootFilesystem: true + requiredDropCapabilities: + - ALL diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/querier/querier-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/querier/querier-dep.yaml new file mode 100644 index 00000000000..f5cd09319ed --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/querier/querier-dep.yaml @@ -0,0 +1,113 @@ +--- +# Source: mimir-distributed/templates/querier/querier-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-oss-values-mimir-querier + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: querier + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: querier + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: querier + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: querier + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=querier" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - querier + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + [] + terminationGracePeriodSeconds: 180 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/querier/querier-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/querier/querier-svc.yaml new file mode 100644 index 00000000000..c1fcf0582bc --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/querier/querier-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/querier/querier-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-querier + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: querier + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: querier diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml new file mode 100644 index 00000000000..c14cc7b880a --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-dep.yaml @@ -0,0 +1,107 @@ +--- +# Source: mimir-distributed/templates/query-frontend/query-frontend-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-oss-values-mimir-query-frontend + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: query-frontend + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: query-frontend + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: query-frontend + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: query-frontend + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=query-frontend" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: runtime-config + mountPath: /var/mimir + - name: config + mountPath: /etc/mimir + - name: storage + mountPath: /data + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchExpressions: + - key: target + operator: In + values: + - query-frontend + topologyKey: kubernetes.io/hostname + weight: 100 + tolerations: + [] + terminationGracePeriodSeconds: 180 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml new file mode 100644 index 00000000000..7295c71edcd --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml @@ -0,0 +1,33 @@ +--- +# Source: mimir-distributed/templates/query-frontend/query-frontend-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-query-frontend-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: query-frontend + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: query-frontend diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml new file mode 100644 index 00000000000..f38983aca6f --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/query-frontend/query-frontend-svc.yaml @@ -0,0 +1,30 @@ +--- +# Source: mimir-distributed/templates/query-frontend/query-frontend-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-query-frontend + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: query-frontend + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: query-frontend diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/role.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/role.yaml new file mode 100644 index 00000000000..744b7b2a513 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/role.yaml @@ -0,0 +1,17 @@ +--- +# Source: mimir-distributed/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: test-oss-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: ['extensions'] + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: [test-oss-values-mimir] diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/rolebinding.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/rolebinding.yaml new file mode 100644 index 00000000000..a2310f0d972 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/rolebinding.yaml @@ -0,0 +1,19 @@ +--- +# Source: mimir-distributed/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: test-oss-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-oss-values-mimir +subjects: +- kind: ServiceAccount + name: test-oss-values-mimir diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml new file mode 100644 index 00000000000..f7ceb3b0491 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml @@ -0,0 +1,103 @@ +--- +# Source: mimir-distributed/templates/ruler/ruler-dep.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-oss-values-mimir-ruler + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ruler + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ruler + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: ruler + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + containers: + - name: ruler + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=ruler" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + subPath: + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 45 + resources: + requests: + cpu: 100m + memory: 128Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: + nodeSelector: + {} + affinity: + {} + tolerations: + [] + terminationGracePeriodSeconds: 180 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-svc.yaml new file mode 100644 index 00000000000..ce75a9acecc --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-svc.yaml @@ -0,0 +1,27 @@ +--- +# Source: mimir-distributed/templates/ruler/ruler-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-ruler + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ruler + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: ruler diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/runtime-configmap.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/runtime-configmap.yaml new file mode 100644 index 00000000000..7c5ea9aa679 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/runtime-configmap.yaml @@ -0,0 +1,16 @@ +--- +# Source: mimir-distributed/templates/runtime-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-oss-values-mimir-runtime + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +data: + runtime.yaml: | + + {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/secret.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/secret.yaml new file mode 100644 index 00000000000..31de04f6fc8 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/secret.yaml @@ -0,0 +1,14 @@ +--- +# Source: mimir-distributed/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: test-oss-values-mimir-config + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +data: + mimir.yaml: bXVsdGl0ZW5hbmN5X2VuYWJsZWQ6IGZhbHNlCgpsaW1pdHM6IHt9CgphY3Rpdml0eV90cmFja2VyOgogIGZpbGVwYXRoOiAvZGF0YS9tZXRyaWNzLWFjdGl2aXR5LmxvZwoKYWxlcnRtYW5hZ2VyOgogIGRhdGFfZGlyOiAnL2RhdGEnCiAgZW5hYmxlX2FwaTogdHJ1ZQogIGV4dGVybmFsX3VybDogJy9hbGVydG1hbmFnZXInCmFsZXJ0bWFuYWdlcl9zdG9yYWdlOgogIGJhY2tlbmQ6IHMzCiAgczM6CiAgICBlbmRwb2ludDogdGVzdC1vc3MtdmFsdWVzLW1pbmlvLmRlZmF1bHQuc3ZjOjkwMDAKICAgIGJ1Y2tldF9uYW1lOiBtaW1pci1ydWxlcgogICAgYWNjZXNzX2tleV9pZDogZ3JhZmFuYS1taW1pcgogICAgc2VjcmV0X2FjY2Vzc19rZXk6IHN1cGVyc2VjcmV0CiAgICBpbnNlY3VyZTogdHJ1ZQoKZnJvbnRlbmRfd29ya2VyOgogIGZyb250ZW5kX2FkZHJlc3M6IHRlc3Qtb3NzLXZhbHVlcy1taW1pci1xdWVyeS1mcm9udGVuZC1oZWFkbGVzcy5kZWZhdWx0LnN2Yzo5MDk1CgpydWxlcjoKICBlbmFibGVfYXBpOiB0cnVlCiAgcnVsZV9wYXRoOiAnL2RhdGEnCiAgYWxlcnRtYW5hZ2VyX3VybDogZG5zc3J2bm9hK2h0dHA6Ly9faHR0cC1tZXRyaWNzLl90Y3AudGVzdC1vc3MtdmFsdWVzLW1pbWlyLWFsZXJ0bWFuYWdlci1oZWFkbGVzcy5kZWZhdWx0LnN2Yy5jbHVzdGVyLmxvY2FsL2FsZXJ0bWFuYWdlcgoKc2VydmVyOgogIGdycGNfc2VydmVyX21heF9yZWN2X21zZ19zaXplOiAxMDQ4NTc2MDAKICBncnBjX3NlcnZlcl9tYXhfc2VuZF9tc2dfc2l6ZTogMTA0ODU3NjAwCiAgZ3JwY19zZXJ2ZXJfbWF4X2NvbmN1cnJlbnRfc3RyZWFtczogMTAwMAoKZnJvbnRlbmQ6CiAgbG9nX3F1ZXJpZXNfbG9uZ2VyX3RoYW46IDEwcwogIGFsaWduX3F1ZXJpZXNfd2l0aF9zdGVwOiB0cnVlCgpjb21wYWN0b3I6CiAgZGF0YV9kaXI6ICIvZGF0YSIKCmluZ2VzdGVyOgogIHJpbmc6CiAgICBmaW5hbF9zbGVlcDogMHMKICAgIG51bV90b2tlbnM6IDUxMgogICAgdW5yZWdpc3Rlcl9vbl9zaHV0ZG93bjogZmFsc2UKCmluZ2VzdGVyX2NsaWVudDoKICBncnBjX2NsaWVudF9jb25maWc6CiAgICBtYXhfcmVjdl9tc2dfc2l6ZTogMTA0ODU3NjAwCiAgICBtYXhfc2VuZF9tc2dfc2l6ZTogMTA0ODU3NjAwCgpydW50aW1lX2NvbmZpZzoKICBmaWxlOiAvdmFyL21pbWlyL3J1bnRpbWUueWFtbAoKbWVtYmVybGlzdDoKICBhYm9ydF9pZl9jbHVzdGVyX2pvaW5fZmFpbHM6IGZhbHNlCiAgY29tcHJlc3Npb25fZW5hYmxlZDogZmFsc2UKICBqb2luX21lbWJlcnM6CiAgLSB0ZXN0LW9zcy12YWx1ZXMtbWltaXItZ29zc2lwLXJpbmcKCiMgVGhpcyBjb25maWd1cmVzIGhvdyB0aGUgc3RvcmUtZ2F0ZXdheSBzeW5jaHJvbml6ZXMgYmxvY2tzIHN0b3JlZCBpbiB0aGUgYnVja2V0LiBJdCB1c2VzIE1pbmlvIGJ5IGRlZmF1bHQgZm9yIGdldHRpbmcgc3RhcnRlZCAoY29uZmlndXJlZCB2aWEgZmxhZ3MpIGJ1dCB0aGlzIHNob3VsZCBiZSBjaGFuZ2VkIGZvciBwcm9kdWN0aW9uIGRlcGxveW1lbnRzLgpibG9ja3Nfc3RvcmFnZToKICBiYWNrZW5kOiBzMwogIHRzZGI6CiAgICBkaXI6IC9kYXRhL3RzZGIKICBidWNrZXRfc3RvcmU6CiAgICBzeW5jX2RpcjogL2RhdGEvdHNkYi1zeW5jCiAgczM6CiAgICBlbmRwb2ludDogdGVzdC1vc3MtdmFsdWVzLW1pbmlvLmRlZmF1bHQuc3ZjOjkwMDAKICAgIGJ1Y2tldF9uYW1lOiBtaW1pci10c2RiCiAgICBhY2Nlc3Nfa2V5X2lkOiBncmFmYW5hLW1pbWlyCiAgICBzZWNyZXRfYWNjZXNzX2tleTogc3VwZXJzZWNyZXQKICAgIGluc2VjdXJlOiB0cnVlCnJ1bGVyX3N0b3JhZ2U6CiAgYmFja2VuZDogczMKICBzMzoKICAgIGVuZHBvaW50OiB0ZXN0LW9zcy12YWx1ZXMtbWluaW8uZGVmYXVsdC5zdmM6OTAwMAogICAgYnVja2V0X25hbWU6IG1pbWlyLXJ1bGVyCiAgICBhY2Nlc3Nfa2V5X2lkOiBncmFmYW5hLW1pbWlyCiAgICBzZWNyZXRfYWNjZXNzX2tleTogc3VwZXJzZWNyZXQKICAgIGluc2VjdXJlOiB0cnVlCg== diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/serviceaccount.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/serviceaccount.yaml new file mode 100644 index 00000000000..c5f94d48a9c --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/serviceaccount.yaml @@ -0,0 +1,14 @@ +--- +# Source: mimir-distributed/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-oss-values-mimir + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml new file mode 100644 index 00000000000..4d990a193d4 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml @@ -0,0 +1,21 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-pdb.yaml +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: test-oss-values-mimir-store-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm +spec: + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway + maxUnavailable: 1 diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml new file mode 100644 index 00000000000..9f9eca6fde0 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml @@ -0,0 +1,108 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-oss-values-mimir-store-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway + updateStrategy: + type: RollingUpdate + serviceName: test-oss-values-mimir-store-gateway-headless + template: + metadata: + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + annotations: + checksum/config: 7b9ed84df22bf324205968cf49b0d824eb15e1748888719ffc3bab3f17d5e2e4 + spec: + serviceAccountName: test-oss-values-mimir + securityContext: + {} + initContainers: + [] + nodeSelector: + {} + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: target + operator: In + values: + - store-gateway + topologyKey: kubernetes.io/hostname + tolerations: + [] + terminationGracePeriodSeconds: 240 + volumes: + - name: config + secret: + secretName: test-oss-values-mimir-config + - name: runtime-config + configMap: + name: test-oss-values-mimir-runtime + - name: storage + emptyDir: {} + containers: + - name: store-gateway + image: "grafana/mimir:2.1.0" + imagePullPolicy: IfNotPresent + args: + - "-target=store-gateway" + - "-config.expand-env=true" + - "-config.file=/etc/mimir/mimir.yaml" + volumeMounts: + - name: config + mountPath: /etc/mimir + - name: runtime-config + mountPath: /var/mimir + - name: storage + mountPath: "/data" + ports: + - name: http-metrics + containerPort: 8080 + protocol: TCP + - name: grpc + containerPort: 9095 + protocol: TCP + - name: memberlist + containerPort: 7946 + protocol: TCP + livenessProbe: + null + readinessProbe: + httpGet: + path: /ready + port: http-metrics + initialDelaySeconds: 60 + resources: + requests: + cpu: 100m + memory: 512Mi + securityContext: + readOnlyRootFilesystem: true + env: + envFrom: diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml new file mode 100644 index 00000000000..07f36435a33 --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml @@ -0,0 +1,29 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-svc-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-store-gateway-headless + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + prometheus.io/service-monitor: "false" + annotations: + {} +spec: + type: ClusterIP + clusterIP: None + ports: + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml new file mode 100644 index 00000000000..288c52639da --- /dev/null +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/store-gateway/store-gateway-svc.yaml @@ -0,0 +1,31 @@ +--- +# Source: mimir-distributed/templates/store-gateway/store-gateway-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: test-oss-values-mimir-store-gateway + labels: + helm.sh/chart: mimir-distributed-2.2.0-weekly.189 + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway + app.kubernetes.io/part-of: memberlist + app.kubernetes.io/version: "2.1.0" + app.kubernetes.io/managed-by: Helm + annotations: + {} +spec: + type: ClusterIP + ports: + - port: 8080 + protocol: TCP + name: http-metrics + targetPort: http-metrics + - port: 9095 + protocol: TCP + name: grpc + targetPort: grpc + selector: + app.kubernetes.io/name: mimir + app.kubernetes.io/instance: test-oss-values + app.kubernetes.io/component: store-gateway diff --git a/operations/k6/load-testing-with-k6.js b/operations/k6/load-testing-with-k6.js index c3874f8ad99..33ad99694cc 100644 --- a/operations/k6/load-testing-with-k6.js +++ b/operations/k6/load-testing-with-k6.js @@ -244,7 +244,7 @@ export function write() { } ); check(res, { - 'write worked': (r) => r.status === 200, + 'write worked': (r) => r.status === 200 || r.status === 202, }, { type: "write" }) || fail(`ERR: write failed. Status: ${res.status}. Body: ${res.body}`); } catch (e) { diff --git a/operations/mimir-mixin-compiled/alerts.yaml b/operations/mimir-mixin-compiled/alerts.yaml index d685790365d..28381bfe5a8 100644 --- a/operations/mimir-mixin-compiled/alerts.yaml +++ b/operations/mimir-mixin-compiled/alerts.yaml @@ -66,18 +66,18 @@ groups: - alert: MimirFrontendQueriesStuck annotations: message: | - There are {{ $value }} queued up queries in {{ $labels.cluster }}/{{ $labels.namespace }} query-frontend. + There are {{ $value }} queued up queries in {{ $labels.cluster }}/{{ $labels.namespace }} {{ $labels.job }}. expr: | - sum by (cluster, namespace) (cortex_query_frontend_queue_length) > 1 + sum by (cluster, namespace, job) (cortex_query_frontend_queue_length) > 1 for: 5m labels: severity: critical - alert: MimirSchedulerQueriesStuck annotations: message: | - There are {{ $value }} queued up queries in {{ $labels.cluster }}/{{ $labels.namespace }} query-scheduler. + There are {{ $value }} queued up queries in {{ $labels.cluster }}/{{ $labels.namespace }} {{ $labels.job }}. expr: | - sum by (cluster, namespace) (cortex_query_scheduler_queue_length) > 1 + sum by (cluster, namespace, job) (cortex_query_scheduler_queue_length) > 1 for: 5m labels: severity: critical @@ -276,7 +276,7 @@ groups: ( container_memory_working_set_bytes{container="ingester"} / - container_spec_memory_limit_bytes{container="ingester"} + ( container_spec_memory_limit_bytes{container="ingester"} > 0 ) ) > 0.65 for: 15m labels: @@ -289,7 +289,7 @@ groups: ( container_memory_working_set_bytes{container="ingester"} / - container_spec_memory_limit_bytes{container="ingester"} + ( container_spec_memory_limit_bytes{container="ingester"} > 0 ) ) > 0.8 for: 15m labels: @@ -327,10 +327,11 @@ groups: message: | Mimir Ruler {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} is experiencing {{ printf "%.2f" $value }}% missed iterations for the rule group {{ $labels.rule_group }}. expr: | + 100 * ( sum by (cluster, namespace, pod, rule_group) (rate(cortex_prometheus_rule_group_iterations_missed_total[1m])) / sum by (cluster, namespace, pod, rule_group) (rate(cortex_prometheus_rule_group_iterations_total[1m])) - > 0.01 + ) > 1 for: 5m labels: severity: warning @@ -351,10 +352,8 @@ groups: message: Mimir instance {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} sees incorrect number of gossip members. expr: | - memberlist_client_cluster_members_count - != on (cluster, namespace) group_left - sum by (cluster, namespace) (up{job=~".+/(compactor|distributor|ingester.*|querier.*|ruler|store-gateway.*|cortex|mimir)"}) - for: 5m + avg by (cluster, namespace) (memberlist_client_cluster_members_count) != sum by (cluster, namespace) (up{job=~".+/(alertmanager|compactor|distributor|ingester.*|querier.*|ruler|store-gateway.*|cortex|mimir)"}) + for: 15m labels: severity: warning - name: etcd_alerts @@ -367,7 +366,7 @@ groups: ( container_memory_working_set_bytes{container="etcd"} / - container_spec_memory_limit_bytes{container="etcd"} + ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.65 for: 15m labels: @@ -380,7 +379,7 @@ groups: ( container_memory_working_set_bytes{container="etcd"} / - container_spec_memory_limit_bytes{container="etcd"} + ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.8 for: 15m labels: @@ -598,7 +597,7 @@ groups: severity: warning - alert: MimirStoreGatewayHasNotSyncTheBucket annotations: - message: Mimir Store Gateway {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace + message: Mimir store-gateway {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} has not successfully synched the bucket since {{ $value | humanizeDuration }}. expr: | @@ -608,6 +607,15 @@ groups: for: 5m labels: severity: critical + - alert: MimirStoreGatewayNoSyncedTenants + annotations: + message: Mimir store-gateway {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace + }} is not syncing any blocks for any tenant. + expr: | + min by(cluster, namespace, pod) (cortex_bucket_stores_tenants_synced{component="store-gateway"}) == 0 + for: 1h + labels: + severity: warning - alert: MimirBucketIndexNotUpdated annotations: message: Mimir bucket index for tenant {{ $labels.user }} in {{ $labels.cluster @@ -670,9 +678,9 @@ groups: message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} has not uploaded any block in the last 24 hours. expr: | - (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/(compactor.*|cortex|mimir)"} > 60 * 60 * 24) + (time() - thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 60 * 60 * 24) and - (thanos_objstore_bucket_last_successful_upload_time{job=~".+/(compactor.*|cortex|mimir)"} > 0) + (thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 0) for: 15m labels: severity: critical @@ -681,7 +689,7 @@ groups: message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} has not uploaded any block in the last 24 hours. expr: | - thanos_objstore_bucket_last_successful_upload_time{job=~".+/(compactor.*|cortex|mimir)"} == 0 + thanos_objstore_bucket_last_successful_upload_time{component="compactor"} == 0 for: 24h labels: severity: critical @@ -690,7 +698,7 @@ groups: message: Mimir Compactor {{ $labels.pod }} in {{ $labels.cluster }}/{{ $labels.namespace }} has found and ignored blocks with out of order chunks. expr: | - increase(cortex_compactor_blocks_marked_for_no_compaction_total{job=~".+/(compactor.*|cortex|mimir)", reason="block-index-out-of-order-chunk"}[5m]) > 0 + increase(cortex_compactor_blocks_marked_for_no_compaction_total{component="compactor", reason="block-index-out-of-order-chunk"}[5m]) > 0 for: 1m labels: severity: warning @@ -732,6 +740,6 @@ groups: message: Mimir continuous test {{ $labels.test }} in {{ $labels.cluster }}/{{ $labels.namespace }} failed when asserting query results. expr: | - sum by(cluster, namespace, test) (rate(mimir_continuous_test_query_result_checks_failed_total[5m])) > 0 + sum by(cluster, namespace, test) (rate(mimir_continuous_test_query_result_checks_failed_total[10m])) > 0 labels: severity: warning diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager-resources.json index d7d747deff2..6ab8d645b3f 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager-resources.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager-resources.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -62,273 +70,6 @@ "span": 4, "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"} > 0)", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"alertmanager\"}[$__rate_interval]))", @@ -390,7 +131,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "id": 2, "legend": { "avg": false, "current": false, @@ -480,7 +221,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "id": 3, "legend": { "avg": false, "current": false, @@ -567,7 +308,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "id": 4, "legend": { "avg": false, "current": false, @@ -657,7 +398,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 5, "legend": { "avg": false, "current": false, @@ -747,7 +488,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "id": 6, "legend": { "avg": false, "current": false, @@ -834,7 +575,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 10, + "id": 7, "legend": { "avg": false, "current": false, @@ -859,7 +600,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"alertmanager.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?alertmanager.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -911,7 +652,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 11, + "id": 8, "legend": { "avg": false, "current": false, @@ -936,7 +677,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"alertmanager.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?alertmanager.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1000,7 +741,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 12, + "id": 9, "legend": { "avg": false, "current": false, @@ -1077,7 +818,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 13, + "id": 10, "legend": { "avg": false, "current": false, @@ -1166,7 +907,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 14, + "id": 11, "legend": { "avg": false, "current": false, @@ -1258,7 +999,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1345,6 +1086,6 @@ }, "timezone": "utc", "title": "Mimir / Alertmanager resources", - "uid": "68b66aed90ccab448009089544a8d6c6", + "uid": "a6883fb22799ac74479c7db872451092", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager.json b/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager.json index a1c5f2c4b29..be5a75321fa 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-alertmanager.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -828,195 +836,6 @@ "title": "Alert notifications", "titleSize": "h6" }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_v1_alerts|alertmanager\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_v1_alerts|alertmanager\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_v1_alerts|alertmanager\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_v1_alerts|alertmanager\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Configuration API (gateway) + Alertmanager UI", - "titleSize": "h6" - }, { "collapse": false, "height": "250px", @@ -1028,7 +847,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 12, + "id": 10, "legend": { "avg": false, "current": false, @@ -1105,7 +924,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 13, + "id": 11, "legend": { "avg": false, "current": false, @@ -1182,7 +1001,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 14, + "id": 12, "legend": { "avg": false, "current": false, @@ -1277,7 +1096,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 15, + "id": 13, "legend": { "avg": false, "current": false, @@ -1384,7 +1203,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 16, + "id": 14, "legend": { "avg": false, "current": false, @@ -1479,7 +1298,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 15, "legend": { "avg": false, "current": false, @@ -1574,7 +1393,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 16, "legend": { "avg": false, "current": false, @@ -1669,7 +1488,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 19, + "id": 17, "legend": { "avg": false, "current": false, @@ -1776,7 +1595,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 20, + "id": 18, "legend": { "avg": false, "current": false, @@ -1853,7 +1672,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 21, + "id": 19, "legend": { "avg": false, "current": false, @@ -1930,7 +1749,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 22, + "id": 20, "legend": { "avg": false, "current": false, @@ -2019,7 +1838,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 23, + "id": 21, "legend": { "avg": false, "current": false, @@ -2105,7 +1924,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 24, + "id": 22, "legend": { "avg": false, "current": false, @@ -2182,7 +2001,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 25, + "id": 23, "legend": { "avg": false, "current": false, @@ -2271,7 +2090,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 26, + "id": 24, "legend": { "avg": false, "current": false, @@ -2348,7 +2167,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 27, + "id": 25, "legend": { "avg": false, "current": false, @@ -2443,7 +2262,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 28, + "id": 26, "legend": { "avg": false, "current": false, @@ -2541,7 +2360,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 29, + "id": 27, "legend": { "avg": false, "current": false, @@ -2627,7 +2446,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 30, + "id": 28, "legend": { "avg": false, "current": false, @@ -2713,7 +2532,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 31, + "id": 29, "legend": { "avg": false, "current": false, @@ -2814,7 +2633,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -2903,6 +2722,6 @@ }, "timezone": "utc", "title": "Mimir / Alertmanager", - "uid": "a76bee5913c97c918d9e56a3cc88cc28", + "uid": "b0d38d318bbddd80476246d4930f9e55", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-compactor-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-compactor-resources.json index 4f1ffd1f965..7861c36cd5c 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-compactor-resources.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-compactor-resources.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -325,7 +333,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -402,7 +410,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"compactor.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?compactor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -712,7 +720,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -801,6 +809,6 @@ }, "timezone": "utc", "title": "Mimir / Compactor resources", - "uid": "df9added6f1f4332f95848cca48ebd99", + "uid": "09a5c49e9cdb2f2b24c6d184574a07fd", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-compactor.json b/operations/mimir-mixin-compiled/dashboards/mimir-compactor.json index f63e1f6bf5d..2db0b60039c 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-compactor.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-compactor.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -2342,7 +2350,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -2431,6 +2439,6 @@ }, "timezone": "utc", "title": "Mimir / Compactor", - "uid": "9c408e1d55681ecb8a22c9fab46875cc", + "uid": "1b3443aea86db629e6efdb7d05c53823", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-config.json b/operations/mimir-mixin-compiled/dashboards/mimir-config.json index 233a64da209..9a6a43d74e5 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-config.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-config.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -214,7 +222,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -303,6 +311,6 @@ }, "timezone": "utc", "title": "Mimir / Config", - "uid": "61bb048ced9817b2d3e07677fb1c6290", + "uid": "5d9d0b4724c0f80d68467088ec61e003", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-object-store.json b/operations/mimir-mixin-compiled/dashboards/mimir-object-store.json index 26fb077351c..f8865c7b714 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-object-store.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-object-store.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -962,7 +970,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1051,6 +1059,6 @@ }, "timezone": "utc", "title": "Mimir / Object Store", - "uid": "d5a3a4489d57c733b5677fb55370a723", + "uid": "e1324ee2a434f4158c00a9ee279d3292", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-overrides.json b/operations/mimir-mixin-compiled/dashboards/mimir-overrides.json index 73d2ea3d1db..2f32b2686f8 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-overrides.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-overrides.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -147,7 +155,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -253,6 +261,6 @@ }, "timezone": "utc", "title": "Mimir / Overrides", - "uid": "b5c95fee2e5e7c4b5930826ff6e89a12", + "uid": "1e2c358600ac53f09faea133f811b5bb", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-queries.json b/operations/mimir-mixin-compiled/dashboards/mimir-queries.json index 1072fe163c9..d65cbf39f3c 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-queries.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-queries.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -676,7 +684,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\", method=\"split_by_interval\"}[1m]))", + "expr": "sum(rate(cortex_frontend_split_queries_total{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\", method=\"split_by_interval\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -753,7 +761,7 @@ "steppedLine": false, "targets": [ { - "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n or\n rate(thanos_cache_memcached_hits_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n)\n/\nsum (\n rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n or\n rate(thanos_cache_memcached_requests_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n)\n", + "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n)\n/\nsum (\n rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n)\n", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -830,7 +838,7 @@ "steppedLine": false, "targets": [ { - "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n or\n rate(thanos_cache_memcached_requests_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n)\n-\nsum (\n rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n or\n rate(thanos_cache_memcached_hits_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[1m])\n)\n", + "expr": "# Query metrics before and after migration to new memcached backend.\nsum (\n rate(cortex_cache_fetched_keys{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_requests_total{name=\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n)\n-\nsum (\n rate(cortex_cache_hits{name=~\"frontend.+\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n or\n rate(thanos_cache_memcached_hits_total{name=~\"frontend-cache\", cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"}[$__rate_interval])\n)\n", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1189,7 +1197,7 @@ "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, - "span": 3, + "span": 4, "stack": false, "steppedLine": false, "targets": [ @@ -1281,99 +1289,7 @@ "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir))\"})) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job:cortex_ingester_queried_chunks_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir))\"})) * 1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "expr": "1 * sum(cluster_job:cortex_ingester_queried_chunks_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir))\"}) / sum(cluster_job:cortex_ingester_queried_chunks_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir))\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Chunks per query", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, + "span": 4, "stack": false, "steppedLine": false, "targets": [ @@ -1445,7 +1361,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 16, "legend": { "avg": false, "current": false, @@ -1465,7 +1381,7 @@ "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, - "span": 3, + "span": 4, "stack": false, "steppedLine": false, "targets": [ @@ -1549,7 +1465,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 17, "legend": { "avg": false, "current": false, @@ -1644,7 +1560,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 19, + "id": 18, "legend": { "avg": false, "current": false, @@ -1739,7 +1655,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 20, + "id": 19, "legend": { "avg": false, "current": false, @@ -1764,7 +1680,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir))\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir))\"}[1m]))", + "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir))\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1828,7 +1744,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 21, + "id": 20, "legend": { "avg": false, "current": false, @@ -1926,7 +1842,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 22, + "id": 21, "legend": { "avg": false, "current": false, @@ -2012,7 +1928,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 23, + "id": 22, "legend": { "avg": false, "current": false, @@ -2119,7 +2035,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 24, + "id": 23, "legend": { "avg": false, "current": false, @@ -2196,7 +2112,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 25, + "id": 24, "legend": { "avg": false, "current": false, @@ -2273,7 +2189,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 26, + "id": 25, "legend": { "avg": false, "current": false, @@ -2362,7 +2278,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 27, + "id": 26, "legend": { "avg": false, "current": false, @@ -2457,7 +2373,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 28, + "id": 27, "legend": { "avg": false, "current": false, @@ -2552,7 +2468,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 29, + "id": 28, "legend": { "avg": false, "current": false, @@ -2641,7 +2557,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 30, + "id": 29, "legend": { "avg": false, "current": false, @@ -2721,7 +2637,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 31, + "id": 30, "legend": { "avg": false, "current": false, @@ -2810,7 +2726,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 32, + "id": 31, "legend": { "avg": false, "current": false, @@ -2908,7 +2824,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 33, + "id": 32, "legend": { "avg": false, "current": false, @@ -2985,7 +2901,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 34, + "id": 33, "legend": { "avg": false, "current": false, @@ -3092,7 +3008,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 35, + "id": 34, "legend": { "avg": false, "current": false, @@ -3169,7 +3085,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 36, + "id": 35, "legend": { "avg": false, "current": false, @@ -3246,7 +3162,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 37, + "id": 36, "legend": { "avg": false, "current": false, @@ -3338,7 +3254,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -3427,6 +3343,6 @@ }, "timezone": "utc", "title": "Mimir / Queries", - "uid": "d9931b1054053c8b972d320774bb8f1d", + "uid": "b3abe8d5c040395cc36615cb4334c92d", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-reads-networking.json b/operations/mimir-mixin-compiled/dashboards/mimir-reads-networking.json index 966e7f312f1..a7b0e8418d8 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-reads-networking.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-reads-networking.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -58,7 +66,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(gateway|cortex-gw|cortex-gw).*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?query-frontend.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -135,7 +143,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(gateway|cortex-gw|cortex-gw).*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?query-frontend.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -210,353 +218,6 @@ "span": 3, "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "avg", - "legendLink": null, - "step": 10 - }, - { - "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "highest", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Inflight requests (per pod)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"}))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "avg", - "legendLink": null, - "step": 10 - }, - { - "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"}))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "highest", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "TCP connections (per pod)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"query-frontend.*\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Receive bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"query-frontend.*\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Transmit bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\"})", @@ -620,7 +281,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 8, + "id": 4, "legend": { "avg": false, "current": false, @@ -727,7 +388,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "id": 5, "legend": { "avg": false, "current": false, @@ -752,7 +413,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"query-scheduler.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?query-scheduler.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -804,7 +465,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 10, + "id": 6, "legend": { "avg": false, "current": false, @@ -829,7 +490,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"query-scheduler.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?query-scheduler.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -881,7 +542,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 11, + "id": 7, "legend": { "avg": false, "current": false, @@ -967,7 +628,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 12, + "id": 8, "legend": { "avg": false, "current": false, @@ -1074,7 +735,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 13, + "id": 9, "legend": { "avg": false, "current": false, @@ -1099,7 +760,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"querier.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?querier.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1151,7 +812,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 14, + "id": 10, "legend": { "avg": false, "current": false, @@ -1176,7 +837,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"querier.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?querier.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1228,7 +889,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 15, + "id": 11, "legend": { "avg": false, "current": false, @@ -1314,7 +975,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 16, + "id": 12, "legend": { "avg": false, "current": false, @@ -1421,7 +1082,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 17, + "id": 13, "legend": { "avg": false, "current": false, @@ -1446,7 +1107,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"store-gateway.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?store-gateway.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1498,7 +1159,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 18, + "id": 14, "legend": { "avg": false, "current": false, @@ -1523,7 +1184,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"store-gateway.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?store-gateway.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1575,7 +1236,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 19, + "id": 15, "legend": { "avg": false, "current": false, @@ -1661,7 +1322,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 20, + "id": 16, "legend": { "avg": false, "current": false, @@ -1768,7 +1429,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 21, + "id": 17, "legend": { "avg": false, "current": false, @@ -1793,7 +1454,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"ruler.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?ruler.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1845,7 +1506,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 22, + "id": 18, "legend": { "avg": false, "current": false, @@ -1870,7 +1531,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"ruler.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?ruler.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1922,7 +1583,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 23, + "id": 19, "legend": { "avg": false, "current": false, @@ -2008,7 +1669,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 24, + "id": 20, "legend": { "avg": false, "current": false, @@ -2118,7 +1779,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -2205,6 +1866,6 @@ }, "timezone": "utc", "title": "Mimir / Reads networking", - "uid": "c0464f0d8bd026f776c9006b05910000", + "uid": "54b2a0a4748b3bd1aefa92ce5559a1c2", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-reads-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-reads-resources.json index c07952884a2..2a4c85a3ec8 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-reads-resources.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-reads-resources.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -62,273 +70,6 @@ "span": 4, "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"} > 0)", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"query-frontend\"}[$__rate_interval]))", @@ -390,7 +131,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "id": 2, "legend": { "avg": false, "current": false, @@ -480,7 +221,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "id": 3, "legend": { "avg": false, "current": false, @@ -567,7 +308,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "id": 4, "legend": { "avg": false, "current": false, @@ -657,7 +398,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 5, "legend": { "avg": false, "current": false, @@ -747,7 +488,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "id": 6, "legend": { "avg": false, "current": false, @@ -834,7 +575,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 10, + "id": 7, "legend": { "avg": false, "current": false, @@ -924,7 +665,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 11, + "id": 8, "legend": { "avg": false, "current": false, @@ -1014,7 +755,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 12, + "id": 9, "legend": { "avg": false, "current": false, @@ -1101,7 +842,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 13, + "id": 10, "legend": { "avg": false, "current": false, @@ -1191,7 +932,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 14, + "id": 11, "legend": { "avg": false, "current": false, @@ -1281,7 +1022,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 15, + "id": 12, "legend": { "avg": false, "current": false, @@ -1368,7 +1109,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 16, + "id": 13, "legend": { "avg": false, "current": false, @@ -1445,7 +1186,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 14, "legend": { "avg": false, "current": false, @@ -1547,7 +1288,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 15, "legend": { "avg": false, "current": false, @@ -1637,7 +1378,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 19, + "id": 16, "legend": { "avg": false, "current": false, @@ -1724,7 +1465,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 20, + "id": 17, "legend": { "avg": false, "current": false, @@ -1814,7 +1555,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 21, + "id": 18, "legend": { "avg": false, "current": false, @@ -1904,7 +1645,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 22, + "id": 19, "legend": { "avg": false, "current": false, @@ -1991,7 +1732,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 23, + "id": 20, "legend": { "avg": false, "current": false, @@ -2068,7 +1809,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 24, + "id": 21, "legend": { "avg": false, "current": false, @@ -2145,7 +1886,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 25, + "id": 22, "legend": { "avg": false, "current": false, @@ -2237,7 +1978,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -2324,6 +2065,6 @@ }, "timezone": "utc", "title": "Mimir / Reads resources", - "uid": "2fd2cda9eea8d8af9fbc0a5960425120", + "uid": "cc86fd5aa9301c6528986572ad974db9", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-reads.json b/operations/mimir-mixin-compiled/dashboards/mimir-reads.json index 2ef50dc4c33..1e584b95a4e 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-reads.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-reads.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -253,264 +261,6 @@ "span": 4, "stack": true, "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "id": 6, - "legend": { - "show": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per pod p99 latency", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, "targets": [ { "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", @@ -565,7 +315,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 5, "legend": { "avg": false, "current": false, @@ -657,7 +407,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 9, + "id": 6, "legend": { "show": false }, @@ -735,7 +485,7 @@ "content": "

\n The query scheduler is an optional service that moves\n the internal queue from the query-frontend into a\n separate component.\n If this service is not deployed,\n these panels will show \"No data.\"\n

\n", "datasource": null, "description": "", - "id": 10, + "id": 7, "mode": "markdown", "span": 4, "title": "", @@ -757,7 +507,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 11, + "id": 8, "legend": { "avg": false, "current": false, @@ -834,7 +584,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 12, + "id": 9, "legend": { "avg": false, "current": false, @@ -941,7 +691,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 13, + "id": 10, "legend": { "avg": false, "current": false, @@ -1018,7 +768,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 14, + "id": 11, "legend": { "avg": false, "current": false, @@ -1110,7 +860,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 15, + "id": 12, "legend": { "avg": false, "current": false, @@ -1225,7 +975,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 16, + "id": 13, "legend": { "avg": false, "current": false, @@ -1302,7 +1052,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 14, "legend": { "avg": false, "current": false, @@ -1394,7 +1144,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 18, + "id": 15, "legend": { "show": false }, @@ -1476,7 +1226,7 @@ "datasource": "$datasource", "description": "### Replicas\nThe minimum, maximum, and current number of querier replicas.\n\n", "fill": 1, - "id": 19, + "id": 16, "legend": { "avg": false, "current": false, @@ -1572,7 +1322,7 @@ "datasource": "$datasource", "description": "### Scaling metric\nThis panel shows the result of the query used as scaling metric and target/threshold used.\nThe desired number of replicas is computed by HPA as: / .\n\n", "fill": 1, - "id": 20, + "id": 17, "legend": { "avg": false, "current": false, @@ -1664,7 +1414,7 @@ "datasource": "$datasource", "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler woudln't work properly.\n\n", "fill": 1, - "id": 21, + "id": 18, "legend": { "avg": false, "current": false, @@ -1761,7 +1511,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 22, + "id": 19, "legend": { "avg": false, "current": false, @@ -1838,7 +1588,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 23, + "id": 20, "legend": { "avg": false, "current": false, @@ -1930,7 +1680,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 24, + "id": 21, "legend": { "show": false }, @@ -2019,7 +1769,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 25, + "id": 22, "legend": { "avg": false, "current": false, @@ -2096,7 +1846,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 26, + "id": 23, "legend": { "avg": false, "current": false, @@ -2188,7 +1938,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 27, + "id": 24, "legend": { "show": false }, @@ -2277,7 +2027,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 28, + "id": 25, "legend": { "avg": false, "current": false, @@ -2354,7 +2104,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 29, + "id": 26, "legend": { "avg": false, "current": false, @@ -2461,7 +2211,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 30, + "id": 27, "legend": { "avg": false, "current": false, @@ -2538,7 +2288,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 31, + "id": 28, "legend": { "avg": false, "current": false, @@ -2634,7 +2384,7 @@ "datasource": "$datasource", "description": "### Hit ratio\nEven if you do not set up memcached for the blocks index cache, you will still see data in this panel because the store-gateway by default has an\nin-memory blocks index cache.\n\n", "fill": 1, - "id": 32, + "id": 29, "legend": { "avg": false, "current": false, @@ -2723,7 +2473,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 33, + "id": 30, "legend": { "avg": false, "current": false, @@ -2800,7 +2550,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 34, + "id": 31, "legend": { "avg": false, "current": false, @@ -2895,7 +2645,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 35, + "id": 32, "legend": { "avg": false, "current": false, @@ -2984,7 +2734,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 36, + "id": 33, "legend": { "avg": false, "current": false, @@ -3061,7 +2811,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 37, + "id": 34, "legend": { "avg": false, "current": false, @@ -3156,7 +2906,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 38, + "id": 35, "legend": { "avg": false, "current": false, @@ -3245,7 +2995,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 39, + "id": 36, "legend": { "avg": false, "current": false, @@ -3322,7 +3072,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 40, + "id": 37, "legend": { "avg": false, "current": false, @@ -3417,7 +3167,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 41, + "id": 38, "legend": { "avg": false, "current": false, @@ -3506,7 +3256,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 42, + "id": 39, "legend": { "avg": false, "current": false, @@ -3583,7 +3333,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 43, + "id": 40, "legend": { "avg": false, "current": false, @@ -3660,7 +3410,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 44, + "id": 41, "legend": { "avg": false, "current": false, @@ -3755,7 +3505,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 45, + "id": 42, "legend": { "avg": false, "current": false, @@ -3862,7 +3612,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 46, + "id": 43, "legend": { "avg": false, "current": false, @@ -3957,7 +3707,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 47, + "id": 44, "legend": { "avg": false, "current": false, @@ -4052,7 +3802,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 48, + "id": 45, "legend": { "avg": false, "current": false, @@ -4147,7 +3897,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 49, + "id": 46, "legend": { "avg": false, "current": false, @@ -4254,7 +4004,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 50, + "id": 47, "legend": { "avg": false, "current": false, @@ -4331,7 +4081,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 51, + "id": 48, "legend": { "avg": false, "current": false, @@ -4408,7 +4158,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 52, + "id": 49, "legend": { "avg": false, "current": false, @@ -4503,7 +4253,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 53, + "id": 50, "legend": { "avg": false, "current": false, @@ -4610,7 +4360,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 54, + "id": 51, "legend": { "avg": false, "current": false, @@ -4705,7 +4455,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 55, + "id": 52, "legend": { "avg": false, "current": false, @@ -4800,7 +4550,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 56, + "id": 53, "legend": { "avg": false, "current": false, @@ -4895,7 +4645,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 57, + "id": 54, "legend": { "avg": false, "current": false, @@ -5005,7 +4755,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -5094,6 +4844,6 @@ }, "timezone": "utc", "title": "Mimir / Reads", - "uid": "8d6ba60eccc4b6eedfa329b24b1bd339", + "uid": "e327503188913dc38ad571c647eef643", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-remote-ruler-reads-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-remote-ruler-reads-resources.json new file mode 100644 index 00000000000..9ba6de5c0fb --- /dev/null +++ b/operations/mimir-mixin-compiled/dashboards/mimir-remote-ruler-reads-resources.json @@ -0,0 +1,937 @@ +{ + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "mimir" + ], + "targetBlank": false, + "title": "Mimir dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"}[$__rate_interval]))", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-frontend\"} > 0)", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query-frontend (dedicated to ruler)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"}[$__rate_interval]))", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-query-scheduler\"} > 0)", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(ruler-query-scheduler.*)\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query-scheduler (dedicated to ruler)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"}[$__rate_interval]))", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "CPU", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "limit", + "color": "#E02F44", + "fill": 0 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + }, + { + "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"ruler-querier\"} > 0)", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (workingset)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\"})", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{pod}}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory (go heap inuse)", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier (dedicated to ruler)", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "mimir" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ ], + "query": "label_values(cortex_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "prod", + "value": "prod" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "namespace", + "multi": false, + "name": "namespace", + "options": [ ], + "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Mimir / Remote ruler reads resources", + "uid": "1940f6ef765a506a171faa2056c956c3", + "version": 0 + } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-remote-ruler-reads.json b/operations/mimir-mixin-compiled/dashboards/mimir-remote-ruler-reads.json new file mode 100644 index 00000000000..35b22fb6bfb --- /dev/null +++ b/operations/mimir-mixin-compiled/dashboards/mimir-remote-ruler-reads.json @@ -0,0 +1,1236 @@ +{ + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "mimir" + ], + "targetBlank": false, + "title": "Mimir dashboards", + "type": "dashboards" + } + ], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "height": "175px", + "panels": [ + { + "content": "

\n This dashboard shows health metrics for the ruler read path when remote operational mode is enabled.\n It is broken into sections for each service on the ruler read path, and organized by the order in which the read request flows.\n
\n For each service, there are three panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service.\n

\n", + "datasource": null, + "description": "", + "id": 1, + "mode": "markdown", + "span": 12, + "title": "", + "transparent": true, + "type": "text" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Remote ruler reads dashboard description", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "100px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "### Evaluations per second\nRate of rule expressions evaluated per second.\n\n", + "fill": 1, + "format": "reqps", + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(\n rate(\n cortex_request_duration_seconds_count{\n cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\",\n route=~\"/httpgrpc.HTTP/Handle\"\n }[$__rate_interval]\n )\n)\n", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "70,80", + "timeFrom": null, + "timeShift": null, + "title": "Evaluations / sec", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "singlestat", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Headlines", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=\"/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests / sec", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=~\"/httpgrpc.HTTP/Handle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "id": 5, + "legend": { + "show": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-query-frontend.*))\", route=\"/httpgrpc.HTTP/Handle\"}[$__rate_interval])))", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Per pod p99 latency", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query-frontend (dedicated to ruler)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler-query-scheduler.*)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests / sec", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler-query-scheduler.*)\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(cortex_query_scheduler_queue_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(ruler-query-scheduler.*)\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "sum(rate(cortex_query_scheduler_queue_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/(ruler-query-scheduler.*)\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_query_scheduler_queue_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(ruler-query-scheduler.*)\"}[$__rate_interval]))", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency (time in queue)", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Query-scheduler (dedicated to ruler)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { + "1xx": "#EAB839", + "2xx": "#7EB26D", + "3xx": "#6ED0E0", + "4xx": "#EF843C", + "5xx": "#E24D42", + "error": "#E24D42", + "success": "#7EB26D" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 10, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 0, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_querier_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{status}}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Requests / sec", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "99th Percentile", + "refId": "A", + "step": 10 + }, + { + "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_querier_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})) * 1e3", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "50th Percentile", + "refId": "B", + "step": 10 + }, + { + "expr": "1e3 * sum(cluster_job_route:cortex_querier_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}) / sum(cluster_job_route:cortex_querier_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Average", + "refId": "C", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Latency", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 0, + "id": 10, + "legend": { + "show": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_querier_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((ruler-querier.*))\", route=~\"(prometheus|api_prom)_api_v1_.+\"}[$__rate_interval])))", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Per pod p99 latency", + "tooltip": { + "sort": 2 + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier (dedicated to ruler)", + "titleSize": "h6" + }, + { + "collapse": false, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "### Replicas\nThe minimum, maximum, and current number of querier replicas.\n\n", + "fill": 1, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kube_horizontalpodautoscaler_spec_min_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=\"keda-hpa-ruler-querier\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Min", + "legendLink": null, + "step": 10 + }, + { + "expr": "kube_horizontalpodautoscaler_spec_max_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=\"keda-hpa-ruler-querier\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Max", + "legendLink": null, + "step": 10 + }, + { + "expr": "kube_horizontalpodautoscaler_status_current_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=\"keda-hpa-ruler-querier\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Current", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Replicas", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "### Scaling metric\nThis panel shows the result of the query that is used as the scaling metric, and the target and threshold used.\nThe desired number of replicas is computed by HPA as: / .\n\n", + "fill": 1, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Target per replica", + "yaxis": 2 + } + ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "keda_metrics_adapter_scaler_metrics_value +\non(metric) group_left\nlabel_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=\"keda-hpa-ruler-querier\"}\n * 0, \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Scaling metric", + "legendLink": null, + "step": 10 + }, + { + "expr": "kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=\"keda-hpa-ruler-querier\"}", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Target per replica", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Scaling metric", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "description": "### Autoscaler failures rate\nThe rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom\nmetrics server is unable to query the scaling metric from Prometheus so the autoscaler does not work properly.\n\n", + "fill": 1, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by(metric) (rate(keda_metrics_adapter_scaler_errors[$__rate_interval])) +\non(metric) group_left\nlabel_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=\"keda-hpa-ruler-querier\"}\n * 0, \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "Failures per second", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Autoscaler failures rate", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Querier (dedicated to ruler) - autoscaling", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "mimir" + ], + "templating": { + "list": [ + { + "current": { + "text": "default", + "value": "default" + }, + "hide": 0, + "label": "Data Source", + "name": "datasource", + "options": [ ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "cluster", + "multi": true, + "name": "cluster", + "options": [ ], + "query": "label_values(cortex_build_info, cluster)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": "All", + "value": "$__all" + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": true, + "label": "namespace", + "multi": true, + "name": "namespace", + "options": [ ], + "query": "label_values(cortex_build_info{cluster=~\"$cluster\"}, namespace)", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Mimir / Remote ruler reads", + "uid": "f103238f7f5ab2f1345ce650cbfbfe2f", + "version": 0 + } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-rollout-progress.json b/operations/mimir-mixin-compiled/dashboards/mimir-rollout-progress.json index 50887116de9..1f869b67ab9 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-rollout-progress.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-rollout-progress.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -98,7 +106,7 @@ "steppedLine": false, "targets": [ { - "expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway.*|ruler|alertmanager.*|overrides-exporter|cortex|mimir\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", + "expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|query-scheduler|querier|compactor|store-gateway|ruler|alertmanager|overrides-exporter|cortex|mimir).*\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_statefulset_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"statefulset\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", "format": null, "intervalFactor": null, "legendFormat": "{{cortex_service}}", @@ -106,7 +114,7 @@ "step": null }, { - "expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",deployment=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway.*|ruler|alertmanager.*|overrides-exporter|cortex|mimir\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", + "expr": "(\n sum by(cortex_service) (\n label_replace(\n kube_deployment_status_replicas_updated{cluster=~\"$cluster\", namespace=~\"$namespace\",deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|query-scheduler|querier|compactor|store-gateway|ruler|alertmanager|overrides-exporter|cortex|mimir).*\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n /\n sum by(cortex_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n) and (\n sum by(cortex_service) (\n label_replace(\n kube_deployment_spec_replicas{cluster=~\"$cluster\", namespace=~\"$namespace\"},\n \"cortex_service\", \"$1\", \"deployment\", \"(.*?)(?:-zone-[a-z])?\"\n )\n )\n > 0\n)\n", "format": null, "intervalFactor": null, "legendFormat": "{{cortex_service}}", @@ -1094,7 +1102,7 @@ "steppedLine": false, "targets": [ { - "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway.*|ruler|alertmanager.*|overrides-exporter|cortex|mimir\"}\n> 0\n", + "expr": "kube_deployment_status_replicas_unavailable{cluster=~\"$cluster\", namespace=~\"$namespace\", deployment=~\".*(cortex-gw|distributor|ingester|query-frontend|query-scheduler|querier|compactor|store-gateway|ruler|alertmanager|overrides-exporter|cortex|mimir).*\"}\n> 0\n", "format": null, "instant": true, "interval": "", @@ -1104,7 +1112,7 @@ "step": null }, { - "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway.*|ruler|alertmanager.*|overrides-exporter|cortex|mimir\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway.*|ruler|alertmanager.*|overrides-exporter|cortex|mimir\"}\n> 0\n", + "expr": "kube_statefulset_status_replicas_current{cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|query-scheduler|querier|compactor|store-gateway|ruler|alertmanager|overrides-exporter|cortex|mimir).*\"} -\nkube_statefulset_status_replicas_ready {cluster=~\"$cluster\", namespace=~\"$namespace\", statefulset=~\".*(cortex-gw|distributor|ingester|query-frontend|query-scheduler|querier|compactor|store-gateway|ruler|alertmanager|overrides-exporter|cortex|mimir).*\"}\n> 0\n", "format": null, "instant": true, "interval": "", @@ -1177,7 +1185,7 @@ "id": 11, "targets": [ { - "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"cortex-gw|distributor|ingester.*|query-frontend.*|query-scheduler.*|querier.*|compactor|store-gateway.*|ruler|alertmanager.*|overrides-exporter|cortex|mimir\"},\n \"version\", \"$1\", \"image\", \".*:(.+)-.*\"\n )\n)\n", + "expr": "count by(container, version) (\n label_replace(\n kube_pod_container_info{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\".*(cortex-gw|distributor|ingester|query-frontend|query-scheduler|querier|compactor|store-gateway|ruler|alertmanager|overrides-exporter|cortex|mimir).*\"},\n \"version\", \"$1\", \"image\", \".*:(.*)\"\n )\n)\n", "instant": true, "legendFormat": "", "refId": "A" @@ -1321,7 +1329,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1408,6 +1416,6 @@ }, "timezone": "utc", "title": "Mimir / Rollout progress", - "uid": "7544a3a62b1be6ffd919fc990ab8ba8f", + "uid": "7f0b5567d543a1698e695b530eb7f5de", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json b/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json index 825f046dd4a..0d1c5e44629 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-ruler.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -211,7 +219,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\", operation=\"/cortex.Ingester/QueryStream\"}[5m]))", + "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\", operation=\"/cortex.Ingester/QueryStream\"}[$__rate_interval]))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -287,7 +295,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\", operation=\"/cortex.Ingester/Push\"}[5m]))", + "expr": "sum(rate(cortex_ingester_client_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\", operation=\"/cortex.Ingester/Push\"}[$__rate_interval]))", "format": "time_series", "instant": true, "intervalFactor": 2, @@ -552,272 +560,6 @@ "renderer": "flot", "seriesOverrides": [ ], "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "QPS", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (route, le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))\"}))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{ route }}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per route p99 latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Configuration API (gateway)", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, "span": 6, "stack": true, "steppedLine": false, @@ -875,7 +617,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 11, + "id": 8, "legend": { "avg": false, "current": false, @@ -990,7 +732,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 12, + "id": 9, "legend": { "avg": false, "current": false, @@ -1067,7 +809,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 13, + "id": 10, "legend": { "avg": false, "current": false, @@ -1182,7 +924,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 14, + "id": 11, "legend": { "avg": false, "current": false, @@ -1259,7 +1001,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 15, + "id": 12, "legend": { "avg": false, "current": false, @@ -1366,7 +1108,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 16, + "id": 13, "legend": { "avg": false, "current": false, @@ -1461,7 +1203,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 17, + "id": 14, "legend": { "avg": false, "current": false, @@ -1556,7 +1298,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 15, "legend": { "avg": false, "current": false, @@ -1581,7 +1323,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\"}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\"}[1m]))", + "expr": "sum(rate(cortex_querier_blocks_consistency_checks_failed_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\"}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{cluster=~\"$cluster\", job=~\"($namespace)/((ruler|cortex|mimir))\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -1645,7 +1387,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 19, + "id": 16, "legend": { "avg": false, "current": false, @@ -1722,7 +1464,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 20, + "id": 17, "legend": { "avg": false, "current": false, @@ -1799,7 +1541,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 21, + "id": 18, "legend": { "avg": false, "current": false, @@ -1888,7 +1630,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 22, + "id": 19, "legend": { "avg": false, "current": false, @@ -1965,7 +1707,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 23, + "id": 20, "legend": { "avg": false, "current": false, @@ -2042,7 +1784,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 24, + "id": 21, "legend": { "avg": false, "current": false, @@ -2131,7 +1873,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 25, + "id": 22, "legend": { "avg": false, "current": false, @@ -2220,7 +1962,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 26, + "id": 23, "legend": { "avg": false, "current": false, @@ -2297,7 +2039,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 27, + "id": 24, "legend": { "avg": false, "current": false, @@ -2374,7 +2116,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 28, + "id": 25, "legend": { "avg": false, "current": false, @@ -2469,7 +2211,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 29, + "id": 26, "legend": { "avg": false, "current": false, @@ -2576,7 +2318,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 30, + "id": 27, "legend": { "avg": false, "current": false, @@ -2671,7 +2413,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 31, + "id": 28, "legend": { "avg": false, "current": false, @@ -2766,7 +2508,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 32, + "id": 29, "legend": { "avg": false, "current": false, @@ -2861,7 +2603,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 33, + "id": 30, "legend": { "avg": false, "current": false, @@ -2971,7 +2713,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -3060,6 +2802,6 @@ }, "timezone": "utc", "title": "Mimir / Ruler", - "uid": "44d12bcb1f95661c6ab6bc946dfc3473", + "uid": "631e15d5d85afb2ca8e35d62984eeaa0", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-scaling.json b/operations/mimir-mixin-compiled/dashboards/mimir-scaling.json index 6ffe5be4a2d..e88c39f2164 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-scaling.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-scaling.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -261,7 +269,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -350,6 +358,6 @@ }, "timezone": "utc", "title": "Mimir / Scaling", - "uid": "88c041017b96856c9176e07cf557bdcf", + "uid": "64bbad83507b7289b514725658e10352", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-slow-queries.json b/operations/mimir-mixin-compiled/dashboards/mimir-slow-queries.json index 9581418d094..3e1576a3f25 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-slow-queries.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-slow-queries.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -171,7 +179,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -305,6 +313,6 @@ }, "timezone": "utc", "title": "Mimir / Slow queries", - "uid": "e6f3091e29d2636e3b8393447e925668", + "uid": "6089e1ce1e678788f46312a0a1e647e6", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-tenants.json b/operations/mimir-mixin-compiled/dashboards/mimir-tenants.json index 1c01242dc79..c35e006aab2 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-tenants.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-tenants.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -470,7 +478,13 @@ "fill": 1, "id": 7, "legend": { - "show": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, "lines": true, "linewidth": 1, @@ -480,7 +494,13 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ ], + "seriesOverrides": [ + { + "alias": "limit", + "dashes": true, + "fill": 0 + } + ], "spaceLength": 10, "span": 3, "stack": false, @@ -494,6 +514,15 @@ "legendFormat": "rate", "legendLink": null, "step": 10 + }, + { + "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/(overrides-exporter)\", limit_name=\"ingestion_rate\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/(overrides-exporter)\", limit_name=\"ingestion_rate\"})\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 } ], "thresholds": [ ], @@ -1202,7 +1231,13 @@ "fill": 1, "id": 16, "legend": { - "show": false + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, "lines": true, "linewidth": 1, @@ -1212,7 +1247,13 @@ "pointradius": 5, "points": false, "renderer": "flot", - "seriesOverrides": [ ], + "seriesOverrides": [ + { + "alias": "limit", + "dashes": true, + "fill": 0 + } + ], "spaceLength": 10, "span": 3, "stack": false, @@ -1226,6 +1267,15 @@ "legendFormat": "groups", "legendLink": null, "step": 10 + }, + { + "expr": "max(cortex_limits_overrides{cluster=~\"$cluster\", job=~\"($namespace)/(overrides-exporter)\", limit_name=\"ruler_max_rule_groups_per_tenant\", user=\"$user\"})\nor\nmax(cortex_limits_defaults{cluster=~\"$cluster\", job=~\"($namespace)/(overrides-exporter)\", limit_name=\"ruler_max_rule_groups_per_tenant\"})\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "limit", + "legendLink": null, + "step": 10 } ], "thresholds": [ ], @@ -1904,7 +1954,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-top-tenants.json b/operations/mimir-mixin-compiled/dashboards/mimir-top-tenants.json index 95f0874fae4..637e17aba3e 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-top-tenants.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-top-tenants.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -548,6 +556,312 @@ "points": false, "renderer": "flot", "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor|cortex|mimir))\"}[$__rate_interval]))\nand\ntopk($limit,\n sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor|cortex|mimir))\"}[$__rate_interval] @ end()))\n -\n sum by (user) (rate(cortex_distributor_received_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((distributor|cortex|mimir))\"}[$__rate_interval] @ start()))\n)\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{ user }}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Top $limit users by received samples rate that grew the most between query range start and query range end", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "By samples rate growth", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "sort": { + "col": 2, + "desc": true + }, + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "hidden" + }, + { + "alias": "samples/s", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": false, + "linkTargetBlank": false, + "linkTooltip": "Drill down", + "linkUrl": "", + "pattern": "Value #A", + "thresholds": [ ], + "type": "number", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "/.*/", + "thresholds": [ ], + "type": "string", + "unit": "short" + } + ], + "targets": [ + { + "expr": "topk($limit, sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir)|(distributor|cortex|mimir))\"}[5m])))", + "format": "table", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Top $limit users by discarded samples rate in last 5m", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "transform": "table", + "type": "table", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "By discarded samples rate", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], + "spaceLength": 10, + "span": 12, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir)|(distributor|cortex|mimir))\"}[$__rate_interval]))\nand\ntopk($limit,\n sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir)|(distributor|cortex|mimir))\"}[$__rate_interval] @ end()))\n -\n sum by (user) (rate(cortex_discarded_samples_total{cluster=~\"$cluster\", job=~\"($namespace)/((ingester.*|cortex|mimir)|(distributor|cortex|mimir))\"}[$__rate_interval] @ start()))\n)\n", + "format": "time_series", + "interval": "15s", + "intervalFactor": 2, + "legendFormat": "{{ user }}", + "legendLink": null, + "step": 10 + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Top $limit users by discarded samples rate that grew the most between query range start and query range end", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "By discarded samples rate growth", + "titleSize": "h6" + }, + { + "collapse": true, + "height": "250px", + "panels": [ + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ ], "sort": { "col": 2, "desc": true @@ -657,7 +971,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "id": 10, "legend": { "avg": false, "current": false, @@ -785,7 +1099,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 11, "legend": { "avg": false, "current": false, @@ -913,7 +1227,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "id": 12, "legend": { "avg": false, "current": false, @@ -1044,7 +1358,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-writes-networking.json b/operations/mimir-mixin-compiled/dashboards/mimir-writes-networking.json index 124b7aaa18d..77b3036a943 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-writes-networking.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-writes-networking.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -58,7 +66,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(gateway|cortex-gw|cortex-gw).*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?distributor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -135,7 +143,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(gateway|cortex-gw|cortex-gw).*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?distributor.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -210,353 +218,6 @@ "span": 3, "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "avg", - "legendLink": null, - "step": 10 - }, - { - "expr": "max(cortex_inflight_requests{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "highest", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Inflight requests (per pod)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"}))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "avg", - "legendLink": null, - "step": 10 - }, - { - "expr": "max(sum by(pod) (cortex_tcp_connections{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"}))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "highest", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(cortex_tcp_connections_limit{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "TCP connections (per pod)", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"distributor.*\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Receive bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"distributor.*\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Transmit bandwidth", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 3, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "avg(cortex_inflight_requests{cluster=~\"$cluster\", job=~\"($namespace)/((distributor|cortex|mimir))\"})", @@ -620,7 +281,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 8, + "id": 4, "legend": { "avg": false, "current": false, @@ -727,7 +388,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 9, + "id": 5, "legend": { "avg": false, "current": false, @@ -752,7 +413,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"ingester.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?ingester.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -804,7 +465,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 10, + "id": 6, "legend": { "avg": false, "current": false, @@ -829,7 +490,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"ingester.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\",pod=~\"(.*-mimir-)?ingester.*\"}[$__rate_interval]))", "format": "time_series", "interval": "15s", "intervalFactor": 2, @@ -881,7 +542,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 11, + "id": 7, "legend": { "avg": false, "current": false, @@ -967,7 +628,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 12, + "id": 8, "legend": { "avg": false, "current": false, @@ -1077,7 +738,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1164,6 +825,6 @@ }, "timezone": "utc", "title": "Mimir / Writes networking", - "uid": "681cd62b680b7154811fe73af55dcfd4", + "uid": "978c1cb452585c96697a238eaac7fe2d", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-writes-resources.json b/operations/mimir-mixin-compiled/dashboards/mimir-writes-resources.json index cbb6f187590..2eb4a47787e 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-writes-resources.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-writes-resources.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -62,273 +70,6 @@ "span": 4, "stack": false, "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"}[$__rate_interval]))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "CPU", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - }, - { - "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"(gateway|cortex-gw|cortex-gw-internal)\"} > 0)", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "limit", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (workingset)", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\"})", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Memory (go heap inuse)", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 - } - ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\",container=~\"distributor\"}[$__rate_interval]))", @@ -390,7 +131,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 5, + "id": 2, "legend": { "avg": false, "current": false, @@ -480,7 +221,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 6, + "id": 3, "legend": { "avg": false, "current": false, @@ -567,7 +308,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 7, + "id": 4, "legend": { "avg": false, "current": false, @@ -642,7 +383,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 8, + "id": 5, "legend": { "avg": false, "current": false, @@ -744,7 +485,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 9, + "id": 6, "legend": { "avg": false, "current": false, @@ -834,7 +575,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 10, + "id": 7, "legend": { "avg": false, "current": false, @@ -921,7 +662,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 11, + "id": 8, "legend": { "avg": false, "current": false, @@ -998,7 +739,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 12, + "id": 9, "legend": { "avg": false, "current": false, @@ -1075,7 +816,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 13, + "id": 10, "legend": { "avg": false, "current": false, @@ -1167,7 +908,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -1254,6 +995,6 @@ }, "timezone": "utc", "title": "Mimir / Writes resources", - "uid": "c0464f0d8bd026f776c9006b0591bb0b", + "uid": "bc9160e50b52e89e0e49c840fea3d379", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/dashboards/mimir-writes.json b/operations/mimir-mixin-compiled/dashboards/mimir-writes.json index da76955b2b5..faf1a79e4a9 100644 --- a/operations/mimir-mixin-compiled/dashboards/mimir-writes.json +++ b/operations/mimir-mixin-compiled/dashboards/mimir-writes.json @@ -1,4 +1,12 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { "list": [ ] }, @@ -431,82 +439,6 @@ "show": false } ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "format": "reqps", - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 2, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"}[5m]))", - "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] } ], "repeat": null, @@ -535,265 +467,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum by (status) (\n label_replace(label_replace(rate(cortex_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "{{status}}", - "refId": "A", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Requests / sec", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 - }, - { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:cortex_request_duration_seconds_bucket:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"})) * 1e3", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 - }, - { - "expr": "1e3 * sum(cluster_job_route:cortex_request_duration_seconds_sum:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"}) / sum(cluster_job_route:cortex_request_duration_seconds_count:sum_rate{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Average", - "refId": "C", - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Latency", - "tooltip": { - "shared": false, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": { }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 0, - "id": 10, - "legend": { - "show": false - }, - "lines": true, - "linewidth": 1, - "links": [ ], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ ], - "spaceLength": 10, - "span": 4, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(0.99, sum by(le, pod) (rate(cortex_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/((gateway|cortex-gw|cortex-gw-internal))\", route=~\"api_(v1|prom)_push\"}[$__rate_interval])))", - "format": "time_series", - "interval": "15s", - "intervalFactor": 2, - "legendFormat": "", - "legendLink": null, - "step": 10 - } - ], - "thresholds": [ ], - "timeFrom": null, - "timeShift": null, - "title": "Per pod p99 latency", - "tooltip": { - "sort": 2 - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - } - ], - "repeat": null, - "repeatIteration": null, - "repeatRowId": null, - "showTitle": true, - "title": "Gateway", - "titleSize": "h6" - }, - { - "collapse": false, - "height": "250px", - "panels": [ - { - "aliasColors": { - "1xx": "#EAB839", - "2xx": "#7EB26D", - "3xx": "#6ED0E0", - "4xx": "#EF843C", - "5xx": "#E24D42", - "error": "#E24D42", - "success": "#7EB26D" - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 10, - "id": 11, + "id": 7, "legend": { "avg": false, "current": false, @@ -870,7 +544,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 12, + "id": 8, "legend": { "avg": false, "current": false, @@ -962,7 +636,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 13, + "id": 9, "legend": { "show": false }, @@ -1051,7 +725,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 14, + "id": 10, "legend": { "avg": false, "current": false, @@ -1128,7 +802,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 15, + "id": 11, "legend": { "avg": false, "current": false, @@ -1220,7 +894,7 @@ "dashes": false, "datasource": "$datasource", "fill": 0, - "id": 16, + "id": 12, "legend": { "show": false }, @@ -1309,7 +983,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 17, + "id": 13, "legend": { "avg": false, "current": false, @@ -1386,7 +1060,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 18, + "id": 14, "legend": { "avg": false, "current": false, @@ -1501,7 +1175,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 19, + "id": 15, "legend": { "avg": false, "current": false, @@ -1578,7 +1252,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 20, + "id": 16, "legend": { "avg": false, "current": false, @@ -1693,7 +1367,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 21, + "id": 17, "legend": { "avg": false, "current": false, @@ -1770,7 +1444,7 @@ "dashes": false, "datasource": "$datasource", "fill": 1, - "id": 22, + "id": 18, "legend": { "avg": false, "current": false, @@ -1881,7 +1555,7 @@ "datasource": "$datasource", "description": "### Uploaded blocks / sec\nThe rate of blocks being uploaded from the ingesters\nto object storage.\n\n", "fill": 10, - "id": 23, + "id": 19, "legend": { "avg": false, "current": false, @@ -1968,7 +1642,7 @@ "datasource": "$datasource", "description": "### Upload latency\nThe average, median (50th percentile), and 99th percentile time\nthe ingesters take to upload blocks to object storage.\n\n", "fill": 1, - "id": 24, + "id": 20, "legend": { "avg": false, "current": false, @@ -2079,7 +1753,7 @@ "datasource": "$datasource", "description": "### Compactions per second\nIngesters maintain a local TSDB per-tenant on disk. Each TSDB maintains a head block for each\nactive time series; these blocks get periodically compacted (by default, every 2h).\nThis panel shows the rate of compaction operations across all TSDBs on all ingesters.\n\n", "fill": 10, - "id": 25, + "id": 21, "legend": { "avg": false, "current": false, @@ -2166,7 +1840,7 @@ "datasource": "$datasource", "description": "### Compaction latency\nThe average, median (50th percentile), and 99th percentile time ingesters take to compact TSDB head blocks\non the local filesystem.\n\n", "fill": 1, - "id": 26, + "id": 22, "legend": { "avg": false, "current": false, @@ -2277,7 +1951,7 @@ "datasource": "$datasource", "description": "### WAL truncations per second\nThe WAL is truncated each time a new TSDB block is written. This panel measures the rate of\ntruncations.\n\n", "fill": 10, - "id": 27, + "id": 23, "legend": { "avg": false, "current": false, @@ -2367,7 +2041,7 @@ "datasource": "$datasource", "description": "### Checkpoints created per second\nCheckpoints are created as part of the WAL truncation process.\nThis metric measures the rate of checkpoint creation.\n\n", "fill": 10, - "id": 28, + "id": 24, "legend": { "avg": false, "current": false, @@ -2454,7 +2128,7 @@ "datasource": "$datasource", "description": "### WAL truncations latency (including checkpointing)\nAverage time taken to perform a full WAL truncation,\nincluding the time taken for the checkpointing to complete.\n\n", "fill": 1, - "id": 29, + "id": 25, "legend": { "avg": false, "current": false, @@ -2534,7 +2208,7 @@ "dashes": false, "datasource": "$datasource", "fill": 10, - "id": 30, + "id": 26, "legend": { "avg": false, "current": false, @@ -2633,7 +2307,7 @@ "datasource": "$datasource", "description": "### Distributor exemplars incoming rate\nThe rate of exemplars that have come in to the distributor, including rejected or deduped exemplars.\n\n", "fill": 1, - "id": 31, + "id": 27, "legend": { "avg": false, "current": false, @@ -2711,7 +2385,7 @@ "datasource": "$datasource", "description": "### Distributor exemplars received rate\nThe rate of received exemplars, excluding rejected and deduped exemplars.\nThis number can be sensibly lower than incoming rate because we dedupe the HA sent exemplars, and then reject based on time, see `cortex_discarded_exemplars_total` for specific reasons rates.\n\n", "fill": 1, - "id": 32, + "id": 28, "legend": { "avg": false, "current": false, @@ -2789,7 +2463,7 @@ "datasource": "$datasource", "description": "### Ingester ingested exemplars rate\nThe rate of exemplars ingested in the ingesters.\nEvery exemplar is sent to the replication factor number of ingesters, so the sum of rates from all ingesters is divided by the replication factor.\nThis ingested exemplars rate should match the distributor's received exemplars rate.\n\n", "fill": 1, - "id": 33, + "id": 29, "legend": { "avg": false, "current": false, @@ -2867,7 +2541,7 @@ "datasource": "$datasource", "description": "### Ingester appended exemplars rate\nThe rate of exemplars appended in the ingesters.\nThis can be lower than ingested exemplars rate since TSDB does not append the same exemplar twice, and those can be frequent.\n\n", "fill": 1, - "id": 34, + "id": 30, "legend": { "avg": false, "current": false, @@ -2959,7 +2633,7 @@ "value": "default" }, "hide": 0, - "label": null, + "label": "Data Source", "name": "datasource", "options": [ ], "query": "prometheus", @@ -3048,6 +2722,6 @@ }, "timezone": "utc", "title": "Mimir / Writes", - "uid": "0156f6d15aa234d452a33a4f13c838e3", + "uid": "8280707b8f16e7b87b840fc1cc92d4c5", "version": 0 } \ No newline at end of file diff --git a/operations/mimir-mixin-compiled/rules.yaml b/operations/mimir-mixin-compiled/rules.yaml index 7de1f25d110..b56f222f923 100644 --- a/operations/mimir-mixin-compiled/rules.yaml +++ b/operations/mimir-mixin-compiled/rules.yaml @@ -225,6 +225,8 @@ groups: - expr: sum(rate(cortex_query_frontend_queue_duration_seconds_count[1m])) by (cluster, job) record: cluster_job:cortex_query_frontend_queue_duration_seconds_count:sum_rate +- name: mimir_ingester_queries + rules: - expr: histogram_quantile(0.99, sum(rate(cortex_ingester_queried_series_bucket[1m])) by (le, cluster, job)) record: cluster_job:cortex_ingester_queried_series:99quantile @@ -458,7 +460,7 @@ groups: sum by (cluster, namespace, deployment) ( label_replace( label_replace( - container_memory_usage_bytes, + container_memory_usage_bytes{image!=""}, "deployment", "$1", "pod", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))" ), # The question mark in "(.*?)" is used to make it non-greedy, otherwise it diff --git a/operations/mimir-mixin-tools/screenshots/Dockerfile b/operations/mimir-mixin-tools/screenshots/Dockerfile index 331d5892bfa..c61b8dd060c 100644 --- a/operations/mimir-mixin-tools/screenshots/Dockerfile +++ b/operations/mimir-mixin-tools/screenshots/Dockerfile @@ -1,8 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only - FROM node:17-bullseye -# Install Chrome dependencies. +# Install chromium along with its dependencies. RUN apt-get update && apt-get install -y \ libnss3 \ libatk1.0-0 \ @@ -15,7 +14,12 @@ RUN apt-get update && apt-get install -y \ libxrandr2 \ libgbm-dev \ libasound2 \ - pngquant + pngquant \ + chromium + +# Tell puppeter to skip chromium installation +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium # Install sources and dependencies. RUN mkdir /app diff --git a/operations/mimir-mixin-tools/screenshots/app.js b/operations/mimir-mixin-tools/screenshots/app.js index 268c5320117..6c988e869bc 100644 --- a/operations/mimir-mixin-tools/screenshots/app.js +++ b/operations/mimir-mixin-tools/screenshots/app.js @@ -23,6 +23,8 @@ const customViewportHeight = { 'mimir-top-tenants': 2000, 'mimir-writes-networking': 1000, 'mimir-writes-resources': 1600, + 'mimir-remote-ruler-reads': 1800, + 'mimir-remote-ruler-reads-resources': 1100, }; // Dashboards for which we're not generating the screenshots because their content diff --git a/operations/mimir-mixin-tools/screenshots/package-lock.json b/operations/mimir-mixin-tools/screenshots/package-lock.json index 119623ae70f..72b357d5a6b 100644 --- a/operations/mimir-mixin-tools/screenshots/package-lock.json +++ b/operations/mimir-mixin-tools/screenshots/package-lock.json @@ -5,7 +5,7 @@ "packages": { "": { "dependencies": { - "puppeteer": "^13.1.2" + "puppeteer": "^14.1.1" } }, "node_modules/@types/node": { @@ -118,10 +118,18 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "node_modules/cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "dependencies": { + "node-fetch": "2.6.7" + } + }, "node_modules/debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, @@ -135,9 +143,9 @@ } }, "node_modules/devtools-protocol": { - "version": "0.0.948846", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.948846.tgz", - "integrity": "sha512-5fGyt9xmMqUl2VI7+rnUkKCiAQIpLns8sfQtTENy5L70ktbNw0Z3TFJ1JoFNYdx/jffz4YXU45VF75wKZD7sZQ==" + "version": "0.0.982423", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.982423.tgz", + "integrity": "sha512-FnVW2nDbjGNw1uD/JRC+9U5768W7e1TfUwqbDTcSsAu1jXFjITSX8w3rkW5FEpHRMPPGpvNSmO1pOpqByiWscA==" }, "node_modules/end-of-stream": { "version": "1.4.4", @@ -230,9 +238,9 @@ } }, "node_modules/https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "dependencies": { "agent-base": "6", "debug": "4" @@ -421,26 +429,26 @@ } }, "node_modules/puppeteer": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-13.1.2.tgz", - "integrity": "sha512-ozVM8Tdg0patMtm/xAr3Uh7rQ28vBpbTHLP+ECmoAxG/s4PKrVLN764H/poLux7Ln77jHThOd8OBJj5mTuA6Iw==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-14.1.1.tgz", + "integrity": "sha512-4dC6GYR5YlXTmNO3TbYEHTdVSdml1cVD2Ok/h/f/xSTp4ITVdbRWkMjiOaEKRAhtIl6GqaP7B89zx+hfhcNGMQ==", "hasInstallScript": true, "dependencies": { - "debug": "4.3.2", - "devtools-protocol": "0.0.948846", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.982423", "extract-zip": "2.0.1", - "https-proxy-agent": "5.0.0", - "node-fetch": "2.6.7", + "https-proxy-agent": "5.0.1", "pkg-dir": "4.2.0", "progress": "2.0.3", "proxy-from-env": "1.1.0", "rimraf": "3.0.2", "tar-fs": "2.1.1", "unbzip2-stream": "1.4.3", - "ws": "8.2.3" + "ws": "8.6.0" }, "engines": { - "node": ">=10.18.1" + "node": ">=14.1.0" } }, "node_modules/readable-stream": { @@ -567,9 +575,9 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "node_modules/ws": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", - "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.6.0.tgz", + "integrity": "sha512-AzmM3aH3gk0aX7/rZLYvjdvZooofDu3fFOzGqcSnQ1tOcTWwhM/o+q++E8mAyVVIyUdajrkzWUGftaVSDLn1bw==", "engines": { "node": ">=10.0.0" }, @@ -673,18 +681,26 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "cross-fetch": { + "version": "3.1.5", + "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", + "integrity": "sha512-lvb1SBsI0Z7GDwmuid+mU3kWVBwTVUbe7S0H52yaaAdQOXq2YktTCZdlAcNKFzE6QtRz0snpw9bNiPeOIkkQvw==", + "requires": { + "node-fetch": "2.6.7" + } + }, "debug": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.2.tgz", - "integrity": "sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==", + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "requires": { "ms": "2.1.2" } }, "devtools-protocol": { - "version": "0.0.948846", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.948846.tgz", - "integrity": "sha512-5fGyt9xmMqUl2VI7+rnUkKCiAQIpLns8sfQtTENy5L70ktbNw0Z3TFJ1JoFNYdx/jffz4YXU45VF75wKZD7sZQ==" + "version": "0.0.982423", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.982423.tgz", + "integrity": "sha512-FnVW2nDbjGNw1uD/JRC+9U5768W7e1TfUwqbDTcSsAu1jXFjITSX8w3rkW5FEpHRMPPGpvNSmO1pOpqByiWscA==" }, "end-of-stream": { "version": "1.4.4", @@ -754,9 +770,9 @@ } }, "https-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", - "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", + "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", "requires": { "agent-base": "6", "debug": "4" @@ -887,22 +903,22 @@ } }, "puppeteer": { - "version": "13.1.2", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-13.1.2.tgz", - "integrity": "sha512-ozVM8Tdg0patMtm/xAr3Uh7rQ28vBpbTHLP+ECmoAxG/s4PKrVLN764H/poLux7Ln77jHThOd8OBJj5mTuA6Iw==", + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-14.1.1.tgz", + "integrity": "sha512-4dC6GYR5YlXTmNO3TbYEHTdVSdml1cVD2Ok/h/f/xSTp4ITVdbRWkMjiOaEKRAhtIl6GqaP7B89zx+hfhcNGMQ==", "requires": { - "debug": "4.3.2", - "devtools-protocol": "0.0.948846", + "cross-fetch": "3.1.5", + "debug": "4.3.4", + "devtools-protocol": "0.0.982423", "extract-zip": "2.0.1", - "https-proxy-agent": "5.0.0", - "node-fetch": "2.6.7", + "https-proxy-agent": "5.0.1", "pkg-dir": "4.2.0", "progress": "2.0.3", "proxy-from-env": "1.1.0", "rimraf": "3.0.2", "tar-fs": "2.1.1", "unbzip2-stream": "1.4.3", - "ws": "8.2.3" + "ws": "8.6.0" } }, "readable-stream": { @@ -1003,9 +1019,9 @@ "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "ws": { - "version": "8.2.3", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.2.3.tgz", - "integrity": "sha512-wBuoj1BDpC6ZQ1B7DWQBYVLphPWkm8i9Y0/3YdHjHKHiohOJ1ws+3OccDWtH+PoC9DZD5WOTrJvNbWvjS6JWaA==", + "version": "8.6.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.6.0.tgz", + "integrity": "sha512-AzmM3aH3gk0aX7/rZLYvjdvZooofDu3fFOzGqcSnQ1tOcTWwhM/o+q++E8mAyVVIyUdajrkzWUGftaVSDLn1bw==", "requires": {} }, "yauzl": { diff --git a/operations/mimir-mixin-tools/screenshots/package.json b/operations/mimir-mixin-tools/screenshots/package.json index 3dc4b69dfcf..09ee1c7c4a3 100644 --- a/operations/mimir-mixin-tools/screenshots/package.json +++ b/operations/mimir-mixin-tools/screenshots/package.json @@ -1,5 +1,5 @@ { "dependencies": { - "puppeteer": "^13.1.2" + "puppeteer": "^14.1.1" } } diff --git a/operations/mimir-mixin/alerts/alerts.libsonnet b/operations/mimir-mixin/alerts/alerts.libsonnet index c6df49aafad..e328a5b3027 100644 --- a/operations/mimir-mixin/alerts/alerts.libsonnet +++ b/operations/mimir-mixin/alerts/alerts.libsonnet @@ -127,7 +127,7 @@ { alert: $.alertName('FrontendQueriesStuck'), expr: ||| - sum by (%s) (cortex_query_frontend_queue_length) > 1 + sum by (%s, job) (cortex_query_frontend_queue_length) > 1 ||| % $._config.alert_aggregation_labels, 'for': '5m', // We don't want to block for longer. labels: { @@ -135,14 +135,14 @@ }, annotations: { message: ||| - There are {{ $value }} queued up queries in %(alert_aggregation_variables)s query-frontend. + There are {{ $value }} queued up queries in %(alert_aggregation_variables)s {{ $labels.job }}. ||| % $._config, }, }, { alert: $.alertName('SchedulerQueriesStuck'), expr: ||| - sum by (%s) (cortex_query_scheduler_queue_length) > 1 + sum by (%s, job) (cortex_query_scheduler_queue_length) > 1 ||| % $._config.alert_aggregation_labels, 'for': '5m', // We don't want to block for longer. labels: { @@ -150,7 +150,7 @@ }, annotations: { message: ||| - There are {{ $value }} queued up queries in %(alert_aggregation_variables)s query-scheduler. + There are {{ $value }} queued up queries in %(alert_aggregation_variables)s {{ $labels.job }}. ||| % $._config, }, }, @@ -453,7 +453,7 @@ ( container_memory_working_set_bytes{container="ingester"} / - container_spec_memory_limit_bytes{container="ingester"} + ( container_spec_memory_limit_bytes{container="ingester"} > 0 ) ) > 0.65 |||, 'for': '15m', @@ -472,7 +472,7 @@ ( container_memory_working_set_bytes{container="ingester"} / - container_spec_memory_limit_bytes{container="ingester"} + ( container_spec_memory_limit_bytes{container="ingester"} > 0 ) ) > 0.8 |||, 'for': '15m', @@ -531,10 +531,11 @@ { alert: $.alertName('RulerMissedEvaluations'), expr: ||| + 100 * ( sum by (%(alert_aggregation_labels)s, %(per_instance_label)s, rule_group) (rate(cortex_prometheus_rule_group_iterations_missed_total[1m])) / sum by (%(alert_aggregation_labels)s, %(per_instance_label)s, rule_group) (rate(cortex_prometheus_rule_group_iterations_total[1m])) - > 0.01 + ) > 1 ||| % $._config, 'for': '5m', labels: { @@ -571,11 +572,9 @@ alert: $.alertName('GossipMembersMismatch'), expr: ||| - memberlist_client_cluster_members_count - != on (%s) group_left - sum by (%s) (up{job=~".+/%s"}) + avg by (%s) (memberlist_client_cluster_members_count) != sum by (%s) (up{job=~".+/%s"}) ||| % [$._config.alert_aggregation_labels, $._config.alert_aggregation_labels, simpleRegexpOpt($._config.job_names.ring_members)], - 'for': '5m', + 'for': '15m', labels: { severity: 'warning', }, @@ -594,7 +593,7 @@ ( container_memory_working_set_bytes{container="etcd"} / - container_spec_memory_limit_bytes{container="etcd"} + ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.65 |||, 'for': '15m', @@ -613,7 +612,7 @@ ( container_memory_working_set_bytes{container="etcd"} / - container_spec_memory_limit_bytes{container="etcd"} + ( container_spec_memory_limit_bytes{container="etcd"} > 0 ) ) > 0.8 |||, 'for': '15m', diff --git a/operations/mimir-mixin/alerts/blocks.libsonnet b/operations/mimir-mixin/alerts/blocks.libsonnet index d6b45e7a063..cc730e1ee39 100644 --- a/operations/mimir-mixin/alerts/blocks.libsonnet +++ b/operations/mimir-mixin/alerts/blocks.libsonnet @@ -206,7 +206,21 @@ severity: 'critical', }, annotations: { - message: '%(product)s Store Gateway %(alert_instance_variable)s in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.' % $._config, + message: '%(product)s store-gateway %(alert_instance_variable)s in %(alert_aggregation_variables)s has not successfully synched the bucket since {{ $value | humanizeDuration }}.' % $._config, + }, + }, + { + // Alert if the store-gateway is not owning any tenant. + alert: $.alertName('StoreGatewayNoSyncedTenants'), + 'for': '1h', + expr: ||| + min by(%(alert_aggregation_labels)s, %(per_instance_label)s) (cortex_bucket_stores_tenants_synced{component="store-gateway"}) == 0 + ||| % $._config, + labels: { + severity: 'warning', + }, + annotations: { + message: '%(product)s store-gateway %(alert_instance_variable)s in %(alert_aggregation_variables)s is not syncing any blocks for any tenant.' % $._config, }, }, { diff --git a/operations/mimir-mixin/alerts/compactor.libsonnet b/operations/mimir-mixin/alerts/compactor.libsonnet index dbf0147269a..7558c7850fc 100644 --- a/operations/mimir-mixin/alerts/compactor.libsonnet +++ b/operations/mimir-mixin/alerts/compactor.libsonnet @@ -65,10 +65,10 @@ alert: $.alertName('CompactorHasNotUploadedBlocks'), 'for': '15m', expr: ||| - (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/(%(compactor)s)"} > 60 * 60 * 24) + (time() - thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 60 * 60 * 24) and - (thanos_objstore_bucket_last_successful_upload_time{job=~".+/(%(compactor)s)"} > 0) - ||| % $._config.job_names, + (thanos_objstore_bucket_last_successful_upload_time{component="compactor"} > 0) + |||, labels: { severity: 'critical', }, @@ -81,8 +81,8 @@ alert: $.alertName('CompactorHasNotUploadedBlocks'), 'for': '24h', expr: ||| - thanos_objstore_bucket_last_successful_upload_time{job=~".+/(%(compactor)s)"} == 0 - ||| % $._config.job_names, + thanos_objstore_bucket_last_successful_upload_time{component="compactor"} == 0 + |||, labels: { severity: 'critical', }, @@ -95,8 +95,8 @@ alert: $.alertName('CompactorSkippedBlocksWithOutOfOrderChunks'), 'for': '1m', expr: ||| - increase(cortex_compactor_blocks_marked_for_no_compaction_total{job=~".+/(%(compactor)s)", reason="block-index-out-of-order-chunk"}[5m]) > 0 - ||| % $._config.job_names, + increase(cortex_compactor_blocks_marked_for_no_compaction_total{component="compactor", reason="block-index-out-of-order-chunk"}[5m]) > 0 + |||, labels: { severity: 'warning', }, diff --git a/operations/mimir-mixin/alerts/continuous-test.libsonnet b/operations/mimir-mixin/alerts/continuous-test.libsonnet index 4e7e717a850..cbdc358733f 100644 --- a/operations/mimir-mixin/alerts/continuous-test.libsonnet +++ b/operations/mimir-mixin/alerts/continuous-test.libsonnet @@ -38,7 +38,7 @@ // should have no "grace period" and alert as soon as the test fails. alert: $.alertName('ContinuousTestFailed'), expr: ||| - sum by(%(alert_aggregation_labels)s, test) (rate(mimir_continuous_test_query_result_checks_failed_total[5m])) > 0 + sum by(%(alert_aggregation_labels)s, test) (rate(mimir_continuous_test_query_result_checks_failed_total[10m])) > 0 ||| % $._config, labels: { severity: 'warning', diff --git a/operations/mimir-mixin/config.libsonnet b/operations/mimir-mixin/config.libsonnet index c02eb76bcb1..e6665145fcf 100644 --- a/operations/mimir-mixin/config.libsonnet +++ b/operations/mimir-mixin/config.libsonnet @@ -24,10 +24,13 @@ ingester: '(ingester.*|cortex|mimir)', // Match also custom and per-zone ingester deployments. distributor: '(distributor|cortex|mimir)', querier: '(querier.*|cortex|mimir)', // Match also custom querier deployments. + ruler_querier: '(ruler-querier.*)', // Match also custom querier deployments. ruler: '(ruler|cortex|mimir)', query_frontend: '(query-frontend.*|cortex|mimir)', // Match also custom query-frontend deployments. + ruler_query_frontend: '(ruler-query-frontend.*)', // Match also custom ruler-query-frontend deployments. query_scheduler: 'query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. - ring_members: ['compactor', 'distributor', 'ingester.*', 'querier.*', 'ruler', 'store-gateway.*', 'cortex', 'mimir'], + ruler_query_scheduler: 'ruler-query-scheduler.*', // Not part of single-binary. Match also custom query-scheduler deployments. + ring_members: ['alertmanager', 'compactor', 'distributor', 'ingester.*', 'querier.*', 'ruler', 'store-gateway.*', 'cortex', 'mimir'], store_gateway: '(store-gateway.*|cortex|mimir)', // Match also per-zone store-gateway deployments. gateway: '(gateway|cortex-gw|cortex-gw-internal)', compactor: 'compactor.*|cortex|mimir', // Match also custom compactor deployments. @@ -47,21 +50,26 @@ // Whether resources dashboards are enabled (based on cAdvisor metrics). resources_dashboards_enabled: true, + // Whether mimir gateway is enabled + gateway_enabled: false, + // The label used to differentiate between different application instances (i.e. 'pod' in a kubernetes install). per_instance_label: 'pod', // Name selectors for different application instances, using the "per_instance_label". instance_names: { - compactor: 'compactor.*', - alertmanager: 'alertmanager.*', - ingester: 'ingester.*', - distributor: 'distributor.*', - querier: 'querier.*', - ruler: 'ruler.*', - query_frontend: 'query-frontend.*', - query_scheduler: 'query-scheduler.*', - store_gateway: 'store-gateway.*', - gateway: '(gateway|cortex-gw|cortex-gw).*', + local helmCompatibleName = function(name) '(.*-mimir-)?%s' % name, + + compactor: helmCompatibleName('compactor.*'), + alertmanager: helmCompatibleName('alertmanager.*'), + ingester: helmCompatibleName('ingester.*'), + distributor: helmCompatibleName('distributor.*'), + querier: helmCompatibleName('querier.*'), + ruler: helmCompatibleName('ruler.*'), + query_frontend: helmCompatibleName('query-frontend.*'), + query_scheduler: helmCompatibleName('query-scheduler.*'), + store_gateway: helmCompatibleName('store-gateway.*'), + gateway: helmCompatibleName('(gateway|cortex-gw|cortex-gw).*'), }, // The label used to differentiate between different nodes (i.e. servers). @@ -79,6 +87,7 @@ autoscaling: { querier_enabled: false, querier_hpa_name: 'keda-hpa-querier', + ruler_querier_hpa_name: 'keda-hpa-ruler-querier', }, // The routes to exclude from alerts. @@ -86,5 +95,6 @@ // The default datasource used for dashboards. dashboard_datasource: 'default', + datasource_regex: '', }, } diff --git a/operations/mimir-mixin/dashboards.libsonnet b/operations/mimir-mixin/dashboards.libsonnet index efe07c0ec71..01ebbc6c5f0 100644 --- a/operations/mimir-mixin/dashboards.libsonnet +++ b/operations/mimir-mixin/dashboards.libsonnet @@ -4,6 +4,7 @@ (import 'dashboards/queries.libsonnet') + (import 'dashboards/reads.libsonnet') + (import 'dashboards/ruler.libsonnet') + + (import 'dashboards/remote-ruler-reads.libsonnet') + (import 'dashboards/alertmanager.libsonnet') + (import 'dashboards/scaling.libsonnet') + (import 'dashboards/writes.libsonnet') + @@ -18,6 +19,7 @@ (if !$._config.resources_dashboards_enabled then {} else (import 'dashboards/reads-resources.libsonnet') + + (import 'dashboards/remote-ruler-reads-resources.libsonnet') + (import 'dashboards/reads-networking.libsonnet') + (import 'dashboards/writes-resources.libsonnet') + (import 'dashboards/writes-networking.libsonnet') + diff --git a/operations/mimir-mixin/dashboards/alertmanager-resources.libsonnet b/operations/mimir-mixin/dashboards/alertmanager-resources.libsonnet index 20385bf1e5c..117a0cd8468 100644 --- a/operations/mimir-mixin/dashboards/alertmanager-resources.libsonnet +++ b/operations/mimir-mixin/dashboards/alertmanager-resources.libsonnet @@ -1,10 +1,12 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-alertmanager-resources.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-alertmanager-resources.json': - ($.dashboard('Alertmanager resources') + { uid: '68b66aed90ccab448009089544a8d6c6' }) + [filename]: + ($.dashboard('Alertmanager resources') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Gateway') .addPanel( $.containerCPUUsagePanel('CPU', $._config.job_names.gateway), diff --git a/operations/mimir-mixin/dashboards/alertmanager.libsonnet b/operations/mimir-mixin/dashboards/alertmanager.libsonnet index 43c2f9392f1..8f7ae832db9 100644 --- a/operations/mimir-mixin/dashboards/alertmanager.libsonnet +++ b/operations/mimir-mixin/dashboards/alertmanager.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-alertmanager.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-alertmanager.json': - ($.dashboard('Alertmanager') + { uid: 'a76bee5913c97c918d9e56a3cc88cc28' }) + [filename]: + ($.dashboard('Alertmanager') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( ($.row('Headlines') + { @@ -88,7 +89,8 @@ local utils = import 'mixin-utils/utils.libsonnet'; $.latencyPanel('cortex_alertmanager_notification_latency_seconds', '{%s}' % $.jobMatcher($._config.job_names.alertmanager)) ) ) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Configuration API (gateway) + Alertmanager UI') .addPanel( $.panel('QPS') + diff --git a/operations/mimir-mixin/dashboards/compactor-resources.libsonnet b/operations/mimir-mixin/dashboards/compactor-resources.libsonnet index 7057cfdefa2..ad48d0945d4 100644 --- a/operations/mimir-mixin/dashboards/compactor-resources.libsonnet +++ b/operations/mimir-mixin/dashboards/compactor-resources.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-compactor-resources.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-compactor-resources.json': - ($.dashboard('Compactor resources') + { uid: 'df9added6f1f4332f95848cca48ebd99' }) + [filename]: + ($.dashboard('Compactor resources') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( $.row('CPU and memory') diff --git a/operations/mimir-mixin/dashboards/compactor.libsonnet b/operations/mimir-mixin/dashboards/compactor.libsonnet index ef8216ccb16..7957895ff87 100644 --- a/operations/mimir-mixin/dashboards/compactor.libsonnet +++ b/operations/mimir-mixin/dashboards/compactor.libsonnet @@ -1,4 +1,5 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-compactor.json'; // This applies to the "longest time since successful run queries" local fixTargetsForTransformations(panel, refIds) = panel { @@ -87,8 +88,8 @@ local fixTargetsForTransformations(panel, refIds) = panel { }), ], - 'mimir-compactor.json': - ($.dashboard('Compactor') + { uid: '9c408e1d55681ecb8a22c9fab46875cc' }) + [filename]: + ($.dashboard('Compactor') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( $.row('Summary') diff --git a/operations/mimir-mixin/dashboards/config.libsonnet b/operations/mimir-mixin/dashboards/config.libsonnet index ce62fa0f3eb..aee56f6c44a 100644 --- a/operations/mimir-mixin/dashboards/config.libsonnet +++ b/operations/mimir-mixin/dashboards/config.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-config.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-config.json': - ($.dashboard('Config') + { uid: '61bb048ced9817b2d3e07677fb1c6290' }) + [filename]: + ($.dashboard('Config') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( $.row('Startup config file') diff --git a/operations/mimir-mixin/dashboards/dashboard-utils.libsonnet b/operations/mimir-mixin/dashboards/dashboard-utils.libsonnet index 12046114b1f..06ca9cac65c 100644 --- a/operations/mimir-mixin/dashboards/dashboard-utils.libsonnet +++ b/operations/mimir-mixin/dashboards/dashboard-utils.libsonnet @@ -4,6 +4,14 @@ local utils = import 'mixin-utils/utils.libsonnet'; _config:: error 'must provide _config', + row(title):: + super.row(title) + { + addPanelIf(condition, panel):: + if condition + then self.addPanel(panel) + else self, + }, + // Override the dashboard constructor to add: // - default tags, // - some links that propagate the selectred cluster. @@ -11,8 +19,18 @@ local utils = import 'mixin-utils/utils.libsonnet'; // Prefix the dashboard title with " /" unless configured otherwise. super.dashboard( title='%(prefix)s%(title)s' % { prefix: $._config.dashboard_prefix, title: title }, - datasource=$._config.dashboard_datasource + datasource=$._config.dashboard_datasource, + datasource_regex=$._config.datasource_regex ) + { + __requires: [ + { + id: 'grafana', + name: 'Grafana', + type: 'grafana', + version: '8.0.0', + }, + ], + addRowIf(condition, row):: if condition then self.addRow(row) @@ -114,6 +132,14 @@ local utils = import 'mixin-utils/utils.libsonnet'; then [utils.selector.noop('%s' % $._config.per_cluster_label), utils.selector.re('job', '$job')] else [utils.selector.re('%s' % $._config.per_cluster_label, '$cluster'), utils.selector.re('job', '($namespace)/(%s)' % job)], + panel(title):: + super.panel(title) + { + tooltip+: { + shared: false, + sort: 0, + }, + }, + queryPanel(queries, legends, legendLink=null):: super.queryPanel(queries, legends, legendLink) + { targets: [ diff --git a/operations/mimir-mixin/dashboards/object-store.libsonnet b/operations/mimir-mixin/dashboards/object-store.libsonnet index c39414fdc2c..c8bb0903541 100644 --- a/operations/mimir-mixin/dashboards/object-store.libsonnet +++ b/operations/mimir-mixin/dashboards/object-store.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-object-store.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-object-store.json': - ($.dashboard('Object Store') + { uid: 'd5a3a4489d57c733b5677fb55370a723' }) + [filename]: + ($.dashboard('Object Store') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( $.row('Components') diff --git a/operations/mimir-mixin/dashboards/overrides.libsonnet b/operations/mimir-mixin/dashboards/overrides.libsonnet index 980ed068e38..3f5226d6fe8 100644 --- a/operations/mimir-mixin/dashboards/overrides.libsonnet +++ b/operations/mimir-mixin/dashboards/overrides.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-overrides.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-overrides.json': - ($.dashboard('Overrides') + { uid: 'b5c95fee2e5e7c4b5930826ff6e89a12' }) + [filename]: + ($.dashboard('Overrides') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) .addRow( $.row('') diff --git a/operations/mimir-mixin/dashboards/queries.libsonnet b/operations/mimir-mixin/dashboards/queries.libsonnet index 67aceaa0c81..84610b000aa 100644 --- a/operations/mimir-mixin/dashboards/queries.libsonnet +++ b/operations/mimir-mixin/dashboards/queries.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-queries.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-queries.json': - ($.dashboard('Queries') + { uid: 'd9931b1054053c8b972d320774bb8f1d' }) + [filename]: + ($.dashboard('Queries') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( $.row('Query-frontend') @@ -55,7 +56,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; $.row('Query-frontend - query splitting and results cache') .addPanel( $.panel('Intervals per Query') + - $.queryPanel('sum(rate(cortex_frontend_split_queries_total{%s}[1m])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{%s, method="split_by_interval"}[1m]))' % [$.jobMatcher($._config.job_names.query_frontend), $.jobMatcher($._config.job_names.query_frontend)], 'splitting rate') + + $.queryPanel('sum(rate(cortex_frontend_split_queries_total{%s}[$__rate_interval])) / sum(rate(cortex_frontend_query_range_duration_seconds_count{%s, method="split_by_interval"}[$__rate_interval]))' % [$.jobMatcher($._config.job_names.query_frontend), $.jobMatcher($._config.job_names.query_frontend)], 'splitting rate') + $.panelDescription( 'Intervals per query', ||| @@ -69,15 +70,15 @@ local utils = import 'mixin-utils/utils.libsonnet'; ||| # Query metrics before and after migration to new memcached backend. sum ( - rate(cortex_cache_hits{name=~"frontend.+", %(frontend)s}[1m]) + rate(cortex_cache_hits{name=~"frontend.+", %(frontend)s}[$__rate_interval]) or - rate(thanos_cache_memcached_hits_total{name="frontend-cache", %(frontend)s}[1m]) + rate(thanos_cache_memcached_hits_total{name="frontend-cache", %(frontend)s}[$__rate_interval]) ) / sum ( - rate(cortex_cache_fetched_keys{name=~"frontend.+", %(frontend)s}[1m]) + rate(cortex_cache_fetched_keys{name=~"frontend.+", %(frontend)s}[$__rate_interval]) or - rate(thanos_cache_memcached_requests_total{name=~"frontend-cache", %(frontend)s}[1m]) + rate(thanos_cache_memcached_requests_total{name=~"frontend-cache", %(frontend)s}[$__rate_interval]) ) ||| % { frontend: $.jobMatcher($._config.job_names.query_frontend), @@ -92,15 +93,15 @@ local utils = import 'mixin-utils/utils.libsonnet'; ||| # Query metrics before and after migration to new memcached backend. sum ( - rate(cortex_cache_fetched_keys{name=~"frontend.+", %(frontend)s}[1m]) + rate(cortex_cache_fetched_keys{name=~"frontend.+", %(frontend)s}[$__rate_interval]) or - rate(thanos_cache_memcached_requests_total{name="frontend-cache", %(frontend)s}[1m]) + rate(thanos_cache_memcached_requests_total{name="frontend-cache", %(frontend)s}[$__rate_interval]) ) - sum ( - rate(cortex_cache_hits{name=~"frontend.+", %(frontend)s}[1m]) + rate(cortex_cache_hits{name=~"frontend.+", %(frontend)s}[$__rate_interval]) or - rate(thanos_cache_memcached_hits_total{name=~"frontend-cache", %(frontend)s}[1m]) + rate(thanos_cache_memcached_hits_total{name=~"frontend-cache", %(frontend)s}[$__rate_interval]) ) ||| % { frontend: $.jobMatcher($._config.job_names.query_frontend), @@ -155,11 +156,6 @@ local utils = import 'mixin-utils/utils.libsonnet'; utils.latencyRecordingRulePanel('cortex_ingester_queried_series', $.jobSelector($._config.job_names.ingester), multiplier=1) + { yaxes: $.yaxes('short') }, ) - .addPanel( - $.panel('Chunks per query') + - utils.latencyRecordingRulePanel('cortex_ingester_queried_chunks', $.jobSelector($._config.job_names.ingester), multiplier=1) + - { yaxes: $.yaxes('short') }, - ) .addPanel( $.panel('Samples per query') + utils.latencyRecordingRulePanel('cortex_ingester_queried_samples', $.jobSelector($._config.job_names.ingester), multiplier=1) + @@ -185,7 +181,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; ) .addPanel( $.panel('Consistency checks failed') + - $.queryPanel('sum(rate(cortex_querier_blocks_consistency_checks_failed_total{%s}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{%s}[1m]))' % [$.jobMatcher($._config.job_names.querier), $.jobMatcher($._config.job_names.querier)], 'Failure Rate') + + $.queryPanel('sum(rate(cortex_querier_blocks_consistency_checks_failed_total{%s}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{%s}[$__rate_interval]))' % [$.jobMatcher($._config.job_names.querier), $.jobMatcher($._config.job_names.querier)], 'Failure Rate') + { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) }, ) ) diff --git a/operations/mimir-mixin/dashboards/reads-networking.libsonnet b/operations/mimir-mixin/dashboards/reads-networking.libsonnet index f8e9cfcd969..b3852098b48 100644 --- a/operations/mimir-mixin/dashboards/reads-networking.libsonnet +++ b/operations/mimir-mixin/dashboards/reads-networking.libsonnet @@ -1,10 +1,11 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-reads-networking.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-reads-networking.json': - ($.dashboard('Reads networking') + { uid: 'c0464f0d8bd026f776c9006b05910000' }) + [filename]: + ($.dashboard('Reads networking') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) - .addRow($.jobNetworkingRow('Gateway', 'gateway')) + .addRowIf($._config.gateway_enabled, $.jobNetworkingRow('Gateway', 'gateway')) .addRow($.jobNetworkingRow('Query-frontend', 'query_frontend')) .addRow($.jobNetworkingRow('Query-scheduler', 'query_scheduler')) .addRow($.jobNetworkingRow('Querier', 'querier')) diff --git a/operations/mimir-mixin/dashboards/reads-resources.libsonnet b/operations/mimir-mixin/dashboards/reads-resources.libsonnet index 7661df45aac..d6c39df55ad 100644 --- a/operations/mimir-mixin/dashboards/reads-resources.libsonnet +++ b/operations/mimir-mixin/dashboards/reads-resources.libsonnet @@ -1,10 +1,12 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-reads-resources.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-reads-resources.json': - ($.dashboard('Reads resources') + { uid: '2fd2cda9eea8d8af9fbc0a5960425120' }) + [filename]: + ($.dashboard('Reads resources') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Gateway') .addPanel( $.containerCPUUsagePanel('CPU', $._config.job_names.gateway), diff --git a/operations/mimir-mixin/dashboards/reads.libsonnet b/operations/mimir-mixin/dashboards/reads.libsonnet index 48a2794f0c9..94d81e3eec4 100644 --- a/operations/mimir-mixin/dashboards/reads.libsonnet +++ b/operations/mimir-mixin/dashboards/reads.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-reads.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-reads.json': - ($.dashboard('Reads') + { uid: '8d6ba60eccc4b6eedfa329b24b1bd339' }) + [filename]: + ($.dashboard('Reads') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRowIf( $._config.show_dashboard_descriptions.reads, @@ -88,7 +89,8 @@ local utils = import 'mixin-utils/utils.libsonnet'; ), ) ) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Gateway') .addPanel( $.panel('Requests / sec') + diff --git a/operations/mimir-mixin/dashboards/remote-ruler-reads-resources.libsonnet b/operations/mimir-mixin/dashboards/remote-ruler-reads-resources.libsonnet new file mode 100644 index 00000000000..6b57c8211e9 --- /dev/null +++ b/operations/mimir-mixin/dashboards/remote-ruler-reads-resources.libsonnet @@ -0,0 +1,53 @@ +local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-remote-ruler-reads-resources.json'; + +(import 'dashboard-utils.libsonnet') { + [filename]: + ($.dashboard('Remote ruler reads resources') + { uid: std.md5(filename) }) + .addClusterSelectorTemplates(false) + .addRow( + $.row('Query-frontend (dedicated to ruler)') + .addPanel( + $.containerCPUUsagePanel('CPU', 'ruler-query-frontend'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'ruler-query-frontend'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.ruler_query_frontend), + ) + ) + .addRow( + $.row('Query-scheduler (dedicated to ruler)') + .addPanel( + $.containerCPUUsagePanel('CPU', 'ruler-query-scheduler'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'ruler-query-scheduler'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.ruler_query_scheduler), + ) + ) + .addRow( + $.row('Querier (dedicated to ruler)') + .addPanel( + $.containerCPUUsagePanel('CPU', 'ruler-querier'), + ) + .addPanel( + $.containerMemoryWorkingSetPanel('Memory (workingset)', 'ruler-querier'), + ) + .addPanel( + $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.ruler_querier), + ) + ) + { + templating+: { + list: [ + // Do not allow to include all clusters/namespaces otherwise this dashboard + // risks to explode because it shows resources per pod. + l + (if (l.name == 'cluster' || l.name == 'namespace') then { includeAll: false } else {}) + for l in super.list + ], + }, + }, +} diff --git a/operations/mimir-mixin/dashboards/remote-ruler-reads.libsonnet b/operations/mimir-mixin/dashboards/remote-ruler-reads.libsonnet new file mode 100644 index 00000000000..702e2873b28 --- /dev/null +++ b/operations/mimir-mixin/dashboards/remote-ruler-reads.libsonnet @@ -0,0 +1,159 @@ +local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-remote-ruler-reads.json'; + +(import 'dashboard-utils.libsonnet') { + [filename]: + ($.dashboard('Remote ruler reads') + { uid: std.md5(filename) }) + .addClusterSelectorTemplates() + .addRowIf( + $._config.show_dashboard_descriptions.reads, + ($.row('Remote ruler reads dashboard description') { height: '175px', showTitle: false }) + .addPanel( + $.textPanel('', ||| +

+ This dashboard shows health metrics for the ruler read path when remote operational mode is enabled. + It is broken into sections for each service on the ruler read path, and organized by the order in which the read request flows. +
+ For each service, there are three panels showing (1) requests per second to that service, (2) average, median, and p99 latency of requests to that service, and (3) p99 latency of requests to each instance of that service. +

+ |||), + ) + ) + .addRow( + ($.row('Headlines') + + { + height: '100px', + showTitle: false, + }) + .addPanel( + $.panel('Evaluations / sec') + + $.statPanel(||| + sum( + rate( + cortex_request_duration_seconds_count{ + %(queryFrontend)s, + route=~"/httpgrpc.HTTP/Handle" + }[$__rate_interval] + ) + ) + ||| % { + queryFrontend: $.jobMatcher($._config.job_names.ruler_query_frontend), + }, format='reqps') + + $.panelDescription( + 'Evaluations per second', + ||| + Rate of rule expressions evaluated per second. + ||| + ), + ) + ) + .addRow( + $.row('Query-frontend (dedicated to ruler)') + .addPanel( + $.panel('Requests / sec') + + $.qpsPanel('cortex_request_duration_seconds_count{%s, route="/httpgrpc.HTTP/Handle"}' % $.jobMatcher($._config.job_names.ruler_query_frontend)) + ) + .addPanel( + $.panel('Latency') + + utils.latencyRecordingRulePanel('cortex_request_duration_seconds', $.jobSelector($._config.job_names.ruler_query_frontend) + [utils.selector.re('route', '/httpgrpc.HTTP/Handle')]) + ) + .addPanel( + $.panel('Per %s p99 latency' % $._config.per_instance_label) + + $.hiddenLegendQueryPanel( + 'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_request_duration_seconds_bucket{%s, route="/httpgrpc.HTTP/Handle"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ruler_query_frontend)], '' + ) + + { yaxes: $.yaxes('s') } + ) + ) + .addRow( + $.row('Query-scheduler (dedicated to ruler)') + .addPanel( + $.panel('Requests / sec') + + $.qpsPanel('cortex_query_scheduler_queue_duration_seconds_count{%s}' % $.jobMatcher($._config.job_names.ruler_query_scheduler)) + ) + .addPanel( + $.panel('Latency (time in queue)') + + $.latencyPanel('cortex_query_scheduler_queue_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.ruler_query_scheduler)) + ) + ) + .addRow( + $.row('Querier (dedicated to ruler)') + .addPanel( + $.panel('Requests / sec') + + $.qpsPanel('cortex_querier_request_duration_seconds_count{%s, route=~"(prometheus|api_prom)_api_v1_.+"}' % $.jobMatcher($._config.job_names.ruler_querier)) + ) + .addPanel( + $.panel('Latency') + + utils.latencyRecordingRulePanel('cortex_querier_request_duration_seconds', $.jobSelector($._config.job_names.ruler_querier) + [utils.selector.re('route', '(prometheus|api_prom)_api_v1_.+')]) + ) + .addPanel( + $.panel('Per %s p99 latency' % $._config.per_instance_label) + + $.hiddenLegendQueryPanel( + 'histogram_quantile(0.99, sum by(le, %s) (rate(cortex_querier_request_duration_seconds_bucket{%s, route=~"(prometheus|api_prom)_api_v1_.+"}[$__rate_interval])))' % [$._config.per_instance_label, $.jobMatcher($._config.job_names.ruler_querier)], '' + ) + + { yaxes: $.yaxes('s') } + ) + ) + .addRowIf( + $._config.autoscaling.querier_enabled, + $.row('Querier (dedicated to ruler) - autoscaling') + .addPanel( + local title = 'Replicas'; + $.panel(title) + + $.queryPanel( + [ + 'kube_horizontalpodautoscaler_spec_min_replicas{%s, horizontalpodautoscaler="%s"}' % [$.namespaceMatcher(), $._config.autoscaling.ruler_querier_hpa_name], + 'kube_horizontalpodautoscaler_spec_max_replicas{%s, horizontalpodautoscaler="%s"}' % [$.namespaceMatcher(), $._config.autoscaling.ruler_querier_hpa_name], + 'kube_horizontalpodautoscaler_status_current_replicas{%s, horizontalpodautoscaler="%s"}' % [$.namespaceMatcher(), $._config.autoscaling.ruler_querier_hpa_name], + ], + [ + 'Min', + 'Max', + 'Current', + ], + ) + + $.panelDescription( + title, + ||| + The minimum, maximum, and current number of querier replicas. + ||| + ), + ) + .addPanel( + local title = 'Scaling metric'; + $.panel(title) + + $.queryPanel( + [ + $.filterKedaMetricByHPA('keda_metrics_adapter_scaler_metrics_value', $._config.autoscaling.ruler_querier_hpa_name), + 'kube_horizontalpodautoscaler_spec_target_metric{%s, horizontalpodautoscaler="%s"}' % [$.namespaceMatcher(), $._config.autoscaling.ruler_querier_hpa_name], + ], [ + 'Scaling metric', + 'Target per replica', + ] + ) + + $.panelDescription( + title, + ||| + This panel shows the result of the query that is used as the scaling metric, and the target and threshold used. + The desired number of replicas is computed by HPA as: / . + ||| + ) + + $.panelAxisPlacement('Target per replica', 'right'), + ) + .addPanel( + local title = 'Autoscaler failures rate'; + $.panel(title) + + $.queryPanel( + $.filterKedaMetricByHPA('sum by(metric) (rate(keda_metrics_adapter_scaler_errors[$__rate_interval]))', $._config.autoscaling.ruler_querier_hpa_name), + 'Failures per second' + ) + + $.panelDescription( + title, + ||| + The rate of failures in the KEDA custom metrics API server. Whenever an error occurs, the KEDA custom + metrics server is unable to query the scaling metric from Prometheus so the autoscaler does not work properly. + ||| + ), + ) + ), +} diff --git a/operations/mimir-mixin/dashboards/rollout-progress.libsonnet b/operations/mimir-mixin/dashboards/rollout-progress.libsonnet index d5c57a3c056..d2a4fe6eac0 100644 --- a/operations/mimir-mixin/dashboards/rollout-progress.libsonnet +++ b/operations/mimir-mixin/dashboards/rollout-progress.libsonnet @@ -1,4 +1,5 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-rollout-progress.json'; (import 'dashboard-utils.libsonnet') { local config = { @@ -7,11 +8,11 @@ local utils = import 'mixin-utils/utils.libsonnet'; gateway_job_matcher: $.jobMatcher($._config.job_names.gateway), gateway_write_routes_regex: 'api_(v1|prom)_push', gateway_read_routes_regex: '(prometheus|api_prom)_api_v1_.+', - all_services_regex: std.join('|', ['cortex-gw', 'distributor', 'ingester.*', 'query-frontend.*', 'query-scheduler.*', 'querier.*', 'compactor', 'store-gateway.*', 'ruler', 'alertmanager.*', 'overrides-exporter', 'cortex', 'mimir']), + all_services_regex: '.*(%s).*' % std.join('|', ['cortex-gw', 'distributor', 'ingester', 'query-frontend', 'query-scheduler', 'querier', 'compactor', 'store-gateway', 'ruler', 'alertmanager', 'overrides-exporter', 'cortex', 'mimir']), }, - 'mimir-rollout-progress.json': - ($.dashboard('Rollout progress') + { uid: '7544a3a62b1be6ffd919fc990ab8ba8f' }) + [filename]: + ($.dashboard('Rollout progress') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) + { // This dashboard uses the new grid system in order to place panels (using gridPos). // Because of this we can't use the mixin's addRow() and addPanel(). @@ -236,7 +237,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; count by(container, version) ( label_replace( kube_pod_container_info{%(namespace_matcher)s,container=~"%(all_services_regex)s"}, - "version", "$1", "image", ".*:(.+)-.*" + "version", "$1", "image", ".*:(.*)" ) ) ||| % config, diff --git a/operations/mimir-mixin/dashboards/ruler.libsonnet b/operations/mimir-mixin/dashboards/ruler.libsonnet index 7bf6bb292a5..1affd02be83 100644 --- a/operations/mimir-mixin/dashboards/ruler.libsonnet +++ b/operations/mimir-mixin/dashboards/ruler.libsonnet @@ -1,4 +1,5 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-ruler.json'; (import 'dashboard-utils.libsonnet') { local ruler_config_api_routes_re = '(prometheus|api_prom)_(rules.*|api_v1_(rules|alerts))', @@ -58,8 +59,8 @@ local utils = import 'mixin-utils/utils.libsonnet'; }, }, - 'mimir-ruler.json': - ($.dashboard('Ruler') + { uid: '44d12bcb1f95661c6ab6bc946dfc3473' }) + [filename]: + ($.dashboard('Ruler') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( ($.row('Headlines') + { @@ -76,11 +77,11 @@ local utils = import 'mixin-utils/utils.libsonnet'; ) .addPanel( $.panel('Read from ingesters - QPS') + - $.statPanel('sum(rate(cortex_ingester_client_request_duration_seconds_count{%s, operation="/cortex.Ingester/QueryStream"}[5m]))' % $.jobMatcher($._config.job_names.ruler), format='reqps') + $.statPanel('sum(rate(cortex_ingester_client_request_duration_seconds_count{%s, operation="/cortex.Ingester/QueryStream"}[$__rate_interval]))' % $.jobMatcher($._config.job_names.ruler), format='reqps') ) .addPanel( $.panel('Write to ingesters - QPS') + - $.statPanel('sum(rate(cortex_ingester_client_request_duration_seconds_count{%s, operation="/cortex.Ingester/Push"}[5m]))' % $.jobMatcher($._config.job_names.ruler), format='reqps') + $.statPanel('sum(rate(cortex_ingester_client_request_duration_seconds_count{%s, operation="/cortex.Ingester/Push"}[$__rate_interval]))' % $.jobMatcher($._config.job_names.ruler), format='reqps') ) ) .addRow( @@ -103,7 +104,8 @@ local utils = import 'mixin-utils/utils.libsonnet'; ), ) ) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Configuration API (gateway)') .addPanel( $.panel('QPS') + @@ -161,7 +163,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; ) .addPanel( $.panel('Consistency checks failed') + - $.queryPanel('sum(rate(cortex_querier_blocks_consistency_checks_failed_total{%s}[1m])) / sum(rate(cortex_querier_blocks_consistency_checks_total{%s}[1m]))' % [$.jobMatcher($._config.job_names.ruler), $.jobMatcher($._config.job_names.ruler)], 'Failure Rate') + + $.queryPanel('sum(rate(cortex_querier_blocks_consistency_checks_failed_total{%s}[$__rate_interval])) / sum(rate(cortex_querier_blocks_consistency_checks_total{%s}[$__rate_interval]))' % [$.jobMatcher($._config.job_names.ruler), $.jobMatcher($._config.job_names.ruler)], 'Failure Rate') + { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) }, ) ) diff --git a/operations/mimir-mixin/dashboards/scaling.libsonnet b/operations/mimir-mixin/dashboards/scaling.libsonnet index 0af07259206..6b9c7d84e33 100644 --- a/operations/mimir-mixin/dashboards/scaling.libsonnet +++ b/operations/mimir-mixin/dashboards/scaling.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-scaling.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-scaling.json': - ($.dashboard('Scaling') + { uid: '88c041017b96856c9176e07cf557bdcf' }) + [filename]: + ($.dashboard('Scaling') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRow( ($.row('Service scaling') + { height: '200px' }) diff --git a/operations/mimir-mixin/dashboards/slow-queries.libsonnet b/operations/mimir-mixin/dashboards/slow-queries.libsonnet index 3bdeb7a096f..a5eea70a8f7 100644 --- a/operations/mimir-mixin/dashboards/slow-queries.libsonnet +++ b/operations/mimir-mixin/dashboards/slow-queries.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-slow-queries.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-slow-queries.json': - ($.dashboard('Slow queries') + { uid: 'e6f3091e29d2636e3b8393447e925668' }) + [filename]: + ($.dashboard('Slow queries') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) .addRow( $.row('') diff --git a/operations/mimir-mixin/dashboards/tenants.libsonnet b/operations/mimir-mixin/dashboards/tenants.libsonnet index e93e41b6ef5..25ae99e988e 100644 --- a/operations/mimir-mixin/dashboards/tenants.libsonnet +++ b/operations/mimir-mixin/dashboards/tenants.libsonnet @@ -1,8 +1,24 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-tenants.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-tenants.json': - ($.dashboard('Tenants') + { uid: '35fa247ce651ba189debf33d7ae41611' }) + local user_limits_overrides_query(limit_name) = ||| + max(cortex_limits_overrides{%(overrides_exporter)s, limit_name="%(limit_name)s", user="$user"}) + or + max(cortex_limits_defaults{%(overrides_exporter)s, limit_name="%(limit_name)s"}) + ||| % { + overrides_exporter: $.jobMatcher($._config.job_names.overrides_exporter), + limit_name: limit_name, + }, + + local limit_style = { + alias: 'limit', + fill: 0, + dashes: true, + }, + + [filename]: + ($.dashboard('Tenants') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addActiveUserSelectorTemplates() .addCustomTemplate('limit', ['10', '50', '100', '500', '1000']) @@ -39,13 +55,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; distributor: $.jobMatcher($._config.job_names.distributor), group_by_cluster: $._config.group_by_cluster, }, - ||| - max(cortex_limits_overrides{%(overrides_exporter)s, limit_name="max_global_series_per_user", user="$user"}) - or - max(cortex_limits_defaults{%(overrides_exporter)s, limit_name="max_global_series_per_user"}) - ||| % { - overrides_exporter: $.jobMatcher($._config.job_names.overrides_exporter), - }, + user_limits_overrides_query('max_global_series_per_user'), ||| sum( cortex_ingester_active_series{%(ingester)s, user="$user"} @@ -76,15 +86,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; 'active ({{ name }})', ], ) + - { - seriesOverrides: [ - { - alias: 'limit', - fill: 0, - dashes: true, - }, - ], - } + + { seriesOverrides: [limit_style] } + $.panelDescription( title, ||| @@ -179,11 +181,17 @@ local utils = import 'mixin-utils/utils.libsonnet'; local title = 'Distributor samples received (accepted) rate'; $.panel(title) + $.queryPanel( - 'sum(rate(cortex_distributor_received_samples_total{%(job)s, user="$user"}[$__rate_interval]))' - % { job: $.jobMatcher($._config.job_names.distributor) }, - 'rate', + [ + 'sum(rate(cortex_distributor_received_samples_total{%(job)s, user="$user"}[$__rate_interval]))' + % { job: $.jobMatcher($._config.job_names.distributor) }, + user_limits_overrides_query('ingestion_rate'), + ], + [ + 'rate', + 'limit', + ], ) + - { legend: { show: false } } + + { seriesOverrides: [limit_style] } + $.panelDescription( title, ||| @@ -358,11 +366,17 @@ local utils = import 'mixin-utils/utils.libsonnet'; local title = 'Number of groups'; $.panel(title) + $.queryPanel( - 'count(sum by (rule_group) (cortex_prometheus_rule_group_rules{%(job)s, user="$user"}))' - % { job: $.jobMatcher($._config.job_names.ruler) }, - 'groups', + [ + 'count(sum by (rule_group) (cortex_prometheus_rule_group_rules{%(job)s, user="$user"}))' + % { job: $.jobMatcher($._config.job_names.ruler) }, + user_limits_overrides_query('ruler_max_rule_groups_per_tenant'), + ], + [ + 'groups', + 'limit', + ] ) + - { legend: { show: false } } + + { seriesOverrides: [limit_style] } + $.panelDescription( title, ||| diff --git a/operations/mimir-mixin/dashboards/top-tenants.libsonnet b/operations/mimir-mixin/dashboards/top-tenants.libsonnet index 0d097c79bc1..d695020649e 100644 --- a/operations/mimir-mixin/dashboards/top-tenants.libsonnet +++ b/operations/mimir-mixin/dashboards/top-tenants.libsonnet @@ -1,4 +1,5 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-top-tenants.json'; (import 'dashboard-utils.libsonnet') { local in_memory_series_per_user_query(at='') = ||| @@ -18,8 +19,8 @@ local utils = import 'mixin-utils/utils.libsonnet'; group_by_cluster: $._config.group_by_cluster, }, - 'mimir-top-tenants.json': - ($.dashboard('Top tenants') + { uid: 'bc6e12d4fe540e4a1785b9d3ca0ffdd9' }) + [filename]: + ($.dashboard('Top tenants') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addCustomTemplate('limit', ['10', '50', '100']) .addRowIf( @@ -111,6 +112,63 @@ local utils = import 'mixin-utils/utils.libsonnet'; ), ) + .addRow( + ($.row('By samples rate growth') + { collapse: true }) + .addPanel( + $.panel('Top $limit users by received samples rate that grew the most between query range start and query range end') + + $.queryPanel( + ||| + sum by (user) (rate(cortex_distributor_received_samples_total{%(job)s}[$__rate_interval])) + and + topk($limit, + sum by (user) (rate(cortex_distributor_received_samples_total{%(job)s}[$__rate_interval] @ end())) + - + sum by (user) (rate(cortex_distributor_received_samples_total{%(job)s}[$__rate_interval] @ start())) + ) + ||| % { + job: $.jobMatcher($._config.job_names.distributor), + }, + '{{ user }}', + ) + ), + ) + + .addRow( + ($.row('By discarded samples rate') + { collapse: true }) + .addPanel( + $.panel('Top $limit users by discarded samples rate in last 5m') + + { sort: { col: 2, desc: true } } + + $.tablePanel( + [ + 'topk($limit, sum by (user) (rate(cortex_discarded_samples_total{%(job)s}[5m])))' + % { job: $.jobMatcher('%s|%s' % [$._config.job_names.ingester, $._config.job_names.distributor]) }, + ], + { 'Value #A': { alias: 'samples/s' } } + ) + ), + ) + + .addRow( + ($.row('By discarded samples rate growth') + { collapse: true }) + .addPanel( + $.panel('Top $limit users by discarded samples rate that grew the most between query range start and query range end') + + $.queryPanel( + ||| + sum by (user) (rate(cortex_discarded_samples_total{%(job)s}[$__rate_interval])) + and + topk($limit, + sum by (user) (rate(cortex_discarded_samples_total{%(job)s}[$__rate_interval] @ end())) + - + sum by (user) (rate(cortex_discarded_samples_total{%(job)s}[$__rate_interval] @ start())) + ) + ||| % { + job: $.jobMatcher('%s|%s' % [$._config.job_names.ingester, $._config.job_names.distributor]), + }, + '{{ user }}', + ) + ), + ) + .addRow( ($.row('By series with exemplars') + { collapse: true }) .addPanel( diff --git a/operations/mimir-mixin/dashboards/writes-networking.libsonnet b/operations/mimir-mixin/dashboards/writes-networking.libsonnet index 2deca121fa8..017fda6f2ec 100644 --- a/operations/mimir-mixin/dashboards/writes-networking.libsonnet +++ b/operations/mimir-mixin/dashboards/writes-networking.libsonnet @@ -1,10 +1,11 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-writes-networking.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-writes-networking.json': - ($.dashboard('Writes networking') + { uid: '681cd62b680b7154811fe73af55dcfd4' }) + [filename]: + ($.dashboard('Writes networking') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) - .addRow($.jobNetworkingRow('Gateway', 'gateway')) + .addRowIf($._config.gateway_enabled, $.jobNetworkingRow('Gateway', 'gateway')) .addRow($.jobNetworkingRow('Distributor', 'distributor')) .addRow($.jobNetworkingRow('Ingester', 'ingester')) + { diff --git a/operations/mimir-mixin/dashboards/writes-resources.libsonnet b/operations/mimir-mixin/dashboards/writes-resources.libsonnet index f2df3f84f96..34b1c8f9375 100644 --- a/operations/mimir-mixin/dashboards/writes-resources.libsonnet +++ b/operations/mimir-mixin/dashboards/writes-resources.libsonnet @@ -1,10 +1,12 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-writes-resources.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-writes-resources.json': - ($.dashboard('Writes resources') + { uid: 'c0464f0d8bd026f776c9006b0591bb0b' }) + [filename]: + ($.dashboard('Writes resources') + { uid: std.md5(filename) }) .addClusterSelectorTemplates(false) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Gateway') .addPanel( $.containerCPUUsagePanel('CPU', $._config.job_names.gateway), diff --git a/operations/mimir-mixin/dashboards/writes.libsonnet b/operations/mimir-mixin/dashboards/writes.libsonnet index 9a112540cd3..7c2f6d9ade2 100644 --- a/operations/mimir-mixin/dashboards/writes.libsonnet +++ b/operations/mimir-mixin/dashboards/writes.libsonnet @@ -1,8 +1,9 @@ local utils = import 'mixin-utils/utils.libsonnet'; +local filename = 'mimir-writes.json'; (import 'dashboard-utils.libsonnet') { - 'mimir-writes.json': - ($.dashboard('Writes') + { uid: '0156f6d15aa234d452a33a4f13c838e3' }) + [filename]: + ($.dashboard('Writes') + { uid: std.md5(filename) }) .addClusterSelectorTemplates() .addRowIf( $._config.show_dashboard_descriptions.writes, @@ -102,12 +103,14 @@ local utils = import 'mixin-utils/utils.libsonnet'; $.panel('Tenants') + $.statPanel('count(count by(user) (cortex_ingester_active_series{%s}))' % $.jobMatcher($._config.job_names.ingester), format='short') ) - .addPanel( + .addPanelIf( + $._config.gateway_enabled, $.panel('Requests / sec') + - $.statPanel('sum(rate(cortex_request_duration_seconds_count{%s, route=~"api_(v1|prom)_push"}[5m]))' % $.jobMatcher($._config.job_names.gateway), format='reqps') + $.statPanel('sum(rate(cortex_request_duration_seconds_count{%s, route=~"api_(v1|prom)_push"}[$__rate_interval]))' % $.jobMatcher($._config.job_names.gateway), format='reqps') ) ) - .addRow( + .addRowIf( + $._config.gateway_enabled, $.row('Gateway') .addPanel( $.panel('Requests / sec') + diff --git a/operations/mimir-mixin/jsonnetfile.lock.json b/operations/mimir-mixin/jsonnetfile.lock.json index a1b021910f4..26e8d13811b 100644 --- a/operations/mimir-mixin/jsonnetfile.lock.json +++ b/operations/mimir-mixin/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "grafana-builder" } }, - "version": "0d13e5ba1b3a4c29015738c203d92ea39f71ebe2", - "sum": "GRf2GvwEU4jhXV+JOonXSZ4wdDv8mnHBPCQ6TUVd+g8=" + "version": "2e980525502eda008cfb88a5672bd70d7d411fda", + "sum": "TieGrr7GyKjURk1+wXHFpdoCiwNaIVfZvyc5mbI9OM0=" }, { "source": { diff --git a/operations/mimir-mixin/recording_rules.libsonnet b/operations/mimir-mixin/recording_rules.libsonnet index c76be953d7c..d06f4f2a769 100644 --- a/operations/mimir-mixin/recording_rules.libsonnet +++ b/operations/mimir-mixin/recording_rules.libsonnet @@ -47,7 +47,11 @@ local utils = import 'mixin-utils/utils.libsonnet'; name: 'mimir_queries', rules: utils.histogramRules('cortex_query_frontend_retries', [$._config.per_cluster_label, 'job']) + - utils.histogramRules('cortex_query_frontend_queue_duration_seconds', [$._config.per_cluster_label, 'job']) + + utils.histogramRules('cortex_query_frontend_queue_duration_seconds', [$._config.per_cluster_label, 'job']), + }, + { + name: 'mimir_ingester_queries', + rules: utils.histogramRules('cortex_ingester_queried_series', [$._config.per_cluster_label, 'job']) + utils.histogramRules('cortex_ingester_queried_samples', [$._config.per_cluster_label, 'job']) + utils.histogramRules('cortex_ingester_queried_exemplars', [$._config.per_cluster_label, 'job']), @@ -335,7 +339,7 @@ local utils = import 'mixin-utils/utils.libsonnet'; sum by (%(alert_aggregation_labels)s, deployment) ( label_replace( label_replace( - container_memory_usage_bytes, + container_memory_usage_bytes{image!=""}, "deployment", "$1", "%(per_instance_label)s", "(.*)-(?:([0-9]+)|([a-z0-9]+)-([a-z0-9]+))" ), # The question mark in "(.*?)" is used to make it non-greedy, otherwise it diff --git a/operations/mimir-mixin/scripts/lint-playbooks.sh b/operations/mimir-mixin/scripts/lint-playbooks.sh deleted file mode 100755 index 915c83c826b..00000000000 --- a/operations/mimir-mixin/scripts/lint-playbooks.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# SPDX-License-Identifier: AGPL-3.0-only - -set -eu -o pipefail - -SCRIPT_DIR=$(realpath "$(dirname "${0}")") - -# List all alerts. -if ! ALERTS=$(yq eval '.groups.[].rules.[].alert' "${SCRIPT_DIR}/../../mimir-mixin-compiled/alerts.yaml" 2> /dev/stdout); then - echo "Unable to list alerts. Got output:" - echo "$ALERTS" - exit 1 -elif [ -z "$ALERTS" ]; then - echo "No alerts found. Something went wrong with the listing." - exit 1 -fi - -# Check if each alert is referenced in the playbooks. -STATUS=0 - -for ALERT in $ALERTS; do - if ! grep -q "${ALERT}$" "${SCRIPT_DIR}/../docs/playbooks.md"; then - echo "Missing playbook for: $ALERT" - STATUS=1 - fi -done - -exit $STATUS diff --git a/operations/mimir-rules-action/Dockerfile b/operations/mimir-rules-action/Dockerfile index a126a76febe..a88a381004f 100644 --- a/operations/mimir-rules-action/Dockerfile +++ b/operations/mimir-rules-action/Dockerfile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: AGPL-3.0-only -FROM grafana/mimirtool:2.0.0 +FROM grafana/mimirtool:2.1.0 COPY entrypoint.sh /entrypoint.sh diff --git a/operations/mimir-tests/build.sh b/operations/mimir-tests/build.sh index 25f0296e40f..0aa0916335d 100755 --- a/operations/mimir-tests/build.sh +++ b/operations/mimir-tests/build.sh @@ -8,7 +8,7 @@ rm -rf jsonnet-tests && mkdir jsonnet-tests cd jsonnet-tests # Initialise the Tanka. -tk init --k8s=1.18 +tk init --k8s=1.21 # Install Mimir jsonnet from this branch. jb install ../operations/mimir diff --git a/operations/mimir-tests/test-autoscaling-generated.yaml b/operations/mimir-tests/test-autoscaling-generated.yaml new file mode 100644 index 00000000000..75fdfde2ce6 --- /dev/null +++ b/operations/mimir-tests/test-autoscaling-generated.yaml @@ -0,0 +1,1788 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: default +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + labels: + name: alertmanager-pdb + name: alertmanager-pdb + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: alertmanager +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + labels: + name: ingester-pdb + name: ingester-pdb + namespace: default +spec: + maxUnavailable: 1 + selector: + matchLabels: + name: ingester +--- +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + labels: + name: store-gateway-pdb + name: store-gateway-pdb + namespace: default +spec: + maxUnavailable: 2 + selector: + matchLabels: + name: store-gateway +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: consul-sidekick + namespace: default +--- +apiVersion: v1 +data: + consul-config.json: '{"leave_on_terminate": true, "raft_snapshot_threshold": 128, + "raft_trailing_logs": 10000, "telemetry": {"dogstatsd_addr": "127.0.0.1:9125"}}' + mapping: | + mappings: + - match: consul.*.runtime.* + name: consul_runtime + labels: + type: $2 + - match: consul.runtime.total_gc_pause_ns + name: consul_runtime_total_gc_pause_ns + labels: + type: $2 + - match: consul.consul.health.service.query-tag.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3 + - match: consul.consul.health.service.query-tag.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4 + - match: consul.consul.health.service.query-tag.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10.$11 + - match: consul.consul.health.service.query-tag.*.*.*.*.*.*.*.*.*.*.*.* + name: consul_health_service_query_tag + labels: + query: $1.$2.$3.$4.$5.$6.$7.$8.$9.$10.$11.$12 + - match: consul.consul.catalog.deregister + name: consul_catalog_deregister + labels: {} + - match: consul.consul.dns.domain_query.*.*.*.*.* + name: consul_dns_domain_query + labels: + query: $1.$2.$3.$4.$5 + - match: consul.consul.health.service.not-found.* + name: consul_health_service_not_found + labels: + query: $1 + - match: consul.consul.health.service.query.* + name: consul_health_service_query + labels: + query: $1 + - match: consul.*.memberlist.health.score + name: consul_memberlist_health_score + labels: {} + - match: consul.serf.queue.* + name: consul_serf_events + labels: + type: $1 + - match: consul.serf.snapshot.appendLine + name: consul_serf_snapshot_appendLine + labels: + type: $1 + - match: consul.serf.coordinate.adjustment-ms + name: consul_serf_coordinate_adjustment_ms + labels: {} + - match: consul.consul.rpc.query + name: consul_rpc_query + labels: {} + - match: consul.*.consul.session_ttl.active + name: consul_session_ttl_active + labels: {} + - match: consul.raft.rpc.* + name: consul_raft_rpc + labels: + type: $1 + - match: consul.raft.rpc.appendEntries.storeLogs + name: consul_raft_rpc_appendEntries_storeLogs + labels: + type: $1 + - match: consul.consul.fsm.persist + name: consul_fsm_persist + labels: {} + - match: consul.raft.fsm.apply + name: consul_raft_fsm_apply + labels: {} + - match: consul.raft.leader.lastContact + name: consul_raft_leader_lastcontact + labels: {} + - match: consul.raft.leader.dispatchLog + name: consul_raft_leader_dispatchLog + labels: {} + - match: consul.raft.commitTime + name: consul_raft_commitTime + labels: {} + - match: consul.raft.replication.appendEntries.logs.*.*.*.* + name: consul_raft_replication_appendEntries_logs + labels: + query: ${1}.${2}.${3}.${4} + - match: consul.raft.replication.appendEntries.rpc.*.*.*.* + name: consul_raft_replication_appendEntries_rpc + labels: + query: ${1}.${2}.${3}.${4} + - match: consul.raft.replication.heartbeat.*.*.*.* + name: consul_raft_replication_heartbeat + labels: + query: ${1}.${2}.${3}.${4} + - match: consul.consul.rpc.request + name: consul_rpc_requests + labels: {} + - match: consul.consul.rpc.accept_conn + name: consul_rpc_accept_conn + labels: {} + - match: consul.memberlist.udp.* + name: consul_memberlist_udp + labels: + type: $1 + - match: consul.memberlist.tcp.* + name: consul_memberlist_tcp + labels: + type: $1 + - match: consul.memberlist.gossip + name: consul_memberlist_gossip + labels: {} + - match: consul.memberlist.probeNode + name: consul_memberlist_probenode + labels: {} + - match: consul.memberlist.pushPullNode + name: consul_memberlist_pushpullnode + labels: {} + - match: consul.http.* + name: consul_http_request + labels: + method: $1 + path: / + - match: consul.http.*.* + name: consul_http_request + labels: + method: $1 + path: /$2 + - match: consul.http.*.*.* + name: consul_http_request + labels: + method: $1 + path: /$2/$3 + - match: consul.http.*.*.*.* + name: consul_http_request + labels: + method: $1 + path: /$2/$3/$4 + - match: consul.http.*.*.*.*.* + name: consul_http_request + labels: + method: $1 + path: /$2/$3/$4/$5 + - match: consul.consul.leader.barrier + name: consul_leader_barrier + labels: {} + - match: consul.consul.leader.reconcileMember + name: consul_leader_reconcileMember + labels: {} + - match: consul.consul.leader.reconcile + name: consul_leader_reconcile + labels: {} + - match: consul.consul.fsm.coordinate.batch-update + name: consul_fsm_coordinate_batch_update + labels: {} + - match: consul.consul.fsm.autopilot + name: consul_fsm_autopilot + labels: {} + - match: consul.consul.fsm.kvs.cas + name: consul_fsm_kvs_cas + labels: {} + - match: consul.consul.fsm.register + name: consul_fsm_register + labels: {} + - match: consul.consul.fsm.deregister + name: consul_fsm_deregister + labels: {} + - match: consul.consul.fsm.tombstone.reap + name: consul_fsm_tombstone_reap + labels: {} + - match: consul.consul.catalog.register + name: consul_catalog_register + labels: {} + - match: consul.consul.catalog.deregister + name: consul_catalog_deregister + labels: {} + - match: consul.consul.leader.reapTombstones + name: consul_leader_reapTombstones + labels: {} +kind: ConfigMap +metadata: + name: consul + namespace: default +--- +apiVersion: v1 +data: + overrides.yaml: | + overrides: {} +kind: ConfigMap +metadata: + name: overrides + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: consul-sidekick + namespace: default +rules: +- apiGroups: + - "" + - extensions + - apps + resources: + - pods + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: consul-sidekick + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: consul-sidekick +subjects: +- kind: ServiceAccount + name: consul-sidekick + namespace: default +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + clusterIP: None + ports: + - name: alertmanager-http-metrics + port: 8080 + targetPort: 8080 + - name: alertmanager-grpc + port: 9095 + targetPort: 9095 + selector: + name: alertmanager +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: consul + name: consul + namespace: default +spec: + ports: + - name: consul-server + port: 8300 + targetPort: 8300 + - name: consul-serf + port: 8301 + targetPort: 8301 + - name: consul-client + port: 8400 + targetPort: 8400 + - name: consul-api + port: 8500 + targetPort: 8500 + - name: statsd-exporter-http-metrics + port: 8000 + targetPort: 8000 + - name: consul-exporter-http-metrics + port: 9107 + targetPort: 9107 + selector: + name: consul +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: distributor + name: distributor + namespace: default +spec: + clusterIP: None + ports: + - name: distributor-http-metrics + port: 8080 + targetPort: 8080 + - name: distributor-grpc + port: 9095 + targetPort: 9095 + selector: + name: distributor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ingester + name: ingester + namespace: default +spec: + ports: + - name: ingester-http-metrics + port: 8080 + targetPort: 8080 + - name: ingester-grpc + port: 9095 + targetPort: 9095 + selector: + name: ingester +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached + name: memcached + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-frontend + name: memcached-frontend + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-index-queries + name: memcached-index-queries + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-index-queries +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: memcached-metadata + name: memcached-metadata + namespace: default +spec: + clusterIP: None + ports: + - name: memcached-client + port: 11211 + targetPort: 11211 + - name: exporter-http-metrics + port: 9150 + targetPort: 9150 + selector: + name: memcached-metadata +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: querier + name: querier + namespace: default +spec: + ports: + - name: querier-http-metrics + port: 8080 + targetPort: 8080 + - name: querier-grpc + port: 9095 + targetPort: 9095 + selector: + name: querier +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-frontend + name: query-frontend + namespace: default +spec: + ports: + - name: query-frontend-http-metrics + port: 8080 + targetPort: 8080 + - name: query-frontend-grpc + port: 9095 + targetPort: 9095 + selector: + name: query-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-frontend + name: query-frontend-discovery + namespace: default +spec: + clusterIP: None + ports: + - name: query-frontend-http-metrics + port: 8080 + targetPort: 8080 + - name: query-frontend-grpc + port: 9095 + targetPort: 9095 + publishNotReadyAddresses: true + selector: + name: query-frontend +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-scheduler + name: query-scheduler + namespace: default +spec: + ports: + - name: query-scheduler-http-metrics + port: 8080 + targetPort: 8080 + - name: query-scheduler-grpc + port: 9095 + targetPort: 9095 + selector: + name: query-scheduler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: query-scheduler + name: query-scheduler-discovery + namespace: default +spec: + clusterIP: None + ports: + - name: query-scheduler-http-metrics + port: 8080 + targetPort: 8080 + - name: query-scheduler-grpc + port: 9095 + targetPort: 9095 + publishNotReadyAddresses: true + selector: + name: query-scheduler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: ruler + name: ruler + namespace: default +spec: + ports: + - name: ruler-http-metrics + port: 8080 + targetPort: 8080 + - name: ruler-grpc + port: 9095 + targetPort: 9095 + selector: + name: ruler +--- +apiVersion: v1 +kind: Service +metadata: + labels: + name: store-gateway + name: store-gateway + namespace: default +spec: + ports: + - name: store-gateway-http-metrics + port: 8080 + targetPort: 8080 + - name: store-gateway-grpc + port: 9095 + targetPort: 9095 + selector: + name: store-gateway +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: consul + namespace: default +spec: + minReadySeconds: 10 + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: consul + template: + metadata: + annotations: + consul-hash: e56ef6821a3557604caccaf6d5820239 + labels: + name: consul + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: consul + topologyKey: kubernetes.io/hostname + - labelSelector: + matchLabels: + name: ingester + namespaces: + - default + topologyKey: kubernetes.io/hostname + containers: + - args: + - agent + - -ui + - -server + - -client=0.0.0.0 + - -config-file=/etc/config/consul-config.json + - -bootstrap-expect=1 + - -ui-content-path=/default/consul/ + env: + - name: CHECKPOINT_DISABLE + value: "1" + image: consul:1.5.3 + imagePullPolicy: IfNotPresent + name: consul + ports: + - containerPort: 8300 + name: server + - containerPort: 8301 + name: serf + - containerPort: 8400 + name: client + - containerPort: 8500 + name: api + resources: + requests: + cpu: "4" + memory: 4Gi + volumeMounts: + - mountPath: /etc/config + name: consul + - mountPath: /consul/data/ + name: data + - args: + - --namespace=$(POD_NAMESPACE) + - --pod-name=$(POD_NAME) + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + image: weaveworks/consul-sidekick:master-f18ad13 + imagePullPolicy: IfNotPresent + name: sidekick + volumeMounts: + - mountPath: /etc/config + name: consul + - mountPath: /consul/data/ + name: data + - args: + - --web.listen-address=:8000 + - --statsd.mapping-config=/etc/config/mapping + image: prom/statsd-exporter:v0.12.2 + imagePullPolicy: IfNotPresent + name: statsd-exporter + ports: + - containerPort: 8000 + name: http-metrics + volumeMounts: + - mountPath: /etc/config + name: consul + - mountPath: /consul/data/ + name: data + - args: + - --consul.server=localhost:8500 + - --web.listen-address=:9107 + - --consul.timeout=1s + - --no-consul.health-summary + - --consul.allow_stale + image: prom/consul-exporter:v0.5.0 + imagePullPolicy: IfNotPresent + name: consul-exporter + ports: + - containerPort: 9107 + name: http-metrics + volumeMounts: + - mountPath: /etc/config + name: consul + - mountPath: /consul/data/ + name: data + serviceAccount: consul-sidekick + volumes: + - configMap: + name: consul + name: consul + - emptyDir: + medium: Memory + name: data +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: distributor + namespace: default +spec: + minReadySeconds: 10 + replicas: 3 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: distributor + strategy: + rollingUpdate: + maxSurge: 5 + maxUnavailable: 1 + template: + metadata: + labels: + name: distributor + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: distributor + topologyKey: kubernetes.io/hostname + containers: + - args: + - -distributor.ha-tracker.enable=true + - -distributor.ha-tracker.enable-for-all-users=true + - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 + - -distributor.ha-tracker.prefix=prom_ha/ + - -distributor.ha-tracker.store=etcd + - -distributor.health-check-ingesters=true + - -distributor.ingestion-burst-size=200000 + - -distributor.ingestion-rate-limit=10000 + - -distributor.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -distributor.ring.prefix= + - -distributor.ring.store=consul + - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=consul + - -mem-ballast-size-bytes=1073741824 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.max-connection-age=2m + - -server.grpc.keepalive.max-connection-age-grace=5m + - -server.grpc.keepalive.max-connection-idle=1m + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=distributor + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: distributor + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 4Gi + requests: + cpu: "2" + memory: 2Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: querier + namespace: default +spec: + minReadySeconds: 10 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: querier + strategy: + rollingUpdate: + maxSurge: 5 + maxUnavailable: 1 + template: + metadata: + labels: + name: querier + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: querier + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.backend=gcs + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -distributor.health-check-ingesters=true + - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=consul + - -mem-ballast-size-bytes=268435456 + - -querier.frontend-client.grpc-max-send-msg-size=104857600 + - -querier.max-concurrent=8 + - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -server.http-write-timeout=1m + - -store-gateway.sharding-ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -store-gateway.sharding-ring.prefix= + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=consul + - -store.max-query-length=768h + - -target=querier + env: + - name: JAEGER_REPORTER_MAX_QUEUE_SIZE + value: "1024" + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: querier + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 24Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-frontend + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: query-frontend + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 1 + template: + metadata: + labels: + name: query-frontend + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: query-frontend + topologyKey: kubernetes.io/hostname + containers: + - args: + - -query-frontend.align-querier-with-step=false + - -query-frontend.cache-results=true + - -query-frontend.max-cache-freshness=10m + - -query-frontend.results-cache.backend=memcached + - -query-frontend.results-cache.memcached.addresses=dnssrvnoa+memcached-frontend.default.svc.cluster.local:11211 + - -query-frontend.results-cache.memcached.timeout=500ms + - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -server.http-write-timeout=1m + - -store.max-query-length=12000h + - -target=query-frontend + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: query-frontend + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 1200Mi + requests: + cpu: "2" + memory: 600Mi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: query-scheduler + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: query-scheduler + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + name: query-scheduler + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: query-scheduler + topologyKey: kubernetes.io/hostname + containers: + - args: + - -query-scheduler.max-outstanding-requests-per-tenant=100 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=query-scheduler + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: query-scheduler + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 2Gi + requests: + cpu: "2" + memory: 1Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ruler + namespace: default +spec: + minReadySeconds: 10 + replicas: 2 + revisionHistoryLimit: 10 + selector: + matchLabels: + name: ruler + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + name: ruler + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: ruler + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.backend=gcs + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -distributor.health-check-ingesters=true + - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.prefix= + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=consul + - -ruler-storage.backend=gcs + - -ruler-storage.gcs.bucket-name=rules-bucket + - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager + - -ruler.max-rule-groups-per-tenant=35 + - -ruler.max-rules-per-rule-group=20 + - -ruler.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -ruler.ring.store=consul + - -ruler.rule-path=/rules + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -store-gateway.sharding-ring.prefix= + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=consul + - -store.max-query-length=768h + - -target=ruler + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: ruler + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + cpu: "16" + memory: 16Gi + requests: + cpu: "1" + memory: 6Gi + volumeMounts: + - mountPath: /etc/mimir + name: overrides + terminationGracePeriodSeconds: 600 + volumes: + - configMap: + name: overrides + name: overrides +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: alertmanager + name: alertmanager + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: alertmanager + serviceName: alertmanager + template: + metadata: + labels: + name: alertmanager + spec: + containers: + - args: + - -alertmanager-storage.backend=gcs + - -alertmanager-storage.gcs.bucket-name=alerts-bucket + - -alertmanager.sharding-ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -alertmanager.sharding-ring.replication-factor=3 + - -alertmanager.sharding-ring.store=consul + - -alertmanager.storage.path=/data + - -alertmanager.web.external-url=http://test/alertmanager + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=alertmanager + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: alertmanager + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 15Gi + requests: + cpu: "2" + memory: 10Gi + volumeMounts: + - mountPath: /data + name: alertmanager-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 900 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: alertmanager-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 1 + selector: + matchLabels: + name: compactor + serviceName: compactor + template: + metadata: + labels: + name: compactor + spec: + containers: + - args: + - -blocks-storage.backend=gcs + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -compactor.block-ranges=2h,12h,24h + - -compactor.blocks-retention-period=0 + - -compactor.cleanup-interval=15m + - -compactor.compaction-concurrency=1 + - -compactor.compaction-interval=30m + - -compactor.compactor-tenant-shard-size=1 + - -compactor.data-dir=/data + - -compactor.deletion-delay=2h + - -compactor.max-closing-blocks-concurrency=2 + - -compactor.max-opening-blocks-concurrency=4 + - -compactor.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -compactor.ring.prefix= + - -compactor.ring.store=consul + - -compactor.ring.wait-stability-min-duration=1m + - -compactor.split-and-merge-shards=0 + - -compactor.split-groups=1 + - -compactor.symbols-flushers-concurrency=4 + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=compactor + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: compactor + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 6Gi + requests: + cpu: 1 + memory: 6Gi + volumeMounts: + - mountPath: /data + name: compactor-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 900 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: compactor-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 250Gi + storageClassName: standard +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: ingester + name: ingester + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + name: ingester + serviceName: ingester + template: + metadata: + labels: + name: ingester + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: ingester + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.backend=gcs + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -blocks-storage.tsdb.block-ranges-period=2h + - -blocks-storage.tsdb.dir=/data/tsdb + - -blocks-storage.tsdb.ship-interval=1m + - -distributor.health-check-ingesters=true + - -ingester.max-global-series-per-metric=20000 + - -ingester.max-global-series-per-user=150000 + - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -ingester.ring.heartbeat-period=15s + - -ingester.ring.heartbeat-timeout=10m + - -ingester.ring.num-tokens=512 + - -ingester.ring.prefix= + - -ingester.ring.readiness-check-ring-health=false + - -ingester.ring.replication-factor=3 + - -ingester.ring.store=consul + - -ingester.ring.tokens-file-path=/data/tokens + - -ingester.ring.unregister-on-shutdown=true + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc-max-concurrent-streams=10000 + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -target=ingester + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: ingester + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 25Gi + requests: + cpu: "4" + memory: 15Gi + volumeMounts: + - mountPath: /data + name: ingester-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 1200 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: ingester-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 100Gi + storageClassName: fast +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached + serviceName: memcached + template: + metadata: + labels: + name: memcached + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 6144 + - -I 1m + - -c 16384 + - -v + image: memcached:1.6.9-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 9Gi + requests: + cpu: 500m + memory: 6552Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.6.0 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-frontend + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached-frontend + serviceName: memcached-frontend + template: + metadata: + labels: + name: memcached-frontend + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-frontend + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 1024 + - -I 5m + - -c 1024 + - -v + image: memcached:1.6.9-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 1536Mi + requests: + cpu: 500m + memory: 1329Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.6.0 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-index-queries + namespace: default +spec: + replicas: 3 + selector: + matchLabels: + name: memcached-index-queries + serviceName: memcached-index-queries + template: + metadata: + labels: + name: memcached-index-queries + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-index-queries + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 1024 + - -I 5m + - -c 16384 + - -v + image: memcached:1.6.9-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 1536Mi + requests: + cpu: 500m + memory: 1329Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.6.0 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: memcached-metadata + namespace: default +spec: + replicas: 1 + selector: + matchLabels: + name: memcached-metadata + serviceName: memcached-metadata + template: + metadata: + labels: + name: memcached-metadata + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: memcached-metadata + topologyKey: kubernetes.io/hostname + containers: + - args: + - -m 512 + - -I 1m + - -c 16384 + - -v + image: memcached:1.6.9-alpine + imagePullPolicy: IfNotPresent + name: memcached + ports: + - containerPort: 11211 + name: client + resources: + limits: + memory: 768Mi + requests: + cpu: 500m + memory: 715Mi + - args: + - --memcached.address=localhost:11211 + - --web.listen-address=0.0.0.0:9150 + image: prom/memcached-exporter:v0.6.0 + imagePullPolicy: IfNotPresent + name: exporter + ports: + - containerPort: 9150 + name: http-metrics + updateStrategy: + type: RollingUpdate +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + name: store-gateway + name: store-gateway + namespace: default +spec: + podManagementPolicy: Parallel + replicas: 3 + selector: + matchLabels: + name: store-gateway + serviceName: store-gateway + template: + metadata: + labels: + name: store-gateway + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + name: store-gateway + topologyKey: kubernetes.io/hostname + containers: + - args: + - -blocks-storage.backend=gcs + - -blocks-storage.bucket-store.chunks-cache.backend=memcached + - -blocks-storage.bucket-store.chunks-cache.memcached.addresses=dnssrvnoa+memcached.default.svc.cluster.local:11211 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 + - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms + - -blocks-storage.bucket-store.index-cache.backend=memcached + - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 + - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.index-cache.memcached.max-idle-connections=100 + - -blocks-storage.bucket-store.index-cache.memcached.max-item-size=5242880 + - -blocks-storage.bucket-store.index-header-lazy-loading-enabled=true + - -blocks-storage.bucket-store.index-header-lazy-loading-idle-timeout=60m + - -blocks-storage.bucket-store.max-chunk-pool-bytes=12884901888 + - -blocks-storage.bucket-store.metadata-cache.backend=memcached + - -blocks-storage.bucket-store.metadata-cache.memcached.addresses=dnssrvnoa+memcached-metadata.default.svc.cluster.local:11211 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-async-concurrency=50 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency=100 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections=100 + - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 + - -blocks-storage.bucket-store.sync-dir=/data/tsdb + - -blocks-storage.bucket-store.sync-interval=15m + - -blocks-storage.gcs.bucket-name=blocks-bucket + - -runtime-config.file=/etc/mimir/overrides.yaml + - -server.grpc.keepalive.min-time-between-pings=10s + - -server.grpc.keepalive.ping-without-stream-allowed=true + - -server.http-listen-port=8080 + - -store-gateway.sharding-ring.consul.hostname=consul.default.svc.cluster.local:8500 + - -store-gateway.sharding-ring.prefix= + - -store-gateway.sharding-ring.replication-factor=3 + - -store-gateway.sharding-ring.store=consul + - -store-gateway.sharding-ring.tokens-file-path=/data/tokens + - -store-gateway.sharding-ring.wait-stability-min-duration=1m + - -target=store-gateway + image: grafana/mimir:2.1.0 + imagePullPolicy: IfNotPresent + name: store-gateway + ports: + - containerPort: 8080 + name: http-metrics + - containerPort: 9095 + name: grpc + readinessProbe: + httpGet: + path: /ready + port: 8080 + initialDelaySeconds: 15 + timeoutSeconds: 1 + resources: + limits: + memory: 18Gi + requests: + cpu: "1" + memory: 12Gi + volumeMounts: + - mountPath: /data + name: store-gateway-data + - mountPath: /etc/mimir + name: overrides + securityContext: + runAsUser: 0 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + name: overrides + name: overrides + updateStrategy: + type: RollingUpdate + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: store-gateway-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi + storageClassName: standard +--- +apiVersion: etcd.database.coreos.com/v1beta2 +kind: EtcdCluster +metadata: + annotations: + etcd.database.coreos.com/scope: clusterwide + name: etcd + namespace: default +spec: + pod: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + etcd_cluster: etcd + topologyKey: kubernetes.io/hostname + annotations: + prometheus.io/port: "2379" + prometheus.io/scrape: "true" + etcdEnv: + - name: ETCD_AUTO_COMPACTION_RETENTION + value: 1h + labels: + name: etcd + resources: + limits: + memory: 512Mi + requests: + cpu: 500m + memory: 512Mi + size: 3 + version: 3.3.13 +--- +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: querier + namespace: default +spec: + maxReplicaCount: 30 + minReplicaCount: 3 + pollingInterval: 10 + scaleTargetRef: + name: querier + triggers: + - metadata: + metricName: cortex_querier_hpa_default + query: sum(max_over_time(cortex_query_scheduler_inflight_requests{container="query-scheduler",namespace="default",quantile="0.75"}[5m])) + serverAddress: http://prometheus.default:9090/prometheus + threshold: "6" + type: prometheus diff --git a/operations/mimir-tests/test-autoscaling.jsonnet b/operations/mimir-tests/test-autoscaling.jsonnet new file mode 100644 index 00000000000..a3bfe081f7c --- /dev/null +++ b/operations/mimir-tests/test-autoscaling.jsonnet @@ -0,0 +1,25 @@ +local mimir = import 'mimir/mimir.libsonnet'; + +mimir { + _config+:: { + namespace: 'default', + external_url: 'http://test', + + blocks_storage_backend: 'gcs', + blocks_storage_bucket_name: 'blocks-bucket', + bucket_index_enabled: true, + query_scheduler_enabled: true, + + ruler_enabled: true, + ruler_client_type: 'gcs', + ruler_storage_bucket_name: 'rules-bucket', + + alertmanager_enabled: true, + alertmanager_client_type: 'gcs', + alertmanager_gcs_bucket_name: 'alerts-bucket', + + autoscaling_querier_enabled: true, + autoscaling_querier_min_replicas: 3, + autoscaling_querier_max_replicas: 30, + }, +} diff --git a/operations/mimir-tests/test-defaults-generated.yaml b/operations/mimir-tests/test-defaults-generated.yaml index 4d5b05c98f3..fa91695fad2 100644 --- a/operations/mimir-tests/test-defaults-generated.yaml +++ b/operations/mimir-tests/test-defaults-generated.yaml @@ -291,6 +291,25 @@ subjects: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -661,7 +680,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -687,7 +705,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -763,8 +781,6 @@ spec: - -querier.frontend-address=query-frontend-discovery.default.svc.cluster.local:9095 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true @@ -779,7 +795,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -844,14 +860,13 @@ spec: - -query-frontend.results-cache.memcached.addresses=dnssrvnoa+memcached-frontend.default.svc.cluster.local:11211 - -query-frontend.results-cache.memcached.timeout=500ms - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -924,7 +939,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1002,7 +1017,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1020,13 +1034,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1320,7 +1332,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1350,7 +1362,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml index b2c7e17ba3e..626101efb9c 100644 --- a/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml +++ b/operations/mimir-tests/test-disable-chunk-streaming-generated.yaml @@ -324,6 +324,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -750,7 +769,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -776,7 +794,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -851,8 +869,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -868,7 +884,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -934,14 +950,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1004,7 +1019,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1070,15 +1085,12 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1088,8 +1100,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1099,7 +1109,7 @@ spec: - -store-gateway.sharding-ring.store=consul - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1166,7 +1176,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1257,7 +1267,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1335,7 +1345,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1353,13 +1362,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1653,7 +1660,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1683,7 +1690,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-gossip-generated.yaml b/operations/mimir-tests/test-gossip-generated.yaml index 0d939de6404..f9ca7f04e73 100644 --- a/operations/mimir-tests/test-gossip-generated.yaml +++ b/operations/mimir-tests/test-gossip-generated.yaml @@ -323,6 +323,28 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -777,7 +799,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -804,7 +825,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -884,8 +905,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -900,7 +919,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -968,14 +987,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1038,7 +1056,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1105,7 +1123,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= @@ -1114,8 +1131,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1124,8 +1139,6 @@ spec: - -ruler.ring.store=memberlist - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1134,7 +1147,7 @@ spec: - -store-gateway.sharding-ring.store=memberlist - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1201,7 +1214,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1295,7 +1308,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1376,7 +1389,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1396,13 +1408,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1699,7 +1709,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1731,7 +1741,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-gossip-multi-zone-generated.yaml b/operations/mimir-tests/test-gossip-multi-zone-generated.yaml index dd84a379c3e..3c35ccae1d2 100644 --- a/operations/mimir-tests/test-gossip-multi-zone-generated.yaml +++ b/operations/mimir-tests/test-gossip-multi-zone-generated.yaml @@ -373,6 +373,28 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -939,7 +961,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -967,7 +988,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -1048,8 +1069,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1065,7 +1084,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -1133,14 +1152,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1203,7 +1221,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1315,7 +1333,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= @@ -1325,8 +1342,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1335,8 +1350,6 @@ spec: - -ruler.ring.store=memberlist - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1346,7 +1359,7 @@ spec: - -store-gateway.sharding-ring.zone-awareness-enabled=true - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1413,7 +1426,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1507,7 +1520,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1599,7 +1612,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1621,13 +1633,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1719,7 +1729,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1741,13 +1750,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1839,7 +1846,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1861,13 +1867,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -2175,7 +2179,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2210,7 +2214,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2306,7 +2310,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2341,7 +2345,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2437,7 +2441,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2472,7 +2476,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-gossip-multikv-generated.yaml b/operations/mimir-tests/test-gossip-multikv-generated.yaml index d26c6f5ee95..c61c8e3f1db 100644 --- a/operations/mimir-tests/test-gossip-multikv-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-generated.yaml @@ -326,6 +326,28 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -780,7 +802,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -813,7 +834,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -896,8 +917,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -915,7 +934,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -983,14 +1002,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1053,7 +1071,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1120,7 +1138,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m @@ -1132,8 +1149,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1145,8 +1160,6 @@ spec: - -ruler.ring.store=multi - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1158,7 +1171,7 @@ spec: - -store-gateway.sharding-ring.store=multi - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1225,7 +1238,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1322,7 +1335,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1403,7 +1416,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1426,13 +1438,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1729,7 +1739,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1764,7 +1774,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml b/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml index 087f2af63c7..5337356f5fc 100644 --- a/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-switch-primary-secondary-generated.yaml @@ -326,6 +326,28 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -780,7 +802,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -813,7 +834,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -896,8 +917,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -915,7 +934,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -983,14 +1002,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1053,7 +1071,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1120,7 +1138,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m @@ -1132,8 +1149,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1145,8 +1160,6 @@ spec: - -ruler.ring.store=multi - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1158,7 +1171,7 @@ spec: - -store-gateway.sharding-ring.store=multi - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1225,7 +1238,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1322,7 +1335,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1403,7 +1416,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1426,13 +1438,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1729,7 +1739,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1764,7 +1774,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml b/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml index 3f4d290b39c..eccf637d9ca 100644 --- a/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml +++ b/operations/mimir-tests/test-gossip-multikv-teardown-generated.yaml @@ -326,6 +326,28 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -780,7 +802,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -807,7 +828,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -887,8 +908,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -903,7 +922,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -971,14 +990,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1041,7 +1059,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1108,7 +1126,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= @@ -1117,8 +1134,6 @@ spec: - -memberlist.abort-if-join-fails=false - -memberlist.bind-port=7946 - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1127,8 +1142,6 @@ spec: - -ruler.ring.store=memberlist - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1137,7 +1150,7 @@ spec: - -store-gateway.sharding-ring.store=memberlist - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1204,7 +1217,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1298,7 +1311,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1379,7 +1392,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1399,13 +1411,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1702,7 +1712,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1734,7 +1744,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml b/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml index a2272142088..2280cccf54a 100644 --- a/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml +++ b/operations/mimir-tests/test-gossip-ruler-disabled-generated.yaml @@ -323,6 +323,28 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + - name: compactor-gossip-ring + port: 7946 + targetPort: 7946 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -759,7 +781,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -786,7 +807,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -866,8 +887,6 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -882,7 +901,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -950,14 +969,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1020,7 +1038,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1085,7 +1103,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1179,7 +1197,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1260,7 +1278,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1280,13 +1297,11 @@ spec: - -memberlist.join=gossip-ring.default.svc.cluster.local:7946 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1583,7 +1598,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1615,7 +1630,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-multi-zone-generated.yaml b/operations/mimir-tests/test-multi-zone-generated.yaml index 3027b5a9476..378afdf2699 100644 --- a/operations/mimir-tests/test-multi-zone-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-generated.yaml @@ -373,6 +373,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -899,7 +918,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -926,7 +944,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -1002,8 +1020,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1020,7 +1036,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -1086,14 +1102,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1156,7 +1171,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1267,7 +1282,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m @@ -1275,8 +1289,6 @@ spec: - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - -ingester.ring.zone-awareness-enabled=true - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1286,8 +1298,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1298,7 +1308,7 @@ spec: - -store-gateway.sharding-ring.zone-awareness-enabled=true - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1365,7 +1375,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1456,7 +1466,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1545,7 +1555,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1565,13 +1574,11 @@ spec: - -ingester.ring.zone-awareness-enabled=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1660,7 +1667,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1680,13 +1686,11 @@ spec: - -ingester.ring.zone-awareness-enabled=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1775,7 +1779,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1795,13 +1798,11 @@ spec: - -ingester.ring.zone-awareness-enabled=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -2106,7 +2107,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2139,7 +2140,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2232,7 +2233,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2265,7 +2266,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2358,7 +2359,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2391,7 +2392,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml b/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml index 6e854a6c968..b90dbede3a9 100644 --- a/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml +++ b/operations/mimir-tests/test-multi-zone-with-ongoing-migration-generated.yaml @@ -399,6 +399,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -961,7 +980,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -988,7 +1006,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -1064,8 +1082,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -1082,7 +1098,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -1148,14 +1164,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1218,7 +1233,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1329,7 +1344,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m @@ -1337,8 +1351,6 @@ spec: - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - -ingester.ring.zone-awareness-enabled=true - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1348,8 +1360,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1360,7 +1370,7 @@ spec: - -store-gateway.sharding-ring.zone-awareness-enabled=true - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1427,7 +1437,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1518,7 +1528,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1596,7 +1606,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1614,13 +1623,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1709,7 +1716,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1729,13 +1735,11 @@ spec: - -ingester.ring.zone-awareness-enabled=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1824,7 +1828,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1844,13 +1847,11 @@ spec: - -ingester.ring.zone-awareness-enabled=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1939,7 +1940,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1959,13 +1959,11 @@ spec: - -ingester.ring.zone-awareness-enabled=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -2259,7 +2257,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2289,7 +2287,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2382,7 +2380,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2415,7 +2413,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2508,7 +2506,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2541,7 +2539,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: @@ -2634,7 +2632,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -2667,7 +2665,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.sharding-ring.zone-awareness-enabled=true - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-query-sharding-generated.yaml b/operations/mimir-tests/test-query-sharding-generated.yaml index b0c64fcaef8..819171e5f2f 100644 --- a/operations/mimir-tests/test-query-sharding-generated.yaml +++ b/operations/mimir-tests/test-query-sharding-generated.yaml @@ -323,6 +323,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -749,7 +768,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -775,7 +793,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -850,8 +868,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=419430400 - -querier.max-concurrent=16 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -867,7 +883,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -944,7 +960,7 @@ spec: - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1007,7 +1023,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1073,15 +1089,12 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1091,8 +1104,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1102,7 +1113,7 @@ spec: - -store-gateway.sharding-ring.store=consul - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1169,7 +1180,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1260,7 +1271,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1338,7 +1349,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1356,13 +1366,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1656,7 +1664,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1686,7 +1694,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-shuffle-sharding-generated.yaml b/operations/mimir-tests/test-shuffle-sharding-generated.yaml index 6183bb464ad..0748befd73a 100644 --- a/operations/mimir-tests/test-shuffle-sharding-generated.yaml +++ b/operations/mimir-tests/test-shuffle-sharding-generated.yaml @@ -323,6 +323,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -749,7 +768,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -776,7 +794,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -852,10 +870,7 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - - -querier.shuffle-sharding-ingesters-lookback-period=13h - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true @@ -871,7 +886,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -938,14 +953,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1009,7 +1023,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1075,7 +1089,6 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -distributor.ingestion-tenant-shard-size=3 - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 @@ -1083,9 +1096,6 @@ spec: - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - - -querier.shuffle-sharding-ingesters-lookback-period=13h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1096,8 +1106,6 @@ spec: - -ruler.rule-path=/rules - -ruler.tenant-shard-size=2 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1108,7 +1116,7 @@ spec: - -store-gateway.tenant-shard-size=3 - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1175,7 +1183,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1266,7 +1274,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1344,7 +1352,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1363,13 +1370,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1663,7 +1668,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1694,7 +1699,7 @@ spec: - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -store-gateway.tenant-shard-size=3 - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-storage-azure-generated.yaml b/operations/mimir-tests/test-storage-azure-generated.yaml index b0a6e99c5a4..28a2d7ac064 100644 --- a/operations/mimir-tests/test-storage-azure-generated.yaml +++ b/operations/mimir-tests/test-storage-azure-generated.yaml @@ -323,6 +323,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -749,7 +768,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -775,7 +793,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -852,8 +870,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -869,7 +885,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -935,14 +951,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1005,7 +1020,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1073,15 +1088,12 @@ spec: - -blocks-storage.bucket-store.metadata-cache.memcached.max-item-size=1048576 - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.azure.account-key=rules-account-key - -ruler-storage.azure.account-name=rules-account-name - -ruler-storage.azure.container-name=rules-bucket @@ -1093,8 +1105,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1104,7 +1114,7 @@ spec: - -store-gateway.sharding-ring.store=consul - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1173,7 +1183,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1266,7 +1276,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1346,7 +1356,6 @@ spec: - -blocks-storage.azure.container-name=blocks-bucket - -blocks-storage.backend=azure - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1364,13 +1373,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1667,7 +1674,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1696,7 +1703,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-storage-gcs-generated.yaml b/operations/mimir-tests/test-storage-gcs-generated.yaml index 1d40fc2a5c1..e64d662b4d1 100644 --- a/operations/mimir-tests/test-storage-gcs-generated.yaml +++ b/operations/mimir-tests/test-storage-gcs-generated.yaml @@ -323,6 +323,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -749,7 +768,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -775,7 +793,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -850,8 +868,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -867,7 +883,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -933,14 +949,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1003,7 +1018,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1069,15 +1084,12 @@ spec: - -blocks-storage.bucket-store.sync-dir=/data/tsdb - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.gcs.bucket-name=blocks-bucket - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=gcs - -ruler-storage.gcs.bucket-name=rules-bucket - -ruler.alertmanager-url=http://alertmanager.default.svc.cluster.local/alertmanager @@ -1087,8 +1099,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1098,7 +1108,7 @@ spec: - -store-gateway.sharding-ring.store=consul - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1165,7 +1175,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1256,7 +1266,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1334,7 +1344,6 @@ spec: - -blocks-storage.backend=gcs - -blocks-storage.gcs.bucket-name=blocks-bucket - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1352,13 +1361,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1652,7 +1659,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1682,7 +1689,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir-tests/test-storage-s3-generated.yaml b/operations/mimir-tests/test-storage-s3-generated.yaml index 7e250a59fb6..ed11f29ca11 100644 --- a/operations/mimir-tests/test-storage-s3-generated.yaml +++ b/operations/mimir-tests/test-storage-s3-generated.yaml @@ -323,6 +323,25 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + name: compactor + name: compactor + namespace: default +spec: + clusterIP: None + ports: + - name: compactor-http-metrics + port: 8080 + targetPort: 8080 + - name: compactor-grpc + port: 9095 + targetPort: 9095 + selector: + name: compactor +--- +apiVersion: v1 +kind: Service metadata: labels: name: consul @@ -749,7 +768,6 @@ spec: topologyKey: kubernetes.io/hostname containers: - args: - - -distributor.extend-writes=true - -distributor.ha-tracker.enable=true - -distributor.ha-tracker.enable-for-all-users=true - -distributor.ha-tracker.etcd.endpoints=etcd-client.default.svc.cluster.local.:2379 @@ -775,7 +793,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=distributor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: distributor ports: @@ -851,8 +869,6 @@ spec: - -mem-ballast-size-bytes=268435456 - -querier.frontend-client.grpc-max-send-msg-size=104857600 - -querier.max-concurrent=8 - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -querier.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc.keepalive.min-time-between-pings=10s @@ -868,7 +884,7 @@ spec: env: - name: JAEGER_REPORTER_MAX_QUEUE_SIZE value: "1024" - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: querier ports: @@ -934,14 +950,13 @@ spec: - -query-frontend.results-cache.memcached.timeout=500ms - -query-frontend.scheduler-address=query-scheduler-discovery.default.svc.cluster.local:9095 - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=104857600 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -server.http-write-timeout=1m - -store.max-query-length=12000h - -target=query-frontend - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-frontend ports: @@ -1004,7 +1019,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=query-scheduler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: query-scheduler ports: @@ -1071,15 +1086,12 @@ spec: - -blocks-storage.bucket-store.sync-interval=15m - -blocks-storage.s3.bucket-name=blocks-bucket - -blocks-storage.s3.endpoint=s3.dualstack.us-east-1.amazonaws.com - - -distributor.extend-writes=true - -distributor.health-check-ingesters=true - -ingester.ring.consul.hostname=consul.default.svc.cluster.local:8500 - -ingester.ring.heartbeat-timeout=10m - -ingester.ring.prefix= - -ingester.ring.replication-factor=3 - -ingester.ring.store=consul - - -querier.query-ingesters-within=13h - - -querier.query-store-after=12h - -ruler-storage.backend=s3 - -ruler-storage.s3.bucket-name=rules-bucket - -ruler-storage.s3.endpoint=s3.dualstack.eu-west-1.amazonaws.com @@ -1091,8 +1103,6 @@ spec: - -ruler.ring.store=consul - -ruler.rule-path=/rules - -runtime-config.file=/etc/mimir/overrides.yaml - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 @@ -1102,7 +1112,7 @@ spec: - -store-gateway.sharding-ring.store=consul - -store.max-query-length=768h - -target=ruler - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ruler ports: @@ -1170,7 +1180,7 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: alertmanager ports: @@ -1262,7 +1272,7 @@ spec: - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=compactor - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: compactor ports: @@ -1341,7 +1351,6 @@ spec: - -blocks-storage.s3.bucket-name=blocks-bucket - -blocks-storage.s3.endpoint=s3.dualstack.us-east-1.amazonaws.com - -blocks-storage.tsdb.block-ranges-period=2h - - -blocks-storage.tsdb.close-idle-tsdb-timeout=13h - -blocks-storage.tsdb.dir=/data/tsdb - -blocks-storage.tsdb.ship-interval=1m - -distributor.health-check-ingesters=true @@ -1359,13 +1368,11 @@ spec: - -ingester.ring.unregister-on-shutdown=true - -runtime-config.file=/etc/mimir/overrides.yaml - -server.grpc-max-concurrent-streams=10000 - - -server.grpc-max-recv-msg-size-bytes=10485760 - - -server.grpc-max-send-msg-size-bytes=10485760 - -server.grpc.keepalive.min-time-between-pings=10s - -server.grpc.keepalive.ping-without-stream-allowed=true - -server.http-listen-port=8080 - -target=ingester - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: ingester ports: @@ -1659,7 +1666,7 @@ spec: - -blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections=100 - -blocks-storage.bucket-store.chunks-cache.memcached.max-item-size=1048576 - - -blocks-storage.bucket-store.ignore-blocks-within=10h + - -blocks-storage.bucket-store.chunks-cache.memcached.timeout=450ms - -blocks-storage.bucket-store.index-cache.backend=memcached - -blocks-storage.bucket-store.index-cache.memcached.addresses=dnssrvnoa+memcached-index-queries.default.svc.cluster.local:11211 - -blocks-storage.bucket-store.index-cache.memcached.max-async-concurrency=50 @@ -1690,7 +1697,7 @@ spec: - -store-gateway.sharding-ring.tokens-file-path=/data/tokens - -store-gateway.sharding-ring.wait-stability-min-duration=1m - -target=store-gateway - image: grafana/mimir:2.0.0 + image: grafana/mimir:2.1.0 imagePullPolicy: IfNotPresent name: store-gateway ports: diff --git a/operations/mimir/README.md b/operations/mimir/README.md index 24eb4e22285..4a5faf61176 100644 --- a/operations/mimir/README.md +++ b/operations/mimir/README.md @@ -28,7 +28,7 @@ set -e # Initialise the Tanka. mkdir jsonnet-example && cd jsonnet-example -tk init --k8s=1.18 +tk init --k8s=1.21 # Install Mimir jsonnet. jb install github.com/grafana/mimir/operations/mimir@main @@ -67,7 +67,7 @@ This can become an issue when playing with Mimir in a single-node kubernetes clu For example: ```jsonnet -local mimir = 'mimir/mimir.libsonnet'; +local mimir = import 'mimir/mimir.libsonnet'; mimir { _config+:: { @@ -92,7 +92,7 @@ For example, if you're just playing with Mimir and want to run it on a small (po local k = import 'github.com/grafana/jsonnet-libs/ksonnet-util/kausal.libsonnet', deployment = k.apps.v1.deployment, statefulSet = k.apps.v1.statefulSet; -local mimir = 'mimir/mimir.libsonnet'; +local mimir = import 'mimir/mimir.libsonnet'; mimir { _config+:: { diff --git a/operations/mimir/alertmanager.libsonnet b/operations/mimir/alertmanager.libsonnet index 497cd1f915f..70bfc2aa0ed 100644 --- a/operations/mimir/alertmanager.libsonnet +++ b/operations/mimir/alertmanager.libsonnet @@ -1,7 +1,6 @@ { local configMap = $.core.v1.configMap, local container = $.core.v1.container, - local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget, local pvc = $.core.v1.persistentVolumeClaim, local service = $.core.v1.service, local statefulSet = $.apps.v1.statefulSet, @@ -66,17 +65,9 @@ alertmanager_statefulset: if $._config.alertmanager_enabled then - statefulSet.new('alertmanager', $._config.alertmanager.replicas, [$.alertmanager_container], $.alertmanager_pvc) + - statefulSet.mixin.spec.withServiceName('alertmanager') + - statefulSet.mixin.metadata.withNamespace($._config.namespace) + - statefulSet.mixin.metadata.withLabels({ name: 'alertmanager' }) + - statefulSet.mixin.spec.template.metadata.withLabels({ name: 'alertmanager' }) + - statefulSet.mixin.spec.selector.withMatchLabels({ name: 'alertmanager' }) + - statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + - statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + $.newMimirStatefulSet('alertmanager', $._config.alertmanager.replicas, $.alertmanager_container, $.alertmanager_pvc, podManagementPolicy=null) + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(900) + $.util.configVolumeMount($._config.overrides_configmap, $._config.overrides_configmap_mountpoint) + - (if !std.isObject($._config.node_selector) then {} else statefulSet.mixin.spec.template.spec.withNodeSelectorMixin($._config.node_selector)) + statefulSet.mixin.spec.template.spec.withVolumesMixin( if hasFallbackConfig then [volume.fromConfigMap('alertmanager-fallback-config', 'alertmanager-fallback-config')] @@ -90,13 +81,6 @@ service.mixin.spec.withClusterIp('None') else {}, - alertmanager_pdb: - if $._config.alertmanager_enabled then - podDisruptionBudget.new('alertmanager-pdb') + - podDisruptionBudget.mixin.metadata.withLabels({ name: 'alertmanager-pdb' }) + - podDisruptionBudget.mixin.spec.selector.withMatchLabels({ - name: $.alertmanager_statefulset.spec.template.metadata.labels.name, - }) + - podDisruptionBudget.mixin.spec.withMaxUnavailable(1) - else {}, + alertmanager_pdb: if !$._config.alertmanager_enabled then null else + $.newMimirPdb('alertmanager'), } diff --git a/operations/mimir/autoscaling.libsonnet b/operations/mimir/autoscaling.libsonnet new file mode 100644 index 00000000000..2c006fe5919 --- /dev/null +++ b/operations/mimir/autoscaling.libsonnet @@ -0,0 +1,122 @@ +{ + _config+:: { + autoscaling_querier_enabled: false, + autoscaling_querier_min_replicas: error 'you must set autoscaling_querier_min_replicas in the _config', + autoscaling_querier_max_replicas: error 'you must set autoscaling_querier_max_replicas in the _config', + autoscaling_prometheus_url: 'http://prometheus.default:9090/prometheus', + }, + + ensure_query_scheduler_is_enabled:: if $._config.autoscaling_querier_enabled && !$._config.query_scheduler_enabled then + error 'you must enable query-scheduler in order to use querier autoscaling' + else + null, + + // The ScaledObject resource is watched by the KEDA operator. When this resource is created, KEDA + // creates the related HPA resource in the namespace. Likewise, then ScaledObject is deleted, KEDA + // deletes the related HPA. + newScaledObject(name, namespace, config):: { + apiVersion: 'keda.sh/v1alpha1', + kind: 'ScaledObject', + metadata: { + name: name, + namespace: namespace, + }, + spec: { + scaleTargetRef: { + name: name, + }, + + // The min/max replica count settings are reflected to HPA too. + maxReplicaCount: config.max_replica_count, + minReplicaCount: config.min_replica_count, + + // The pollingInterval defines how frequently KEDA will run the queries defined in triggers. + // This setting is only effective when scaling from 0->N because the scale up from 0 is managed + // by KEDA, while scaling from 1+->N is managed by HPA (and this setting doesn't apply to HPA). + pollingInterval: 10, + + triggers: [ + { + type: 'prometheus', + metadata: { + serverAddress: $._config.autoscaling_prometheus_url, + query: config.query, + + // The metric name uniquely identify a metric in the KEDA metrics server. + metricName: config.metric_name, + + // The threshold value is set to the HPA's targetAverageValue. The number of desired replicas is computed + // by HPA as: + // desiredReplicas = ceil[currentReplicas * ( currentMetricValue / desiredMetricValue )] + // + // Where: + // - currentMetricValue = / + // - desiredMetricValue = + // + // Read more: + // https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#algorithm-details + threshold: config.threshold, + }, + }, + ], + }, + }, + + newQuerierScaledObject(name, query_scheduler_container, querier_max_concurrent, min_replicas, max_replicas):: self.newScaledObject(name, $._config.namespace, { + min_replica_count: min_replicas, + max_replica_count: max_replicas, + metric_name: 'cortex_%s_hpa_%s' % [std.strReplace(name, '-', '_'), $._config.namespace], + + // Each query scheduler tracks *at regular intervals* the number of inflight requests + // (both enqueued and processing queries) as a summary. With the following query we target + // to have enough querier workers to run the max observed inflight requests 75% of time. + // + // Instead of measuring it as instant query, we look at the max 75th percentile over the last + // 5 minutes. This allows us to scale up quickly, but scale down slowly (and not too early + // if within the next 5 minutes after a scale up we have further spikes). + query: 'sum(max_over_time(cortex_query_scheduler_inflight_requests{container="%s",namespace="%s",quantile="0.75"}[5m]))' % [query_scheduler_container, $._config.namespace], + + // Target to utilize 75% querier workers on peak traffic (as measured by query above), + // so we have 25% room for higher peaks. + local targetUtilization = 0.75, + threshold: '%d' % (querier_max_concurrent * targetUtilization), + }), + + // When querier autoscaling is enabled the querier's "replicas" is missing from the spec + // so the query sharding jsonnet can't compute the min number of replicas. To fix it, we + // override it and we compute it based on the max number of replicas (worst case scenario). + // This pessimistic logic can be removed once we'll migrate query-frontend to autoscaling too. + local queryFrontendReplicas(querier_max_replicas) = { + spec+: { + local min_replicas = std.max(std.floor(0.2 * querier_max_replicas), 2), + + replicas: std.max(super.replicas, min_replicas), + }, + }, + + local removeReplicasFromSpec = { + spec+: { + // Remove the "replicas" field so that Flux doesn't reconcile it. + replicas+:: null, + }, + }, + + // + // Queriers + // + + querier_scaled_object: if !$._config.autoscaling_querier_enabled then null else + self.newQuerierScaledObject( + name='querier', + query_scheduler_container='query-scheduler', + querier_max_concurrent=$.querier_args['querier.max-concurrent'], + min_replicas=$._config.autoscaling_querier_min_replicas, + max_replicas=$._config.autoscaling_querier_max_replicas, + ), + + querier_deployment+: if !$._config.autoscaling_querier_enabled then {} else + removeReplicasFromSpec, + + query_frontend_deployment+: if !$._config.query_sharding_enabled || !$._config.autoscaling_querier_enabled then {} else + queryFrontendReplicas($._config.autoscaling_querier_max_replicas), +} diff --git a/operations/mimir/common.libsonnet b/operations/mimir/common.libsonnet index b714914e95d..176d68c077b 100644 --- a/operations/mimir/common.libsonnet +++ b/operations/mimir/common.libsonnet @@ -18,4 +18,44 @@ container.mixin.readinessProbe.withInitialDelaySeconds(15) + container.mixin.readinessProbe.withTimeoutSeconds(1), }, + + // Utility to create an headless service used to discover replicas of a Mimir deployment. + newMimirDiscoveryService(name, deployment):: + local service = $.core.v1.service; + + $.util.serviceFor(deployment, $._config.service_ignored_labels) + + service.mixin.spec.withPublishNotReadyAddresses(true) + + service.mixin.spec.withClusterIp('None') + + service.mixin.metadata.withName(name), + + // Utility to create a PodDisruptionBudget for a Mimir deployment. + newMimirPdb(deploymentName, maxUnavailable=1):: + local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget; + local pdbName = '%s-pdb' % deploymentName; + + podDisruptionBudget.new() + + podDisruptionBudget.mixin.metadata.withName(pdbName) + + podDisruptionBudget.mixin.metadata.withLabels({ name: pdbName }) + + podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: deploymentName }) + + podDisruptionBudget.mixin.spec.withMaxUnavailable(maxUnavailable), + + // Utility to create a StatefulSet for a Mimir component. + // + // By default, it uses the Parallel pod management policy which parallelly + // scale up/down instances instead of starting them one by one. This does NOT + // affect rolling updates: they will continue to be rolled out one by one + // (the next pod will be rolled out once the previous is ready). + newMimirStatefulSet(name, replicas, container, pvc, podManagementPolicy='Parallel'):: + local statefulSet = $.apps.v1.statefulSet; + + statefulSet.new(name, replicas, [container], pvc) + + statefulSet.mixin.spec.withServiceName(name) + + statefulSet.mixin.metadata.withNamespace($._config.namespace) + + statefulSet.mixin.metadata.withLabels({ name: name }) + + statefulSet.mixin.spec.template.metadata.withLabels({ name: name }) + + statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) + + statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + + statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + (if podManagementPolicy != null then statefulSet.mixin.spec.withPodManagementPolicy(podManagementPolicy) else {}) + + (if !std.isObject($._config.node_selector) then {} else statefulSet.mixin.spec.template.spec.withNodeSelectorMixin($._config.node_selector)), } diff --git a/operations/mimir/compactor.libsonnet b/operations/mimir/compactor.libsonnet index e8e7db76144..d353b5e5a18 100644 --- a/operations/mimir/compactor.libsonnet +++ b/operations/mimir/compactor.libsonnet @@ -83,8 +83,6 @@ 'runtime-config.file': '%s/overrides.yaml' % $._config.overrides_configmap_mountpoint, }, - // The compactor runs a statefulset with a single replica, because - // it does not support horizontal scalability yet. local compactor_data_pvc = pvc.new() + pvc.mixin.spec.resources.withRequests({ storage: $._config.compactor_data_disk_size }) + @@ -106,23 +104,16 @@ $.jaeger_mixin, newCompactorStatefulSet(name, container):: - statefulSet.new(name, 1, [container], compactor_data_pvc) + - statefulSet.mixin.spec.withServiceName(name) + - statefulSet.mixin.metadata.withNamespace($._config.namespace) + - statefulSet.mixin.metadata.withLabels({ name: name }) + - statefulSet.mixin.spec.template.metadata.withLabels({ name: name }) + - statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) + - statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + - statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + $.newMimirStatefulSet(name, 1, container, compactor_data_pvc) + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(900) + - (if !std.isObject($._config.node_selector) then {} else statefulSet.mixin.spec.template.spec.withNodeSelectorMixin($._config.node_selector)) + - // Parallelly scale up/down compactor instances instead of starting them - // one by one. This does NOT affect rolling updates: they will continue to be - // rolled out one by one (the next pod will be rolled out once the previous is - // ready). - statefulSet.mixin.spec.withPodManagementPolicy('Parallel') + $.util.configVolumeMount($._config.overrides_configmap, $._config.overrides_configmap_mountpoint), compactor_statefulset: $.newCompactorStatefulSet('compactor', $.compactor_container), + + compactor_service: + local service = $.core.v1.service; + + $.util.serviceFor($.compactor_statefulset, $._config.service_ignored_labels) + + service.mixin.spec.withClusterIp('None'), } diff --git a/operations/mimir/config.libsonnet b/operations/mimir/config.libsonnet index 4862e93baba..31fde0bcf9b 100644 --- a/operations/mimir/config.libsonnet +++ b/operations/mimir/config.libsonnet @@ -133,12 +133,6 @@ // type queries. 32 days to allow for comparision over the last month (31d) and // then some. 'store.max-query-length': '768h', - - // Ingesters don't have data older than 13h, no need to ask them. - 'querier.query-ingesters-within': '13h', - - // No need to look at store for data younger than 12h, as ingesters have all of it. - 'querier.query-store-after': '12h', }, // PromQL query engine config (shared between all services running PromQL engine, like the ruler and querier). @@ -188,8 +182,6 @@ alertmanager: { replicas: 3, - sharding_enabled: false, - gossip_port: 9094, fallback_config: {}, ring_store: 'consul', ring_hostname: 'consul.%s.svc.cluster.local:8500' % $._config.namespace, @@ -419,6 +411,7 @@ 'blocks-storage.bucket-store.chunks-cache.memcached.addresses': 'dnssrvnoa+memcached.%(namespace)s.svc.cluster.local:11211' % $._config, 'blocks-storage.bucket-store.chunks-cache.memcached.max-item-size': $._config.memcached_chunks_max_item_size_mb * 1024 * 1024, 'blocks-storage.bucket-store.chunks-cache.memcached.max-async-concurrency': '50', + 'blocks-storage.bucket-store.chunks-cache.memcached.timeout': '450ms', } else {} ), diff --git a/operations/mimir/continuous-test.libsonnet b/operations/mimir/continuous-test.libsonnet index befb9045e31..6d8a6396a98 100644 --- a/operations/mimir/continuous-test.libsonnet +++ b/operations/mimir/continuous-test.libsonnet @@ -16,6 +16,8 @@ local k = import 'ksonnet-util/kausal.libsonnet'; 'tests.write-endpoint': $._config.continuous_test_write_endpoint, 'tests.read-endpoint': $._config.continuous_test_read_endpoint, 'tests.tenant-id': $._config.continuous_test_tenant_id, + 'tests.write-read-series-test.num-series': 1000, + 'tests.write-read-series-test.max-query-age': '48h', }, continuous_test_container:: @@ -25,7 +27,8 @@ local k = import 'ksonnet-util/kausal.libsonnet'; k.core.v1.containerPort.new('http-metrics', 9900), ]) + k.util.resourcesRequests('1', '512Mi') + - k.util.resourcesLimits(null, '1Gi'), + k.util.resourcesLimits(null, '1Gi') + + $.jaeger_mixin, continuous_test_deployment: if !$._config.continuous_test_enabled then null else deployment.new('continuous-test', 1, [ diff --git a/operations/mimir/distributor.libsonnet b/operations/mimir/distributor.libsonnet index e1b23746c61..5b4296dd2e3 100644 --- a/operations/mimir/distributor.libsonnet +++ b/operations/mimir/distributor.libsonnet @@ -31,10 +31,6 @@ 'distributor.ring.store': 'consul', 'distributor.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace, 'distributor.ring.prefix': '', - - // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown" - // is set to false. - 'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown, }, distributor_ports:: $.util.defaultPorts, diff --git a/operations/mimir/getting-started.sh b/operations/mimir/getting-started.sh index b5c31dfe517..6885142687d 100755 --- a/operations/mimir/getting-started.sh +++ b/operations/mimir/getting-started.sh @@ -5,7 +5,7 @@ set -e # Initialise the Tanka. mkdir jsonnet-example && cd jsonnet-example -tk init --k8s=1.18 +tk init --k8s=1.21 # Install Mimir jsonnet. jb install github.com/grafana/mimir/operations/mimir@main diff --git a/operations/mimir/images.libsonnet b/operations/mimir/images.libsonnet index 66cba370d1c..1c394235007 100644 --- a/operations/mimir/images.libsonnet +++ b/operations/mimir/images.libsonnet @@ -5,7 +5,7 @@ memcachedExporter: 'prom/memcached-exporter:v0.6.0', // Our services. - mimir: 'grafana/mimir:2.0.0', + mimir: 'grafana/mimir:2.1.0', alertmanager: self.mimir, distributor: self.mimir, @@ -19,7 +19,7 @@ query_scheduler: self.mimir, overrides_exporter: self.mimir, - query_tee: 'grafana/query-tee:2.0.0', + query_tee: 'grafana/query-tee:2.1.0', continuous_test: 'grafana/mimir-continuous-test:main-8a8648e81', // See: https://github.com/grafana/rollout-operator diff --git a/operations/mimir/ingester.libsonnet b/operations/mimir/ingester.libsonnet index 50b19cfaa61..c64521a3d4c 100644 --- a/operations/mimir/ingester.libsonnet +++ b/operations/mimir/ingester.libsonnet @@ -1,6 +1,5 @@ { local container = $.core.v1.container, - local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget, local pvc = $.core.v1.persistentVolumeClaim, local statefulSet = $.apps.v1.statefulSet, local volumeMount = $.core.v1.volumeMount, @@ -29,17 +28,12 @@ // Limits config. 'runtime-config.file': '%s/overrides.yaml' % $._config.overrides_configmap_mountpoint, 'server.grpc-max-concurrent-streams': 10000, - 'server.grpc-max-send-msg-size-bytes': 10 * 1024 * 1024, - 'server.grpc-max-recv-msg-size-bytes': 10 * 1024 * 1024, // Blocks storage. 'blocks-storage.tsdb.dir': '/data/tsdb', 'blocks-storage.tsdb.block-ranges-period': '2h', 'blocks-storage.tsdb.ship-interval': '1m', - // Close idle TSDBs. - 'blocks-storage.tsdb.close-idle-tsdb-timeout': $._config.queryConfig['querier.query-ingesters-within'], - // Persist ring tokens so that when the ingester will be restarted // it will pick the same tokens 'ingester.ring.tokens-file-path': '/data/tokens', @@ -68,29 +62,16 @@ pvc.mixin.metadata.withName('ingester-data'), newIngesterStatefulSet(name, container, with_anti_affinity=true):: - statefulSet.new(name, 3, [ - container + $.core.v1.container.withVolumeMountsMixin([ - volumeMount.new('ingester-data', '/data'), - ]), - ], ingester_data_pvc) + - statefulSet.mixin.spec.withServiceName(name) + - statefulSet.mixin.metadata.withNamespace($._config.namespace) + - statefulSet.mixin.metadata.withLabels({ name: name }) + - statefulSet.mixin.spec.template.metadata.withLabels({ name: name }) + - statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) + - statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + + local ingesterContainer = container + $.core.v1.container.withVolumeMountsMixin([ + volumeMount.new('ingester-data', '/data'), + ]); + + $.newMimirStatefulSet(name, 3, ingesterContainer, ingester_data_pvc) + // When the ingester needs to flush blocks to the storage, it may take quite a lot of time. // For this reason, we grant an high termination period (80 minutes). statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(1200) + - statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + $.util.configVolumeMount($._config.overrides_configmap, $._config.overrides_configmap_mountpoint) + $.util.podPriority('high') + - // Parallelly scale up/down ingester instances instead of starting them - // one by one. This does NOT affect rolling updates: they will continue to be - // rolled out one by one (the next pod will be rolled out once the previous is - // ready). - statefulSet.mixin.spec.withPodManagementPolicy('Parallel') + - (if !std.isObject($._config.node_selector) then {} else statefulSet.mixin.spec.template.spec.withNodeSelectorMixin($._config.node_selector)) + (if with_anti_affinity then $.util.antiAffinity else {}), ingester_statefulset: self.newIngesterStatefulSet('ingester', $.ingester_container, !$._config.ingester_allow_multiple_replicas_on_same_node), @@ -98,12 +79,8 @@ ingester_service: $.util.serviceFor($.ingester_statefulset, $._config.service_ignored_labels), - newIngesterPdb(pdbName, ingesterName):: - podDisruptionBudget.new() + - podDisruptionBudget.mixin.metadata.withName(pdbName) + - podDisruptionBudget.mixin.metadata.withLabels({ name: pdbName }) + - podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: ingesterName }) + - podDisruptionBudget.mixin.spec.withMaxUnavailable(1), + newIngesterPdb(ingesterName):: + $.newMimirPdb(ingesterName), - ingester_pdb: self.newIngesterPdb('ingester-pdb', name), + ingester_pdb: self.newIngesterPdb(name), } diff --git a/operations/mimir/mimir.libsonnet b/operations/mimir/mimir.libsonnet index 89a824f2150..4840888c966 100644 --- a/operations/mimir/mimir.libsonnet +++ b/operations/mimir/mimir.libsonnet @@ -25,4 +25,7 @@ (import 'query-sharding.libsonnet') + (import 'multi-zone.libsonnet') + (import 'memberlist.libsonnet') + -(import 'continuous-test.libsonnet') +(import 'continuous-test.libsonnet') + + +// Import autoscaling at the end because it overrides deployments. +(import 'autoscaling.libsonnet') diff --git a/operations/mimir/multi-zone.libsonnet b/operations/mimir/multi-zone.libsonnet index 4b98ee3aa59..5cdca9be8ad 100644 --- a/operations/mimir/multi-zone.libsonnet +++ b/operations/mimir/multi-zone.libsonnet @@ -182,18 +182,24 @@ // Multi-zone store-gateways. // - newStoreGatewayZoneContainer(zone):: + store_gateway_zone_a_args:: {}, + store_gateway_zone_b_args:: {}, + store_gateway_zone_c_args:: {}, + + newStoreGatewayZoneContainer(zone, zone_args):: $.store_gateway_container + - container.withArgs($.util.mapToFlags($.store_gateway_args { - 'store-gateway.sharding-ring.instance-availability-zone': 'zone-%s' % zone, - 'store-gateway.sharding-ring.zone-awareness-enabled': true, + container.withArgs($.util.mapToFlags( + $.store_gateway_args + zone_args + { + 'store-gateway.sharding-ring.instance-availability-zone': 'zone-%s' % zone, + 'store-gateway.sharding-ring.zone-awareness-enabled': true, - // Use a different prefix so that both single-zone and multi-zone store-gateway rings can co-exists. - 'store-gateway.sharding-ring.prefix': 'multi-zone/', + // Use a different prefix so that both single-zone and multi-zone store-gateway rings can co-exists. + 'store-gateway.sharding-ring.prefix': 'multi-zone/', - // Do not unregister from ring at shutdown, so that no blocks re-shuffling occurs during rollouts. - 'store-gateway.sharding-ring.unregister-on-shutdown': false, - })), + // Do not unregister from ring at shutdown, so that no blocks re-shuffling occurs during rollouts. + 'store-gateway.sharding-ring.unregister-on-shutdown': false, + } + )), newStoreGatewayZoneStatefulSet(zone, container):: local name = 'store-gateway-zone-%s' % zone; @@ -237,7 +243,7 @@ }, store_gateway_zone_a_container:: if !$._config.multi_zone_store_gateway_enabled then null else - self.newStoreGatewayZoneContainer('a'), + self.newStoreGatewayZoneContainer('a', $.store_gateway_zone_a_args), store_gateway_zone_a_statefulset: if !$._config.multi_zone_store_gateway_enabled then null else (self + nonRetainablePVCs).newStoreGatewayZoneStatefulSet('a', $.store_gateway_zone_a_container), @@ -246,7 +252,7 @@ self.newStoreGatewayZoneService($.store_gateway_zone_a_statefulset), store_gateway_zone_b_container:: if !$._config.multi_zone_store_gateway_enabled then null else - self.newStoreGatewayZoneContainer('b'), + self.newStoreGatewayZoneContainer('b', $.store_gateway_zone_b_args), store_gateway_zone_b_statefulset: if !$._config.multi_zone_store_gateway_enabled then null else (self + nonRetainablePVCs).newStoreGatewayZoneStatefulSet('b', $.store_gateway_zone_b_container), @@ -255,7 +261,7 @@ self.newStoreGatewayZoneService($.store_gateway_zone_b_statefulset), store_gateway_zone_c_container:: if !$._config.multi_zone_store_gateway_enabled then null else - self.newStoreGatewayZoneContainer('c'), + self.newStoreGatewayZoneContainer('c', $.store_gateway_zone_c_args), store_gateway_zone_c_statefulset: if !$._config.multi_zone_store_gateway_enabled then null else (self + nonRetainablePVCs).newStoreGatewayZoneStatefulSet('c', $.store_gateway_zone_c_container), diff --git a/operations/mimir/querier.libsonnet b/operations/mimir/querier.libsonnet index ed02f3673f0..44bd312f500 100644 --- a/operations/mimir/querier.libsonnet +++ b/operations/mimir/querier.libsonnet @@ -37,16 +37,19 @@ JAEGER_REPORTER_MAX_QUEUE_SIZE: '1024', // Default is 100. }, - querier_container:: - container.new('querier', $._images.querier) + + newQuerierContainer(name, args):: + container.new(name, $._images.querier) + container.withPorts($.querier_ports) + - container.withArgsMixin($.util.mapToFlags($.querier_args)) + + container.withArgsMixin($.util.mapToFlags(args)) + $.jaeger_mixin + $.util.readinessProbe + container.withEnvMap($.querier_env_map) + $.util.resourcesRequests('1', '12Gi') + $.util.resourcesLimits(null, '24Gi'), + querier_container:: + self.newQuerierContainer('querier', $.querier_args), + local deployment = $.apps.v1.deployment, newQuerierDeployment(name, container):: diff --git a/operations/mimir/query-frontend.libsonnet b/operations/mimir/query-frontend.libsonnet index 5252b166af1..4c0846bbebd 100644 --- a/operations/mimir/query-frontend.libsonnet +++ b/operations/mimir/query-frontend.libsonnet @@ -22,23 +22,23 @@ // So that exporters like cloudwatch can still send in data and be un-cached. 'query-frontend.max-cache-freshness': '10m', - // So it can receive big responses from the querier. - 'server.grpc-max-recv-msg-size-bytes': 100 << 20, - // Limit queries to 500 days, allow this to be override per-user. 'store.max-query-length': '12000h', // 500 Days 'runtime-config.file': '%s/overrides.yaml' % $._config.overrides_configmap_mountpoint, }, - query_frontend_container:: - container.new('query-frontend', $._images.query_frontend) + + newQueryFrontendContainer(name, args):: + container.new(name, $._images.query_frontend) + container.withPorts($.util.defaultPorts) + - container.withArgsMixin($.util.mapToFlags($.query_frontend_args)) + + container.withArgsMixin($.util.mapToFlags(args)) + $.jaeger_mixin + $.util.readinessProbe + $.util.resourcesRequests('2', '600Mi') + $.util.resourcesLimits(null, '1200Mi'), + query_frontend_container:: + self.newQueryFrontendContainer('query-frontend', $.query_frontend_args), + local deployment = $.apps.v1.deployment, newQueryFrontendDeployment(name, container):: @@ -51,18 +51,13 @@ query_frontend_deployment: self.newQueryFrontendDeployment('query-frontend', $.query_frontend_container), - local service = $.core.v1.service, - query_frontend_service: $.util.serviceFor($.query_frontend_deployment, $._config.service_ignored_labels), query_frontend_discovery_service: - $.util.serviceFor($.query_frontend_deployment, $._config.service_ignored_labels) + // Make sure that query frontend worker, running in the querier, do resolve // each query-frontend pod IP and NOT the service IP. To make it, we do NOT // use the service cluster IP so that when the service DNS is resolved it // returns the set of query-frontend IPs. - service.mixin.spec.withPublishNotReadyAddresses(true) + - service.mixin.spec.withClusterIp('None') + - service.mixin.metadata.withName('query-frontend-discovery'), + $.newMimirDiscoveryService('query-frontend-discovery', $.query_frontend_deployment), } diff --git a/operations/mimir/query-scheduler.libsonnet b/operations/mimir/query-scheduler.libsonnet index 8b71e4146f7..7c06d2362f0 100644 --- a/operations/mimir/query-scheduler.libsonnet +++ b/operations/mimir/query-scheduler.libsonnet @@ -3,7 +3,6 @@ { local container = $.core.v1.container, local deployment = $.apps.v1.deployment, - local service = $.core.v1.service, query_scheduler_args+:: $._config.grpcConfig @@ -13,15 +12,18 @@ 'query-scheduler.max-outstanding-requests-per-tenant': 100, }, - query_scheduler_container:: - container.new('query-scheduler', $._images.query_scheduler) + + newQuerySchedulerContainer(name, args):: + container.new(name, $._images.query_scheduler) + container.withPorts($.util.defaultPorts) + - container.withArgsMixin($.util.mapToFlags($.query_scheduler_args)) + + container.withArgsMixin($.util.mapToFlags(args)) + $.jaeger_mixin + $.util.readinessProbe + $.util.resourcesRequests('2', '1Gi') + $.util.resourcesLimits(null, '2Gi'), + query_scheduler_container:: + self.newQuerySchedulerContainer('query-scheduler', $.query_scheduler_args), + newQuerySchedulerDeployment(name, container):: deployment.new(name, 2, [container]) + $.util.configVolumeMount($._config.overrides_configmap, $._config.overrides_configmap_mountpoint) + @@ -37,20 +39,32 @@ query_scheduler_service: if !$._config.query_scheduler_enabled then {} else $.util.serviceFor($.query_scheduler_deployment, $._config.service_ignored_labels), + local discoveryServiceName(prefix) = '%s-discovery' % prefix, + // Headless to make sure resolution gets IP address of target pods, and not service IP. + newQuerySchedulerDiscoveryService(name, deployment):: + $.newMimirDiscoveryService(discoveryServiceName(name), deployment), + query_scheduler_discovery_service: if !$._config.query_scheduler_enabled then {} else - $.util.serviceFor($.query_scheduler_deployment, $._config.service_ignored_labels) + - service.mixin.spec.withPublishNotReadyAddresses(true) + - service.mixin.spec.withClusterIp('None') + - service.mixin.metadata.withName('query-scheduler-discovery'), + self.newQuerySchedulerDiscoveryService('query-scheduler', $.query_scheduler_deployment), // Reconfigure querier and query-frontend to use scheduler. - querier_args+:: if !$._config.query_scheduler_enabled then {} else { + + local querySchedulerAddress(name) = + '%s.%s.svc.cluster.local:9095' % [discoveryServiceName(name), $._config.namespace], + + querierUseQuerySchedulerArgs(name):: { 'querier.frontend-address': null, - 'querier.scheduler-address': 'query-scheduler-discovery.%(namespace)s.svc.cluster.local:9095' % $._config, + 'querier.scheduler-address': querySchedulerAddress(name), }, - query_frontend_args+:: if !$._config.query_scheduler_enabled then {} else { - 'query-frontend.scheduler-address': 'query-scheduler-discovery.%(namespace)s.svc.cluster.local:9095' % $._config, + queryFrontendUseQuerySchedulerArgs(name):: { + 'query-frontend.scheduler-address': querySchedulerAddress(name), }, + + querier_args+:: if !$._config.query_scheduler_enabled then {} else + self.querierUseQuerySchedulerArgs('query-scheduler'), + + query_frontend_args+:: if !$._config.query_scheduler_enabled then {} else + self.queryFrontendUseQuerySchedulerArgs('query-scheduler'), } diff --git a/operations/mimir/query-sharding.libsonnet b/operations/mimir/query-sharding.libsonnet index 94511633417..b53bab38c35 100644 --- a/operations/mimir/query-sharding.libsonnet +++ b/operations/mimir/query-sharding.libsonnet @@ -50,7 +50,12 @@ 'query-frontend.query-sharding-max-sharded-queries': 128, - 'server.grpc-max-recv-msg-size-bytes': super['server.grpc-max-recv-msg-size-bytes'] * $._config.query_sharding_msg_size_factor, + 'server.grpc-max-recv-msg-size-bytes': ( + if 'server.grpc-max-recv-msg-size-bytes' in super then + super['server.grpc-max-recv-msg-size-bytes'] + else + 100 * 1024 * 1024 + ) * $._config.query_sharding_msg_size_factor, }, query_scheduler_args+:: if !$._config.query_sharding_enabled then {} else { diff --git a/operations/mimir/ruler.libsonnet b/operations/mimir/ruler.libsonnet index f4245d46c10..6489f90e1ba 100644 --- a/operations/mimir/ruler.libsonnet +++ b/operations/mimir/ruler.libsonnet @@ -26,15 +26,7 @@ 'ruler.ring.store': 'consul', 'ruler.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace, - // Limits - 'server.grpc-max-send-msg-size-bytes': 10 * 1024 * 1024, - 'server.grpc-max-recv-msg-size-bytes': 10 * 1024 * 1024, - 'server.http-listen-port': $._config.server_http_port, - - // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown" - // is set to false. - 'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown, }, ruler_container:: diff --git a/operations/mimir/shuffle-sharding.libsonnet b/operations/mimir/shuffle-sharding.libsonnet index 150933d8cdd..3fa96feb300 100644 --- a/operations/mimir/shuffle-sharding.libsonnet +++ b/operations/mimir/shuffle-sharding.libsonnet @@ -102,7 +102,6 @@ } ) + ( if !$._config.shuffle_sharding.ingester_read_path_enabled then {} else { - 'querier.shuffle-sharding-ingesters-lookback-period': $._config.queryConfig['querier.query-ingesters-within'], 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, } ), @@ -122,7 +121,6 @@ } ) + ( if !$._config.shuffle_sharding.ingester_read_path_enabled then {} else { - 'querier.shuffle-sharding-ingesters-lookback-period': $._config.queryConfig['querier.query-ingesters-within'], 'distributor.ingestion-tenant-shard-size': $._config.shuffle_sharding.ingester_shard_size, } ) + ( diff --git a/operations/mimir/store-gateway.libsonnet b/operations/mimir/store-gateway.libsonnet index 84da071c2ce..87555ec9770 100644 --- a/operations/mimir/store-gateway.libsonnet +++ b/operations/mimir/store-gateway.libsonnet @@ -1,6 +1,5 @@ { local container = $.core.v1.container, - local podDisruptionBudget = $.policy.v1beta1.podDisruptionBudget, local pvc = $.core.v1.persistentVolumeClaim, local statefulSet = $.apps.v1.statefulSet, local volumeMount = $.core.v1.volumeMount, @@ -49,11 +48,6 @@ 'blocks-storage.bucket-store.index-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.index-cache.memcached.max-get-multi-concurrency'], 'blocks-storage.bucket-store.chunks-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.chunks-cache.memcached.max-get-multi-concurrency'], 'blocks-storage.bucket-store.metadata-cache.memcached.max-idle-connections': $.store_gateway_args['blocks-storage.bucket-store.metadata-cache.memcached.max-get-multi-concurrency'], - - // Queriers will not query store for data younger than 12h (see -querier.query-store-after). - // Store-gateways don't need to load blocks with very most recent data. We use 2h buffer to - // make sure that blocks are ready for querying when needed. - 'blocks-storage.bucket-store.ignore-blocks-within': '10h', } + $.blocks_chunks_caching_config + $.blocks_metadata_caching_config + @@ -72,21 +66,8 @@ $.jaeger_mixin, newStoreGatewayStatefulSet(name, container, with_anti_affinity=false):: - statefulSet.new(name, 3, [container], store_gateway_data_pvc) + - statefulSet.mixin.spec.withServiceName(name) + - statefulSet.mixin.metadata.withNamespace($._config.namespace) + - statefulSet.mixin.metadata.withLabels({ name: name }) + - statefulSet.mixin.spec.template.metadata.withLabels({ name: name }) + - statefulSet.mixin.spec.selector.withMatchLabels({ name: name }) + - statefulSet.mixin.spec.template.spec.securityContext.withRunAsUser(0) + - (if !std.isObject($._config.node_selector) then {} else statefulSet.mixin.spec.template.spec.withNodeSelectorMixin($._config.node_selector)) + - statefulSet.mixin.spec.updateStrategy.withType('RollingUpdate') + + $.newMimirStatefulSet(name, 3, container, store_gateway_data_pvc) + statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) + - // Parallelly scale up/down store-gateway instances instead of starting them - // one by one. This does NOT affect rolling updates: they will continue to be - // rolled out one by one (the next pod will be rolled out once the previous is - // ready). - statefulSet.mixin.spec.withPodManagementPolicy('Parallel') + $.util.configVolumeMount($._config.overrides_configmap, $._config.overrides_configmap_mountpoint) + (if with_anti_affinity then $.util.antiAffinity else {}), @@ -96,11 +77,8 @@ $.util.serviceFor($.store_gateway_statefulset, $._config.service_ignored_labels), store_gateway_pdb: - podDisruptionBudget.new() + - podDisruptionBudget.mixin.metadata.withName('store-gateway-pdb') + - podDisruptionBudget.mixin.metadata.withLabels({ name: 'store-gateway-pdb' }) + - podDisruptionBudget.mixin.spec.selector.withMatchLabels({ name: 'store-gateway' }) + // To avoid any disruption in the read path we need at least 1 replica of each // block available, so the disruption budget depends on the blocks replication factor. - podDisruptionBudget.mixin.spec.withMaxUnavailable(if $._config.store_gateway_replication_factor > 1 then $._config.store_gateway_replication_factor - 1 else 1), + local maxUnavailable = if $._config.store_gateway_replication_factor > 1 then $._config.store_gateway_replication_factor - 1 else 1; + $.newMimirPdb('store-gateway', maxUnavailable), } diff --git a/pkg/alertmanager/alertmanager_metrics.go b/pkg/alertmanager/alertmanager_metrics.go index f7c1128ea39..4e1baf7e505 100644 --- a/pkg/alertmanager/alertmanager_metrics.go +++ b/pkg/alertmanager/alertmanager_metrics.go @@ -296,10 +296,10 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCountersPerUser(out, m.alertsReceived, "alertmanager_alerts_received_total") data.SendSumOfCountersPerUser(out, m.alertsInvalid, "alertmanager_alerts_invalid_total") - data.SendSumOfCountersPerUserWithLabels(out, m.numNotifications, "alertmanager_notifications_total", "integration") - data.SendSumOfCountersPerUserWithLabels(out, m.numFailedNotifications, "alertmanager_notifications_failed_total", "integration") - data.SendSumOfCountersPerUserWithLabels(out, m.numNotificationRequestsTotal, "alertmanager_notification_requests_total", "integration") - data.SendSumOfCountersPerUserWithLabels(out, m.numNotificationRequestsFailedTotal, "alertmanager_notification_requests_failed_total", "integration") + data.SendSumOfCountersPerUser(out, m.numNotifications, "alertmanager_notifications_total", util.WithLabels("integration"), util.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerUser(out, m.numFailedNotifications, "alertmanager_notifications_failed_total", util.WithLabels("integration"), util.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerUser(out, m.numNotificationRequestsTotal, "alertmanager_notification_requests_total", util.WithLabels("integration"), util.WithSkipZeroValueMetrics) + data.SendSumOfCountersPerUser(out, m.numNotificationRequestsFailedTotal, "alertmanager_notification_requests_failed_total", util.WithLabels("integration"), util.WithSkipZeroValueMetrics) data.SendSumOfHistograms(out, m.notificationLatencySeconds, "alertmanager_notification_latency_seconds") data.SendSumOfGaugesPerUserWithLabels(out, m.markerAlerts, "alertmanager_alerts", "state") @@ -334,7 +334,7 @@ func (m *alertmanagerMetrics) Collect(out chan<- prometheus.Metric) { data.SendSumOfCounters(out, m.persistTotal, "alertmanager_state_persist_total") data.SendSumOfCounters(out, m.persistFailed, "alertmanager_state_persist_failed_total") - data.SendSumOfCountersPerUserWithLabels(out, m.notificationRateLimited, "alertmanager_notification_rate_limited_total", "integration") + data.SendSumOfCountersPerUser(out, m.notificationRateLimited, "alertmanager_notification_rate_limited_total", util.WithLabels("integration"), util.WithSkipZeroValueMetrics) data.SendSumOfCountersPerUser(out, m.dispatcherAggregationGroupsLimitReached, "alertmanager_dispatcher_aggregation_group_limit_reached_total") data.SendSumOfCountersPerUser(out, m.insertAlertFailures, "alertmanager_alerts_insert_limited_total") data.SendSumOfGaugesPerUser(out, m.alertsLimiterAlertsCount, "alertmanager_alerts_limiter_current_alerts") diff --git a/pkg/alertmanager/alertmanager_metrics_test.go b/pkg/alertmanager/alertmanager_metrics_test.go index 7b822a06e12..ead75ed331d 100644 --- a/pkg/alertmanager/alertmanager_metrics_test.go +++ b/pkg/alertmanager/alertmanager_metrics_test.go @@ -110,9 +110,6 @@ func TestAlertmanagerMetricsStore(t *testing.T) { cortex_alertmanager_notification_latency_seconds_count 27 # HELP cortex_alertmanager_notifications_failed_total The total number of failed notifications. # TYPE cortex_alertmanager_notifications_failed_total counter - cortex_alertmanager_notifications_failed_total{integration="email",user="user1"} 0 - cortex_alertmanager_notifications_failed_total{integration="email",user="user2"} 0 - cortex_alertmanager_notifications_failed_total{integration="email",user="user3"} 0 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user3"} 500 @@ -139,9 +136,6 @@ func TestAlertmanagerMetricsStore(t *testing.T) { cortex_alertmanager_notifications_failed_total{integration="sns",user="user3"} 800 # HELP cortex_alertmanager_notification_requests_total The total number of attempted notification requests. # TYPE cortex_alertmanager_notification_requests_total counter - cortex_alertmanager_notification_requests_total{integration="email",user="user1"} 0 - cortex_alertmanager_notification_requests_total{integration="email",user="user2"} 0 - cortex_alertmanager_notification_requests_total{integration="email",user="user3"} 0 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user3"} 500 @@ -168,9 +162,6 @@ func TestAlertmanagerMetricsStore(t *testing.T) { cortex_alertmanager_notification_requests_total{integration="sns",user="user3"} 800 # HELP cortex_alertmanager_notification_requests_failed_total The total number of failed notification requests. # TYPE cortex_alertmanager_notification_requests_failed_total counter - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user1"} 0 - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user2"} 0 - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user3"} 0 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user3"} 500 @@ -197,9 +188,6 @@ func TestAlertmanagerMetricsStore(t *testing.T) { cortex_alertmanager_notification_requests_failed_total{integration="sns",user="user3"} 800 # HELP cortex_alertmanager_notifications_total The total number of attempted notifications. # TYPE cortex_alertmanager_notifications_total counter - cortex_alertmanager_notifications_total{integration="email",user="user1"} 0 - cortex_alertmanager_notifications_total{integration="email",user="user2"} 0 - cortex_alertmanager_notifications_total{integration="email",user="user3"} 0 cortex_alertmanager_notifications_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notifications_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notifications_total{integration="opsgenie",user="user3"} 500 @@ -411,9 +399,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notification_requests_failed_total The total number of failed notification requests. # TYPE cortex_alertmanager_notification_requests_failed_total counter - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user1"} 0 - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user2"} 0 - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user3"} 0 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user3"} 500 @@ -441,9 +426,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notification_requests_total The total number of attempted notification requests. # TYPE cortex_alertmanager_notification_requests_total counter - cortex_alertmanager_notification_requests_total{integration="email",user="user1"} 0 - cortex_alertmanager_notification_requests_total{integration="email",user="user2"} 0 - cortex_alertmanager_notification_requests_total{integration="email",user="user3"} 0 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user3"} 500 @@ -471,9 +453,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notifications_failed_total The total number of failed notifications. # TYPE cortex_alertmanager_notifications_failed_total counter - cortex_alertmanager_notifications_failed_total{integration="email",user="user1"} 0 - cortex_alertmanager_notifications_failed_total{integration="email",user="user2"} 0 - cortex_alertmanager_notifications_failed_total{integration="email",user="user3"} 0 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user3"} 500 @@ -501,9 +480,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notifications_total The total number of attempted notifications. # TYPE cortex_alertmanager_notifications_total counter - cortex_alertmanager_notifications_total{integration="email",user="user1"} 0 - cortex_alertmanager_notifications_total{integration="email",user="user2"} 0 - cortex_alertmanager_notifications_total{integration="email",user="user3"} 0 cortex_alertmanager_notifications_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notifications_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notifications_total{integration="opsgenie",user="user3"} 500 @@ -705,8 +681,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notification_requests_failed_total The total number of failed notification requests. # TYPE cortex_alertmanager_notification_requests_failed_total counter - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user1"} 0 - cortex_alertmanager_notification_requests_failed_total{integration="email",user="user2"} 0 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notification_requests_failed_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notification_requests_failed_total{integration="pagerduty",user="user1"} 1 @@ -726,8 +700,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notification_requests_total The total number of attempted notification requests. # TYPE cortex_alertmanager_notification_requests_total counter - cortex_alertmanager_notification_requests_total{integration="email",user="user1"} 0 - cortex_alertmanager_notification_requests_total{integration="email",user="user2"} 0 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notification_requests_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notification_requests_total{integration="pagerduty",user="user1"} 1 @@ -747,8 +719,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notifications_failed_total The total number of failed notifications. # TYPE cortex_alertmanager_notifications_failed_total counter - cortex_alertmanager_notifications_failed_total{integration="email",user="user1"} 0 - cortex_alertmanager_notifications_failed_total{integration="email",user="user2"} 0 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notifications_failed_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notifications_failed_total{integration="pagerduty",user="user1"} 1 @@ -768,8 +738,6 @@ func TestAlertmanagerMetricsRemoval(t *testing.T) { # HELP cortex_alertmanager_notifications_total The total number of attempted notifications. # TYPE cortex_alertmanager_notifications_total counter - cortex_alertmanager_notifications_total{integration="email",user="user1"} 0 - cortex_alertmanager_notifications_total{integration="email",user="user2"} 0 cortex_alertmanager_notifications_total{integration="opsgenie",user="user1"} 5 cortex_alertmanager_notifications_total{integration="opsgenie",user="user2"} 50 cortex_alertmanager_notifications_total{integration="pagerduty",user="user1"} 1 diff --git a/pkg/alertmanager/alertmanager_ring.go b/pkg/alertmanager/alertmanager_ring.go index c66dfd23678..482759919a3 100644 --- a/pkg/alertmanager/alertmanager_ring.go +++ b/pkg/alertmanager/alertmanager_ring.go @@ -114,6 +114,7 @@ func (cfg *RingConfig) ToLifecyclerConfig(logger log.Logger) (ring.BasicLifecycl ID: cfg.InstanceID, Addr: fmt.Sprintf("%s:%d", instanceAddr, instancePort), HeartbeatPeriod: cfg.HeartbeatPeriod, + HeartbeatTimeout: cfg.HeartbeatTimeout, TokensObservePeriod: 0, Zone: cfg.InstanceZone, NumTokens: RingNumTokens, diff --git a/pkg/alertmanager/alertstore/config.go b/pkg/alertmanager/alertstore/config.go index a6a6aedb556..29ef3f3fb48 100644 --- a/pkg/alertmanager/alertstore/config.go +++ b/pkg/alertmanager/alertstore/config.go @@ -26,13 +26,3 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { cfg.Local.RegisterFlagsWithPrefix(prefix, f) cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "alertmanager", f) } - -// IsFullStateSupported returns if the given configuration supports access to FullState objects. -func (cfg *Config) IsFullStateSupported() bool { - for _, backend := range bucket.SupportedBackends { - if cfg.Backend == backend { - return true - } - } - return false -} diff --git a/pkg/alertmanager/alertstore/local/store.go b/pkg/alertmanager/alertstore/local/store.go index 824fe08f757..9cac1f8198c 100644 --- a/pkg/alertmanager/alertstore/local/store.go +++ b/pkg/alertmanager/alertstore/local/store.go @@ -108,12 +108,12 @@ func (f *Store) DeleteAlertConfig(_ context.Context, user string) error { // ListUsersWithFullState implements alertstore.AlertStore. func (f *Store) ListUsersWithFullState(ctx context.Context) ([]string, error) { - return nil, errState + return []string{}, nil } // GetFullState implements alertstore.AlertStore. func (f *Store) GetFullState(ctx context.Context, user string) (alertspb.FullStateDesc, error) { - return alertspb.FullStateDesc{}, errState + return alertspb.FullStateDesc{}, alertspb.ErrNotFound } // SetFullState implements alertstore.AlertStore. diff --git a/pkg/alertmanager/alertstore/local/store_test.go b/pkg/alertmanager/alertstore/local/store_test.go index 55c3aced632..9a70ded471a 100644 --- a/pkg/alertmanager/alertstore/local/store_test.go +++ b/pkg/alertmanager/alertstore/local/store_test.go @@ -108,6 +108,30 @@ func TestStore_GetAlertConfigs(t *testing.T) { } } +func TestStore_FullState(t *testing.T) { + ctx := context.Background() + store, _ := prepareLocalStore(t) + + // FullState not persisted - List always returns no users. + + configs, err := store.ListUsersWithFullState(ctx) + require.NoError(t, err) + assert.Empty(t, configs) + + // FullState not persisted - Get always returns NotFound. + + _, err = store.GetFullState(ctx, "user-1") + require.ErrorIs(t, err, alertspb.ErrNotFound) + + // Any attempt to write the store fails. + + err = store.SetFullState(ctx, "user-1", alertspb.FullStateDesc{}) + require.ErrorIs(t, err, errState) + + err = store.DeleteFullState(ctx, "user-1") + require.ErrorIs(t, err, errState) +} + func prepareLocalStore(t *testing.T) (store *Store, storeDir string) { var err error diff --git a/pkg/alertmanager/alertstore/store.go b/pkg/alertmanager/alertstore/store.go index ef646e9c0ea..f42d7475cf0 100644 --- a/pkg/alertmanager/alertstore/store.go +++ b/pkg/alertmanager/alertstore/store.go @@ -55,6 +55,7 @@ type AlertStore interface { // NewAlertStore returns a alertmanager store backend client based on the provided cfg. func NewAlertStore(ctx context.Context, cfg Config, cfgProvider bucket.TenantConfigProvider, logger log.Logger, reg prometheus.Registerer) (AlertStore, error) { if cfg.Backend == local.Name { + level.Warn(logger).Log("msg", "-alertmanager-storage.backend=local is not suitable for persisting alertmanager state between replicas (silences, notifications); you should switch to an external object store for production use") return local.NewStore(cfg.Local) } diff --git a/pkg/alertmanager/api_test.go b/pkg/alertmanager/api_test.go index 44a3704e4c1..7b8b8f7eb12 100644 --- a/pkg/alertmanager/api_test.go +++ b/pkg/alertmanager/api_test.go @@ -155,7 +155,7 @@ template_files: "good.tpl": "good-templ" ".": "bad-template" `, - err: fmt.Errorf("error validating Alertmanager config: unable to store template file '.'"), + err: fmt.Errorf("error validating Alertmanager config: invalid template name \".\""), }, { name: "Should return error if the referenced template contains the root /", diff --git a/pkg/alertmanager/multitenant.go b/pkg/alertmanager/multitenant.go index bf2edd6b2cd..7ba28645447 100644 --- a/pkg/alertmanager/multitenant.go +++ b/pkg/alertmanager/multitenant.go @@ -58,7 +58,6 @@ const ( var ( errEmptyExternalURL = errors.New("-alertmanager.web.external-url cannot be empty") errInvalidExternalURL = errors.New("the configured external URL is invalid: should not end with /") - errShardingUnsupportedStorage = errors.New("the configured alertmanager storage backend is not supported when sharding is enabled") errZoneAwarenessEnabledWithoutZoneInfo = errors.New("the configured alertmanager has zone awareness enabled but zone is not set") errNotUploadingFallback = errors.New("not uploading fallback configuration") ) @@ -129,9 +128,6 @@ func (cfg *MultitenantAlertmanagerConfig) Validate(storageCfg alertstore.Config) return err } - if !storageCfg.IsFullStateSupported() { - return errShardingUnsupportedStorage - } if cfg.ShardingRing.ZoneAwarenessEnabled && cfg.ShardingRing.InstanceZone == "" { return errZoneAwarenessEnabledWithoutZoneInfo } @@ -1262,6 +1258,15 @@ func safeTemplateFilepath(dir, templateName string) (string, error) { return "", err } + // If actualPath is same as containerDir, it's likely that actualPath was empty, or just ".". + if containerDir == actualPath { + return "", fmt.Errorf("invalid template name %q", templateName) + } + + if !strings.HasSuffix(containerDir, string(os.PathSeparator)) { + containerDir = containerDir + string(os.PathSeparator) + } + // Ensure the actual path of the template is within the expected directory. // This check is a counter-measure to make sure the tenant is not trying to // escape its own directory on disk. diff --git a/pkg/alertmanager/multitenant_test.go b/pkg/alertmanager/multitenant_test.go index a81e87d6864..174ac337df4 100644 --- a/pkg/alertmanager/multitenant_test.go +++ b/pkg/alertmanager/multitenant_test.go @@ -165,11 +165,11 @@ func TestMultitenantAlertmanagerConfig_Validate(t *testing.T) { }, expected: nil, }, - "should fail if new storage store configuration given with local type": { + "should succeed if new storage store configuration given with local type": { setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) { storageCfg.Backend = "local" }, - expected: errShardingUnsupportedStorage, + expected: nil, }, "should fail if zone aware is enabled but zone is not set": { setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) { @@ -1969,6 +1969,37 @@ func TestSafeTemplateFilepath(t *testing.T) { template: "../test.tmpl", expectedErr: errors.New(`invalid template name "../test.tmpl": the template filepath is escaping the per-tenant local directory`), }, + "template name starting with /": { + dir: "/tmp", + template: "/file", + expectedErr: nil, + expectedPath: "/tmp/file", + }, + "escaping template name that has prefix of dir (tmp is prefix of tmpfile)": { + dir: "/sub/tmp", + template: "../tmpfile", + expectedErr: errors.New(`invalid template name "../tmpfile": the template filepath is escaping the per-tenant local directory`), + }, + "empty template name": { + dir: "/tmp", + template: "", + expectedErr: errors.New(`invalid template name ""`), + }, + "dot template name": { + dir: "/tmp", + template: ".", + expectedErr: errors.New(`invalid template name "."`), + }, + "root dir": { + dir: "/", + template: "file", + expectedPath: "/file", + }, + "root dir 2": { + dir: "/", + template: "/subdir/file", + expectedPath: "/subdir/file", + }, } for testName, testData := range tests { diff --git a/pkg/api/api.go b/pkg/api/api.go index b3bd60837ac..b8dcb5e9a51 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -13,11 +13,11 @@ import ( "strings" "time" - "github.com/NYTimes/gziphandler" "github.com/felixge/fgprof" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/gorilla/mux" + "github.com/grafana/dskit/kv/memberlist" "github.com/prometheus/prometheus/storage" "github.com/weaveworks/common/middleware" "github.com/weaveworks/common/server" @@ -40,6 +40,7 @@ import ( "github.com/grafana/mimir/pkg/scheduler/schedulerpb" "github.com/grafana/mimir/pkg/storegateway" "github.com/grafana/mimir/pkg/storegateway/storegatewaypb" + "github.com/grafana/mimir/pkg/util/gziphandler" util_log "github.com/grafana/mimir/pkg/util/log" "github.com/grafana/mimir/pkg/util/push" ) @@ -182,8 +183,7 @@ func (a *API) newRoute(path string, handler http.Handler, isPrefix, auth, gzip b return route } -// RegisterAlertmanager registers endpoints associated with the alertmanager. It will only -// serve endpoints using the legacy http-prefix if it is not run as a single binary. +// RegisterAlertmanager registers endpoints that are associated with the alertmanager. func (a *API) RegisterAlertmanager(am *alertmanager.MultitenantAlertmanager, apiEnabled bool, buildInfoHandler http.Handler) { alertmanagerpb.RegisterAlertmanagerServer(a.server.GRPC, am) @@ -422,9 +422,9 @@ func (a *API) RegisterServiceMapHandler(handler http.Handler) { a.RegisterRoute("/services", handler, false, true, "GET") } -func (a *API) RegisterMemberlistKV(handler http.Handler) { +func (a *API) RegisterMemberlistKV(pathPrefix string, kvs *memberlist.KVInitService) { a.indexPage.AddLinks(memberlistWeight, "Memberlist", []IndexPageLink{ {Desc: "Status", Path: "/memberlist"}, }) - a.RegisterRoute("/memberlist", handler, false, true, "GET") + a.RegisterRoute("/memberlist", memberlistStatusHandler(pathPrefix, kvs), false, true, "GET") } diff --git a/pkg/api/api_test.go b/pkg/api/api_test.go index e4630451af5..e5fe4c4d063 100644 --- a/pkg/api/api_test.go +++ b/pkg/api/api_test.go @@ -6,11 +6,18 @@ package api import ( + "fmt" + "net" + "net/http" + "strconv" "testing" + "github.com/go-kit/log" "github.com/gorilla/mux" "github.com/stretchr/testify/require" "github.com/weaveworks/common/server" + + "github.com/grafana/mimir/pkg/util/gziphandler" ) type FakeLogger struct{} @@ -62,3 +69,144 @@ func TestNewApiWithInvalidSourceIPExtractor(t *testing.T) { require.Error(t, err) require.Nil(t, api) } + +func TestApiGzip(t *testing.T) { + cfg := Config{} + serverCfg := getServerConfig(t) + srv, err := server.New(serverCfg) + require.NoError(t, err) + go func() { _ = srv.Run() }() + t.Cleanup(srv.Stop) + + api, err := New(cfg, serverCfg, srv, log.NewNopLogger()) + require.NoError(t, err) + + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + size, err := strconv.Atoi(r.URL.Query().Get("respBodySize")) + if err != nil { + http.Error(w, fmt.Sprintf("respBodySize invalid: %s", err), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write(make([]byte, size)) + }) + + api.RegisterRoute("/gzip_enabled", handler, false, true, http.MethodGet) + api.RegisterRoute("/gzip_disabled", handler, false, false, http.MethodGet) + + for _, tc := range []struct { + name string + endpoint string + respBodySize int + acceptEncodingHeader string + expectedGzip bool + }{ + { + name: "happy case gzip", + endpoint: "gzip_enabled", + respBodySize: gziphandler.DefaultMinSize + 1, + acceptEncodingHeader: "gzip", + expectedGzip: true, + }, + { + name: "gzip with priority header", + endpoint: "gzip_enabled", + respBodySize: gziphandler.DefaultMinSize + 1, + acceptEncodingHeader: "gzip;q=1", + expectedGzip: true, + }, + { + name: "gzip because any is accepted", + endpoint: "gzip_enabled", + respBodySize: gziphandler.DefaultMinSize + 1, + acceptEncodingHeader: "*", + expectedGzip: true, + }, + { + name: "no gzip because no header", + endpoint: "gzip_enabled", + respBodySize: gziphandler.DefaultMinSize + 1, + acceptEncodingHeader: "", + expectedGzip: false, + }, + { + name: "no gzip because not accepted", + endpoint: "gzip_enabled", + respBodySize: gziphandler.DefaultMinSize + 1, + acceptEncodingHeader: "identity", + expectedGzip: false, + }, + { + name: "no gzip because small payload", + endpoint: "gzip_enabled", + respBodySize: 1, + acceptEncodingHeader: "gzip", + expectedGzip: false, + }, + { + name: "forced gzip with small payload", + endpoint: "gzip_enabled", + respBodySize: 1, + acceptEncodingHeader: "gzip;q=1, *;q=0", + expectedGzip: true, + }, + { + name: "gzip disabled endpoint", + endpoint: "gzip_disabled", + respBodySize: gziphandler.DefaultMinSize + 1, + acceptEncodingHeader: "gzip", + expectedGzip: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + u := fmt.Sprintf("http://%s:%d/%s?respBodySize=%d", serverCfg.HTTPListenAddress, serverCfg.HTTPListenPort, tc.endpoint, tc.respBodySize) + req, err := http.NewRequest(http.MethodGet, u, nil) + require.NoError(t, err) + if tc.acceptEncodingHeader != "" { + req.Header.Set("Accept-Encoding", tc.acceptEncodingHeader) + } + + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + + require.Equal(t, http.StatusOK, res.StatusCode) + if tc.expectedGzip { + require.Equal(t, "gzip", res.Header.Get("Content-Encoding"), "Invalid Content-Encoding header value") + } else { + require.Empty(t, res.Header.Get("Content-Encoding"), "Invalid Content-Encoding header value") + } + }) + } + + t.Run("compressed with gzip", func(t *testing.T) { + }) +} + +// Generates server config, with gRPC listening on random port. +func getServerConfig(t *testing.T) server.Config { + grpcHost, grpcPortNum := getHostnameAndRandomPort(t) + httpHost, httpPortNum := getHostnameAndRandomPort(t) + + return server.Config{ + HTTPListenAddress: httpHost, + HTTPListenPort: httpPortNum, + + GRPCListenAddress: grpcHost, + GRPCListenPort: grpcPortNum, + + GPRCServerMaxRecvMsgSize: 1024, + } +} + +func getHostnameAndRandomPort(t *testing.T) (string, int) { + listen, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + host, port, err := net.SplitHostPort(listen.Addr().String()) + require.NoError(t, err) + require.NoError(t, listen.Close()) + + portNum, err := strconv.Atoi(port) + require.NoError(t, err) + return host, portNum +} diff --git a/pkg/api/handlers.go b/pkg/api/handlers.go index 844c09c7c6b..1527bd42539 100644 --- a/pkg/api/handlers.go +++ b/pkg/api/handlers.go @@ -12,10 +12,12 @@ import ( "net/http" "path" "sort" + "strings" "sync" "github.com/go-kit/log" "github.com/gorilla/mux" + "github.com/grafana/dskit/kv/memberlist" "github.com/grafana/regexp" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" @@ -229,7 +231,7 @@ func NewQuerierHandler( router := mux.NewRouter() // Use a separate metric for the querier in order to differentiate requests from the query-frontend when - // running Mimir as a single binary. + // running Mimir in monolithic mode. instrumentMiddleware := middleware.Instrument{ RouteMatcher: router, Duration: querierRequestDuration, @@ -248,8 +250,7 @@ func NewQuerierHandler( // TODO(gotjosh): This custom handler is temporary until we're able to vendor the changes in: // https://github.com/prometheus/prometheus/pull/7125/files router.Path(path.Join(prefix, "/api/v1/metadata")).Handler(querier.MetadataHandler(distributor)) - router.Path(path.Join(prefix, "/api/v1/read")).Handler(querier.RemoteReadHandler(queryable, logger)) - router.Path(path.Join(prefix, "/api/v1/read")).Methods("POST").Handler(promRouter) + router.Path(path.Join(prefix, "/api/v1/read")).Methods("POST").Handler(querier.RemoteReadHandler(queryable, logger)) router.Path(path.Join(prefix, "/api/v1/query")).Methods("GET", "POST").Handler(promRouter) router.Path(path.Join(prefix, "/api/v1/query_range")).Methods("GET", "POST").Handler(promRouter) router.Path(path.Join(prefix, "/api/v1/query_exemplars")).Methods("GET", "POST").Handler(promRouter) @@ -263,3 +264,16 @@ func NewQuerierHandler( // Track execution time. return stats.NewWallTimeMiddleware().Wrap(router) } + +//go:embed memberlist_status.gohtml +var memberlistStatusPageHTML string + +func memberlistStatusHandler(httpPathPrefix string, kvs *memberlist.KVInitService) http.Handler { + templ := template.New("memberlist_status") + templ.Funcs(map[string]interface{}{ + "AddPathPrefix": func(link string) string { return path.Join(httpPathPrefix, link) }, + "StringsJoin": strings.Join, + }) + template.Must(templ.Parse(memberlistStatusPageHTML)) + return memberlist.NewHTTPStatusHandler(kvs, templ) +} diff --git a/pkg/api/index.gohtml b/pkg/api/index.gohtml index 7303fd5b93f..d112b90d370 100644 --- a/pkg/api/index.gohtml +++ b/pkg/api/index.gohtml @@ -8,35 +8,34 @@ Grafana Mimir - - + + -
- ',trigger:"hover focus",title:"",delay:0,html:!1,container:!1,viewport:{selector:"body",padding:0},sanitize:!0,sanitizeFn:null,whiteList:t},m.prototype.init=function(t,e,i){if(this.enabled=!0,this.type=t,this.$element=g(e),this.options=this.getOptions(i),this.$viewport=this.options.viewport&&g(document).find(g.isFunction(this.options.viewport)?this.options.viewport.call(this,this.$element):this.options.viewport.selector||this.options.viewport),this.inState={click:!1,hover:!1,focus:!1},this.$element[0]instanceof document.constructor&&!this.options.selector)throw new Error("`selector` option must be specified when initializing "+this.type+" on the window.document object!");for(var o=this.options.trigger.split(" "),n=o.length;n--;){var s=o[n];if("click"==s)this.$element.on("click."+this.type,this.options.selector,g.proxy(this.toggle,this));else if("manual"!=s){var a="hover"==s?"mouseenter":"focusin",r="hover"==s?"mouseleave":"focusout";this.$element.on(a+"."+this.type,this.options.selector,g.proxy(this.enter,this)),this.$element.on(r+"."+this.type,this.options.selector,g.proxy(this.leave,this))}}this.options.selector?this._options=g.extend({},this.options,{trigger:"manual",selector:""}):this.fixTitle()},m.prototype.getDefaults=function(){return m.DEFAULTS},m.prototype.getOptions=function(t){var e=this.$element.data();for(var i in e)e.hasOwnProperty(i)&&-1!==g.inArray(i,o)&&delete e[i];return(t=g.extend({},this.getDefaults(),e,t)).delay&&"number"==typeof t.delay&&(t.delay={show:t.delay,hide:t.delay}),t.sanitize&&(t.template=n(t.template,t.whiteList,t.sanitizeFn)),t},m.prototype.getDelegateOptions=function(){var i={},o=this.getDefaults();return this._options&&g.each(this._options,function(t,e){o[t]!=e&&(i[t]=e)}),i},m.prototype.enter=function(t){var e=t instanceof this.constructor?t:g(t.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e)),t instanceof g.Event&&(e.inState["focusin"==t.type?"focus":"hover"]=!0),e.tip().hasClass("in")||"in"==e.hoverState)e.hoverState="in";else{if(clearTimeout(e.timeout),e.hoverState="in",!e.options.delay||!e.options.delay.show)return e.show();e.timeout=setTimeout(function(){"in"==e.hoverState&&e.show()},e.options.delay.show)}},m.prototype.isInStateTrue=function(){for(var t in this.inState)if(this.inState[t])return!0;return!1},m.prototype.leave=function(t){var e=t instanceof this.constructor?t:g(t.currentTarget).data("bs."+this.type);if(e||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e)),t instanceof g.Event&&(e.inState["focusout"==t.type?"focus":"hover"]=!1),!e.isInStateTrue()){if(clearTimeout(e.timeout),e.hoverState="out",!e.options.delay||!e.options.delay.hide)return e.hide();e.timeout=setTimeout(function(){"out"==e.hoverState&&e.hide()},e.options.delay.hide)}},m.prototype.show=function(){var t=g.Event("show.bs."+this.type);if(this.hasContent()&&this.enabled){this.$element.trigger(t);var e=g.contains(this.$element[0].ownerDocument.documentElement,this.$element[0]);if(t.isDefaultPrevented()||!e)return;var i=this,o=this.tip(),n=this.getUID(this.type);this.setContent(),o.attr("id",n),this.$element.attr("aria-describedby",n),this.options.animation&&o.addClass("fade");var s="function"==typeof this.options.placement?this.options.placement.call(this,o[0],this.$element[0]):this.options.placement,a=/\s?auto?\s?/i,r=a.test(s);r&&(s=s.replace(a,"")||"top"),o.detach().css({top:0,left:0,display:"block"}).addClass(s).data("bs."+this.type,this),this.options.container?o.appendTo(g(document).find(this.options.container)):o.insertAfter(this.$element),this.$element.trigger("inserted.bs."+this.type);var l=this.getPosition(),h=o[0].offsetWidth,d=o[0].offsetHeight;if(r){var p=s,c=this.getPosition(this.$viewport);s="bottom"==s&&l.bottom+d>c.bottom?"top":"top"==s&&l.top-dc.width?"left":"left"==s&&l.left-ha.top+a.height&&(n.top=a.top+a.height-l)}else{var h=e.left-s,d=e.left+s+i;ha.right&&(n.left=a.left+a.width-d)}return n},m.prototype.getTitle=function(){var t=this.$element,e=this.options;return t.attr("data-original-title")||("function"==typeof e.title?e.title.call(t[0]):e.title)},m.prototype.getUID=function(t){for(;t+=~~(1e6*Math.random()),document.getElementById(t););return t},m.prototype.tip=function(){if(!this.$tip&&(this.$tip=g(this.options.template),1!=this.$tip.length))throw new Error(this.type+" `template` option must consist of exactly 1 top-level element!");return this.$tip},m.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".tooltip-arrow")},m.prototype.enable=function(){this.enabled=!0},m.prototype.disable=function(){this.enabled=!1},m.prototype.toggleEnabled=function(){this.enabled=!this.enabled},m.prototype.toggle=function(t){var e=this;t&&((e=g(t.currentTarget).data("bs."+this.type))||(e=new this.constructor(t.currentTarget,this.getDelegateOptions()),g(t.currentTarget).data("bs."+this.type,e))),t?(e.inState.click=!e.inState.click,e.isInStateTrue()?e.enter(e):e.leave(e)):e.tip().hasClass("in")?e.leave(e):e.enter(e)},m.prototype.destroy=function(){var t=this;clearTimeout(this.timeout),this.hide(function(){t.$element.off("."+t.type).removeData("bs."+t.type),t.$tip&&t.$tip.detach(),t.$tip=null,t.$arrow=null,t.$viewport=null,t.$element=null})},m.prototype.sanitizeHtml=function(t){return n(t,this.options.whiteList,this.options.sanitizeFn)};var e=g.fn.tooltip;g.fn.tooltip=function i(o){return this.each(function(){var t=g(this),e=t.data("bs.tooltip"),i="object"==typeof o&&o;!e&&/destroy|hide/.test(o)||(e||t.data("bs.tooltip",e=new m(this,i)),"string"==typeof o&&e[o]())})},g.fn.tooltip.Constructor=m,g.fn.tooltip.noConflict=function(){return g.fn.tooltip=e,this}}(jQuery),function(n){"use strict";var s=function(t,e){this.init("popover",t,e)};if(!n.fn.tooltip)throw new Error("Popover requires tooltip.js");s.VERSION="3.4.1",s.DEFAULTS=n.extend({},n.fn.tooltip.Constructor.DEFAULTS,{placement:"right",trigger:"click",content:"",template:''}),((s.prototype=n.extend({},n.fn.tooltip.Constructor.prototype)).constructor=s).prototype.getDefaults=function(){return s.DEFAULTS},s.prototype.setContent=function(){var t=this.tip(),e=this.getTitle(),i=this.getContent();if(this.options.html){var o=typeof i;this.options.sanitize&&(e=this.sanitizeHtml(e),"string"===o&&(i=this.sanitizeHtml(i))),t.find(".popover-title").html(e),t.find(".popover-content").children().detach().end()["string"===o?"html":"append"](i)}else t.find(".popover-title").text(e),t.find(".popover-content").children().detach().end().text(i);t.removeClass("fade top bottom left right in"),t.find(".popover-title").html()||t.find(".popover-title").hide()},s.prototype.hasContent=function(){return this.getTitle()||this.getContent()},s.prototype.getContent=function(){var t=this.$element,e=this.options;return t.attr("data-content")||("function"==typeof e.content?e.content.call(t[0]):e.content)},s.prototype.arrow=function(){return this.$arrow=this.$arrow||this.tip().find(".arrow")};var t=n.fn.popover;n.fn.popover=function e(o){return this.each(function(){var t=n(this),e=t.data("bs.popover"),i="object"==typeof o&&o;!e&&/destroy|hide/.test(o)||(e||t.data("bs.popover",e=new s(this,i)),"string"==typeof o&&e[o]())})},n.fn.popover.Constructor=s,n.fn.popover.noConflict=function(){return n.fn.popover=t,this}}(jQuery),function(s){"use strict";function n(t,e){this.$body=s(document.body),this.$scrollElement=s(t).is(document.body)?s(window):s(t),this.options=s.extend({},n.DEFAULTS,e),this.selector=(this.options.target||"")+" .nav li > a",this.offsets=[],this.targets=[],this.activeTarget=null,this.scrollHeight=0,this.$scrollElement.on("scroll.bs.scrollspy",s.proxy(this.process,this)),this.refresh(),this.process()}function e(o){return this.each(function(){var t=s(this),e=t.data("bs.scrollspy"),i="object"==typeof o&&o;e||t.data("bs.scrollspy",e=new n(this,i)),"string"==typeof o&&e[o]()})}n.VERSION="3.4.1",n.DEFAULTS={offset:10},n.prototype.getScrollHeight=function(){return this.$scrollElement[0].scrollHeight||Math.max(this.$body[0].scrollHeight,document.documentElement.scrollHeight)},n.prototype.refresh=function(){var t=this,o="offset",n=0;this.offsets=[],this.targets=[],this.scrollHeight=this.getScrollHeight(),s.isWindow(this.$scrollElement[0])||(o="position",n=this.$scrollElement.scrollTop()),this.$body.find(this.selector).map(function(){var t=s(this),e=t.data("target")||t.attr("href"),i=/^#./.test(e)&&s(e);return i&&i.length&&i.is(":visible")&&[[i[o]().top+n,e]]||null}).sort(function(t,e){return t[0]-e[0]}).each(function(){t.offsets.push(this[0]),t.targets.push(this[1])})},n.prototype.process=function(){var t,e=this.$scrollElement.scrollTop()+this.options.offset,i=this.getScrollHeight(),o=this.options.offset+i-this.$scrollElement.height(),n=this.offsets,s=this.targets,a=this.activeTarget;if(this.scrollHeight!=i&&this.refresh(),o<=e)return a!=(t=s[s.length-1])&&this.activate(t);if(a&&e=n[t]&&(n[t+1]===undefined||e .active"),n=i&&r.support.transition&&(o.length&&o.hasClass("fade")||!!e.find("> .fade").length);function s(){o.removeClass("active").find("> .dropdown-menu > .active").removeClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!1),t.addClass("active").find('[data-toggle="tab"]').attr("aria-expanded",!0),n?(t[0].offsetWidth,t.addClass("in")):t.removeClass("fade"),t.parent(".dropdown-menu").length&&t.closest("li.dropdown").addClass("active").end().find('[data-toggle="tab"]').attr("aria-expanded",!0),i&&i()}o.length&&n?o.one("bsTransitionEnd",s).emulateTransitionEnd(a.TRANSITION_DURATION):s(),o.removeClass("in")};var t=r.fn.tab;r.fn.tab=e,r.fn.tab.Constructor=a,r.fn.tab.noConflict=function(){return r.fn.tab=t,this};var i=function(t){t.preventDefault(),e.call(r(this),"show")};r(document).on("click.bs.tab.data-api",'[data-toggle="tab"]',i).on("click.bs.tab.data-api",'[data-toggle="pill"]',i)}(jQuery),function(l){"use strict";var h=function(t,e){this.options=l.extend({},h.DEFAULTS,e);var i=this.options.target===h.DEFAULTS.target?l(this.options.target):l(document).find(this.options.target);this.$target=i.on("scroll.bs.affix.data-api",l.proxy(this.checkPosition,this)).on("click.bs.affix.data-api",l.proxy(this.checkPositionWithEventLoop,this)),this.$element=l(t),this.affixed=null,this.unpin=null,this.pinnedOffset=null,this.checkPosition()};function i(o){return this.each(function(){var t=l(this),e=t.data("bs.affix"),i="object"==typeof o&&o;e||t.data("bs.affix",e=new h(this,i)),"string"==typeof o&&e[o]()})}h.VERSION="3.4.1",h.RESET="affix affix-top affix-bottom",h.DEFAULTS={offset:0,target:window},h.prototype.getState=function(t,e,i,o){var n=this.$target.scrollTop(),s=this.$element.offset(),a=this.$target.height();if(null!=i&&"top"==this.affixed)return n{let e=t.getAttribute("data-bs-target");if(!e||"#"===e){let i=t.getAttribute("href");if(!i||!i.includes("#")&&!i.startsWith("."))return null;i.includes("#")&&!i.startsWith("#")&&(i=`#${i.split("#")[1]}`),e=i&&"#"!==i?i.trim():null}return e},i=t=>{const i=e(t);return i&&document.querySelector(i)?i:null},n=t=>{const i=e(t);return i?document.querySelector(i):null},s=e=>{e.dispatchEvent(new Event(t))},o=t=>!(!t||"object"!=typeof t)&&(void 0!==t.jquery&&(t=t[0]),void 0!==t.nodeType),r=t=>o(t)?t.jquery?t[0]:t:"string"==typeof t&&t.length>0?document.querySelector(t):null,a=(t,e,i)=>{Object.keys(i).forEach((n=>{const s=i[n],r=e[n],a=r&&o(r)?"element":null==(l=r)?`${l}`:{}.toString.call(l).match(/\s([a-z]+)/i)[1].toLowerCase();var l;if(!new RegExp(s).test(a))throw new TypeError(`${t.toUpperCase()}: Option "${n}" provided type "${a}" but expected type "${s}".`)}))},l=t=>!(!o(t)||0===t.getClientRects().length)&&"visible"===getComputedStyle(t).getPropertyValue("visibility"),c=t=>!t||t.nodeType!==Node.ELEMENT_NODE||!!t.classList.contains("disabled")||(void 0!==t.disabled?t.disabled:t.hasAttribute("disabled")&&"false"!==t.getAttribute("disabled")),h=t=>{if(!document.documentElement.attachShadow)return null;if("function"==typeof t.getRootNode){const e=t.getRootNode();return e instanceof ShadowRoot?e:null}return t instanceof ShadowRoot?t:t.parentNode?h(t.parentNode):null},d=()=>{},u=t=>{t.offsetHeight},f=()=>{const{jQuery:t}=window;return t&&!document.body.hasAttribute("data-bs-no-jquery")?t:null},p=[],m=()=>"rtl"===document.documentElement.dir,g=t=>{var e;e=()=>{const e=f();if(e){const i=t.NAME,n=e.fn[i];e.fn[i]=t.jQueryInterface,e.fn[i].Constructor=t,e.fn[i].noConflict=()=>(e.fn[i]=n,t.jQueryInterface)}},"loading"===document.readyState?(p.length||document.addEventListener("DOMContentLoaded",(()=>{p.forEach((t=>t()))})),p.push(e)):e()},_=t=>{"function"==typeof t&&t()},b=(e,i,n=!0)=>{if(!n)return void _(e);const o=(t=>{if(!t)return 0;let{transitionDuration:e,transitionDelay:i}=window.getComputedStyle(t);const n=Number.parseFloat(e),s=Number.parseFloat(i);return n||s?(e=e.split(",")[0],i=i.split(",")[0],1e3*(Number.parseFloat(e)+Number.parseFloat(i))):0})(i)+5;let r=!1;const a=({target:n})=>{n===i&&(r=!0,i.removeEventListener(t,a),_(e))};i.addEventListener(t,a),setTimeout((()=>{r||s(i)}),o)},v=(t,e,i,n)=>{let s=t.indexOf(e);if(-1===s)return t[!i&&n?t.length-1:0];const o=t.length;return s+=i?1:-1,n&&(s=(s+o)%o),t[Math.max(0,Math.min(s,o-1))]},y=/[^.]*(?=\..*)\.|.*/,w=/\..*/,E=/::\d+$/,A={};let T=1;const O={mouseenter:"mouseover",mouseleave:"mouseout"},C=/^(mouseenter|mouseleave)/i,k=new Set(["click","dblclick","mouseup","mousedown","contextmenu","mousewheel","DOMMouseScroll","mouseover","mouseout","mousemove","selectstart","selectend","keydown","keypress","keyup","orientationchange","touchstart","touchmove","touchend","touchcancel","pointerdown","pointermove","pointerup","pointerleave","pointercancel","gesturestart","gesturechange","gestureend","focus","blur","change","reset","select","submit","focusin","focusout","load","unload","beforeunload","resize","move","DOMContentLoaded","readystatechange","error","abort","scroll"]);function L(t,e){return e&&`${e}::${T++}`||t.uidEvent||T++}function x(t){const e=L(t);return t.uidEvent=e,A[e]=A[e]||{},A[e]}function D(t,e,i=null){const n=Object.keys(t);for(let s=0,o=n.length;sfunction(e){if(!e.relatedTarget||e.relatedTarget!==e.delegateTarget&&!e.delegateTarget.contains(e.relatedTarget))return t.call(this,e)};n?n=t(n):i=t(i)}const[o,r,a]=S(e,i,n),l=x(t),c=l[a]||(l[a]={}),h=D(c,r,o?i:null);if(h)return void(h.oneOff=h.oneOff&&s);const d=L(r,e.replace(y,"")),u=o?function(t,e,i){return function n(s){const o=t.querySelectorAll(e);for(let{target:r}=s;r&&r!==this;r=r.parentNode)for(let a=o.length;a--;)if(o[a]===r)return s.delegateTarget=r,n.oneOff&&j.off(t,s.type,e,i),i.apply(r,[s]);return null}}(t,i,n):function(t,e){return function i(n){return n.delegateTarget=t,i.oneOff&&j.off(t,n.type,e),e.apply(t,[n])}}(t,i);u.delegationSelector=o?i:null,u.originalHandler=r,u.oneOff=s,u.uidEvent=d,c[d]=u,t.addEventListener(a,u,o)}function I(t,e,i,n,s){const o=D(e[i],n,s);o&&(t.removeEventListener(i,o,Boolean(s)),delete e[i][o.uidEvent])}function P(t){return t=t.replace(w,""),O[t]||t}const j={on(t,e,i,n){N(t,e,i,n,!1)},one(t,e,i,n){N(t,e,i,n,!0)},off(t,e,i,n){if("string"!=typeof e||!t)return;const[s,o,r]=S(e,i,n),a=r!==e,l=x(t),c=e.startsWith(".");if(void 0!==o){if(!l||!l[r])return;return void I(t,l,r,o,s?i:null)}c&&Object.keys(l).forEach((i=>{!function(t,e,i,n){const s=e[i]||{};Object.keys(s).forEach((o=>{if(o.includes(n)){const n=s[o];I(t,e,i,n.originalHandler,n.delegationSelector)}}))}(t,l,i,e.slice(1))}));const h=l[r]||{};Object.keys(h).forEach((i=>{const n=i.replace(E,"");if(!a||e.includes(n)){const e=h[i];I(t,l,r,e.originalHandler,e.delegationSelector)}}))},trigger(t,e,i){if("string"!=typeof e||!t)return null;const n=f(),s=P(e),o=e!==s,r=k.has(s);let a,l=!0,c=!0,h=!1,d=null;return o&&n&&(a=n.Event(e,i),n(t).trigger(a),l=!a.isPropagationStopped(),c=!a.isImmediatePropagationStopped(),h=a.isDefaultPrevented()),r?(d=document.createEvent("HTMLEvents"),d.initEvent(s,l,!0)):d=new CustomEvent(e,{bubbles:l,cancelable:!0}),void 0!==i&&Object.keys(i).forEach((t=>{Object.defineProperty(d,t,{get:()=>i[t]})})),h&&d.preventDefault(),c&&t.dispatchEvent(d),d.defaultPrevented&&void 0!==a&&a.preventDefault(),d}},M=new Map,H={set(t,e,i){M.has(t)||M.set(t,new Map);const n=M.get(t);n.has(e)||0===n.size?n.set(e,i):console.error(`Bootstrap doesn't allow more than one instance per element. Bound instance: ${Array.from(n.keys())[0]}.`)},get:(t,e)=>M.has(t)&&M.get(t).get(e)||null,remove(t,e){if(!M.has(t))return;const i=M.get(t);i.delete(e),0===i.size&&M.delete(t)}};class B{constructor(t){(t=r(t))&&(this._element=t,H.set(this._element,this.constructor.DATA_KEY,this))}dispose(){H.remove(this._element,this.constructor.DATA_KEY),j.off(this._element,this.constructor.EVENT_KEY),Object.getOwnPropertyNames(this).forEach((t=>{this[t]=null}))}_queueCallback(t,e,i=!0){b(t,e,i)}static getInstance(t){return H.get(r(t),this.DATA_KEY)}static getOrCreateInstance(t,e={}){return this.getInstance(t)||new this(t,"object"==typeof e?e:null)}static get VERSION(){return"5.1.3"}static get NAME(){throw new Error('You have to implement the static method "NAME", for each component!')}static get DATA_KEY(){return`bs.${this.NAME}`}static get EVENT_KEY(){return`.${this.DATA_KEY}`}}const R=(t,e="hide")=>{const i=`click.dismiss${t.EVENT_KEY}`,s=t.NAME;j.on(document,i,`[data-bs-dismiss="${s}"]`,(function(i){if(["A","AREA"].includes(this.tagName)&&i.preventDefault(),c(this))return;const o=n(this)||this.closest(`.${s}`);t.getOrCreateInstance(o)[e]()}))};class W extends B{static get NAME(){return"alert"}close(){if(j.trigger(this._element,"close.bs.alert").defaultPrevented)return;this._element.classList.remove("show");const t=this._element.classList.contains("fade");this._queueCallback((()=>this._destroyElement()),this._element,t)}_destroyElement(){this._element.remove(),j.trigger(this._element,"closed.bs.alert"),this.dispose()}static jQueryInterface(t){return this.each((function(){const e=W.getOrCreateInstance(this);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}R(W,"close"),g(W);const $='[data-bs-toggle="button"]';class z extends B{static get NAME(){return"button"}toggle(){this._element.setAttribute("aria-pressed",this._element.classList.toggle("active"))}static jQueryInterface(t){return this.each((function(){const e=z.getOrCreateInstance(this);"toggle"===t&&e[t]()}))}}function q(t){return"true"===t||"false"!==t&&(t===Number(t).toString()?Number(t):""===t||"null"===t?null:t)}function F(t){return t.replace(/[A-Z]/g,(t=>`-${t.toLowerCase()}`))}j.on(document,"click.bs.button.data-api",$,(t=>{t.preventDefault();const e=t.target.closest($);z.getOrCreateInstance(e).toggle()})),g(z);const U={setDataAttribute(t,e,i){t.setAttribute(`data-bs-${F(e)}`,i)},removeDataAttribute(t,e){t.removeAttribute(`data-bs-${F(e)}`)},getDataAttributes(t){if(!t)return{};const e={};return Object.keys(t.dataset).filter((t=>t.startsWith("bs"))).forEach((i=>{let n=i.replace(/^bs/,"");n=n.charAt(0).toLowerCase()+n.slice(1,n.length),e[n]=q(t.dataset[i])})),e},getDataAttribute:(t,e)=>q(t.getAttribute(`data-bs-${F(e)}`)),offset(t){const e=t.getBoundingClientRect();return{top:e.top+window.pageYOffset,left:e.left+window.pageXOffset}},position:t=>({top:t.offsetTop,left:t.offsetLeft})},V={find:(t,e=document.documentElement)=>[].concat(...Element.prototype.querySelectorAll.call(e,t)),findOne:(t,e=document.documentElement)=>Element.prototype.querySelector.call(e,t),children:(t,e)=>[].concat(...t.children).filter((t=>t.matches(e))),parents(t,e){const i=[];let n=t.parentNode;for(;n&&n.nodeType===Node.ELEMENT_NODE&&3!==n.nodeType;)n.matches(e)&&i.push(n),n=n.parentNode;return i},prev(t,e){let i=t.previousElementSibling;for(;i;){if(i.matches(e))return[i];i=i.previousElementSibling}return[]},next(t,e){let i=t.nextElementSibling;for(;i;){if(i.matches(e))return[i];i=i.nextElementSibling}return[]},focusableChildren(t){const e=["a","button","input","textarea","select","details","[tabindex]",'[contenteditable="true"]'].map((t=>`${t}:not([tabindex^="-"])`)).join(", ");return this.find(e,t).filter((t=>!c(t)&&l(t)))}},K="carousel",X={interval:5e3,keyboard:!0,slide:!1,pause:"hover",wrap:!0,touch:!0},Y={interval:"(number|boolean)",keyboard:"boolean",slide:"(boolean|string)",pause:"(string|boolean)",wrap:"boolean",touch:"boolean"},Q="next",G="prev",Z="left",J="right",tt={ArrowLeft:J,ArrowRight:Z},et="slid.bs.carousel",it="active",nt=".active.carousel-item";class st extends B{constructor(t,e){super(t),this._items=null,this._interval=null,this._activeElement=null,this._isPaused=!1,this._isSliding=!1,this.touchTimeout=null,this.touchStartX=0,this.touchDeltaX=0,this._config=this._getConfig(e),this._indicatorsElement=V.findOne(".carousel-indicators",this._element),this._touchSupported="ontouchstart"in document.documentElement||navigator.maxTouchPoints>0,this._pointerEvent=Boolean(window.PointerEvent),this._addEventListeners()}static get Default(){return X}static get NAME(){return K}next(){this._slide(Q)}nextWhenVisible(){!document.hidden&&l(this._element)&&this.next()}prev(){this._slide(G)}pause(t){t||(this._isPaused=!0),V.findOne(".carousel-item-next, .carousel-item-prev",this._element)&&(s(this._element),this.cycle(!0)),clearInterval(this._interval),this._interval=null}cycle(t){t||(this._isPaused=!1),this._interval&&(clearInterval(this._interval),this._interval=null),this._config&&this._config.interval&&!this._isPaused&&(this._updateInterval(),this._interval=setInterval((document.visibilityState?this.nextWhenVisible:this.next).bind(this),this._config.interval))}to(t){this._activeElement=V.findOne(nt,this._element);const e=this._getItemIndex(this._activeElement);if(t>this._items.length-1||t<0)return;if(this._isSliding)return void j.one(this._element,et,(()=>this.to(t)));if(e===t)return this.pause(),void this.cycle();const i=t>e?Q:G;this._slide(i,this._items[t])}_getConfig(t){return t={...X,...U.getDataAttributes(this._element),..."object"==typeof t?t:{}},a(K,t,Y),t}_handleSwipe(){const t=Math.abs(this.touchDeltaX);if(t<=40)return;const e=t/this.touchDeltaX;this.touchDeltaX=0,e&&this._slide(e>0?J:Z)}_addEventListeners(){this._config.keyboard&&j.on(this._element,"keydown.bs.carousel",(t=>this._keydown(t))),"hover"===this._config.pause&&(j.on(this._element,"mouseenter.bs.carousel",(t=>this.pause(t))),j.on(this._element,"mouseleave.bs.carousel",(t=>this.cycle(t)))),this._config.touch&&this._touchSupported&&this._addTouchEventListeners()}_addTouchEventListeners(){const t=t=>this._pointerEvent&&("pen"===t.pointerType||"touch"===t.pointerType),e=e=>{t(e)?this.touchStartX=e.clientX:this._pointerEvent||(this.touchStartX=e.touches[0].clientX)},i=t=>{this.touchDeltaX=t.touches&&t.touches.length>1?0:t.touches[0].clientX-this.touchStartX},n=e=>{t(e)&&(this.touchDeltaX=e.clientX-this.touchStartX),this._handleSwipe(),"hover"===this._config.pause&&(this.pause(),this.touchTimeout&&clearTimeout(this.touchTimeout),this.touchTimeout=setTimeout((t=>this.cycle(t)),500+this._config.interval))};V.find(".carousel-item img",this._element).forEach((t=>{j.on(t,"dragstart.bs.carousel",(t=>t.preventDefault()))})),this._pointerEvent?(j.on(this._element,"pointerdown.bs.carousel",(t=>e(t))),j.on(this._element,"pointerup.bs.carousel",(t=>n(t))),this._element.classList.add("pointer-event")):(j.on(this._element,"touchstart.bs.carousel",(t=>e(t))),j.on(this._element,"touchmove.bs.carousel",(t=>i(t))),j.on(this._element,"touchend.bs.carousel",(t=>n(t))))}_keydown(t){if(/input|textarea/i.test(t.target.tagName))return;const e=tt[t.key];e&&(t.preventDefault(),this._slide(e))}_getItemIndex(t){return this._items=t&&t.parentNode?V.find(".carousel-item",t.parentNode):[],this._items.indexOf(t)}_getItemByOrder(t,e){const i=t===Q;return v(this._items,e,i,this._config.wrap)}_triggerSlideEvent(t,e){const i=this._getItemIndex(t),n=this._getItemIndex(V.findOne(nt,this._element));return j.trigger(this._element,"slide.bs.carousel",{relatedTarget:t,direction:e,from:n,to:i})}_setActiveIndicatorElement(t){if(this._indicatorsElement){const e=V.findOne(".active",this._indicatorsElement);e.classList.remove(it),e.removeAttribute("aria-current");const i=V.find("[data-bs-target]",this._indicatorsElement);for(let e=0;e{j.trigger(this._element,et,{relatedTarget:o,direction:d,from:s,to:r})};if(this._element.classList.contains("slide")){o.classList.add(h),u(o),n.classList.add(c),o.classList.add(c);const t=()=>{o.classList.remove(c,h),o.classList.add(it),n.classList.remove(it,h,c),this._isSliding=!1,setTimeout(f,0)};this._queueCallback(t,n,!0)}else n.classList.remove(it),o.classList.add(it),this._isSliding=!1,f();a&&this.cycle()}_directionToOrder(t){return[J,Z].includes(t)?m()?t===Z?G:Q:t===Z?Q:G:t}_orderToDirection(t){return[Q,G].includes(t)?m()?t===G?Z:J:t===G?J:Z:t}static carouselInterface(t,e){const i=st.getOrCreateInstance(t,e);let{_config:n}=i;"object"==typeof e&&(n={...n,...e});const s="string"==typeof e?e:n.slide;if("number"==typeof e)i.to(e);else if("string"==typeof s){if(void 0===i[s])throw new TypeError(`No method named "${s}"`);i[s]()}else n.interval&&n.ride&&(i.pause(),i.cycle())}static jQueryInterface(t){return this.each((function(){st.carouselInterface(this,t)}))}static dataApiClickHandler(t){const e=n(this);if(!e||!e.classList.contains("carousel"))return;const i={...U.getDataAttributes(e),...U.getDataAttributes(this)},s=this.getAttribute("data-bs-slide-to");s&&(i.interval=!1),st.carouselInterface(e,i),s&&st.getInstance(e).to(s),t.preventDefault()}}j.on(document,"click.bs.carousel.data-api","[data-bs-slide], [data-bs-slide-to]",st.dataApiClickHandler),j.on(window,"load.bs.carousel.data-api",(()=>{const t=V.find('[data-bs-ride="carousel"]');for(let e=0,i=t.length;et===this._element));null!==s&&o.length&&(this._selector=s,this._triggerArray.push(e))}this._initializeChildren(),this._config.parent||this._addAriaAndCollapsedClass(this._triggerArray,this._isShown()),this._config.toggle&&this.toggle()}static get Default(){return rt}static get NAME(){return ot}toggle(){this._isShown()?this.hide():this.show()}show(){if(this._isTransitioning||this._isShown())return;let t,e=[];if(this._config.parent){const t=V.find(ut,this._config.parent);e=V.find(".collapse.show, .collapse.collapsing",this._config.parent).filter((e=>!t.includes(e)))}const i=V.findOne(this._selector);if(e.length){const n=e.find((t=>i!==t));if(t=n?pt.getInstance(n):null,t&&t._isTransitioning)return}if(j.trigger(this._element,"show.bs.collapse").defaultPrevented)return;e.forEach((e=>{i!==e&&pt.getOrCreateInstance(e,{toggle:!1}).hide(),t||H.set(e,"bs.collapse",null)}));const n=this._getDimension();this._element.classList.remove(ct),this._element.classList.add(ht),this._element.style[n]=0,this._addAriaAndCollapsedClass(this._triggerArray,!0),this._isTransitioning=!0;const s=`scroll${n[0].toUpperCase()+n.slice(1)}`;this._queueCallback((()=>{this._isTransitioning=!1,this._element.classList.remove(ht),this._element.classList.add(ct,lt),this._element.style[n]="",j.trigger(this._element,"shown.bs.collapse")}),this._element,!0),this._element.style[n]=`${this._element[s]}px`}hide(){if(this._isTransitioning||!this._isShown())return;if(j.trigger(this._element,"hide.bs.collapse").defaultPrevented)return;const t=this._getDimension();this._element.style[t]=`${this._element.getBoundingClientRect()[t]}px`,u(this._element),this._element.classList.add(ht),this._element.classList.remove(ct,lt);const e=this._triggerArray.length;for(let t=0;t{this._isTransitioning=!1,this._element.classList.remove(ht),this._element.classList.add(ct),j.trigger(this._element,"hidden.bs.collapse")}),this._element,!0)}_isShown(t=this._element){return t.classList.contains(lt)}_getConfig(t){return(t={...rt,...U.getDataAttributes(this._element),...t}).toggle=Boolean(t.toggle),t.parent=r(t.parent),a(ot,t,at),t}_getDimension(){return this._element.classList.contains("collapse-horizontal")?"width":"height"}_initializeChildren(){if(!this._config.parent)return;const t=V.find(ut,this._config.parent);V.find(ft,this._config.parent).filter((e=>!t.includes(e))).forEach((t=>{const e=n(t);e&&this._addAriaAndCollapsedClass([t],this._isShown(e))}))}_addAriaAndCollapsedClass(t,e){t.length&&t.forEach((t=>{e?t.classList.remove(dt):t.classList.add(dt),t.setAttribute("aria-expanded",e)}))}static jQueryInterface(t){return this.each((function(){const e={};"string"==typeof t&&/show|hide/.test(t)&&(e.toggle=!1);const i=pt.getOrCreateInstance(this,e);if("string"==typeof t){if(void 0===i[t])throw new TypeError(`No method named "${t}"`);i[t]()}}))}}j.on(document,"click.bs.collapse.data-api",ft,(function(t){("A"===t.target.tagName||t.delegateTarget&&"A"===t.delegateTarget.tagName)&&t.preventDefault();const e=i(this);V.find(e).forEach((t=>{pt.getOrCreateInstance(t,{toggle:!1}).toggle()}))})),g(pt);var mt="top",gt="bottom",_t="right",bt="left",vt="auto",yt=[mt,gt,_t,bt],wt="start",Et="end",At="clippingParents",Tt="viewport",Ot="popper",Ct="reference",kt=yt.reduce((function(t,e){return t.concat([e+"-"+wt,e+"-"+Et])}),[]),Lt=[].concat(yt,[vt]).reduce((function(t,e){return t.concat([e,e+"-"+wt,e+"-"+Et])}),[]),xt="beforeRead",Dt="read",St="afterRead",Nt="beforeMain",It="main",Pt="afterMain",jt="beforeWrite",Mt="write",Ht="afterWrite",Bt=[xt,Dt,St,Nt,It,Pt,jt,Mt,Ht];function Rt(t){return t?(t.nodeName||"").toLowerCase():null}function Wt(t){if(null==t)return window;if("[object Window]"!==t.toString()){var e=t.ownerDocument;return e&&e.defaultView||window}return t}function $t(t){return t instanceof Wt(t).Element||t instanceof Element}function zt(t){return t instanceof Wt(t).HTMLElement||t instanceof HTMLElement}function qt(t){return"undefined"!=typeof ShadowRoot&&(t instanceof Wt(t).ShadowRoot||t instanceof ShadowRoot)}const Ft={name:"applyStyles",enabled:!0,phase:"write",fn:function(t){var e=t.state;Object.keys(e.elements).forEach((function(t){var i=e.styles[t]||{},n=e.attributes[t]||{},s=e.elements[t];zt(s)&&Rt(s)&&(Object.assign(s.style,i),Object.keys(n).forEach((function(t){var e=n[t];!1===e?s.removeAttribute(t):s.setAttribute(t,!0===e?"":e)})))}))},effect:function(t){var e=t.state,i={popper:{position:e.options.strategy,left:"0",top:"0",margin:"0"},arrow:{position:"absolute"},reference:{}};return Object.assign(e.elements.popper.style,i.popper),e.styles=i,e.elements.arrow&&Object.assign(e.elements.arrow.style,i.arrow),function(){Object.keys(e.elements).forEach((function(t){var n=e.elements[t],s=e.attributes[t]||{},o=Object.keys(e.styles.hasOwnProperty(t)?e.styles[t]:i[t]).reduce((function(t,e){return t[e]="",t}),{});zt(n)&&Rt(n)&&(Object.assign(n.style,o),Object.keys(s).forEach((function(t){n.removeAttribute(t)})))}))}},requires:["computeStyles"]};function Ut(t){return t.split("-")[0]}function Vt(t,e){var i=t.getBoundingClientRect();return{width:i.width/1,height:i.height/1,top:i.top/1,right:i.right/1,bottom:i.bottom/1,left:i.left/1,x:i.left/1,y:i.top/1}}function Kt(t){var e=Vt(t),i=t.offsetWidth,n=t.offsetHeight;return Math.abs(e.width-i)<=1&&(i=e.width),Math.abs(e.height-n)<=1&&(n=e.height),{x:t.offsetLeft,y:t.offsetTop,width:i,height:n}}function Xt(t,e){var i=e.getRootNode&&e.getRootNode();if(t.contains(e))return!0;if(i&&qt(i)){var n=e;do{if(n&&t.isSameNode(n))return!0;n=n.parentNode||n.host}while(n)}return!1}function Yt(t){return Wt(t).getComputedStyle(t)}function Qt(t){return["table","td","th"].indexOf(Rt(t))>=0}function Gt(t){return(($t(t)?t.ownerDocument:t.document)||window.document).documentElement}function Zt(t){return"html"===Rt(t)?t:t.assignedSlot||t.parentNode||(qt(t)?t.host:null)||Gt(t)}function Jt(t){return zt(t)&&"fixed"!==Yt(t).position?t.offsetParent:null}function te(t){for(var e=Wt(t),i=Jt(t);i&&Qt(i)&&"static"===Yt(i).position;)i=Jt(i);return i&&("html"===Rt(i)||"body"===Rt(i)&&"static"===Yt(i).position)?e:i||function(t){var e=-1!==navigator.userAgent.toLowerCase().indexOf("firefox");if(-1!==navigator.userAgent.indexOf("Trident")&&zt(t)&&"fixed"===Yt(t).position)return null;for(var i=Zt(t);zt(i)&&["html","body"].indexOf(Rt(i))<0;){var n=Yt(i);if("none"!==n.transform||"none"!==n.perspective||"paint"===n.contain||-1!==["transform","perspective"].indexOf(n.willChange)||e&&"filter"===n.willChange||e&&n.filter&&"none"!==n.filter)return i;i=i.parentNode}return null}(t)||e}function ee(t){return["top","bottom"].indexOf(t)>=0?"x":"y"}var ie=Math.max,ne=Math.min,se=Math.round;function oe(t,e,i){return ie(t,ne(e,i))}function re(t){return Object.assign({},{top:0,right:0,bottom:0,left:0},t)}function ae(t,e){return e.reduce((function(e,i){return e[i]=t,e}),{})}const le={name:"arrow",enabled:!0,phase:"main",fn:function(t){var e,i=t.state,n=t.name,s=t.options,o=i.elements.arrow,r=i.modifiersData.popperOffsets,a=Ut(i.placement),l=ee(a),c=[bt,_t].indexOf(a)>=0?"height":"width";if(o&&r){var h=function(t,e){return re("number"!=typeof(t="function"==typeof t?t(Object.assign({},e.rects,{placement:e.placement})):t)?t:ae(t,yt))}(s.padding,i),d=Kt(o),u="y"===l?mt:bt,f="y"===l?gt:_t,p=i.rects.reference[c]+i.rects.reference[l]-r[l]-i.rects.popper[c],m=r[l]-i.rects.reference[l],g=te(o),_=g?"y"===l?g.clientHeight||0:g.clientWidth||0:0,b=p/2-m/2,v=h[u],y=_-d[c]-h[f],w=_/2-d[c]/2+b,E=oe(v,w,y),A=l;i.modifiersData[n]=((e={})[A]=E,e.centerOffset=E-w,e)}},effect:function(t){var e=t.state,i=t.options.element,n=void 0===i?"[data-popper-arrow]":i;null!=n&&("string"!=typeof n||(n=e.elements.popper.querySelector(n)))&&Xt(e.elements.popper,n)&&(e.elements.arrow=n)},requires:["popperOffsets"],requiresIfExists:["preventOverflow"]};function ce(t){return t.split("-")[1]}var he={top:"auto",right:"auto",bottom:"auto",left:"auto"};function de(t){var e,i=t.popper,n=t.popperRect,s=t.placement,o=t.variation,r=t.offsets,a=t.position,l=t.gpuAcceleration,c=t.adaptive,h=t.roundOffsets,d=!0===h?function(t){var e=t.x,i=t.y,n=window.devicePixelRatio||1;return{x:se(se(e*n)/n)||0,y:se(se(i*n)/n)||0}}(r):"function"==typeof h?h(r):r,u=d.x,f=void 0===u?0:u,p=d.y,m=void 0===p?0:p,g=r.hasOwnProperty("x"),_=r.hasOwnProperty("y"),b=bt,v=mt,y=window;if(c){var w=te(i),E="clientHeight",A="clientWidth";w===Wt(i)&&"static"!==Yt(w=Gt(i)).position&&"absolute"===a&&(E="scrollHeight",A="scrollWidth"),w=w,s!==mt&&(s!==bt&&s!==_t||o!==Et)||(v=gt,m-=w[E]-n.height,m*=l?1:-1),s!==bt&&(s!==mt&&s!==gt||o!==Et)||(b=_t,f-=w[A]-n.width,f*=l?1:-1)}var T,O=Object.assign({position:a},c&&he);return l?Object.assign({},O,((T={})[v]=_?"0":"",T[b]=g?"0":"",T.transform=(y.devicePixelRatio||1)<=1?"translate("+f+"px, "+m+"px)":"translate3d("+f+"px, "+m+"px, 0)",T)):Object.assign({},O,((e={})[v]=_?m+"px":"",e[b]=g?f+"px":"",e.transform="",e))}const ue={name:"computeStyles",enabled:!0,phase:"beforeWrite",fn:function(t){var e=t.state,i=t.options,n=i.gpuAcceleration,s=void 0===n||n,o=i.adaptive,r=void 0===o||o,a=i.roundOffsets,l=void 0===a||a,c={placement:Ut(e.placement),variation:ce(e.placement),popper:e.elements.popper,popperRect:e.rects.popper,gpuAcceleration:s};null!=e.modifiersData.popperOffsets&&(e.styles.popper=Object.assign({},e.styles.popper,de(Object.assign({},c,{offsets:e.modifiersData.popperOffsets,position:e.options.strategy,adaptive:r,roundOffsets:l})))),null!=e.modifiersData.arrow&&(e.styles.arrow=Object.assign({},e.styles.arrow,de(Object.assign({},c,{offsets:e.modifiersData.arrow,position:"absolute",adaptive:!1,roundOffsets:l})))),e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-placement":e.placement})},data:{}};var fe={passive:!0};const pe={name:"eventListeners",enabled:!0,phase:"write",fn:function(){},effect:function(t){var e=t.state,i=t.instance,n=t.options,s=n.scroll,o=void 0===s||s,r=n.resize,a=void 0===r||r,l=Wt(e.elements.popper),c=[].concat(e.scrollParents.reference,e.scrollParents.popper);return o&&c.forEach((function(t){t.addEventListener("scroll",i.update,fe)})),a&&l.addEventListener("resize",i.update,fe),function(){o&&c.forEach((function(t){t.removeEventListener("scroll",i.update,fe)})),a&&l.removeEventListener("resize",i.update,fe)}},data:{}};var me={left:"right",right:"left",bottom:"top",top:"bottom"};function ge(t){return t.replace(/left|right|bottom|top/g,(function(t){return me[t]}))}var _e={start:"end",end:"start"};function be(t){return t.replace(/start|end/g,(function(t){return _e[t]}))}function ve(t){var e=Wt(t);return{scrollLeft:e.pageXOffset,scrollTop:e.pageYOffset}}function ye(t){return Vt(Gt(t)).left+ve(t).scrollLeft}function we(t){var e=Yt(t),i=e.overflow,n=e.overflowX,s=e.overflowY;return/auto|scroll|overlay|hidden/.test(i+s+n)}function Ee(t){return["html","body","#document"].indexOf(Rt(t))>=0?t.ownerDocument.body:zt(t)&&we(t)?t:Ee(Zt(t))}function Ae(t,e){var i;void 0===e&&(e=[]);var n=Ee(t),s=n===(null==(i=t.ownerDocument)?void 0:i.body),o=Wt(n),r=s?[o].concat(o.visualViewport||[],we(n)?n:[]):n,a=e.concat(r);return s?a:a.concat(Ae(Zt(r)))}function Te(t){return Object.assign({},t,{left:t.x,top:t.y,right:t.x+t.width,bottom:t.y+t.height})}function Oe(t,e){return e===Tt?Te(function(t){var e=Wt(t),i=Gt(t),n=e.visualViewport,s=i.clientWidth,o=i.clientHeight,r=0,a=0;return n&&(s=n.width,o=n.height,/^((?!chrome|android).)*safari/i.test(navigator.userAgent)||(r=n.offsetLeft,a=n.offsetTop)),{width:s,height:o,x:r+ye(t),y:a}}(t)):zt(e)?function(t){var e=Vt(t);return e.top=e.top+t.clientTop,e.left=e.left+t.clientLeft,e.bottom=e.top+t.clientHeight,e.right=e.left+t.clientWidth,e.width=t.clientWidth,e.height=t.clientHeight,e.x=e.left,e.y=e.top,e}(e):Te(function(t){var e,i=Gt(t),n=ve(t),s=null==(e=t.ownerDocument)?void 0:e.body,o=ie(i.scrollWidth,i.clientWidth,s?s.scrollWidth:0,s?s.clientWidth:0),r=ie(i.scrollHeight,i.clientHeight,s?s.scrollHeight:0,s?s.clientHeight:0),a=-n.scrollLeft+ye(t),l=-n.scrollTop;return"rtl"===Yt(s||i).direction&&(a+=ie(i.clientWidth,s?s.clientWidth:0)-o),{width:o,height:r,x:a,y:l}}(Gt(t)))}function Ce(t){var e,i=t.reference,n=t.element,s=t.placement,o=s?Ut(s):null,r=s?ce(s):null,a=i.x+i.width/2-n.width/2,l=i.y+i.height/2-n.height/2;switch(o){case mt:e={x:a,y:i.y-n.height};break;case gt:e={x:a,y:i.y+i.height};break;case _t:e={x:i.x+i.width,y:l};break;case bt:e={x:i.x-n.width,y:l};break;default:e={x:i.x,y:i.y}}var c=o?ee(o):null;if(null!=c){var h="y"===c?"height":"width";switch(r){case wt:e[c]=e[c]-(i[h]/2-n[h]/2);break;case Et:e[c]=e[c]+(i[h]/2-n[h]/2)}}return e}function ke(t,e){void 0===e&&(e={});var i=e,n=i.placement,s=void 0===n?t.placement:n,o=i.boundary,r=void 0===o?At:o,a=i.rootBoundary,l=void 0===a?Tt:a,c=i.elementContext,h=void 0===c?Ot:c,d=i.altBoundary,u=void 0!==d&&d,f=i.padding,p=void 0===f?0:f,m=re("number"!=typeof p?p:ae(p,yt)),g=h===Ot?Ct:Ot,_=t.rects.popper,b=t.elements[u?g:h],v=function(t,e,i){var n="clippingParents"===e?function(t){var e=Ae(Zt(t)),i=["absolute","fixed"].indexOf(Yt(t).position)>=0&&zt(t)?te(t):t;return $t(i)?e.filter((function(t){return $t(t)&&Xt(t,i)&&"body"!==Rt(t)})):[]}(t):[].concat(e),s=[].concat(n,[i]),o=s[0],r=s.reduce((function(e,i){var n=Oe(t,i);return e.top=ie(n.top,e.top),e.right=ne(n.right,e.right),e.bottom=ne(n.bottom,e.bottom),e.left=ie(n.left,e.left),e}),Oe(t,o));return r.width=r.right-r.left,r.height=r.bottom-r.top,r.x=r.left,r.y=r.top,r}($t(b)?b:b.contextElement||Gt(t.elements.popper),r,l),y=Vt(t.elements.reference),w=Ce({reference:y,element:_,strategy:"absolute",placement:s}),E=Te(Object.assign({},_,w)),A=h===Ot?E:y,T={top:v.top-A.top+m.top,bottom:A.bottom-v.bottom+m.bottom,left:v.left-A.left+m.left,right:A.right-v.right+m.right},O=t.modifiersData.offset;if(h===Ot&&O){var C=O[s];Object.keys(T).forEach((function(t){var e=[_t,gt].indexOf(t)>=0?1:-1,i=[mt,gt].indexOf(t)>=0?"y":"x";T[t]+=C[i]*e}))}return T}function Le(t,e){void 0===e&&(e={});var i=e,n=i.placement,s=i.boundary,o=i.rootBoundary,r=i.padding,a=i.flipVariations,l=i.allowedAutoPlacements,c=void 0===l?Lt:l,h=ce(n),d=h?a?kt:kt.filter((function(t){return ce(t)===h})):yt,u=d.filter((function(t){return c.indexOf(t)>=0}));0===u.length&&(u=d);var f=u.reduce((function(e,i){return e[i]=ke(t,{placement:i,boundary:s,rootBoundary:o,padding:r})[Ut(i)],e}),{});return Object.keys(f).sort((function(t,e){return f[t]-f[e]}))}const xe={name:"flip",enabled:!0,phase:"main",fn:function(t){var e=t.state,i=t.options,n=t.name;if(!e.modifiersData[n]._skip){for(var s=i.mainAxis,o=void 0===s||s,r=i.altAxis,a=void 0===r||r,l=i.fallbackPlacements,c=i.padding,h=i.boundary,d=i.rootBoundary,u=i.altBoundary,f=i.flipVariations,p=void 0===f||f,m=i.allowedAutoPlacements,g=e.options.placement,_=Ut(g),b=l||(_!==g&&p?function(t){if(Ut(t)===vt)return[];var e=ge(t);return[be(t),e,be(e)]}(g):[ge(g)]),v=[g].concat(b).reduce((function(t,i){return t.concat(Ut(i)===vt?Le(e,{placement:i,boundary:h,rootBoundary:d,padding:c,flipVariations:p,allowedAutoPlacements:m}):i)}),[]),y=e.rects.reference,w=e.rects.popper,E=new Map,A=!0,T=v[0],O=0;O=0,D=x?"width":"height",S=ke(e,{placement:C,boundary:h,rootBoundary:d,altBoundary:u,padding:c}),N=x?L?_t:bt:L?gt:mt;y[D]>w[D]&&(N=ge(N));var I=ge(N),P=[];if(o&&P.push(S[k]<=0),a&&P.push(S[N]<=0,S[I]<=0),P.every((function(t){return t}))){T=C,A=!1;break}E.set(C,P)}if(A)for(var j=function(t){var e=v.find((function(e){var i=E.get(e);if(i)return i.slice(0,t).every((function(t){return t}))}));if(e)return T=e,"break"},M=p?3:1;M>0&&"break"!==j(M);M--);e.placement!==T&&(e.modifiersData[n]._skip=!0,e.placement=T,e.reset=!0)}},requiresIfExists:["offset"],data:{_skip:!1}};function De(t,e,i){return void 0===i&&(i={x:0,y:0}),{top:t.top-e.height-i.y,right:t.right-e.width+i.x,bottom:t.bottom-e.height+i.y,left:t.left-e.width-i.x}}function Se(t){return[mt,_t,gt,bt].some((function(e){return t[e]>=0}))}const Ne={name:"hide",enabled:!0,phase:"main",requiresIfExists:["preventOverflow"],fn:function(t){var e=t.state,i=t.name,n=e.rects.reference,s=e.rects.popper,o=e.modifiersData.preventOverflow,r=ke(e,{elementContext:"reference"}),a=ke(e,{altBoundary:!0}),l=De(r,n),c=De(a,s,o),h=Se(l),d=Se(c);e.modifiersData[i]={referenceClippingOffsets:l,popperEscapeOffsets:c,isReferenceHidden:h,hasPopperEscaped:d},e.attributes.popper=Object.assign({},e.attributes.popper,{"data-popper-reference-hidden":h,"data-popper-escaped":d})}},Ie={name:"offset",enabled:!0,phase:"main",requires:["popperOffsets"],fn:function(t){var e=t.state,i=t.options,n=t.name,s=i.offset,o=void 0===s?[0,0]:s,r=Lt.reduce((function(t,i){return t[i]=function(t,e,i){var n=Ut(t),s=[bt,mt].indexOf(n)>=0?-1:1,o="function"==typeof i?i(Object.assign({},e,{placement:t})):i,r=o[0],a=o[1];return r=r||0,a=(a||0)*s,[bt,_t].indexOf(n)>=0?{x:a,y:r}:{x:r,y:a}}(i,e.rects,o),t}),{}),a=r[e.placement],l=a.x,c=a.y;null!=e.modifiersData.popperOffsets&&(e.modifiersData.popperOffsets.x+=l,e.modifiersData.popperOffsets.y+=c),e.modifiersData[n]=r}},Pe={name:"popperOffsets",enabled:!0,phase:"read",fn:function(t){var e=t.state,i=t.name;e.modifiersData[i]=Ce({reference:e.rects.reference,element:e.rects.popper,strategy:"absolute",placement:e.placement})},data:{}},je={name:"preventOverflow",enabled:!0,phase:"main",fn:function(t){var e=t.state,i=t.options,n=t.name,s=i.mainAxis,o=void 0===s||s,r=i.altAxis,a=void 0!==r&&r,l=i.boundary,c=i.rootBoundary,h=i.altBoundary,d=i.padding,u=i.tether,f=void 0===u||u,p=i.tetherOffset,m=void 0===p?0:p,g=ke(e,{boundary:l,rootBoundary:c,padding:d,altBoundary:h}),_=Ut(e.placement),b=ce(e.placement),v=!b,y=ee(_),w="x"===y?"y":"x",E=e.modifiersData.popperOffsets,A=e.rects.reference,T=e.rects.popper,O="function"==typeof m?m(Object.assign({},e.rects,{placement:e.placement})):m,C={x:0,y:0};if(E){if(o||a){var k="y"===y?mt:bt,L="y"===y?gt:_t,x="y"===y?"height":"width",D=E[y],S=E[y]+g[k],N=E[y]-g[L],I=f?-T[x]/2:0,P=b===wt?A[x]:T[x],j=b===wt?-T[x]:-A[x],M=e.elements.arrow,H=f&&M?Kt(M):{width:0,height:0},B=e.modifiersData["arrow#persistent"]?e.modifiersData["arrow#persistent"].padding:{top:0,right:0,bottom:0,left:0},R=B[k],W=B[L],$=oe(0,A[x],H[x]),z=v?A[x]/2-I-$-R-O:P-$-R-O,q=v?-A[x]/2+I+$+W+O:j+$+W+O,F=e.elements.arrow&&te(e.elements.arrow),U=F?"y"===y?F.clientTop||0:F.clientLeft||0:0,V=e.modifiersData.offset?e.modifiersData.offset[e.placement][y]:0,K=E[y]+z-V-U,X=E[y]+q-V;if(o){var Y=oe(f?ne(S,K):S,D,f?ie(N,X):N);E[y]=Y,C[y]=Y-D}if(a){var Q="x"===y?mt:bt,G="x"===y?gt:_t,Z=E[w],J=Z+g[Q],tt=Z-g[G],et=oe(f?ne(J,K):J,Z,f?ie(tt,X):tt);E[w]=et,C[w]=et-Z}}e.modifiersData[n]=C}},requiresIfExists:["offset"]};function Me(t,e,i){void 0===i&&(i=!1);var n=zt(e);zt(e)&&function(t){var e=t.getBoundingClientRect();e.width,t.offsetWidth,e.height,t.offsetHeight}(e);var s,o,r=Gt(e),a=Vt(t),l={scrollLeft:0,scrollTop:0},c={x:0,y:0};return(n||!n&&!i)&&(("body"!==Rt(e)||we(r))&&(l=(s=e)!==Wt(s)&&zt(s)?{scrollLeft:(o=s).scrollLeft,scrollTop:o.scrollTop}:ve(s)),zt(e)?((c=Vt(e)).x+=e.clientLeft,c.y+=e.clientTop):r&&(c.x=ye(r))),{x:a.left+l.scrollLeft-c.x,y:a.top+l.scrollTop-c.y,width:a.width,height:a.height}}function He(t){var e=new Map,i=new Set,n=[];function s(t){i.add(t.name),[].concat(t.requires||[],t.requiresIfExists||[]).forEach((function(t){if(!i.has(t)){var n=e.get(t);n&&s(n)}})),n.push(t)}return t.forEach((function(t){e.set(t.name,t)})),t.forEach((function(t){i.has(t.name)||s(t)})),n}var Be={placement:"bottom",modifiers:[],strategy:"absolute"};function Re(){for(var t=arguments.length,e=new Array(t),i=0;ij.on(t,"mouseover",d))),this._element.focus(),this._element.setAttribute("aria-expanded",!0),this._menu.classList.add(Je),this._element.classList.add(Je),j.trigger(this._element,"shown.bs.dropdown",t)}hide(){if(c(this._element)||!this._isShown(this._menu))return;const t={relatedTarget:this._element};this._completeHide(t)}dispose(){this._popper&&this._popper.destroy(),super.dispose()}update(){this._inNavbar=this._detectNavbar(),this._popper&&this._popper.update()}_completeHide(t){j.trigger(this._element,"hide.bs.dropdown",t).defaultPrevented||("ontouchstart"in document.documentElement&&[].concat(...document.body.children).forEach((t=>j.off(t,"mouseover",d))),this._popper&&this._popper.destroy(),this._menu.classList.remove(Je),this._element.classList.remove(Je),this._element.setAttribute("aria-expanded","false"),U.removeDataAttribute(this._menu,"popper"),j.trigger(this._element,"hidden.bs.dropdown",t))}_getConfig(t){if(t={...this.constructor.Default,...U.getDataAttributes(this._element),...t},a(Ue,t,this.constructor.DefaultType),"object"==typeof t.reference&&!o(t.reference)&&"function"!=typeof t.reference.getBoundingClientRect)throw new TypeError(`${Ue.toUpperCase()}: Option "reference" provided type "object" without a required "getBoundingClientRect" method.`);return t}_createPopper(t){if(void 0===Fe)throw new TypeError("Bootstrap's dropdowns require Popper (https://popper.js.org)");let e=this._element;"parent"===this._config.reference?e=t:o(this._config.reference)?e=r(this._config.reference):"object"==typeof this._config.reference&&(e=this._config.reference);const i=this._getPopperConfig(),n=i.modifiers.find((t=>"applyStyles"===t.name&&!1===t.enabled));this._popper=qe(e,this._menu,i),n&&U.setDataAttribute(this._menu,"popper","static")}_isShown(t=this._element){return t.classList.contains(Je)}_getMenuElement(){return V.next(this._element,ei)[0]}_getPlacement(){const t=this._element.parentNode;if(t.classList.contains("dropend"))return ri;if(t.classList.contains("dropstart"))return ai;const e="end"===getComputedStyle(this._menu).getPropertyValue("--bs-position").trim();return t.classList.contains("dropup")?e?ni:ii:e?oi:si}_detectNavbar(){return null!==this._element.closest(".navbar")}_getOffset(){const{offset:t}=this._config;return"string"==typeof t?t.split(",").map((t=>Number.parseInt(t,10))):"function"==typeof t?e=>t(e,this._element):t}_getPopperConfig(){const t={placement:this._getPlacement(),modifiers:[{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"offset",options:{offset:this._getOffset()}}]};return"static"===this._config.display&&(t.modifiers=[{name:"applyStyles",enabled:!1}]),{...t,..."function"==typeof this._config.popperConfig?this._config.popperConfig(t):this._config.popperConfig}}_selectMenuItem({key:t,target:e}){const i=V.find(".dropdown-menu .dropdown-item:not(.disabled):not(:disabled)",this._menu).filter(l);i.length&&v(i,e,t===Ye,!i.includes(e)).focus()}static jQueryInterface(t){return this.each((function(){const e=hi.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}static clearMenus(t){if(t&&(2===t.button||"keyup"===t.type&&"Tab"!==t.key))return;const e=V.find(ti);for(let i=0,n=e.length;ie+t)),this._setElementAttributes(di,"paddingRight",(e=>e+t)),this._setElementAttributes(ui,"marginRight",(e=>e-t))}_disableOverFlow(){this._saveInitialAttribute(this._element,"overflow"),this._element.style.overflow="hidden"}_setElementAttributes(t,e,i){const n=this.getWidth();this._applyManipulationCallback(t,(t=>{if(t!==this._element&&window.innerWidth>t.clientWidth+n)return;this._saveInitialAttribute(t,e);const s=window.getComputedStyle(t)[e];t.style[e]=`${i(Number.parseFloat(s))}px`}))}reset(){this._resetElementAttributes(this._element,"overflow"),this._resetElementAttributes(this._element,"paddingRight"),this._resetElementAttributes(di,"paddingRight"),this._resetElementAttributes(ui,"marginRight")}_saveInitialAttribute(t,e){const i=t.style[e];i&&U.setDataAttribute(t,e,i)}_resetElementAttributes(t,e){this._applyManipulationCallback(t,(t=>{const i=U.getDataAttribute(t,e);void 0===i?t.style.removeProperty(e):(U.removeDataAttribute(t,e),t.style[e]=i)}))}_applyManipulationCallback(t,e){o(t)?e(t):V.find(t,this._element).forEach(e)}isOverflowing(){return this.getWidth()>0}}const pi={className:"modal-backdrop",isVisible:!0,isAnimated:!1,rootElement:"body",clickCallback:null},mi={className:"string",isVisible:"boolean",isAnimated:"boolean",rootElement:"(element|string)",clickCallback:"(function|null)"},gi="show",_i="mousedown.bs.backdrop";class bi{constructor(t){this._config=this._getConfig(t),this._isAppended=!1,this._element=null}show(t){this._config.isVisible?(this._append(),this._config.isAnimated&&u(this._getElement()),this._getElement().classList.add(gi),this._emulateAnimation((()=>{_(t)}))):_(t)}hide(t){this._config.isVisible?(this._getElement().classList.remove(gi),this._emulateAnimation((()=>{this.dispose(),_(t)}))):_(t)}_getElement(){if(!this._element){const t=document.createElement("div");t.className=this._config.className,this._config.isAnimated&&t.classList.add("fade"),this._element=t}return this._element}_getConfig(t){return(t={...pi,..."object"==typeof t?t:{}}).rootElement=r(t.rootElement),a("backdrop",t,mi),t}_append(){this._isAppended||(this._config.rootElement.append(this._getElement()),j.on(this._getElement(),_i,(()=>{_(this._config.clickCallback)})),this._isAppended=!0)}dispose(){this._isAppended&&(j.off(this._element,_i),this._element.remove(),this._isAppended=!1)}_emulateAnimation(t){b(t,this._getElement(),this._config.isAnimated)}}const vi={trapElement:null,autofocus:!0},yi={trapElement:"element",autofocus:"boolean"},wi=".bs.focustrap",Ei="backward";class Ai{constructor(t){this._config=this._getConfig(t),this._isActive=!1,this._lastTabNavDirection=null}activate(){const{trapElement:t,autofocus:e}=this._config;this._isActive||(e&&t.focus(),j.off(document,wi),j.on(document,"focusin.bs.focustrap",(t=>this._handleFocusin(t))),j.on(document,"keydown.tab.bs.focustrap",(t=>this._handleKeydown(t))),this._isActive=!0)}deactivate(){this._isActive&&(this._isActive=!1,j.off(document,wi))}_handleFocusin(t){const{target:e}=t,{trapElement:i}=this._config;if(e===document||e===i||i.contains(e))return;const n=V.focusableChildren(i);0===n.length?i.focus():this._lastTabNavDirection===Ei?n[n.length-1].focus():n[0].focus()}_handleKeydown(t){"Tab"===t.key&&(this._lastTabNavDirection=t.shiftKey?Ei:"forward")}_getConfig(t){return t={...vi,..."object"==typeof t?t:{}},a("focustrap",t,yi),t}}const Ti="modal",Oi="Escape",Ci={backdrop:!0,keyboard:!0,focus:!0},ki={backdrop:"(boolean|string)",keyboard:"boolean",focus:"boolean"},Li="hidden.bs.modal",xi="show.bs.modal",Di="resize.bs.modal",Si="click.dismiss.bs.modal",Ni="keydown.dismiss.bs.modal",Ii="mousedown.dismiss.bs.modal",Pi="modal-open",ji="show",Mi="modal-static";class Hi extends B{constructor(t,e){super(t),this._config=this._getConfig(e),this._dialog=V.findOne(".modal-dialog",this._element),this._backdrop=this._initializeBackDrop(),this._focustrap=this._initializeFocusTrap(),this._isShown=!1,this._ignoreBackdropClick=!1,this._isTransitioning=!1,this._scrollBar=new fi}static get Default(){return Ci}static get NAME(){return Ti}toggle(t){return this._isShown?this.hide():this.show(t)}show(t){this._isShown||this._isTransitioning||j.trigger(this._element,xi,{relatedTarget:t}).defaultPrevented||(this._isShown=!0,this._isAnimated()&&(this._isTransitioning=!0),this._scrollBar.hide(),document.body.classList.add(Pi),this._adjustDialog(),this._setEscapeEvent(),this._setResizeEvent(),j.on(this._dialog,Ii,(()=>{j.one(this._element,"mouseup.dismiss.bs.modal",(t=>{t.target===this._element&&(this._ignoreBackdropClick=!0)}))})),this._showBackdrop((()=>this._showElement(t))))}hide(){if(!this._isShown||this._isTransitioning)return;if(j.trigger(this._element,"hide.bs.modal").defaultPrevented)return;this._isShown=!1;const t=this._isAnimated();t&&(this._isTransitioning=!0),this._setEscapeEvent(),this._setResizeEvent(),this._focustrap.deactivate(),this._element.classList.remove(ji),j.off(this._element,Si),j.off(this._dialog,Ii),this._queueCallback((()=>this._hideModal()),this._element,t)}dispose(){[window,this._dialog].forEach((t=>j.off(t,".bs.modal"))),this._backdrop.dispose(),this._focustrap.deactivate(),super.dispose()}handleUpdate(){this._adjustDialog()}_initializeBackDrop(){return new bi({isVisible:Boolean(this._config.backdrop),isAnimated:this._isAnimated()})}_initializeFocusTrap(){return new Ai({trapElement:this._element})}_getConfig(t){return t={...Ci,...U.getDataAttributes(this._element),..."object"==typeof t?t:{}},a(Ti,t,ki),t}_showElement(t){const e=this._isAnimated(),i=V.findOne(".modal-body",this._dialog);this._element.parentNode&&this._element.parentNode.nodeType===Node.ELEMENT_NODE||document.body.append(this._element),this._element.style.display="block",this._element.removeAttribute("aria-hidden"),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.scrollTop=0,i&&(i.scrollTop=0),e&&u(this._element),this._element.classList.add(ji),this._queueCallback((()=>{this._config.focus&&this._focustrap.activate(),this._isTransitioning=!1,j.trigger(this._element,"shown.bs.modal",{relatedTarget:t})}),this._dialog,e)}_setEscapeEvent(){this._isShown?j.on(this._element,Ni,(t=>{this._config.keyboard&&t.key===Oi?(t.preventDefault(),this.hide()):this._config.keyboard||t.key!==Oi||this._triggerBackdropTransition()})):j.off(this._element,Ni)}_setResizeEvent(){this._isShown?j.on(window,Di,(()=>this._adjustDialog())):j.off(window,Di)}_hideModal(){this._element.style.display="none",this._element.setAttribute("aria-hidden",!0),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._isTransitioning=!1,this._backdrop.hide((()=>{document.body.classList.remove(Pi),this._resetAdjustments(),this._scrollBar.reset(),j.trigger(this._element,Li)}))}_showBackdrop(t){j.on(this._element,Si,(t=>{this._ignoreBackdropClick?this._ignoreBackdropClick=!1:t.target===t.currentTarget&&(!0===this._config.backdrop?this.hide():"static"===this._config.backdrop&&this._triggerBackdropTransition())})),this._backdrop.show(t)}_isAnimated(){return this._element.classList.contains("fade")}_triggerBackdropTransition(){if(j.trigger(this._element,"hidePrevented.bs.modal").defaultPrevented)return;const{classList:t,scrollHeight:e,style:i}=this._element,n=e>document.documentElement.clientHeight;!n&&"hidden"===i.overflowY||t.contains(Mi)||(n||(i.overflowY="hidden"),t.add(Mi),this._queueCallback((()=>{t.remove(Mi),n||this._queueCallback((()=>{i.overflowY=""}),this._dialog)}),this._dialog),this._element.focus())}_adjustDialog(){const t=this._element.scrollHeight>document.documentElement.clientHeight,e=this._scrollBar.getWidth(),i=e>0;(!i&&t&&!m()||i&&!t&&m())&&(this._element.style.paddingLeft=`${e}px`),(i&&!t&&!m()||!i&&t&&m())&&(this._element.style.paddingRight=`${e}px`)}_resetAdjustments(){this._element.style.paddingLeft="",this._element.style.paddingRight=""}static jQueryInterface(t,e){return this.each((function(){const i=Hi.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===i[t])throw new TypeError(`No method named "${t}"`);i[t](e)}}))}}j.on(document,"click.bs.modal.data-api",'[data-bs-toggle="modal"]',(function(t){const e=n(this);["A","AREA"].includes(this.tagName)&&t.preventDefault(),j.one(e,xi,(t=>{t.defaultPrevented||j.one(e,Li,(()=>{l(this)&&this.focus()}))}));const i=V.findOne(".modal.show");i&&Hi.getInstance(i).hide(),Hi.getOrCreateInstance(e).toggle(this)})),R(Hi),g(Hi);const Bi="offcanvas",Ri={backdrop:!0,keyboard:!0,scroll:!1},Wi={backdrop:"boolean",keyboard:"boolean",scroll:"boolean"},$i="show",zi=".offcanvas.show",qi="hidden.bs.offcanvas";class Fi extends B{constructor(t,e){super(t),this._config=this._getConfig(e),this._isShown=!1,this._backdrop=this._initializeBackDrop(),this._focustrap=this._initializeFocusTrap(),this._addEventListeners()}static get NAME(){return Bi}static get Default(){return Ri}toggle(t){return this._isShown?this.hide():this.show(t)}show(t){this._isShown||j.trigger(this._element,"show.bs.offcanvas",{relatedTarget:t}).defaultPrevented||(this._isShown=!0,this._element.style.visibility="visible",this._backdrop.show(),this._config.scroll||(new fi).hide(),this._element.removeAttribute("aria-hidden"),this._element.setAttribute("aria-modal",!0),this._element.setAttribute("role","dialog"),this._element.classList.add($i),this._queueCallback((()=>{this._config.scroll||this._focustrap.activate(),j.trigger(this._element,"shown.bs.offcanvas",{relatedTarget:t})}),this._element,!0))}hide(){this._isShown&&(j.trigger(this._element,"hide.bs.offcanvas").defaultPrevented||(this._focustrap.deactivate(),this._element.blur(),this._isShown=!1,this._element.classList.remove($i),this._backdrop.hide(),this._queueCallback((()=>{this._element.setAttribute("aria-hidden",!0),this._element.removeAttribute("aria-modal"),this._element.removeAttribute("role"),this._element.style.visibility="hidden",this._config.scroll||(new fi).reset(),j.trigger(this._element,qi)}),this._element,!0)))}dispose(){this._backdrop.dispose(),this._focustrap.deactivate(),super.dispose()}_getConfig(t){return t={...Ri,...U.getDataAttributes(this._element),..."object"==typeof t?t:{}},a(Bi,t,Wi),t}_initializeBackDrop(){return new bi({className:"offcanvas-backdrop",isVisible:this._config.backdrop,isAnimated:!0,rootElement:this._element.parentNode,clickCallback:()=>this.hide()})}_initializeFocusTrap(){return new Ai({trapElement:this._element})}_addEventListeners(){j.on(this._element,"keydown.dismiss.bs.offcanvas",(t=>{this._config.keyboard&&"Escape"===t.key&&this.hide()}))}static jQueryInterface(t){return this.each((function(){const e=Fi.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t]||t.startsWith("_")||"constructor"===t)throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}j.on(document,"click.bs.offcanvas.data-api",'[data-bs-toggle="offcanvas"]',(function(t){const e=n(this);if(["A","AREA"].includes(this.tagName)&&t.preventDefault(),c(this))return;j.one(e,qi,(()=>{l(this)&&this.focus()}));const i=V.findOne(zi);i&&i!==e&&Fi.getInstance(i).hide(),Fi.getOrCreateInstance(e).toggle(this)})),j.on(window,"load.bs.offcanvas.data-api",(()=>V.find(zi).forEach((t=>Fi.getOrCreateInstance(t).show())))),R(Fi),g(Fi);const Ui=new Set(["background","cite","href","itemtype","longdesc","poster","src","xlink:href"]),Vi=/^(?:(?:https?|mailto|ftp|tel|file|sms):|[^#&/:?]*(?:[#/?]|$))/i,Ki=/^data:(?:image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp)|video\/(?:mpeg|mp4|ogg|webm)|audio\/(?:mp3|oga|ogg|opus));base64,[\d+/a-z]+=*$/i,Xi=(t,e)=>{const i=t.nodeName.toLowerCase();if(e.includes(i))return!Ui.has(i)||Boolean(Vi.test(t.nodeValue)||Ki.test(t.nodeValue));const n=e.filter((t=>t instanceof RegExp));for(let t=0,e=n.length;t{Xi(t,r)||i.removeAttribute(t.nodeName)}))}return n.body.innerHTML}const Qi="tooltip",Gi=new Set(["sanitize","allowList","sanitizeFn"]),Zi={animation:"boolean",template:"string",title:"(string|element|function)",trigger:"string",delay:"(number|object)",html:"boolean",selector:"(string|boolean)",placement:"(string|function)",offset:"(array|string|function)",container:"(string|element|boolean)",fallbackPlacements:"array",boundary:"(string|element)",customClass:"(string|function)",sanitize:"boolean",sanitizeFn:"(null|function)",allowList:"object",popperConfig:"(null|object|function)"},Ji={AUTO:"auto",TOP:"top",RIGHT:m()?"left":"right",BOTTOM:"bottom",LEFT:m()?"right":"left"},tn={animation:!0,template:'',trigger:"hover focus",title:"",delay:0,html:!1,selector:!1,placement:"top",offset:[0,0],container:!1,fallbackPlacements:["top","right","bottom","left"],boundary:"clippingParents",customClass:"",sanitize:!0,sanitizeFn:null,allowList:{"*":["class","dir","id","lang","role",/^aria-[\w-]*$/i],a:["target","href","title","rel"],area:[],b:[],br:[],col:[],code:[],div:[],em:[],hr:[],h1:[],h2:[],h3:[],h4:[],h5:[],h6:[],i:[],img:["src","srcset","alt","title","width","height"],li:[],ol:[],p:[],pre:[],s:[],small:[],span:[],sub:[],sup:[],strong:[],u:[],ul:[]},popperConfig:null},en={HIDE:"hide.bs.tooltip",HIDDEN:"hidden.bs.tooltip",SHOW:"show.bs.tooltip",SHOWN:"shown.bs.tooltip",INSERTED:"inserted.bs.tooltip",CLICK:"click.bs.tooltip",FOCUSIN:"focusin.bs.tooltip",FOCUSOUT:"focusout.bs.tooltip",MOUSEENTER:"mouseenter.bs.tooltip",MOUSELEAVE:"mouseleave.bs.tooltip"},nn="fade",sn="show",on="show",rn="out",an=".tooltip-inner",ln=".modal",cn="hide.bs.modal",hn="hover",dn="focus";class un extends B{constructor(t,e){if(void 0===Fe)throw new TypeError("Bootstrap's tooltips require Popper (https://popper.js.org)");super(t),this._isEnabled=!0,this._timeout=0,this._hoverState="",this._activeTrigger={},this._popper=null,this._config=this._getConfig(e),this.tip=null,this._setListeners()}static get Default(){return tn}static get NAME(){return Qi}static get Event(){return en}static get DefaultType(){return Zi}enable(){this._isEnabled=!0}disable(){this._isEnabled=!1}toggleEnabled(){this._isEnabled=!this._isEnabled}toggle(t){if(this._isEnabled)if(t){const e=this._initializeOnDelegatedTarget(t);e._activeTrigger.click=!e._activeTrigger.click,e._isWithActiveTrigger()?e._enter(null,e):e._leave(null,e)}else{if(this.getTipElement().classList.contains(sn))return void this._leave(null,this);this._enter(null,this)}}dispose(){clearTimeout(this._timeout),j.off(this._element.closest(ln),cn,this._hideModalHandler),this.tip&&this.tip.remove(),this._disposePopper(),super.dispose()}show(){if("none"===this._element.style.display)throw new Error("Please use show on visible elements");if(!this.isWithContent()||!this._isEnabled)return;const t=j.trigger(this._element,this.constructor.Event.SHOW),e=h(this._element),i=null===e?this._element.ownerDocument.documentElement.contains(this._element):e.contains(this._element);if(t.defaultPrevented||!i)return;"tooltip"===this.constructor.NAME&&this.tip&&this.getTitle()!==this.tip.querySelector(an).innerHTML&&(this._disposePopper(),this.tip.remove(),this.tip=null);const n=this.getTipElement(),s=(t=>{do{t+=Math.floor(1e6*Math.random())}while(document.getElementById(t));return t})(this.constructor.NAME);n.setAttribute("id",s),this._element.setAttribute("aria-describedby",s),this._config.animation&&n.classList.add(nn);const o="function"==typeof this._config.placement?this._config.placement.call(this,n,this._element):this._config.placement,r=this._getAttachment(o);this._addAttachmentClass(r);const{container:a}=this._config;H.set(n,this.constructor.DATA_KEY,this),this._element.ownerDocument.documentElement.contains(this.tip)||(a.append(n),j.trigger(this._element,this.constructor.Event.INSERTED)),this._popper?this._popper.update():this._popper=qe(this._element,n,this._getPopperConfig(r)),n.classList.add(sn);const l=this._resolvePossibleFunction(this._config.customClass);l&&n.classList.add(...l.split(" ")),"ontouchstart"in document.documentElement&&[].concat(...document.body.children).forEach((t=>{j.on(t,"mouseover",d)}));const c=this.tip.classList.contains(nn);this._queueCallback((()=>{const t=this._hoverState;this._hoverState=null,j.trigger(this._element,this.constructor.Event.SHOWN),t===rn&&this._leave(null,this)}),this.tip,c)}hide(){if(!this._popper)return;const t=this.getTipElement();if(j.trigger(this._element,this.constructor.Event.HIDE).defaultPrevented)return;t.classList.remove(sn),"ontouchstart"in document.documentElement&&[].concat(...document.body.children).forEach((t=>j.off(t,"mouseover",d))),this._activeTrigger.click=!1,this._activeTrigger.focus=!1,this._activeTrigger.hover=!1;const e=this.tip.classList.contains(nn);this._queueCallback((()=>{this._isWithActiveTrigger()||(this._hoverState!==on&&t.remove(),this._cleanTipClass(),this._element.removeAttribute("aria-describedby"),j.trigger(this._element,this.constructor.Event.HIDDEN),this._disposePopper())}),this.tip,e),this._hoverState=""}update(){null!==this._popper&&this._popper.update()}isWithContent(){return Boolean(this.getTitle())}getTipElement(){if(this.tip)return this.tip;const t=document.createElement("div");t.innerHTML=this._config.template;const e=t.children[0];return this.setContent(e),e.classList.remove(nn,sn),this.tip=e,this.tip}setContent(t){this._sanitizeAndSetContent(t,this.getTitle(),an)}_sanitizeAndSetContent(t,e,i){const n=V.findOne(i,t);e||!n?this.setElementContent(n,e):n.remove()}setElementContent(t,e){if(null!==t)return o(e)?(e=r(e),void(this._config.html?e.parentNode!==t&&(t.innerHTML="",t.append(e)):t.textContent=e.textContent)):void(this._config.html?(this._config.sanitize&&(e=Yi(e,this._config.allowList,this._config.sanitizeFn)),t.innerHTML=e):t.textContent=e)}getTitle(){const t=this._element.getAttribute("data-bs-original-title")||this._config.title;return this._resolvePossibleFunction(t)}updateAttachment(t){return"right"===t?"end":"left"===t?"start":t}_initializeOnDelegatedTarget(t,e){return e||this.constructor.getOrCreateInstance(t.delegateTarget,this._getDelegateConfig())}_getOffset(){const{offset:t}=this._config;return"string"==typeof t?t.split(",").map((t=>Number.parseInt(t,10))):"function"==typeof t?e=>t(e,this._element):t}_resolvePossibleFunction(t){return"function"==typeof t?t.call(this._element):t}_getPopperConfig(t){const e={placement:t,modifiers:[{name:"flip",options:{fallbackPlacements:this._config.fallbackPlacements}},{name:"offset",options:{offset:this._getOffset()}},{name:"preventOverflow",options:{boundary:this._config.boundary}},{name:"arrow",options:{element:`.${this.constructor.NAME}-arrow`}},{name:"onChange",enabled:!0,phase:"afterWrite",fn:t=>this._handlePopperPlacementChange(t)}],onFirstUpdate:t=>{t.options.placement!==t.placement&&this._handlePopperPlacementChange(t)}};return{...e,..."function"==typeof this._config.popperConfig?this._config.popperConfig(e):this._config.popperConfig}}_addAttachmentClass(t){this.getTipElement().classList.add(`${this._getBasicClassPrefix()}-${this.updateAttachment(t)}`)}_getAttachment(t){return Ji[t.toUpperCase()]}_setListeners(){this._config.trigger.split(" ").forEach((t=>{if("click"===t)j.on(this._element,this.constructor.Event.CLICK,this._config.selector,(t=>this.toggle(t)));else if("manual"!==t){const e=t===hn?this.constructor.Event.MOUSEENTER:this.constructor.Event.FOCUSIN,i=t===hn?this.constructor.Event.MOUSELEAVE:this.constructor.Event.FOCUSOUT;j.on(this._element,e,this._config.selector,(t=>this._enter(t))),j.on(this._element,i,this._config.selector,(t=>this._leave(t)))}})),this._hideModalHandler=()=>{this._element&&this.hide()},j.on(this._element.closest(ln),cn,this._hideModalHandler),this._config.selector?this._config={...this._config,trigger:"manual",selector:""}:this._fixTitle()}_fixTitle(){const t=this._element.getAttribute("title"),e=typeof this._element.getAttribute("data-bs-original-title");(t||"string"!==e)&&(this._element.setAttribute("data-bs-original-title",t||""),!t||this._element.getAttribute("aria-label")||this._element.textContent||this._element.setAttribute("aria-label",t),this._element.setAttribute("title",""))}_enter(t,e){e=this._initializeOnDelegatedTarget(t,e),t&&(e._activeTrigger["focusin"===t.type?dn:hn]=!0),e.getTipElement().classList.contains(sn)||e._hoverState===on?e._hoverState=on:(clearTimeout(e._timeout),e._hoverState=on,e._config.delay&&e._config.delay.show?e._timeout=setTimeout((()=>{e._hoverState===on&&e.show()}),e._config.delay.show):e.show())}_leave(t,e){e=this._initializeOnDelegatedTarget(t,e),t&&(e._activeTrigger["focusout"===t.type?dn:hn]=e._element.contains(t.relatedTarget)),e._isWithActiveTrigger()||(clearTimeout(e._timeout),e._hoverState=rn,e._config.delay&&e._config.delay.hide?e._timeout=setTimeout((()=>{e._hoverState===rn&&e.hide()}),e._config.delay.hide):e.hide())}_isWithActiveTrigger(){for(const t in this._activeTrigger)if(this._activeTrigger[t])return!0;return!1}_getConfig(t){const e=U.getDataAttributes(this._element);return Object.keys(e).forEach((t=>{Gi.has(t)&&delete e[t]})),(t={...this.constructor.Default,...e,..."object"==typeof t&&t?t:{}}).container=!1===t.container?document.body:r(t.container),"number"==typeof t.delay&&(t.delay={show:t.delay,hide:t.delay}),"number"==typeof t.title&&(t.title=t.title.toString()),"number"==typeof t.content&&(t.content=t.content.toString()),a(Qi,t,this.constructor.DefaultType),t.sanitize&&(t.template=Yi(t.template,t.allowList,t.sanitizeFn)),t}_getDelegateConfig(){const t={};for(const e in this._config)this.constructor.Default[e]!==this._config[e]&&(t[e]=this._config[e]);return t}_cleanTipClass(){const t=this.getTipElement(),e=new RegExp(`(^|\\s)${this._getBasicClassPrefix()}\\S+`,"g"),i=t.getAttribute("class").match(e);null!==i&&i.length>0&&i.map((t=>t.trim())).forEach((e=>t.classList.remove(e)))}_getBasicClassPrefix(){return"bs-tooltip"}_handlePopperPlacementChange(t){const{state:e}=t;e&&(this.tip=e.elements.popper,this._cleanTipClass(),this._addAttachmentClass(this._getAttachment(e.placement)))}_disposePopper(){this._popper&&(this._popper.destroy(),this._popper=null)}static jQueryInterface(t){return this.each((function(){const e=un.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}g(un);const fn={...un.Default,placement:"right",offset:[0,8],trigger:"click",content:"",template:''},pn={...un.DefaultType,content:"(string|element|function)"},mn={HIDE:"hide.bs.popover",HIDDEN:"hidden.bs.popover",SHOW:"show.bs.popover",SHOWN:"shown.bs.popover",INSERTED:"inserted.bs.popover",CLICK:"click.bs.popover",FOCUSIN:"focusin.bs.popover",FOCUSOUT:"focusout.bs.popover",MOUSEENTER:"mouseenter.bs.popover",MOUSELEAVE:"mouseleave.bs.popover"};class gn extends un{static get Default(){return fn}static get NAME(){return"popover"}static get Event(){return mn}static get DefaultType(){return pn}isWithContent(){return this.getTitle()||this._getContent()}setContent(t){this._sanitizeAndSetContent(t,this.getTitle(),".popover-header"),this._sanitizeAndSetContent(t,this._getContent(),".popover-body")}_getContent(){return this._resolvePossibleFunction(this._config.content)}_getBasicClassPrefix(){return"bs-popover"}static jQueryInterface(t){return this.each((function(){const e=gn.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}g(gn);const _n="scrollspy",bn={offset:10,method:"auto",target:""},vn={offset:"number",method:"string",target:"(string|element)"},yn="active",wn=".nav-link, .list-group-item, .dropdown-item",En="position";class An extends B{constructor(t,e){super(t),this._scrollElement="BODY"===this._element.tagName?window:this._element,this._config=this._getConfig(e),this._offsets=[],this._targets=[],this._activeTarget=null,this._scrollHeight=0,j.on(this._scrollElement,"scroll.bs.scrollspy",(()=>this._process())),this.refresh(),this._process()}static get Default(){return bn}static get NAME(){return _n}refresh(){const t=this._scrollElement===this._scrollElement.window?"offset":En,e="auto"===this._config.method?t:this._config.method,n=e===En?this._getScrollTop():0;this._offsets=[],this._targets=[],this._scrollHeight=this._getScrollHeight(),V.find(wn,this._config.target).map((t=>{const s=i(t),o=s?V.findOne(s):null;if(o){const t=o.getBoundingClientRect();if(t.width||t.height)return[U[e](o).top+n,s]}return null})).filter((t=>t)).sort(((t,e)=>t[0]-e[0])).forEach((t=>{this._offsets.push(t[0]),this._targets.push(t[1])}))}dispose(){j.off(this._scrollElement,".bs.scrollspy"),super.dispose()}_getConfig(t){return(t={...bn,...U.getDataAttributes(this._element),..."object"==typeof t&&t?t:{}}).target=r(t.target)||document.documentElement,a(_n,t,vn),t}_getScrollTop(){return this._scrollElement===window?this._scrollElement.pageYOffset:this._scrollElement.scrollTop}_getScrollHeight(){return this._scrollElement.scrollHeight||Math.max(document.body.scrollHeight,document.documentElement.scrollHeight)}_getOffsetHeight(){return this._scrollElement===window?window.innerHeight:this._scrollElement.getBoundingClientRect().height}_process(){const t=this._getScrollTop()+this._config.offset,e=this._getScrollHeight(),i=this._config.offset+e-this._getOffsetHeight();if(this._scrollHeight!==e&&this.refresh(),t>=i){const t=this._targets[this._targets.length-1];this._activeTarget!==t&&this._activate(t)}else{if(this._activeTarget&&t0)return this._activeTarget=null,void this._clear();for(let e=this._offsets.length;e--;)this._activeTarget!==this._targets[e]&&t>=this._offsets[e]&&(void 0===this._offsets[e+1]||t`${e}[data-bs-target="${t}"],${e}[href="${t}"]`)),i=V.findOne(e.join(","),this._config.target);i.classList.add(yn),i.classList.contains("dropdown-item")?V.findOne(".dropdown-toggle",i.closest(".dropdown")).classList.add(yn):V.parents(i,".nav, .list-group").forEach((t=>{V.prev(t,".nav-link, .list-group-item").forEach((t=>t.classList.add(yn))),V.prev(t,".nav-item").forEach((t=>{V.children(t,".nav-link").forEach((t=>t.classList.add(yn)))}))})),j.trigger(this._scrollElement,"activate.bs.scrollspy",{relatedTarget:t})}_clear(){V.find(wn,this._config.target).filter((t=>t.classList.contains(yn))).forEach((t=>t.classList.remove(yn)))}static jQueryInterface(t){return this.each((function(){const e=An.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}j.on(window,"load.bs.scrollspy.data-api",(()=>{V.find('[data-bs-spy="scroll"]').forEach((t=>new An(t)))})),g(An);const Tn="active",On="fade",Cn="show",kn=".active",Ln=":scope > li > .active";class xn extends B{static get NAME(){return"tab"}show(){if(this._element.parentNode&&this._element.parentNode.nodeType===Node.ELEMENT_NODE&&this._element.classList.contains(Tn))return;let t;const e=n(this._element),i=this._element.closest(".nav, .list-group");if(i){const e="UL"===i.nodeName||"OL"===i.nodeName?Ln:kn;t=V.find(e,i),t=t[t.length-1]}const s=t?j.trigger(t,"hide.bs.tab",{relatedTarget:this._element}):null;if(j.trigger(this._element,"show.bs.tab",{relatedTarget:t}).defaultPrevented||null!==s&&s.defaultPrevented)return;this._activate(this._element,i);const o=()=>{j.trigger(t,"hidden.bs.tab",{relatedTarget:this._element}),j.trigger(this._element,"shown.bs.tab",{relatedTarget:t})};e?this._activate(e,e.parentNode,o):o()}_activate(t,e,i){const n=(!e||"UL"!==e.nodeName&&"OL"!==e.nodeName?V.children(e,kn):V.find(Ln,e))[0],s=i&&n&&n.classList.contains(On),o=()=>this._transitionComplete(t,n,i);n&&s?(n.classList.remove(Cn),this._queueCallback(o,t,!0)):o()}_transitionComplete(t,e,i){if(e){e.classList.remove(Tn);const t=V.findOne(":scope > .dropdown-menu .active",e.parentNode);t&&t.classList.remove(Tn),"tab"===e.getAttribute("role")&&e.setAttribute("aria-selected",!1)}t.classList.add(Tn),"tab"===t.getAttribute("role")&&t.setAttribute("aria-selected",!0),u(t),t.classList.contains(On)&&t.classList.add(Cn);let n=t.parentNode;if(n&&"LI"===n.nodeName&&(n=n.parentNode),n&&n.classList.contains("dropdown-menu")){const e=t.closest(".dropdown");e&&V.find(".dropdown-toggle",e).forEach((t=>t.classList.add(Tn))),t.setAttribute("aria-expanded",!0)}i&&i()}static jQueryInterface(t){return this.each((function(){const e=xn.getOrCreateInstance(this);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t]()}}))}}j.on(document,"click.bs.tab.data-api",'[data-bs-toggle="tab"], [data-bs-toggle="pill"], [data-bs-toggle="list"]',(function(t){["A","AREA"].includes(this.tagName)&&t.preventDefault(),c(this)||xn.getOrCreateInstance(this).show()})),g(xn);const Dn="toast",Sn="hide",Nn="show",In="showing",Pn={animation:"boolean",autohide:"boolean",delay:"number"},jn={animation:!0,autohide:!0,delay:5e3};class Mn extends B{constructor(t,e){super(t),this._config=this._getConfig(e),this._timeout=null,this._hasMouseInteraction=!1,this._hasKeyboardInteraction=!1,this._setListeners()}static get DefaultType(){return Pn}static get Default(){return jn}static get NAME(){return Dn}show(){j.trigger(this._element,"show.bs.toast").defaultPrevented||(this._clearTimeout(),this._config.animation&&this._element.classList.add("fade"),this._element.classList.remove(Sn),u(this._element),this._element.classList.add(Nn),this._element.classList.add(In),this._queueCallback((()=>{this._element.classList.remove(In),j.trigger(this._element,"shown.bs.toast"),this._maybeScheduleHide()}),this._element,this._config.animation))}hide(){this._element.classList.contains(Nn)&&(j.trigger(this._element,"hide.bs.toast").defaultPrevented||(this._element.classList.add(In),this._queueCallback((()=>{this._element.classList.add(Sn),this._element.classList.remove(In),this._element.classList.remove(Nn),j.trigger(this._element,"hidden.bs.toast")}),this._element,this._config.animation)))}dispose(){this._clearTimeout(),this._element.classList.contains(Nn)&&this._element.classList.remove(Nn),super.dispose()}_getConfig(t){return t={...jn,...U.getDataAttributes(this._element),..."object"==typeof t&&t?t:{}},a(Dn,t,this.constructor.DefaultType),t}_maybeScheduleHide(){this._config.autohide&&(this._hasMouseInteraction||this._hasKeyboardInteraction||(this._timeout=setTimeout((()=>{this.hide()}),this._config.delay)))}_onInteraction(t,e){switch(t.type){case"mouseover":case"mouseout":this._hasMouseInteraction=e;break;case"focusin":case"focusout":this._hasKeyboardInteraction=e}if(e)return void this._clearTimeout();const i=t.relatedTarget;this._element===i||this._element.contains(i)||this._maybeScheduleHide()}_setListeners(){j.on(this._element,"mouseover.bs.toast",(t=>this._onInteraction(t,!0))),j.on(this._element,"mouseout.bs.toast",(t=>this._onInteraction(t,!1))),j.on(this._element,"focusin.bs.toast",(t=>this._onInteraction(t,!0))),j.on(this._element,"focusout.bs.toast",(t=>this._onInteraction(t,!1)))}_clearTimeout(){clearTimeout(this._timeout),this._timeout=null}static jQueryInterface(t){return this.each((function(){const e=Mn.getOrCreateInstance(this,t);if("string"==typeof t){if(void 0===e[t])throw new TypeError(`No method named "${t}"`);e[t](this)}}))}}return R(Mn),g(Mn),{Alert:W,Button:z,Carousel:st,Collapse:pt,Dropdown:hi,Modal:Hi,Offcanvas:Fi,Popover:gn,ScrollSpy:An,Tab:xn,Toast:Mn,Tooltip:un}})); +//# sourceMappingURL=bootstrap.bundle.min.js.map \ No newline at end of file diff --git a/pkg/api/static/bootstrap-5.1.3.min.css b/pkg/api/static/bootstrap-5.1.3.min.css new file mode 100644 index 00000000000..1472dec059b --- /dev/null +++ b/pkg/api/static/bootstrap-5.1.3.min.css @@ -0,0 +1,7 @@ +@charset "UTF-8";/*! + * Bootstrap v5.1.3 (https://getbootstrap.com/) + * Copyright 2011-2021 The Bootstrap Authors + * Copyright 2011-2021 Twitter, Inc. + * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE) + */:root{--bs-blue:#0d6efd;--bs-indigo:#6610f2;--bs-purple:#6f42c1;--bs-pink:#d63384;--bs-red:#dc3545;--bs-orange:#fd7e14;--bs-yellow:#ffc107;--bs-green:#198754;--bs-teal:#20c997;--bs-cyan:#0dcaf0;--bs-white:#fff;--bs-gray:#6c757d;--bs-gray-dark:#343a40;--bs-gray-100:#f8f9fa;--bs-gray-200:#e9ecef;--bs-gray-300:#dee2e6;--bs-gray-400:#ced4da;--bs-gray-500:#adb5bd;--bs-gray-600:#6c757d;--bs-gray-700:#495057;--bs-gray-800:#343a40;--bs-gray-900:#212529;--bs-primary:#0d6efd;--bs-secondary:#6c757d;--bs-success:#198754;--bs-info:#0dcaf0;--bs-warning:#ffc107;--bs-danger:#dc3545;--bs-light:#f8f9fa;--bs-dark:#212529;--bs-primary-rgb:13,110,253;--bs-secondary-rgb:108,117,125;--bs-success-rgb:25,135,84;--bs-info-rgb:13,202,240;--bs-warning-rgb:255,193,7;--bs-danger-rgb:220,53,69;--bs-light-rgb:248,249,250;--bs-dark-rgb:33,37,41;--bs-white-rgb:255,255,255;--bs-black-rgb:0,0,0;--bs-body-color-rgb:33,37,41;--bs-body-bg-rgb:255,255,255;--bs-font-sans-serif:system-ui,-apple-system,"Segoe UI",Roboto,"Helvetica Neue",Arial,"Noto Sans","Liberation Sans",sans-serif,"Apple Color Emoji","Segoe UI Emoji","Segoe UI Symbol","Noto Color Emoji";--bs-font-monospace:SFMono-Regular,Menlo,Monaco,Consolas,"Liberation Mono","Courier New",monospace;--bs-gradient:linear-gradient(180deg, rgba(255, 255, 255, 0.15), rgba(255, 255, 255, 0));--bs-body-font-family:var(--bs-font-sans-serif);--bs-body-font-size:1rem;--bs-body-font-weight:400;--bs-body-line-height:1.5;--bs-body-color:#212529;--bs-body-bg:#fff}*,::after,::before{box-sizing:border-box}@media (prefers-reduced-motion:no-preference){:root{scroll-behavior:smooth}}body{margin:0;font-family:var(--bs-body-font-family);font-size:var(--bs-body-font-size);font-weight:var(--bs-body-font-weight);line-height:var(--bs-body-line-height);color:var(--bs-body-color);text-align:var(--bs-body-text-align);background-color:var(--bs-body-bg);-webkit-text-size-adjust:100%;-webkit-tap-highlight-color:transparent}hr{margin:1rem 0;color:inherit;background-color:currentColor;border:0;opacity:.25}hr:not([size]){height:1px}.h1,.h2,.h3,.h4,.h5,.h6,h1,h2,h3,h4,h5,h6{margin-top:0;margin-bottom:.5rem;font-weight:500;line-height:1.2}.h1,h1{font-size:calc(1.375rem + 1.5vw)}@media (min-width:1200px){.h1,h1{font-size:2.5rem}}.h2,h2{font-size:calc(1.325rem + .9vw)}@media (min-width:1200px){.h2,h2{font-size:2rem}}.h3,h3{font-size:calc(1.3rem + .6vw)}@media (min-width:1200px){.h3,h3{font-size:1.75rem}}.h4,h4{font-size:calc(1.275rem + .3vw)}@media (min-width:1200px){.h4,h4{font-size:1.5rem}}.h5,h5{font-size:1.25rem}.h6,h6{font-size:1rem}p{margin-top:0;margin-bottom:1rem}abbr[data-bs-original-title],abbr[title]{-webkit-text-decoration:underline dotted;text-decoration:underline dotted;cursor:help;-webkit-text-decoration-skip-ink:none;text-decoration-skip-ink:none}address{margin-bottom:1rem;font-style:normal;line-height:inherit}ol,ul{padding-left:2rem}dl,ol,ul{margin-top:0;margin-bottom:1rem}ol ol,ol ul,ul ol,ul ul{margin-bottom:0}dt{font-weight:700}dd{margin-bottom:.5rem;margin-left:0}blockquote{margin:0 0 1rem}b,strong{font-weight:bolder}.small,small{font-size:.875em}.mark,mark{padding:.2em;background-color:#fcf8e3}sub,sup{position:relative;font-size:.75em;line-height:0;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}a{color:#0d6efd;text-decoration:underline}a:hover{color:#0a58ca}a:not([href]):not([class]),a:not([href]):not([class]):hover{color:inherit;text-decoration:none}code,kbd,pre,samp{font-family:var(--bs-font-monospace);font-size:1em;direction:ltr;unicode-bidi:bidi-override}pre{display:block;margin-top:0;margin-bottom:1rem;overflow:auto;font-size:.875em}pre code{font-size:inherit;color:inherit;word-break:normal}code{font-size:.875em;color:#d63384;word-wrap:break-word}a>code{color:inherit}kbd{padding:.2rem .4rem;font-size:.875em;color:#fff;background-color:#212529;border-radius:.2rem}kbd kbd{padding:0;font-size:1em;font-weight:700}figure{margin:0 0 1rem}img,svg{vertical-align:middle}table{caption-side:bottom;border-collapse:collapse}caption{padding-top:.5rem;padding-bottom:.5rem;color:#6c757d;text-align:left}th{text-align:inherit;text-align:-webkit-match-parent}tbody,td,tfoot,th,thead,tr{border-color:inherit;border-style:solid;border-width:0}label{display:inline-block}button{border-radius:0}button:focus:not(:focus-visible){outline:0}button,input,optgroup,select,textarea{margin:0;font-family:inherit;font-size:inherit;line-height:inherit}button,select{text-transform:none}[role=button]{cursor:pointer}select{word-wrap:normal}select:disabled{opacity:1}[list]::-webkit-calendar-picker-indicator{display:none}[type=button],[type=reset],[type=submit],button{-webkit-appearance:button}[type=button]:not(:disabled),[type=reset]:not(:disabled),[type=submit]:not(:disabled),button:not(:disabled){cursor:pointer}::-moz-focus-inner{padding:0;border-style:none}textarea{resize:vertical}fieldset{min-width:0;padding:0;margin:0;border:0}legend{float:left;width:100%;padding:0;margin-bottom:.5rem;font-size:calc(1.275rem + .3vw);line-height:inherit}@media (min-width:1200px){legend{font-size:1.5rem}}legend+*{clear:left}::-webkit-datetime-edit-day-field,::-webkit-datetime-edit-fields-wrapper,::-webkit-datetime-edit-hour-field,::-webkit-datetime-edit-minute,::-webkit-datetime-edit-month-field,::-webkit-datetime-edit-text,::-webkit-datetime-edit-year-field{padding:0}::-webkit-inner-spin-button{height:auto}[type=search]{outline-offset:-2px;-webkit-appearance:textfield}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-color-swatch-wrapper{padding:0}::-webkit-file-upload-button{font:inherit}::file-selector-button{font:inherit}::-webkit-file-upload-button{font:inherit;-webkit-appearance:button}output{display:inline-block}iframe{border:0}summary{display:list-item;cursor:pointer}progress{vertical-align:baseline}[hidden]{display:none!important}.lead{font-size:1.25rem;font-weight:300}.display-1{font-size:calc(1.625rem + 4.5vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-1{font-size:5rem}}.display-2{font-size:calc(1.575rem + 3.9vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-2{font-size:4.5rem}}.display-3{font-size:calc(1.525rem + 3.3vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-3{font-size:4rem}}.display-4{font-size:calc(1.475rem + 2.7vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-4{font-size:3.5rem}}.display-5{font-size:calc(1.425rem + 2.1vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-5{font-size:3rem}}.display-6{font-size:calc(1.375rem + 1.5vw);font-weight:300;line-height:1.2}@media (min-width:1200px){.display-6{font-size:2.5rem}}.list-unstyled{padding-left:0;list-style:none}.list-inline{padding-left:0;list-style:none}.list-inline-item{display:inline-block}.list-inline-item:not(:last-child){margin-right:.5rem}.initialism{font-size:.875em;text-transform:uppercase}.blockquote{margin-bottom:1rem;font-size:1.25rem}.blockquote>:last-child{margin-bottom:0}.blockquote-footer{margin-top:-1rem;margin-bottom:1rem;font-size:.875em;color:#6c757d}.blockquote-footer::before{content:"— "}.img-fluid{max-width:100%;height:auto}.img-thumbnail{padding:.25rem;background-color:#fff;border:1px solid #dee2e6;border-radius:.25rem;max-width:100%;height:auto}.figure{display:inline-block}.figure-img{margin-bottom:.5rem;line-height:1}.figure-caption{font-size:.875em;color:#6c757d}.container,.container-fluid,.container-lg,.container-md,.container-sm,.container-xl,.container-xxl{width:100%;padding-right:var(--bs-gutter-x,.75rem);padding-left:var(--bs-gutter-x,.75rem);margin-right:auto;margin-left:auto}@media (min-width:576px){.container,.container-sm{max-width:540px}}@media (min-width:768px){.container,.container-md,.container-sm{max-width:720px}}@media (min-width:992px){.container,.container-lg,.container-md,.container-sm{max-width:960px}}@media (min-width:1200px){.container,.container-lg,.container-md,.container-sm,.container-xl{max-width:1140px}}@media (min-width:1400px){.container,.container-lg,.container-md,.container-sm,.container-xl,.container-xxl{max-width:1320px}}.row{--bs-gutter-x:1.5rem;--bs-gutter-y:0;display:flex;flex-wrap:wrap;margin-top:calc(-1 * var(--bs-gutter-y));margin-right:calc(-.5 * var(--bs-gutter-x));margin-left:calc(-.5 * var(--bs-gutter-x))}.row>*{flex-shrink:0;width:100%;max-width:100%;padding-right:calc(var(--bs-gutter-x) * .5);padding-left:calc(var(--bs-gutter-x) * .5);margin-top:var(--bs-gutter-y)}.col{flex:1 0 0%}.row-cols-auto>*{flex:0 0 auto;width:auto}.row-cols-1>*{flex:0 0 auto;width:100%}.row-cols-2>*{flex:0 0 auto;width:50%}.row-cols-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-4>*{flex:0 0 auto;width:25%}.row-cols-5>*{flex:0 0 auto;width:20%}.row-cols-6>*{flex:0 0 auto;width:16.6666666667%}.col-auto{flex:0 0 auto;width:auto}.col-1{flex:0 0 auto;width:8.33333333%}.col-2{flex:0 0 auto;width:16.66666667%}.col-3{flex:0 0 auto;width:25%}.col-4{flex:0 0 auto;width:33.33333333%}.col-5{flex:0 0 auto;width:41.66666667%}.col-6{flex:0 0 auto;width:50%}.col-7{flex:0 0 auto;width:58.33333333%}.col-8{flex:0 0 auto;width:66.66666667%}.col-9{flex:0 0 auto;width:75%}.col-10{flex:0 0 auto;width:83.33333333%}.col-11{flex:0 0 auto;width:91.66666667%}.col-12{flex:0 0 auto;width:100%}.offset-1{margin-left:8.33333333%}.offset-2{margin-left:16.66666667%}.offset-3{margin-left:25%}.offset-4{margin-left:33.33333333%}.offset-5{margin-left:41.66666667%}.offset-6{margin-left:50%}.offset-7{margin-left:58.33333333%}.offset-8{margin-left:66.66666667%}.offset-9{margin-left:75%}.offset-10{margin-left:83.33333333%}.offset-11{margin-left:91.66666667%}.g-0,.gx-0{--bs-gutter-x:0}.g-0,.gy-0{--bs-gutter-y:0}.g-1,.gx-1{--bs-gutter-x:0.25rem}.g-1,.gy-1{--bs-gutter-y:0.25rem}.g-2,.gx-2{--bs-gutter-x:0.5rem}.g-2,.gy-2{--bs-gutter-y:0.5rem}.g-3,.gx-3{--bs-gutter-x:1rem}.g-3,.gy-3{--bs-gutter-y:1rem}.g-4,.gx-4{--bs-gutter-x:1.5rem}.g-4,.gy-4{--bs-gutter-y:1.5rem}.g-5,.gx-5{--bs-gutter-x:3rem}.g-5,.gy-5{--bs-gutter-y:3rem}@media (min-width:576px){.col-sm{flex:1 0 0%}.row-cols-sm-auto>*{flex:0 0 auto;width:auto}.row-cols-sm-1>*{flex:0 0 auto;width:100%}.row-cols-sm-2>*{flex:0 0 auto;width:50%}.row-cols-sm-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-sm-4>*{flex:0 0 auto;width:25%}.row-cols-sm-5>*{flex:0 0 auto;width:20%}.row-cols-sm-6>*{flex:0 0 auto;width:16.6666666667%}.col-sm-auto{flex:0 0 auto;width:auto}.col-sm-1{flex:0 0 auto;width:8.33333333%}.col-sm-2{flex:0 0 auto;width:16.66666667%}.col-sm-3{flex:0 0 auto;width:25%}.col-sm-4{flex:0 0 auto;width:33.33333333%}.col-sm-5{flex:0 0 auto;width:41.66666667%}.col-sm-6{flex:0 0 auto;width:50%}.col-sm-7{flex:0 0 auto;width:58.33333333%}.col-sm-8{flex:0 0 auto;width:66.66666667%}.col-sm-9{flex:0 0 auto;width:75%}.col-sm-10{flex:0 0 auto;width:83.33333333%}.col-sm-11{flex:0 0 auto;width:91.66666667%}.col-sm-12{flex:0 0 auto;width:100%}.offset-sm-0{margin-left:0}.offset-sm-1{margin-left:8.33333333%}.offset-sm-2{margin-left:16.66666667%}.offset-sm-3{margin-left:25%}.offset-sm-4{margin-left:33.33333333%}.offset-sm-5{margin-left:41.66666667%}.offset-sm-6{margin-left:50%}.offset-sm-7{margin-left:58.33333333%}.offset-sm-8{margin-left:66.66666667%}.offset-sm-9{margin-left:75%}.offset-sm-10{margin-left:83.33333333%}.offset-sm-11{margin-left:91.66666667%}.g-sm-0,.gx-sm-0{--bs-gutter-x:0}.g-sm-0,.gy-sm-0{--bs-gutter-y:0}.g-sm-1,.gx-sm-1{--bs-gutter-x:0.25rem}.g-sm-1,.gy-sm-1{--bs-gutter-y:0.25rem}.g-sm-2,.gx-sm-2{--bs-gutter-x:0.5rem}.g-sm-2,.gy-sm-2{--bs-gutter-y:0.5rem}.g-sm-3,.gx-sm-3{--bs-gutter-x:1rem}.g-sm-3,.gy-sm-3{--bs-gutter-y:1rem}.g-sm-4,.gx-sm-4{--bs-gutter-x:1.5rem}.g-sm-4,.gy-sm-4{--bs-gutter-y:1.5rem}.g-sm-5,.gx-sm-5{--bs-gutter-x:3rem}.g-sm-5,.gy-sm-5{--bs-gutter-y:3rem}}@media (min-width:768px){.col-md{flex:1 0 0%}.row-cols-md-auto>*{flex:0 0 auto;width:auto}.row-cols-md-1>*{flex:0 0 auto;width:100%}.row-cols-md-2>*{flex:0 0 auto;width:50%}.row-cols-md-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-md-4>*{flex:0 0 auto;width:25%}.row-cols-md-5>*{flex:0 0 auto;width:20%}.row-cols-md-6>*{flex:0 0 auto;width:16.6666666667%}.col-md-auto{flex:0 0 auto;width:auto}.col-md-1{flex:0 0 auto;width:8.33333333%}.col-md-2{flex:0 0 auto;width:16.66666667%}.col-md-3{flex:0 0 auto;width:25%}.col-md-4{flex:0 0 auto;width:33.33333333%}.col-md-5{flex:0 0 auto;width:41.66666667%}.col-md-6{flex:0 0 auto;width:50%}.col-md-7{flex:0 0 auto;width:58.33333333%}.col-md-8{flex:0 0 auto;width:66.66666667%}.col-md-9{flex:0 0 auto;width:75%}.col-md-10{flex:0 0 auto;width:83.33333333%}.col-md-11{flex:0 0 auto;width:91.66666667%}.col-md-12{flex:0 0 auto;width:100%}.offset-md-0{margin-left:0}.offset-md-1{margin-left:8.33333333%}.offset-md-2{margin-left:16.66666667%}.offset-md-3{margin-left:25%}.offset-md-4{margin-left:33.33333333%}.offset-md-5{margin-left:41.66666667%}.offset-md-6{margin-left:50%}.offset-md-7{margin-left:58.33333333%}.offset-md-8{margin-left:66.66666667%}.offset-md-9{margin-left:75%}.offset-md-10{margin-left:83.33333333%}.offset-md-11{margin-left:91.66666667%}.g-md-0,.gx-md-0{--bs-gutter-x:0}.g-md-0,.gy-md-0{--bs-gutter-y:0}.g-md-1,.gx-md-1{--bs-gutter-x:0.25rem}.g-md-1,.gy-md-1{--bs-gutter-y:0.25rem}.g-md-2,.gx-md-2{--bs-gutter-x:0.5rem}.g-md-2,.gy-md-2{--bs-gutter-y:0.5rem}.g-md-3,.gx-md-3{--bs-gutter-x:1rem}.g-md-3,.gy-md-3{--bs-gutter-y:1rem}.g-md-4,.gx-md-4{--bs-gutter-x:1.5rem}.g-md-4,.gy-md-4{--bs-gutter-y:1.5rem}.g-md-5,.gx-md-5{--bs-gutter-x:3rem}.g-md-5,.gy-md-5{--bs-gutter-y:3rem}}@media (min-width:992px){.col-lg{flex:1 0 0%}.row-cols-lg-auto>*{flex:0 0 auto;width:auto}.row-cols-lg-1>*{flex:0 0 auto;width:100%}.row-cols-lg-2>*{flex:0 0 auto;width:50%}.row-cols-lg-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-lg-4>*{flex:0 0 auto;width:25%}.row-cols-lg-5>*{flex:0 0 auto;width:20%}.row-cols-lg-6>*{flex:0 0 auto;width:16.6666666667%}.col-lg-auto{flex:0 0 auto;width:auto}.col-lg-1{flex:0 0 auto;width:8.33333333%}.col-lg-2{flex:0 0 auto;width:16.66666667%}.col-lg-3{flex:0 0 auto;width:25%}.col-lg-4{flex:0 0 auto;width:33.33333333%}.col-lg-5{flex:0 0 auto;width:41.66666667%}.col-lg-6{flex:0 0 auto;width:50%}.col-lg-7{flex:0 0 auto;width:58.33333333%}.col-lg-8{flex:0 0 auto;width:66.66666667%}.col-lg-9{flex:0 0 auto;width:75%}.col-lg-10{flex:0 0 auto;width:83.33333333%}.col-lg-11{flex:0 0 auto;width:91.66666667%}.col-lg-12{flex:0 0 auto;width:100%}.offset-lg-0{margin-left:0}.offset-lg-1{margin-left:8.33333333%}.offset-lg-2{margin-left:16.66666667%}.offset-lg-3{margin-left:25%}.offset-lg-4{margin-left:33.33333333%}.offset-lg-5{margin-left:41.66666667%}.offset-lg-6{margin-left:50%}.offset-lg-7{margin-left:58.33333333%}.offset-lg-8{margin-left:66.66666667%}.offset-lg-9{margin-left:75%}.offset-lg-10{margin-left:83.33333333%}.offset-lg-11{margin-left:91.66666667%}.g-lg-0,.gx-lg-0{--bs-gutter-x:0}.g-lg-0,.gy-lg-0{--bs-gutter-y:0}.g-lg-1,.gx-lg-1{--bs-gutter-x:0.25rem}.g-lg-1,.gy-lg-1{--bs-gutter-y:0.25rem}.g-lg-2,.gx-lg-2{--bs-gutter-x:0.5rem}.g-lg-2,.gy-lg-2{--bs-gutter-y:0.5rem}.g-lg-3,.gx-lg-3{--bs-gutter-x:1rem}.g-lg-3,.gy-lg-3{--bs-gutter-y:1rem}.g-lg-4,.gx-lg-4{--bs-gutter-x:1.5rem}.g-lg-4,.gy-lg-4{--bs-gutter-y:1.5rem}.g-lg-5,.gx-lg-5{--bs-gutter-x:3rem}.g-lg-5,.gy-lg-5{--bs-gutter-y:3rem}}@media (min-width:1200px){.col-xl{flex:1 0 0%}.row-cols-xl-auto>*{flex:0 0 auto;width:auto}.row-cols-xl-1>*{flex:0 0 auto;width:100%}.row-cols-xl-2>*{flex:0 0 auto;width:50%}.row-cols-xl-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-xl-4>*{flex:0 0 auto;width:25%}.row-cols-xl-5>*{flex:0 0 auto;width:20%}.row-cols-xl-6>*{flex:0 0 auto;width:16.6666666667%}.col-xl-auto{flex:0 0 auto;width:auto}.col-xl-1{flex:0 0 auto;width:8.33333333%}.col-xl-2{flex:0 0 auto;width:16.66666667%}.col-xl-3{flex:0 0 auto;width:25%}.col-xl-4{flex:0 0 auto;width:33.33333333%}.col-xl-5{flex:0 0 auto;width:41.66666667%}.col-xl-6{flex:0 0 auto;width:50%}.col-xl-7{flex:0 0 auto;width:58.33333333%}.col-xl-8{flex:0 0 auto;width:66.66666667%}.col-xl-9{flex:0 0 auto;width:75%}.col-xl-10{flex:0 0 auto;width:83.33333333%}.col-xl-11{flex:0 0 auto;width:91.66666667%}.col-xl-12{flex:0 0 auto;width:100%}.offset-xl-0{margin-left:0}.offset-xl-1{margin-left:8.33333333%}.offset-xl-2{margin-left:16.66666667%}.offset-xl-3{margin-left:25%}.offset-xl-4{margin-left:33.33333333%}.offset-xl-5{margin-left:41.66666667%}.offset-xl-6{margin-left:50%}.offset-xl-7{margin-left:58.33333333%}.offset-xl-8{margin-left:66.66666667%}.offset-xl-9{margin-left:75%}.offset-xl-10{margin-left:83.33333333%}.offset-xl-11{margin-left:91.66666667%}.g-xl-0,.gx-xl-0{--bs-gutter-x:0}.g-xl-0,.gy-xl-0{--bs-gutter-y:0}.g-xl-1,.gx-xl-1{--bs-gutter-x:0.25rem}.g-xl-1,.gy-xl-1{--bs-gutter-y:0.25rem}.g-xl-2,.gx-xl-2{--bs-gutter-x:0.5rem}.g-xl-2,.gy-xl-2{--bs-gutter-y:0.5rem}.g-xl-3,.gx-xl-3{--bs-gutter-x:1rem}.g-xl-3,.gy-xl-3{--bs-gutter-y:1rem}.g-xl-4,.gx-xl-4{--bs-gutter-x:1.5rem}.g-xl-4,.gy-xl-4{--bs-gutter-y:1.5rem}.g-xl-5,.gx-xl-5{--bs-gutter-x:3rem}.g-xl-5,.gy-xl-5{--bs-gutter-y:3rem}}@media (min-width:1400px){.col-xxl{flex:1 0 0%}.row-cols-xxl-auto>*{flex:0 0 auto;width:auto}.row-cols-xxl-1>*{flex:0 0 auto;width:100%}.row-cols-xxl-2>*{flex:0 0 auto;width:50%}.row-cols-xxl-3>*{flex:0 0 auto;width:33.3333333333%}.row-cols-xxl-4>*{flex:0 0 auto;width:25%}.row-cols-xxl-5>*{flex:0 0 auto;width:20%}.row-cols-xxl-6>*{flex:0 0 auto;width:16.6666666667%}.col-xxl-auto{flex:0 0 auto;width:auto}.col-xxl-1{flex:0 0 auto;width:8.33333333%}.col-xxl-2{flex:0 0 auto;width:16.66666667%}.col-xxl-3{flex:0 0 auto;width:25%}.col-xxl-4{flex:0 0 auto;width:33.33333333%}.col-xxl-5{flex:0 0 auto;width:41.66666667%}.col-xxl-6{flex:0 0 auto;width:50%}.col-xxl-7{flex:0 0 auto;width:58.33333333%}.col-xxl-8{flex:0 0 auto;width:66.66666667%}.col-xxl-9{flex:0 0 auto;width:75%}.col-xxl-10{flex:0 0 auto;width:83.33333333%}.col-xxl-11{flex:0 0 auto;width:91.66666667%}.col-xxl-12{flex:0 0 auto;width:100%}.offset-xxl-0{margin-left:0}.offset-xxl-1{margin-left:8.33333333%}.offset-xxl-2{margin-left:16.66666667%}.offset-xxl-3{margin-left:25%}.offset-xxl-4{margin-left:33.33333333%}.offset-xxl-5{margin-left:41.66666667%}.offset-xxl-6{margin-left:50%}.offset-xxl-7{margin-left:58.33333333%}.offset-xxl-8{margin-left:66.66666667%}.offset-xxl-9{margin-left:75%}.offset-xxl-10{margin-left:83.33333333%}.offset-xxl-11{margin-left:91.66666667%}.g-xxl-0,.gx-xxl-0{--bs-gutter-x:0}.g-xxl-0,.gy-xxl-0{--bs-gutter-y:0}.g-xxl-1,.gx-xxl-1{--bs-gutter-x:0.25rem}.g-xxl-1,.gy-xxl-1{--bs-gutter-y:0.25rem}.g-xxl-2,.gx-xxl-2{--bs-gutter-x:0.5rem}.g-xxl-2,.gy-xxl-2{--bs-gutter-y:0.5rem}.g-xxl-3,.gx-xxl-3{--bs-gutter-x:1rem}.g-xxl-3,.gy-xxl-3{--bs-gutter-y:1rem}.g-xxl-4,.gx-xxl-4{--bs-gutter-x:1.5rem}.g-xxl-4,.gy-xxl-4{--bs-gutter-y:1.5rem}.g-xxl-5,.gx-xxl-5{--bs-gutter-x:3rem}.g-xxl-5,.gy-xxl-5{--bs-gutter-y:3rem}}.table{--bs-table-bg:transparent;--bs-table-accent-bg:transparent;--bs-table-striped-color:#212529;--bs-table-striped-bg:rgba(0, 0, 0, 0.05);--bs-table-active-color:#212529;--bs-table-active-bg:rgba(0, 0, 0, 0.1);--bs-table-hover-color:#212529;--bs-table-hover-bg:rgba(0, 0, 0, 0.075);width:100%;margin-bottom:1rem;color:#212529;vertical-align:top;border-color:#dee2e6}.table>:not(caption)>*>*{padding:.5rem .5rem;background-color:var(--bs-table-bg);border-bottom-width:1px;box-shadow:inset 0 0 0 9999px var(--bs-table-accent-bg)}.table>tbody{vertical-align:inherit}.table>thead{vertical-align:bottom}.table>:not(:first-child){border-top:2px solid currentColor}.caption-top{caption-side:top}.table-sm>:not(caption)>*>*{padding:.25rem .25rem}.table-bordered>:not(caption)>*{border-width:1px 0}.table-bordered>:not(caption)>*>*{border-width:0 1px}.table-borderless>:not(caption)>*>*{border-bottom-width:0}.table-borderless>:not(:first-child){border-top-width:0}.table-striped>tbody>tr:nth-of-type(odd)>*{--bs-table-accent-bg:var(--bs-table-striped-bg);color:var(--bs-table-striped-color)}.table-active{--bs-table-accent-bg:var(--bs-table-active-bg);color:var(--bs-table-active-color)}.table-hover>tbody>tr:hover>*{--bs-table-accent-bg:var(--bs-table-hover-bg);color:var(--bs-table-hover-color)}.table-primary{--bs-table-bg:#cfe2ff;--bs-table-striped-bg:#c5d7f2;--bs-table-striped-color:#000;--bs-table-active-bg:#bacbe6;--bs-table-active-color:#000;--bs-table-hover-bg:#bfd1ec;--bs-table-hover-color:#000;color:#000;border-color:#bacbe6}.table-secondary{--bs-table-bg:#e2e3e5;--bs-table-striped-bg:#d7d8da;--bs-table-striped-color:#000;--bs-table-active-bg:#cbccce;--bs-table-active-color:#000;--bs-table-hover-bg:#d1d2d4;--bs-table-hover-color:#000;color:#000;border-color:#cbccce}.table-success{--bs-table-bg:#d1e7dd;--bs-table-striped-bg:#c7dbd2;--bs-table-striped-color:#000;--bs-table-active-bg:#bcd0c7;--bs-table-active-color:#000;--bs-table-hover-bg:#c1d6cc;--bs-table-hover-color:#000;color:#000;border-color:#bcd0c7}.table-info{--bs-table-bg:#cff4fc;--bs-table-striped-bg:#c5e8ef;--bs-table-striped-color:#000;--bs-table-active-bg:#badce3;--bs-table-active-color:#000;--bs-table-hover-bg:#bfe2e9;--bs-table-hover-color:#000;color:#000;border-color:#badce3}.table-warning{--bs-table-bg:#fff3cd;--bs-table-striped-bg:#f2e7c3;--bs-table-striped-color:#000;--bs-table-active-bg:#e6dbb9;--bs-table-active-color:#000;--bs-table-hover-bg:#ece1be;--bs-table-hover-color:#000;color:#000;border-color:#e6dbb9}.table-danger{--bs-table-bg:#f8d7da;--bs-table-striped-bg:#eccccf;--bs-table-striped-color:#000;--bs-table-active-bg:#dfc2c4;--bs-table-active-color:#000;--bs-table-hover-bg:#e5c7ca;--bs-table-hover-color:#000;color:#000;border-color:#dfc2c4}.table-light{--bs-table-bg:#f8f9fa;--bs-table-striped-bg:#ecedee;--bs-table-striped-color:#000;--bs-table-active-bg:#dfe0e1;--bs-table-active-color:#000;--bs-table-hover-bg:#e5e6e7;--bs-table-hover-color:#000;color:#000;border-color:#dfe0e1}.table-dark{--bs-table-bg:#212529;--bs-table-striped-bg:#2c3034;--bs-table-striped-color:#fff;--bs-table-active-bg:#373b3e;--bs-table-active-color:#fff;--bs-table-hover-bg:#323539;--bs-table-hover-color:#fff;color:#fff;border-color:#373b3e}.table-responsive{overflow-x:auto;-webkit-overflow-scrolling:touch}@media (max-width:575.98px){.table-responsive-sm{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:767.98px){.table-responsive-md{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:991.98px){.table-responsive-lg{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:1199.98px){.table-responsive-xl{overflow-x:auto;-webkit-overflow-scrolling:touch}}@media (max-width:1399.98px){.table-responsive-xxl{overflow-x:auto;-webkit-overflow-scrolling:touch}}.form-label{margin-bottom:.5rem}.col-form-label{padding-top:calc(.375rem + 1px);padding-bottom:calc(.375rem + 1px);margin-bottom:0;font-size:inherit;line-height:1.5}.col-form-label-lg{padding-top:calc(.5rem + 1px);padding-bottom:calc(.5rem + 1px);font-size:1.25rem}.col-form-label-sm{padding-top:calc(.25rem + 1px);padding-bottom:calc(.25rem + 1px);font-size:.875rem}.form-text{margin-top:.25rem;font-size:.875em;color:#6c757d}.form-control{display:block;width:100%;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#212529;background-color:#fff;background-clip:padding-box;border:1px solid #ced4da;-webkit-appearance:none;-moz-appearance:none;appearance:none;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control{transition:none}}.form-control[type=file]{overflow:hidden}.form-control[type=file]:not(:disabled):not([readonly]){cursor:pointer}.form-control:focus{color:#212529;background-color:#fff;border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-control::-webkit-date-and-time-value{height:1.5em}.form-control::-moz-placeholder{color:#6c757d;opacity:1}.form-control::placeholder{color:#6c757d;opacity:1}.form-control:disabled,.form-control[readonly]{background-color:#e9ecef;opacity:1}.form-control::-webkit-file-upload-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:#212529;background-color:#e9ecef;pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:1px;border-radius:0;-webkit-transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}.form-control::file-selector-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:#212529;background-color:#e9ecef;pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:1px;border-radius:0;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control::-webkit-file-upload-button{-webkit-transition:none;transition:none}.form-control::file-selector-button{transition:none}}.form-control:hover:not(:disabled):not([readonly])::-webkit-file-upload-button{background-color:#dde0e3}.form-control:hover:not(:disabled):not([readonly])::file-selector-button{background-color:#dde0e3}.form-control::-webkit-file-upload-button{padding:.375rem .75rem;margin:-.375rem -.75rem;-webkit-margin-end:.75rem;margin-inline-end:.75rem;color:#212529;background-color:#e9ecef;pointer-events:none;border-color:inherit;border-style:solid;border-width:0;border-inline-end-width:1px;border-radius:0;-webkit-transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-control::-webkit-file-upload-button{-webkit-transition:none;transition:none}}.form-control:hover:not(:disabled):not([readonly])::-webkit-file-upload-button{background-color:#dde0e3}.form-control-plaintext{display:block;width:100%;padding:.375rem 0;margin-bottom:0;line-height:1.5;color:#212529;background-color:transparent;border:solid transparent;border-width:1px 0}.form-control-plaintext.form-control-lg,.form-control-plaintext.form-control-sm{padding-right:0;padding-left:0}.form-control-sm{min-height:calc(1.5em + .5rem + 2px);padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.form-control-sm::-webkit-file-upload-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-sm::file-selector-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-sm::-webkit-file-upload-button{padding:.25rem .5rem;margin:-.25rem -.5rem;-webkit-margin-end:.5rem;margin-inline-end:.5rem}.form-control-lg{min-height:calc(1.5em + 1rem + 2px);padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.form-control-lg::-webkit-file-upload-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}.form-control-lg::file-selector-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}.form-control-lg::-webkit-file-upload-button{padding:.5rem 1rem;margin:-.5rem -1rem;-webkit-margin-end:1rem;margin-inline-end:1rem}textarea.form-control{min-height:calc(1.5em + .75rem + 2px)}textarea.form-control-sm{min-height:calc(1.5em + .5rem + 2px)}textarea.form-control-lg{min-height:calc(1.5em + 1rem + 2px)}.form-control-color{width:3rem;height:auto;padding:.375rem}.form-control-color:not(:disabled):not([readonly]){cursor:pointer}.form-control-color::-moz-color-swatch{height:1.5em;border-radius:.25rem}.form-control-color::-webkit-color-swatch{height:1.5em;border-radius:.25rem}.form-select{display:block;width:100%;padding:.375rem 2.25rem .375rem .75rem;-moz-padding-start:calc(0.75rem - 3px);font-size:1rem;font-weight:400;line-height:1.5;color:#212529;background-color:#fff;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right .75rem center;background-size:16px 12px;border:1px solid #ced4da;border-radius:.25rem;transition:border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.form-select{transition:none}}.form-select:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-select[multiple],.form-select[size]:not([size="1"]){padding-right:.75rem;background-image:none}.form-select:disabled{background-color:#e9ecef}.form-select:-moz-focusring{color:transparent;text-shadow:0 0 0 #212529}.form-select-sm{padding-top:.25rem;padding-bottom:.25rem;padding-left:.5rem;font-size:.875rem;border-radius:.2rem}.form-select-lg{padding-top:.5rem;padding-bottom:.5rem;padding-left:1rem;font-size:1.25rem;border-radius:.3rem}.form-check{display:block;min-height:1.5rem;padding-left:1.5em;margin-bottom:.125rem}.form-check .form-check-input{float:left;margin-left:-1.5em}.form-check-input{width:1em;height:1em;margin-top:.25em;vertical-align:top;background-color:#fff;background-repeat:no-repeat;background-position:center;background-size:contain;border:1px solid rgba(0,0,0,.25);-webkit-appearance:none;-moz-appearance:none;appearance:none;-webkit-print-color-adjust:exact;color-adjust:exact}.form-check-input[type=checkbox]{border-radius:.25em}.form-check-input[type=radio]{border-radius:50%}.form-check-input:active{filter:brightness(90%)}.form-check-input:focus{border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.form-check-input:checked{background-color:#0d6efd;border-color:#0d6efd}.form-check-input:checked[type=checkbox]{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10l3 3l6-6'/%3e%3c/svg%3e")}.form-check-input:checked[type=radio]{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='2' fill='%23fff'/%3e%3c/svg%3e")}.form-check-input[type=checkbox]:indeterminate{background-color:#0d6efd;border-color:#0d6efd;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 20 20'%3e%3cpath fill='none' stroke='%23fff' stroke-linecap='round' stroke-linejoin='round' stroke-width='3' d='M6 10h8'/%3e%3c/svg%3e")}.form-check-input:disabled{pointer-events:none;filter:none;opacity:.5}.form-check-input:disabled~.form-check-label,.form-check-input[disabled]~.form-check-label{opacity:.5}.form-switch{padding-left:2.5em}.form-switch .form-check-input{width:2em;margin-left:-2.5em;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='rgba%280, 0, 0, 0.25%29'/%3e%3c/svg%3e");background-position:left center;border-radius:2em;transition:background-position .15s ease-in-out}@media (prefers-reduced-motion:reduce){.form-switch .form-check-input{transition:none}}.form-switch .form-check-input:focus{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%2386b7fe'/%3e%3c/svg%3e")}.form-switch .form-check-input:checked{background-position:right center;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='-4 -4 8 8'%3e%3ccircle r='3' fill='%23fff'/%3e%3c/svg%3e")}.form-check-inline{display:inline-block;margin-right:1rem}.btn-check{position:absolute;clip:rect(0,0,0,0);pointer-events:none}.btn-check:disabled+.btn,.btn-check[disabled]+.btn{pointer-events:none;filter:none;opacity:.65}.form-range{width:100%;height:1.5rem;padding:0;background-color:transparent;-webkit-appearance:none;-moz-appearance:none;appearance:none}.form-range:focus{outline:0}.form-range:focus::-webkit-slider-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(13,110,253,.25)}.form-range:focus::-moz-range-thumb{box-shadow:0 0 0 1px #fff,0 0 0 .25rem rgba(13,110,253,.25)}.form-range::-moz-focus-outer{border:0}.form-range::-webkit-slider-thumb{width:1rem;height:1rem;margin-top:-.25rem;background-color:#0d6efd;border:0;border-radius:1rem;-webkit-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-webkit-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.form-range::-webkit-slider-thumb{-webkit-transition:none;transition:none}}.form-range::-webkit-slider-thumb:active{background-color:#b6d4fe}.form-range::-webkit-slider-runnable-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.form-range::-moz-range-thumb{width:1rem;height:1rem;background-color:#0d6efd;border:0;border-radius:1rem;-moz-transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;transition:background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;-moz-appearance:none;appearance:none}@media (prefers-reduced-motion:reduce){.form-range::-moz-range-thumb{-moz-transition:none;transition:none}}.form-range::-moz-range-thumb:active{background-color:#b6d4fe}.form-range::-moz-range-track{width:100%;height:.5rem;color:transparent;cursor:pointer;background-color:#dee2e6;border-color:transparent;border-radius:1rem}.form-range:disabled{pointer-events:none}.form-range:disabled::-webkit-slider-thumb{background-color:#adb5bd}.form-range:disabled::-moz-range-thumb{background-color:#adb5bd}.form-floating{position:relative}.form-floating>.form-control,.form-floating>.form-select{height:calc(3.5rem + 2px);line-height:1.25}.form-floating>label{position:absolute;top:0;left:0;height:100%;padding:1rem .75rem;pointer-events:none;border:1px solid transparent;transform-origin:0 0;transition:opacity .1s ease-in-out,transform .1s ease-in-out}@media (prefers-reduced-motion:reduce){.form-floating>label{transition:none}}.form-floating>.form-control{padding:1rem .75rem}.form-floating>.form-control::-moz-placeholder{color:transparent}.form-floating>.form-control::placeholder{color:transparent}.form-floating>.form-control:not(:-moz-placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:focus,.form-floating>.form-control:not(:placeholder-shown){padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:-webkit-autofill{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-select{padding-top:1.625rem;padding-bottom:.625rem}.form-floating>.form-control:not(:-moz-placeholder-shown)~label{opacity:.65;transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control:focus~label,.form-floating>.form-control:not(:placeholder-shown)~label,.form-floating>.form-select~label{opacity:.65;transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.form-floating>.form-control:-webkit-autofill~label{opacity:.65;transform:scale(.85) translateY(-.5rem) translateX(.15rem)}.input-group{position:relative;display:flex;flex-wrap:wrap;align-items:stretch;width:100%}.input-group>.form-control,.input-group>.form-select{position:relative;flex:1 1 auto;width:1%;min-width:0}.input-group>.form-control:focus,.input-group>.form-select:focus{z-index:3}.input-group .btn{position:relative;z-index:2}.input-group .btn:focus{z-index:3}.input-group-text{display:flex;align-items:center;padding:.375rem .75rem;font-size:1rem;font-weight:400;line-height:1.5;color:#212529;text-align:center;white-space:nowrap;background-color:#e9ecef;border:1px solid #ced4da;border-radius:.25rem}.input-group-lg>.btn,.input-group-lg>.form-control,.input-group-lg>.form-select,.input-group-lg>.input-group-text{padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.input-group-sm>.btn,.input-group-sm>.form-control,.input-group-sm>.form-select,.input-group-sm>.input-group-text{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.input-group-lg>.form-select,.input-group-sm>.form-select{padding-right:3rem}.input-group:not(.has-validation)>.dropdown-toggle:nth-last-child(n+3),.input-group:not(.has-validation)>:not(:last-child):not(.dropdown-toggle):not(.dropdown-menu){border-top-right-radius:0;border-bottom-right-radius:0}.input-group.has-validation>.dropdown-toggle:nth-last-child(n+4),.input-group.has-validation>:nth-last-child(n+3):not(.dropdown-toggle):not(.dropdown-menu){border-top-right-radius:0;border-bottom-right-radius:0}.input-group>:not(:first-child):not(.dropdown-menu):not(.valid-tooltip):not(.valid-feedback):not(.invalid-tooltip):not(.invalid-feedback){margin-left:-1px;border-top-left-radius:0;border-bottom-left-radius:0}.valid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:#198754}.valid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:rgba(25,135,84,.9);border-radius:.25rem}.is-valid~.valid-feedback,.is-valid~.valid-tooltip,.was-validated :valid~.valid-feedback,.was-validated :valid~.valid-tooltip{display:block}.form-control.is-valid,.was-validated .form-control:valid{border-color:#198754;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-valid:focus,.was-validated .form-control:valid:focus{border-color:#198754;box-shadow:0 0 0 .25rem rgba(25,135,84,.25)}.was-validated textarea.form-control:valid,textarea.form-control.is-valid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.form-select.is-valid,.was-validated .form-select:valid{border-color:#198754}.form-select.is-valid:not([multiple]):not([size]),.form-select.is-valid:not([multiple])[size="1"],.was-validated .form-select:valid:not([multiple]):not([size]),.was-validated .form-select:valid:not([multiple])[size="1"]{padding-right:4.125rem;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e"),url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 8 8'%3e%3cpath fill='%23198754' d='M2.3 6.73L.6 4.53c-.4-1.04.46-1.4 1.1-.8l1.1 1.4 3.4-3.8c.6-.63 1.6-.27 1.2.7l-4 4.6c-.43.5-.8.4-1.1.1z'/%3e%3c/svg%3e");background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.form-select.is-valid:focus,.was-validated .form-select:valid:focus{border-color:#198754;box-shadow:0 0 0 .25rem rgba(25,135,84,.25)}.form-check-input.is-valid,.was-validated .form-check-input:valid{border-color:#198754}.form-check-input.is-valid:checked,.was-validated .form-check-input:valid:checked{background-color:#198754}.form-check-input.is-valid:focus,.was-validated .form-check-input:valid:focus{box-shadow:0 0 0 .25rem rgba(25,135,84,.25)}.form-check-input.is-valid~.form-check-label,.was-validated .form-check-input:valid~.form-check-label{color:#198754}.form-check-inline .form-check-input~.valid-feedback{margin-left:.5em}.input-group .form-control.is-valid,.input-group .form-select.is-valid,.was-validated .input-group .form-control:valid,.was-validated .input-group .form-select:valid{z-index:1}.input-group .form-control.is-valid:focus,.input-group .form-select.is-valid:focus,.was-validated .input-group .form-control:valid:focus,.was-validated .input-group .form-select:valid:focus{z-index:3}.invalid-feedback{display:none;width:100%;margin-top:.25rem;font-size:.875em;color:#dc3545}.invalid-tooltip{position:absolute;top:100%;z-index:5;display:none;max-width:100%;padding:.25rem .5rem;margin-top:.1rem;font-size:.875rem;color:#fff;background-color:rgba(220,53,69,.9);border-radius:.25rem}.is-invalid~.invalid-feedback,.is-invalid~.invalid-tooltip,.was-validated :invalid~.invalid-feedback,.was-validated :invalid~.invalid-tooltip{display:block}.form-control.is-invalid,.was-validated .form-control:invalid{border-color:#dc3545;padding-right:calc(1.5em + .75rem);background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-repeat:no-repeat;background-position:right calc(.375em + .1875rem) center;background-size:calc(.75em + .375rem) calc(.75em + .375rem)}.form-control.is-invalid:focus,.was-validated .form-control:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .25rem rgba(220,53,69,.25)}.was-validated textarea.form-control:invalid,textarea.form-control.is-invalid{padding-right:calc(1.5em + .75rem);background-position:top calc(.375em + .1875rem) right calc(.375em + .1875rem)}.form-select.is-invalid,.was-validated .form-select:invalid{border-color:#dc3545}.form-select.is-invalid:not([multiple]):not([size]),.form-select.is-invalid:not([multiple])[size="1"],.was-validated .form-select:invalid:not([multiple]):not([size]),.was-validated .form-select:invalid:not([multiple])[size="1"]{padding-right:4.125rem;background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e"),url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 12 12' width='12' height='12' fill='none' stroke='%23dc3545'%3e%3ccircle cx='6' cy='6' r='4.5'/%3e%3cpath stroke-linejoin='round' d='M5.8 3.6h.4L6 6.5z'/%3e%3ccircle cx='6' cy='8.2' r='.6' fill='%23dc3545' stroke='none'/%3e%3c/svg%3e");background-position:right .75rem center,center right 2.25rem;background-size:16px 12px,calc(.75em + .375rem) calc(.75em + .375rem)}.form-select.is-invalid:focus,.was-validated .form-select:invalid:focus{border-color:#dc3545;box-shadow:0 0 0 .25rem rgba(220,53,69,.25)}.form-check-input.is-invalid,.was-validated .form-check-input:invalid{border-color:#dc3545}.form-check-input.is-invalid:checked,.was-validated .form-check-input:invalid:checked{background-color:#dc3545}.form-check-input.is-invalid:focus,.was-validated .form-check-input:invalid:focus{box-shadow:0 0 0 .25rem rgba(220,53,69,.25)}.form-check-input.is-invalid~.form-check-label,.was-validated .form-check-input:invalid~.form-check-label{color:#dc3545}.form-check-inline .form-check-input~.invalid-feedback{margin-left:.5em}.input-group .form-control.is-invalid,.input-group .form-select.is-invalid,.was-validated .input-group .form-control:invalid,.was-validated .input-group .form-select:invalid{z-index:2}.input-group .form-control.is-invalid:focus,.input-group .form-select.is-invalid:focus,.was-validated .input-group .form-control:invalid:focus,.was-validated .input-group .form-select:invalid:focus{z-index:3}.btn{display:inline-block;font-weight:400;line-height:1.5;color:#212529;text-align:center;text-decoration:none;vertical-align:middle;cursor:pointer;-webkit-user-select:none;-moz-user-select:none;user-select:none;background-color:transparent;border:1px solid transparent;padding:.375rem .75rem;font-size:1rem;border-radius:.25rem;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.btn{transition:none}}.btn:hover{color:#212529}.btn-check:focus+.btn,.btn:focus{outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.btn.disabled,.btn:disabled,fieldset:disabled .btn{pointer-events:none;opacity:.65}.btn-primary{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-primary:hover{color:#fff;background-color:#0b5ed7;border-color:#0a58ca}.btn-check:focus+.btn-primary,.btn-primary:focus{color:#fff;background-color:#0b5ed7;border-color:#0a58ca;box-shadow:0 0 0 .25rem rgba(49,132,253,.5)}.btn-check:active+.btn-primary,.btn-check:checked+.btn-primary,.btn-primary.active,.btn-primary:active,.show>.btn-primary.dropdown-toggle{color:#fff;background-color:#0a58ca;border-color:#0a53be}.btn-check:active+.btn-primary:focus,.btn-check:checked+.btn-primary:focus,.btn-primary.active:focus,.btn-primary:active:focus,.show>.btn-primary.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(49,132,253,.5)}.btn-primary.disabled,.btn-primary:disabled{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-secondary{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-secondary:hover{color:#fff;background-color:#5c636a;border-color:#565e64}.btn-check:focus+.btn-secondary,.btn-secondary:focus{color:#fff;background-color:#5c636a;border-color:#565e64;box-shadow:0 0 0 .25rem rgba(130,138,145,.5)}.btn-check:active+.btn-secondary,.btn-check:checked+.btn-secondary,.btn-secondary.active,.btn-secondary:active,.show>.btn-secondary.dropdown-toggle{color:#fff;background-color:#565e64;border-color:#51585e}.btn-check:active+.btn-secondary:focus,.btn-check:checked+.btn-secondary:focus,.btn-secondary.active:focus,.btn-secondary:active:focus,.show>.btn-secondary.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(130,138,145,.5)}.btn-secondary.disabled,.btn-secondary:disabled{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-success{color:#fff;background-color:#198754;border-color:#198754}.btn-success:hover{color:#fff;background-color:#157347;border-color:#146c43}.btn-check:focus+.btn-success,.btn-success:focus{color:#fff;background-color:#157347;border-color:#146c43;box-shadow:0 0 0 .25rem rgba(60,153,110,.5)}.btn-check:active+.btn-success,.btn-check:checked+.btn-success,.btn-success.active,.btn-success:active,.show>.btn-success.dropdown-toggle{color:#fff;background-color:#146c43;border-color:#13653f}.btn-check:active+.btn-success:focus,.btn-check:checked+.btn-success:focus,.btn-success.active:focus,.btn-success:active:focus,.show>.btn-success.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(60,153,110,.5)}.btn-success.disabled,.btn-success:disabled{color:#fff;background-color:#198754;border-color:#198754}.btn-info{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-info:hover{color:#000;background-color:#31d2f2;border-color:#25cff2}.btn-check:focus+.btn-info,.btn-info:focus{color:#000;background-color:#31d2f2;border-color:#25cff2;box-shadow:0 0 0 .25rem rgba(11,172,204,.5)}.btn-check:active+.btn-info,.btn-check:checked+.btn-info,.btn-info.active,.btn-info:active,.show>.btn-info.dropdown-toggle{color:#000;background-color:#3dd5f3;border-color:#25cff2}.btn-check:active+.btn-info:focus,.btn-check:checked+.btn-info:focus,.btn-info.active:focus,.btn-info:active:focus,.show>.btn-info.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(11,172,204,.5)}.btn-info.disabled,.btn-info:disabled{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-warning{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-warning:hover{color:#000;background-color:#ffca2c;border-color:#ffc720}.btn-check:focus+.btn-warning,.btn-warning:focus{color:#000;background-color:#ffca2c;border-color:#ffc720;box-shadow:0 0 0 .25rem rgba(217,164,6,.5)}.btn-check:active+.btn-warning,.btn-check:checked+.btn-warning,.btn-warning.active,.btn-warning:active,.show>.btn-warning.dropdown-toggle{color:#000;background-color:#ffcd39;border-color:#ffc720}.btn-check:active+.btn-warning:focus,.btn-check:checked+.btn-warning:focus,.btn-warning.active:focus,.btn-warning:active:focus,.show>.btn-warning.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(217,164,6,.5)}.btn-warning.disabled,.btn-warning:disabled{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-danger{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-danger:hover{color:#fff;background-color:#bb2d3b;border-color:#b02a37}.btn-check:focus+.btn-danger,.btn-danger:focus{color:#fff;background-color:#bb2d3b;border-color:#b02a37;box-shadow:0 0 0 .25rem rgba(225,83,97,.5)}.btn-check:active+.btn-danger,.btn-check:checked+.btn-danger,.btn-danger.active,.btn-danger:active,.show>.btn-danger.dropdown-toggle{color:#fff;background-color:#b02a37;border-color:#a52834}.btn-check:active+.btn-danger:focus,.btn-check:checked+.btn-danger:focus,.btn-danger.active:focus,.btn-danger:active:focus,.show>.btn-danger.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(225,83,97,.5)}.btn-danger.disabled,.btn-danger:disabled{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-light{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-light:hover{color:#000;background-color:#f9fafb;border-color:#f9fafb}.btn-check:focus+.btn-light,.btn-light:focus{color:#000;background-color:#f9fafb;border-color:#f9fafb;box-shadow:0 0 0 .25rem rgba(211,212,213,.5)}.btn-check:active+.btn-light,.btn-check:checked+.btn-light,.btn-light.active,.btn-light:active,.show>.btn-light.dropdown-toggle{color:#000;background-color:#f9fafb;border-color:#f9fafb}.btn-check:active+.btn-light:focus,.btn-check:checked+.btn-light:focus,.btn-light.active:focus,.btn-light:active:focus,.show>.btn-light.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(211,212,213,.5)}.btn-light.disabled,.btn-light:disabled{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-dark{color:#fff;background-color:#212529;border-color:#212529}.btn-dark:hover{color:#fff;background-color:#1c1f23;border-color:#1a1e21}.btn-check:focus+.btn-dark,.btn-dark:focus{color:#fff;background-color:#1c1f23;border-color:#1a1e21;box-shadow:0 0 0 .25rem rgba(66,70,73,.5)}.btn-check:active+.btn-dark,.btn-check:checked+.btn-dark,.btn-dark.active,.btn-dark:active,.show>.btn-dark.dropdown-toggle{color:#fff;background-color:#1a1e21;border-color:#191c1f}.btn-check:active+.btn-dark:focus,.btn-check:checked+.btn-dark:focus,.btn-dark.active:focus,.btn-dark:active:focus,.show>.btn-dark.dropdown-toggle:focus{box-shadow:0 0 0 .25rem rgba(66,70,73,.5)}.btn-dark.disabled,.btn-dark:disabled{color:#fff;background-color:#212529;border-color:#212529}.btn-outline-primary{color:#0d6efd;border-color:#0d6efd}.btn-outline-primary:hover{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-check:focus+.btn-outline-primary,.btn-outline-primary:focus{box-shadow:0 0 0 .25rem rgba(13,110,253,.5)}.btn-check:active+.btn-outline-primary,.btn-check:checked+.btn-outline-primary,.btn-outline-primary.active,.btn-outline-primary.dropdown-toggle.show,.btn-outline-primary:active{color:#fff;background-color:#0d6efd;border-color:#0d6efd}.btn-check:active+.btn-outline-primary:focus,.btn-check:checked+.btn-outline-primary:focus,.btn-outline-primary.active:focus,.btn-outline-primary.dropdown-toggle.show:focus,.btn-outline-primary:active:focus{box-shadow:0 0 0 .25rem rgba(13,110,253,.5)}.btn-outline-primary.disabled,.btn-outline-primary:disabled{color:#0d6efd;background-color:transparent}.btn-outline-secondary{color:#6c757d;border-color:#6c757d}.btn-outline-secondary:hover{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-check:focus+.btn-outline-secondary,.btn-outline-secondary:focus{box-shadow:0 0 0 .25rem rgba(108,117,125,.5)}.btn-check:active+.btn-outline-secondary,.btn-check:checked+.btn-outline-secondary,.btn-outline-secondary.active,.btn-outline-secondary.dropdown-toggle.show,.btn-outline-secondary:active{color:#fff;background-color:#6c757d;border-color:#6c757d}.btn-check:active+.btn-outline-secondary:focus,.btn-check:checked+.btn-outline-secondary:focus,.btn-outline-secondary.active:focus,.btn-outline-secondary.dropdown-toggle.show:focus,.btn-outline-secondary:active:focus{box-shadow:0 0 0 .25rem rgba(108,117,125,.5)}.btn-outline-secondary.disabled,.btn-outline-secondary:disabled{color:#6c757d;background-color:transparent}.btn-outline-success{color:#198754;border-color:#198754}.btn-outline-success:hover{color:#fff;background-color:#198754;border-color:#198754}.btn-check:focus+.btn-outline-success,.btn-outline-success:focus{box-shadow:0 0 0 .25rem rgba(25,135,84,.5)}.btn-check:active+.btn-outline-success,.btn-check:checked+.btn-outline-success,.btn-outline-success.active,.btn-outline-success.dropdown-toggle.show,.btn-outline-success:active{color:#fff;background-color:#198754;border-color:#198754}.btn-check:active+.btn-outline-success:focus,.btn-check:checked+.btn-outline-success:focus,.btn-outline-success.active:focus,.btn-outline-success.dropdown-toggle.show:focus,.btn-outline-success:active:focus{box-shadow:0 0 0 .25rem rgba(25,135,84,.5)}.btn-outline-success.disabled,.btn-outline-success:disabled{color:#198754;background-color:transparent}.btn-outline-info{color:#0dcaf0;border-color:#0dcaf0}.btn-outline-info:hover{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-check:focus+.btn-outline-info,.btn-outline-info:focus{box-shadow:0 0 0 .25rem rgba(13,202,240,.5)}.btn-check:active+.btn-outline-info,.btn-check:checked+.btn-outline-info,.btn-outline-info.active,.btn-outline-info.dropdown-toggle.show,.btn-outline-info:active{color:#000;background-color:#0dcaf0;border-color:#0dcaf0}.btn-check:active+.btn-outline-info:focus,.btn-check:checked+.btn-outline-info:focus,.btn-outline-info.active:focus,.btn-outline-info.dropdown-toggle.show:focus,.btn-outline-info:active:focus{box-shadow:0 0 0 .25rem rgba(13,202,240,.5)}.btn-outline-info.disabled,.btn-outline-info:disabled{color:#0dcaf0;background-color:transparent}.btn-outline-warning{color:#ffc107;border-color:#ffc107}.btn-outline-warning:hover{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-check:focus+.btn-outline-warning,.btn-outline-warning:focus{box-shadow:0 0 0 .25rem rgba(255,193,7,.5)}.btn-check:active+.btn-outline-warning,.btn-check:checked+.btn-outline-warning,.btn-outline-warning.active,.btn-outline-warning.dropdown-toggle.show,.btn-outline-warning:active{color:#000;background-color:#ffc107;border-color:#ffc107}.btn-check:active+.btn-outline-warning:focus,.btn-check:checked+.btn-outline-warning:focus,.btn-outline-warning.active:focus,.btn-outline-warning.dropdown-toggle.show:focus,.btn-outline-warning:active:focus{box-shadow:0 0 0 .25rem rgba(255,193,7,.5)}.btn-outline-warning.disabled,.btn-outline-warning:disabled{color:#ffc107;background-color:transparent}.btn-outline-danger{color:#dc3545;border-color:#dc3545}.btn-outline-danger:hover{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-check:focus+.btn-outline-danger,.btn-outline-danger:focus{box-shadow:0 0 0 .25rem rgba(220,53,69,.5)}.btn-check:active+.btn-outline-danger,.btn-check:checked+.btn-outline-danger,.btn-outline-danger.active,.btn-outline-danger.dropdown-toggle.show,.btn-outline-danger:active{color:#fff;background-color:#dc3545;border-color:#dc3545}.btn-check:active+.btn-outline-danger:focus,.btn-check:checked+.btn-outline-danger:focus,.btn-outline-danger.active:focus,.btn-outline-danger.dropdown-toggle.show:focus,.btn-outline-danger:active:focus{box-shadow:0 0 0 .25rem rgba(220,53,69,.5)}.btn-outline-danger.disabled,.btn-outline-danger:disabled{color:#dc3545;background-color:transparent}.btn-outline-light{color:#f8f9fa;border-color:#f8f9fa}.btn-outline-light:hover{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-check:focus+.btn-outline-light,.btn-outline-light:focus{box-shadow:0 0 0 .25rem rgba(248,249,250,.5)}.btn-check:active+.btn-outline-light,.btn-check:checked+.btn-outline-light,.btn-outline-light.active,.btn-outline-light.dropdown-toggle.show,.btn-outline-light:active{color:#000;background-color:#f8f9fa;border-color:#f8f9fa}.btn-check:active+.btn-outline-light:focus,.btn-check:checked+.btn-outline-light:focus,.btn-outline-light.active:focus,.btn-outline-light.dropdown-toggle.show:focus,.btn-outline-light:active:focus{box-shadow:0 0 0 .25rem rgba(248,249,250,.5)}.btn-outline-light.disabled,.btn-outline-light:disabled{color:#f8f9fa;background-color:transparent}.btn-outline-dark{color:#212529;border-color:#212529}.btn-outline-dark:hover{color:#fff;background-color:#212529;border-color:#212529}.btn-check:focus+.btn-outline-dark,.btn-outline-dark:focus{box-shadow:0 0 0 .25rem rgba(33,37,41,.5)}.btn-check:active+.btn-outline-dark,.btn-check:checked+.btn-outline-dark,.btn-outline-dark.active,.btn-outline-dark.dropdown-toggle.show,.btn-outline-dark:active{color:#fff;background-color:#212529;border-color:#212529}.btn-check:active+.btn-outline-dark:focus,.btn-check:checked+.btn-outline-dark:focus,.btn-outline-dark.active:focus,.btn-outline-dark.dropdown-toggle.show:focus,.btn-outline-dark:active:focus{box-shadow:0 0 0 .25rem rgba(33,37,41,.5)}.btn-outline-dark.disabled,.btn-outline-dark:disabled{color:#212529;background-color:transparent}.btn-link{font-weight:400;color:#0d6efd;text-decoration:underline}.btn-link:hover{color:#0a58ca}.btn-link.disabled,.btn-link:disabled{color:#6c757d}.btn-group-lg>.btn,.btn-lg{padding:.5rem 1rem;font-size:1.25rem;border-radius:.3rem}.btn-group-sm>.btn,.btn-sm{padding:.25rem .5rem;font-size:.875rem;border-radius:.2rem}.fade{transition:opacity .15s linear}@media (prefers-reduced-motion:reduce){.fade{transition:none}}.fade:not(.show){opacity:0}.collapse:not(.show){display:none}.collapsing{height:0;overflow:hidden;transition:height .35s ease}@media (prefers-reduced-motion:reduce){.collapsing{transition:none}}.collapsing.collapse-horizontal{width:0;height:auto;transition:width .35s ease}@media (prefers-reduced-motion:reduce){.collapsing.collapse-horizontal{transition:none}}.dropdown,.dropend,.dropstart,.dropup{position:relative}.dropdown-toggle{white-space:nowrap}.dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid;border-right:.3em solid transparent;border-bottom:0;border-left:.3em solid transparent}.dropdown-toggle:empty::after{margin-left:0}.dropdown-menu{position:absolute;z-index:1000;display:none;min-width:10rem;padding:.5rem 0;margin:0;font-size:1rem;color:#212529;text-align:left;list-style:none;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.15);border-radius:.25rem}.dropdown-menu[data-bs-popper]{top:100%;left:0;margin-top:.125rem}.dropdown-menu-start{--bs-position:start}.dropdown-menu-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-end{--bs-position:end}.dropdown-menu-end[data-bs-popper]{right:0;left:auto}@media (min-width:576px){.dropdown-menu-sm-start{--bs-position:start}.dropdown-menu-sm-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-sm-end{--bs-position:end}.dropdown-menu-sm-end[data-bs-popper]{right:0;left:auto}}@media (min-width:768px){.dropdown-menu-md-start{--bs-position:start}.dropdown-menu-md-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-md-end{--bs-position:end}.dropdown-menu-md-end[data-bs-popper]{right:0;left:auto}}@media (min-width:992px){.dropdown-menu-lg-start{--bs-position:start}.dropdown-menu-lg-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-lg-end{--bs-position:end}.dropdown-menu-lg-end[data-bs-popper]{right:0;left:auto}}@media (min-width:1200px){.dropdown-menu-xl-start{--bs-position:start}.dropdown-menu-xl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xl-end{--bs-position:end}.dropdown-menu-xl-end[data-bs-popper]{right:0;left:auto}}@media (min-width:1400px){.dropdown-menu-xxl-start{--bs-position:start}.dropdown-menu-xxl-start[data-bs-popper]{right:auto;left:0}.dropdown-menu-xxl-end{--bs-position:end}.dropdown-menu-xxl-end[data-bs-popper]{right:0;left:auto}}.dropup .dropdown-menu[data-bs-popper]{top:auto;bottom:100%;margin-top:0;margin-bottom:.125rem}.dropup .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:0;border-right:.3em solid transparent;border-bottom:.3em solid;border-left:.3em solid transparent}.dropup .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-menu[data-bs-popper]{top:0;right:auto;left:100%;margin-top:0;margin-left:.125rem}.dropend .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:0;border-bottom:.3em solid transparent;border-left:.3em solid}.dropend .dropdown-toggle:empty::after{margin-left:0}.dropend .dropdown-toggle::after{vertical-align:0}.dropstart .dropdown-menu[data-bs-popper]{top:0;right:100%;left:auto;margin-top:0;margin-right:.125rem}.dropstart .dropdown-toggle::after{display:inline-block;margin-left:.255em;vertical-align:.255em;content:""}.dropstart .dropdown-toggle::after{display:none}.dropstart .dropdown-toggle::before{display:inline-block;margin-right:.255em;vertical-align:.255em;content:"";border-top:.3em solid transparent;border-right:.3em solid;border-bottom:.3em solid transparent}.dropstart .dropdown-toggle:empty::after{margin-left:0}.dropstart .dropdown-toggle::before{vertical-align:0}.dropdown-divider{height:0;margin:.5rem 0;overflow:hidden;border-top:1px solid rgba(0,0,0,.15)}.dropdown-item{display:block;width:100%;padding:.25rem 1rem;clear:both;font-weight:400;color:#212529;text-align:inherit;text-decoration:none;white-space:nowrap;background-color:transparent;border:0}.dropdown-item:focus,.dropdown-item:hover{color:#1e2125;background-color:#e9ecef}.dropdown-item.active,.dropdown-item:active{color:#fff;text-decoration:none;background-color:#0d6efd}.dropdown-item.disabled,.dropdown-item:disabled{color:#adb5bd;pointer-events:none;background-color:transparent}.dropdown-menu.show{display:block}.dropdown-header{display:block;padding:.5rem 1rem;margin-bottom:0;font-size:.875rem;color:#6c757d;white-space:nowrap}.dropdown-item-text{display:block;padding:.25rem 1rem;color:#212529}.dropdown-menu-dark{color:#dee2e6;background-color:#343a40;border-color:rgba(0,0,0,.15)}.dropdown-menu-dark .dropdown-item{color:#dee2e6}.dropdown-menu-dark .dropdown-item:focus,.dropdown-menu-dark .dropdown-item:hover{color:#fff;background-color:rgba(255,255,255,.15)}.dropdown-menu-dark .dropdown-item.active,.dropdown-menu-dark .dropdown-item:active{color:#fff;background-color:#0d6efd}.dropdown-menu-dark .dropdown-item.disabled,.dropdown-menu-dark .dropdown-item:disabled{color:#adb5bd}.dropdown-menu-dark .dropdown-divider{border-color:rgba(0,0,0,.15)}.dropdown-menu-dark .dropdown-item-text{color:#dee2e6}.dropdown-menu-dark .dropdown-header{color:#adb5bd}.btn-group,.btn-group-vertical{position:relative;display:inline-flex;vertical-align:middle}.btn-group-vertical>.btn,.btn-group>.btn{position:relative;flex:1 1 auto}.btn-group-vertical>.btn-check:checked+.btn,.btn-group-vertical>.btn-check:focus+.btn,.btn-group-vertical>.btn.active,.btn-group-vertical>.btn:active,.btn-group-vertical>.btn:focus,.btn-group-vertical>.btn:hover,.btn-group>.btn-check:checked+.btn,.btn-group>.btn-check:focus+.btn,.btn-group>.btn.active,.btn-group>.btn:active,.btn-group>.btn:focus,.btn-group>.btn:hover{z-index:1}.btn-toolbar{display:flex;flex-wrap:wrap;justify-content:flex-start}.btn-toolbar .input-group{width:auto}.btn-group>.btn-group:not(:first-child),.btn-group>.btn:not(:first-child){margin-left:-1px}.btn-group>.btn-group:not(:last-child)>.btn,.btn-group>.btn:not(:last-child):not(.dropdown-toggle){border-top-right-radius:0;border-bottom-right-radius:0}.btn-group>.btn-group:not(:first-child)>.btn,.btn-group>.btn:nth-child(n+3),.btn-group>:not(.btn-check)+.btn{border-top-left-radius:0;border-bottom-left-radius:0}.dropdown-toggle-split{padding-right:.5625rem;padding-left:.5625rem}.dropdown-toggle-split::after,.dropend .dropdown-toggle-split::after,.dropup .dropdown-toggle-split::after{margin-left:0}.dropstart .dropdown-toggle-split::before{margin-right:0}.btn-group-sm>.btn+.dropdown-toggle-split,.btn-sm+.dropdown-toggle-split{padding-right:.375rem;padding-left:.375rem}.btn-group-lg>.btn+.dropdown-toggle-split,.btn-lg+.dropdown-toggle-split{padding-right:.75rem;padding-left:.75rem}.btn-group-vertical{flex-direction:column;align-items:flex-start;justify-content:center}.btn-group-vertical>.btn,.btn-group-vertical>.btn-group{width:100%}.btn-group-vertical>.btn-group:not(:first-child),.btn-group-vertical>.btn:not(:first-child){margin-top:-1px}.btn-group-vertical>.btn-group:not(:last-child)>.btn,.btn-group-vertical>.btn:not(:last-child):not(.dropdown-toggle){border-bottom-right-radius:0;border-bottom-left-radius:0}.btn-group-vertical>.btn-group:not(:first-child)>.btn,.btn-group-vertical>.btn~.btn{border-top-left-radius:0;border-top-right-radius:0}.nav{display:flex;flex-wrap:wrap;padding-left:0;margin-bottom:0;list-style:none}.nav-link{display:block;padding:.5rem 1rem;color:#0d6efd;text-decoration:none;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out}@media (prefers-reduced-motion:reduce){.nav-link{transition:none}}.nav-link:focus,.nav-link:hover{color:#0a58ca}.nav-link.disabled{color:#6c757d;pointer-events:none;cursor:default}.nav-tabs{border-bottom:1px solid #dee2e6}.nav-tabs .nav-link{margin-bottom:-1px;background:0 0;border:1px solid transparent;border-top-left-radius:.25rem;border-top-right-radius:.25rem}.nav-tabs .nav-link:focus,.nav-tabs .nav-link:hover{border-color:#e9ecef #e9ecef #dee2e6;isolation:isolate}.nav-tabs .nav-link.disabled{color:#6c757d;background-color:transparent;border-color:transparent}.nav-tabs .nav-item.show .nav-link,.nav-tabs .nav-link.active{color:#495057;background-color:#fff;border-color:#dee2e6 #dee2e6 #fff}.nav-tabs .dropdown-menu{margin-top:-1px;border-top-left-radius:0;border-top-right-radius:0}.nav-pills .nav-link{background:0 0;border:0;border-radius:.25rem}.nav-pills .nav-link.active,.nav-pills .show>.nav-link{color:#fff;background-color:#0d6efd}.nav-fill .nav-item,.nav-fill>.nav-link{flex:1 1 auto;text-align:center}.nav-justified .nav-item,.nav-justified>.nav-link{flex-basis:0;flex-grow:1;text-align:center}.nav-fill .nav-item .nav-link,.nav-justified .nav-item .nav-link{width:100%}.tab-content>.tab-pane{display:none}.tab-content>.active{display:block}.navbar{position:relative;display:flex;flex-wrap:wrap;align-items:center;justify-content:space-between;padding-top:.5rem;padding-bottom:.5rem}.navbar>.container,.navbar>.container-fluid,.navbar>.container-lg,.navbar>.container-md,.navbar>.container-sm,.navbar>.container-xl,.navbar>.container-xxl{display:flex;flex-wrap:inherit;align-items:center;justify-content:space-between}.navbar-brand{padding-top:.3125rem;padding-bottom:.3125rem;margin-right:1rem;font-size:1.25rem;text-decoration:none;white-space:nowrap}.navbar-nav{display:flex;flex-direction:column;padding-left:0;margin-bottom:0;list-style:none}.navbar-nav .nav-link{padding-right:0;padding-left:0}.navbar-nav .dropdown-menu{position:static}.navbar-text{padding-top:.5rem;padding-bottom:.5rem}.navbar-collapse{flex-basis:100%;flex-grow:1;align-items:center}.navbar-toggler{padding:.25rem .75rem;font-size:1.25rem;line-height:1;background-color:transparent;border:1px solid transparent;border-radius:.25rem;transition:box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.navbar-toggler{transition:none}}.navbar-toggler:hover{text-decoration:none}.navbar-toggler:focus{text-decoration:none;outline:0;box-shadow:0 0 0 .25rem}.navbar-toggler-icon{display:inline-block;width:1.5em;height:1.5em;vertical-align:middle;background-repeat:no-repeat;background-position:center;background-size:100%}.navbar-nav-scroll{max-height:var(--bs-scroll-height,75vh);overflow-y:auto}@media (min-width:576px){.navbar-expand-sm{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-sm .navbar-nav{flex-direction:row}.navbar-expand-sm .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-sm .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-sm .navbar-nav-scroll{overflow:visible}.navbar-expand-sm .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-sm .navbar-toggler{display:none}.navbar-expand-sm .offcanvas-header{display:none}.navbar-expand-sm .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-sm .offcanvas-bottom,.navbar-expand-sm .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-sm .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:768px){.navbar-expand-md{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-md .navbar-nav{flex-direction:row}.navbar-expand-md .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-md .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-md .navbar-nav-scroll{overflow:visible}.navbar-expand-md .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-md .navbar-toggler{display:none}.navbar-expand-md .offcanvas-header{display:none}.navbar-expand-md .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-md .offcanvas-bottom,.navbar-expand-md .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-md .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:992px){.navbar-expand-lg{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-lg .navbar-nav{flex-direction:row}.navbar-expand-lg .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-lg .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-lg .navbar-nav-scroll{overflow:visible}.navbar-expand-lg .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-lg .navbar-toggler{display:none}.navbar-expand-lg .offcanvas-header{display:none}.navbar-expand-lg .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-lg .offcanvas-bottom,.navbar-expand-lg .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-lg .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:1200px){.navbar-expand-xl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xl .navbar-nav{flex-direction:row}.navbar-expand-xl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xl .navbar-nav-scroll{overflow:visible}.navbar-expand-xl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xl .navbar-toggler{display:none}.navbar-expand-xl .offcanvas-header{display:none}.navbar-expand-xl .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-xl .offcanvas-bottom,.navbar-expand-xl .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-xl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}@media (min-width:1400px){.navbar-expand-xxl{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand-xxl .navbar-nav{flex-direction:row}.navbar-expand-xxl .navbar-nav .dropdown-menu{position:absolute}.navbar-expand-xxl .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand-xxl .navbar-nav-scroll{overflow:visible}.navbar-expand-xxl .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand-xxl .navbar-toggler{display:none}.navbar-expand-xxl .offcanvas-header{display:none}.navbar-expand-xxl .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand-xxl .offcanvas-bottom,.navbar-expand-xxl .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand-xxl .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}}.navbar-expand{flex-wrap:nowrap;justify-content:flex-start}.navbar-expand .navbar-nav{flex-direction:row}.navbar-expand .navbar-nav .dropdown-menu{position:absolute}.navbar-expand .navbar-nav .nav-link{padding-right:.5rem;padding-left:.5rem}.navbar-expand .navbar-nav-scroll{overflow:visible}.navbar-expand .navbar-collapse{display:flex!important;flex-basis:auto}.navbar-expand .navbar-toggler{display:none}.navbar-expand .offcanvas-header{display:none}.navbar-expand .offcanvas{position:inherit;bottom:0;z-index:1000;flex-grow:1;visibility:visible!important;background-color:transparent;border-right:0;border-left:0;transition:none;transform:none}.navbar-expand .offcanvas-bottom,.navbar-expand .offcanvas-top{height:auto;border-top:0;border-bottom:0}.navbar-expand .offcanvas-body{display:flex;flex-grow:0;padding:0;overflow-y:visible}.navbar-light .navbar-brand{color:rgba(0,0,0,.9)}.navbar-light .navbar-brand:focus,.navbar-light .navbar-brand:hover{color:rgba(0,0,0,.9)}.navbar-light .navbar-nav .nav-link{color:rgba(0,0,0,.55)}.navbar-light .navbar-nav .nav-link:focus,.navbar-light .navbar-nav .nav-link:hover{color:rgba(0,0,0,.7)}.navbar-light .navbar-nav .nav-link.disabled{color:rgba(0,0,0,.3)}.navbar-light .navbar-nav .nav-link.active,.navbar-light .navbar-nav .show>.nav-link{color:rgba(0,0,0,.9)}.navbar-light .navbar-toggler{color:rgba(0,0,0,.55);border-color:rgba(0,0,0,.1)}.navbar-light .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%280, 0, 0, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-light .navbar-text{color:rgba(0,0,0,.55)}.navbar-light .navbar-text a,.navbar-light .navbar-text a:focus,.navbar-light .navbar-text a:hover{color:rgba(0,0,0,.9)}.navbar-dark .navbar-brand{color:#fff}.navbar-dark .navbar-brand:focus,.navbar-dark .navbar-brand:hover{color:#fff}.navbar-dark .navbar-nav .nav-link{color:rgba(255,255,255,.55)}.navbar-dark .navbar-nav .nav-link:focus,.navbar-dark .navbar-nav .nav-link:hover{color:rgba(255,255,255,.75)}.navbar-dark .navbar-nav .nav-link.disabled{color:rgba(255,255,255,.25)}.navbar-dark .navbar-nav .nav-link.active,.navbar-dark .navbar-nav .show>.nav-link{color:#fff}.navbar-dark .navbar-toggler{color:rgba(255,255,255,.55);border-color:rgba(255,255,255,.1)}.navbar-dark .navbar-toggler-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 30 30'%3e%3cpath stroke='rgba%28255, 255, 255, 0.55%29' stroke-linecap='round' stroke-miterlimit='10' stroke-width='2' d='M4 7h22M4 15h22M4 23h22'/%3e%3c/svg%3e")}.navbar-dark .navbar-text{color:rgba(255,255,255,.55)}.navbar-dark .navbar-text a,.navbar-dark .navbar-text a:focus,.navbar-dark .navbar-text a:hover{color:#fff}.card{position:relative;display:flex;flex-direction:column;min-width:0;word-wrap:break-word;background-color:#fff;background-clip:border-box;border:1px solid rgba(0,0,0,.125);border-radius:.25rem}.card>hr{margin-right:0;margin-left:0}.card>.list-group{border-top:inherit;border-bottom:inherit}.card>.list-group:first-child{border-top-width:0;border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card>.list-group:last-child{border-bottom-width:0;border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card>.card-header+.list-group,.card>.list-group+.card-footer{border-top:0}.card-body{flex:1 1 auto;padding:1rem 1rem}.card-title{margin-bottom:.5rem}.card-subtitle{margin-top:-.25rem;margin-bottom:0}.card-text:last-child{margin-bottom:0}.card-link+.card-link{margin-left:1rem}.card-header{padding:.5rem 1rem;margin-bottom:0;background-color:rgba(0,0,0,.03);border-bottom:1px solid rgba(0,0,0,.125)}.card-header:first-child{border-radius:calc(.25rem - 1px) calc(.25rem - 1px) 0 0}.card-footer{padding:.5rem 1rem;background-color:rgba(0,0,0,.03);border-top:1px solid rgba(0,0,0,.125)}.card-footer:last-child{border-radius:0 0 calc(.25rem - 1px) calc(.25rem - 1px)}.card-header-tabs{margin-right:-.5rem;margin-bottom:-.5rem;margin-left:-.5rem;border-bottom:0}.card-header-pills{margin-right:-.5rem;margin-left:-.5rem}.card-img-overlay{position:absolute;top:0;right:0;bottom:0;left:0;padding:1rem;border-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom,.card-img-top{width:100%}.card-img,.card-img-top{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.card-img,.card-img-bottom{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.card-group>.card{margin-bottom:.75rem}@media (min-width:576px){.card-group{display:flex;flex-flow:row wrap}.card-group>.card{flex:1 0 0%;margin-bottom:0}.card-group>.card+.card{margin-left:0;border-left:0}.card-group>.card:not(:last-child){border-top-right-radius:0;border-bottom-right-radius:0}.card-group>.card:not(:last-child) .card-header,.card-group>.card:not(:last-child) .card-img-top{border-top-right-radius:0}.card-group>.card:not(:last-child) .card-footer,.card-group>.card:not(:last-child) .card-img-bottom{border-bottom-right-radius:0}.card-group>.card:not(:first-child){border-top-left-radius:0;border-bottom-left-radius:0}.card-group>.card:not(:first-child) .card-header,.card-group>.card:not(:first-child) .card-img-top{border-top-left-radius:0}.card-group>.card:not(:first-child) .card-footer,.card-group>.card:not(:first-child) .card-img-bottom{border-bottom-left-radius:0}}.accordion-button{position:relative;display:flex;align-items:center;width:100%;padding:1rem 1.25rem;font-size:1rem;color:#212529;text-align:left;background-color:#fff;border:0;border-radius:0;overflow-anchor:none;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out,border-radius .15s ease}@media (prefers-reduced-motion:reduce){.accordion-button{transition:none}}.accordion-button:not(.collapsed){color:#0c63e4;background-color:#e7f1ff;box-shadow:inset 0 -1px 0 rgba(0,0,0,.125)}.accordion-button:not(.collapsed)::after{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%230c63e4'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");transform:rotate(-180deg)}.accordion-button::after{flex-shrink:0;width:1.25rem;height:1.25rem;margin-left:auto;content:"";background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23212529'%3e%3cpath fill-rule='evenodd' d='M1.646 4.646a.5.5 0 0 1 .708 0L8 10.293l5.646-5.647a.5.5 0 0 1 .708.708l-6 6a.5.5 0 0 1-.708 0l-6-6a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e");background-repeat:no-repeat;background-size:1.25rem;transition:transform .2s ease-in-out}@media (prefers-reduced-motion:reduce){.accordion-button::after{transition:none}}.accordion-button:hover{z-index:2}.accordion-button:focus{z-index:3;border-color:#86b7fe;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.accordion-header{margin-bottom:0}.accordion-item{background-color:#fff;border:1px solid rgba(0,0,0,.125)}.accordion-item:first-of-type{border-top-left-radius:.25rem;border-top-right-radius:.25rem}.accordion-item:first-of-type .accordion-button{border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.accordion-item:not(:first-of-type){border-top:0}.accordion-item:last-of-type{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.accordion-item:last-of-type .accordion-button.collapsed{border-bottom-right-radius:calc(.25rem - 1px);border-bottom-left-radius:calc(.25rem - 1px)}.accordion-item:last-of-type .accordion-collapse{border-bottom-right-radius:.25rem;border-bottom-left-radius:.25rem}.accordion-body{padding:1rem 1.25rem}.accordion-flush .accordion-collapse{border-width:0}.accordion-flush .accordion-item{border-right:0;border-left:0;border-radius:0}.accordion-flush .accordion-item:first-child{border-top:0}.accordion-flush .accordion-item:last-child{border-bottom:0}.accordion-flush .accordion-item .accordion-button{border-radius:0}.breadcrumb{display:flex;flex-wrap:wrap;padding:0 0;margin-bottom:1rem;list-style:none}.breadcrumb-item+.breadcrumb-item{padding-left:.5rem}.breadcrumb-item+.breadcrumb-item::before{float:left;padding-right:.5rem;color:#6c757d;content:var(--bs-breadcrumb-divider, "/")}.breadcrumb-item.active{color:#6c757d}.pagination{display:flex;padding-left:0;list-style:none}.page-link{position:relative;display:block;color:#0d6efd;text-decoration:none;background-color:#fff;border:1px solid #dee2e6;transition:color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out}@media (prefers-reduced-motion:reduce){.page-link{transition:none}}.page-link:hover{z-index:2;color:#0a58ca;background-color:#e9ecef;border-color:#dee2e6}.page-link:focus{z-index:3;color:#0a58ca;background-color:#e9ecef;outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25)}.page-item:not(:first-child) .page-link{margin-left:-1px}.page-item.active .page-link{z-index:3;color:#fff;background-color:#0d6efd;border-color:#0d6efd}.page-item.disabled .page-link{color:#6c757d;pointer-events:none;background-color:#fff;border-color:#dee2e6}.page-link{padding:.375rem .75rem}.page-item:first-child .page-link{border-top-left-radius:.25rem;border-bottom-left-radius:.25rem}.page-item:last-child .page-link{border-top-right-radius:.25rem;border-bottom-right-radius:.25rem}.pagination-lg .page-link{padding:.75rem 1.5rem;font-size:1.25rem}.pagination-lg .page-item:first-child .page-link{border-top-left-radius:.3rem;border-bottom-left-radius:.3rem}.pagination-lg .page-item:last-child .page-link{border-top-right-radius:.3rem;border-bottom-right-radius:.3rem}.pagination-sm .page-link{padding:.25rem .5rem;font-size:.875rem}.pagination-sm .page-item:first-child .page-link{border-top-left-radius:.2rem;border-bottom-left-radius:.2rem}.pagination-sm .page-item:last-child .page-link{border-top-right-radius:.2rem;border-bottom-right-radius:.2rem}.badge{display:inline-block;padding:.35em .65em;font-size:.75em;font-weight:700;line-height:1;color:#fff;text-align:center;white-space:nowrap;vertical-align:baseline;border-radius:.25rem}.badge:empty{display:none}.btn .badge{position:relative;top:-1px}.alert{position:relative;padding:1rem 1rem;margin-bottom:1rem;border:1px solid transparent;border-radius:.25rem}.alert-heading{color:inherit}.alert-link{font-weight:700}.alert-dismissible{padding-right:3rem}.alert-dismissible .btn-close{position:absolute;top:0;right:0;z-index:2;padding:1.25rem 1rem}.alert-primary{color:#084298;background-color:#cfe2ff;border-color:#b6d4fe}.alert-primary .alert-link{color:#06357a}.alert-secondary{color:#41464b;background-color:#e2e3e5;border-color:#d3d6d8}.alert-secondary .alert-link{color:#34383c}.alert-success{color:#0f5132;background-color:#d1e7dd;border-color:#badbcc}.alert-success .alert-link{color:#0c4128}.alert-info{color:#055160;background-color:#cff4fc;border-color:#b6effb}.alert-info .alert-link{color:#04414d}.alert-warning{color:#664d03;background-color:#fff3cd;border-color:#ffecb5}.alert-warning .alert-link{color:#523e02}.alert-danger{color:#842029;background-color:#f8d7da;border-color:#f5c2c7}.alert-danger .alert-link{color:#6a1a21}.alert-light{color:#636464;background-color:#fefefe;border-color:#fdfdfe}.alert-light .alert-link{color:#4f5050}.alert-dark{color:#141619;background-color:#d3d3d4;border-color:#bcbebf}.alert-dark .alert-link{color:#101214}@-webkit-keyframes progress-bar-stripes{0%{background-position-x:1rem}}@keyframes progress-bar-stripes{0%{background-position-x:1rem}}.progress{display:flex;height:1rem;overflow:hidden;font-size:.75rem;background-color:#e9ecef;border-radius:.25rem}.progress-bar{display:flex;flex-direction:column;justify-content:center;overflow:hidden;color:#fff;text-align:center;white-space:nowrap;background-color:#0d6efd;transition:width .6s ease}@media (prefers-reduced-motion:reduce){.progress-bar{transition:none}}.progress-bar-striped{background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-size:1rem 1rem}.progress-bar-animated{-webkit-animation:1s linear infinite progress-bar-stripes;animation:1s linear infinite progress-bar-stripes}@media (prefers-reduced-motion:reduce){.progress-bar-animated{-webkit-animation:none;animation:none}}.list-group{display:flex;flex-direction:column;padding-left:0;margin-bottom:0;border-radius:.25rem}.list-group-numbered{list-style-type:none;counter-reset:section}.list-group-numbered>li::before{content:counters(section, ".") ". ";counter-increment:section}.list-group-item-action{width:100%;color:#495057;text-align:inherit}.list-group-item-action:focus,.list-group-item-action:hover{z-index:1;color:#495057;text-decoration:none;background-color:#f8f9fa}.list-group-item-action:active{color:#212529;background-color:#e9ecef}.list-group-item{position:relative;display:block;padding:.5rem 1rem;color:#212529;text-decoration:none;background-color:#fff;border:1px solid rgba(0,0,0,.125)}.list-group-item:first-child{border-top-left-radius:inherit;border-top-right-radius:inherit}.list-group-item:last-child{border-bottom-right-radius:inherit;border-bottom-left-radius:inherit}.list-group-item.disabled,.list-group-item:disabled{color:#6c757d;pointer-events:none;background-color:#fff}.list-group-item.active{z-index:2;color:#fff;background-color:#0d6efd;border-color:#0d6efd}.list-group-item+.list-group-item{border-top-width:0}.list-group-item+.list-group-item.active{margin-top:-1px;border-top-width:1px}.list-group-horizontal{flex-direction:row}.list-group-horizontal>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal>.list-group-item.active{margin-top:0}.list-group-horizontal>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}@media (min-width:576px){.list-group-horizontal-sm{flex-direction:row}.list-group-horizontal-sm>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-sm>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-sm>.list-group-item.active{margin-top:0}.list-group-horizontal-sm>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-sm>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:768px){.list-group-horizontal-md{flex-direction:row}.list-group-horizontal-md>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-md>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-md>.list-group-item.active{margin-top:0}.list-group-horizontal-md>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-md>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:992px){.list-group-horizontal-lg{flex-direction:row}.list-group-horizontal-lg>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-lg>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-lg>.list-group-item.active{margin-top:0}.list-group-horizontal-lg>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-lg>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1200px){.list-group-horizontal-xl{flex-direction:row}.list-group-horizontal-xl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xl>.list-group-item.active{margin-top:0}.list-group-horizontal-xl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}@media (min-width:1400px){.list-group-horizontal-xxl{flex-direction:row}.list-group-horizontal-xxl>.list-group-item:first-child{border-bottom-left-radius:.25rem;border-top-right-radius:0}.list-group-horizontal-xxl>.list-group-item:last-child{border-top-right-radius:.25rem;border-bottom-left-radius:0}.list-group-horizontal-xxl>.list-group-item.active{margin-top:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item{border-top-width:1px;border-left-width:0}.list-group-horizontal-xxl>.list-group-item+.list-group-item.active{margin-left:-1px;border-left-width:1px}}.list-group-flush{border-radius:0}.list-group-flush>.list-group-item{border-width:0 0 1px}.list-group-flush>.list-group-item:last-child{border-bottom-width:0}.list-group-item-primary{color:#084298;background-color:#cfe2ff}.list-group-item-primary.list-group-item-action:focus,.list-group-item-primary.list-group-item-action:hover{color:#084298;background-color:#bacbe6}.list-group-item-primary.list-group-item-action.active{color:#fff;background-color:#084298;border-color:#084298}.list-group-item-secondary{color:#41464b;background-color:#e2e3e5}.list-group-item-secondary.list-group-item-action:focus,.list-group-item-secondary.list-group-item-action:hover{color:#41464b;background-color:#cbccce}.list-group-item-secondary.list-group-item-action.active{color:#fff;background-color:#41464b;border-color:#41464b}.list-group-item-success{color:#0f5132;background-color:#d1e7dd}.list-group-item-success.list-group-item-action:focus,.list-group-item-success.list-group-item-action:hover{color:#0f5132;background-color:#bcd0c7}.list-group-item-success.list-group-item-action.active{color:#fff;background-color:#0f5132;border-color:#0f5132}.list-group-item-info{color:#055160;background-color:#cff4fc}.list-group-item-info.list-group-item-action:focus,.list-group-item-info.list-group-item-action:hover{color:#055160;background-color:#badce3}.list-group-item-info.list-group-item-action.active{color:#fff;background-color:#055160;border-color:#055160}.list-group-item-warning{color:#664d03;background-color:#fff3cd}.list-group-item-warning.list-group-item-action:focus,.list-group-item-warning.list-group-item-action:hover{color:#664d03;background-color:#e6dbb9}.list-group-item-warning.list-group-item-action.active{color:#fff;background-color:#664d03;border-color:#664d03}.list-group-item-danger{color:#842029;background-color:#f8d7da}.list-group-item-danger.list-group-item-action:focus,.list-group-item-danger.list-group-item-action:hover{color:#842029;background-color:#dfc2c4}.list-group-item-danger.list-group-item-action.active{color:#fff;background-color:#842029;border-color:#842029}.list-group-item-light{color:#636464;background-color:#fefefe}.list-group-item-light.list-group-item-action:focus,.list-group-item-light.list-group-item-action:hover{color:#636464;background-color:#e5e5e5}.list-group-item-light.list-group-item-action.active{color:#fff;background-color:#636464;border-color:#636464}.list-group-item-dark{color:#141619;background-color:#d3d3d4}.list-group-item-dark.list-group-item-action:focus,.list-group-item-dark.list-group-item-action:hover{color:#141619;background-color:#bebebf}.list-group-item-dark.list-group-item-action.active{color:#fff;background-color:#141619;border-color:#141619}.btn-close{box-sizing:content-box;width:1em;height:1em;padding:.25em .25em;color:#000;background:transparent url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23000'%3e%3cpath d='M.293.293a1 1 0 011.414 0L8 6.586 14.293.293a1 1 0 111.414 1.414L9.414 8l6.293 6.293a1 1 0 01-1.414 1.414L8 9.414l-6.293 6.293a1 1 0 01-1.414-1.414L6.586 8 .293 1.707a1 1 0 010-1.414z'/%3e%3c/svg%3e") center/1em auto no-repeat;border:0;border-radius:.25rem;opacity:.5}.btn-close:hover{color:#000;text-decoration:none;opacity:.75}.btn-close:focus{outline:0;box-shadow:0 0 0 .25rem rgba(13,110,253,.25);opacity:1}.btn-close.disabled,.btn-close:disabled{pointer-events:none;-webkit-user-select:none;-moz-user-select:none;user-select:none;opacity:.25}.btn-close-white{filter:invert(1) grayscale(100%) brightness(200%)}.toast{width:350px;max-width:100%;font-size:.875rem;pointer-events:auto;background-color:rgba(255,255,255,.85);background-clip:padding-box;border:1px solid rgba(0,0,0,.1);box-shadow:0 .5rem 1rem rgba(0,0,0,.15);border-radius:.25rem}.toast.showing{opacity:0}.toast:not(.show){display:none}.toast-container{width:-webkit-max-content;width:-moz-max-content;width:max-content;max-width:100%;pointer-events:none}.toast-container>:not(:last-child){margin-bottom:.75rem}.toast-header{display:flex;align-items:center;padding:.5rem .75rem;color:#6c757d;background-color:rgba(255,255,255,.85);background-clip:padding-box;border-bottom:1px solid rgba(0,0,0,.05);border-top-left-radius:calc(.25rem - 1px);border-top-right-radius:calc(.25rem - 1px)}.toast-header .btn-close{margin-right:-.375rem;margin-left:.75rem}.toast-body{padding:.75rem;word-wrap:break-word}.modal{position:fixed;top:0;left:0;z-index:1055;display:none;width:100%;height:100%;overflow-x:hidden;overflow-y:auto;outline:0}.modal-dialog{position:relative;width:auto;margin:.5rem;pointer-events:none}.modal.fade .modal-dialog{transition:transform .3s ease-out;transform:translate(0,-50px)}@media (prefers-reduced-motion:reduce){.modal.fade .modal-dialog{transition:none}}.modal.show .modal-dialog{transform:none}.modal.modal-static .modal-dialog{transform:scale(1.02)}.modal-dialog-scrollable{height:calc(100% - 1rem)}.modal-dialog-scrollable .modal-content{max-height:100%;overflow:hidden}.modal-dialog-scrollable .modal-body{overflow-y:auto}.modal-dialog-centered{display:flex;align-items:center;min-height:calc(100% - 1rem)}.modal-content{position:relative;display:flex;flex-direction:column;width:100%;pointer-events:auto;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem;outline:0}.modal-backdrop{position:fixed;top:0;left:0;z-index:1050;width:100vw;height:100vh;background-color:#000}.modal-backdrop.fade{opacity:0}.modal-backdrop.show{opacity:.5}.modal-header{display:flex;flex-shrink:0;align-items:center;justify-content:space-between;padding:1rem 1rem;border-bottom:1px solid #dee2e6;border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.modal-header .btn-close{padding:.5rem .5rem;margin:-.5rem -.5rem -.5rem auto}.modal-title{margin-bottom:0;line-height:1.5}.modal-body{position:relative;flex:1 1 auto;padding:1rem}.modal-footer{display:flex;flex-wrap:wrap;flex-shrink:0;align-items:center;justify-content:flex-end;padding:.75rem;border-top:1px solid #dee2e6;border-bottom-right-radius:calc(.3rem - 1px);border-bottom-left-radius:calc(.3rem - 1px)}.modal-footer>*{margin:.25rem}@media (min-width:576px){.modal-dialog{max-width:500px;margin:1.75rem auto}.modal-dialog-scrollable{height:calc(100% - 3.5rem)}.modal-dialog-centered{min-height:calc(100% - 3.5rem)}.modal-sm{max-width:300px}}@media (min-width:992px){.modal-lg,.modal-xl{max-width:800px}}@media (min-width:1200px){.modal-xl{max-width:1140px}}.modal-fullscreen{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen .modal-header{border-radius:0}.modal-fullscreen .modal-body{overflow-y:auto}.modal-fullscreen .modal-footer{border-radius:0}@media (max-width:575.98px){.modal-fullscreen-sm-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-sm-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-sm-down .modal-header{border-radius:0}.modal-fullscreen-sm-down .modal-body{overflow-y:auto}.modal-fullscreen-sm-down .modal-footer{border-radius:0}}@media (max-width:767.98px){.modal-fullscreen-md-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-md-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-md-down .modal-header{border-radius:0}.modal-fullscreen-md-down .modal-body{overflow-y:auto}.modal-fullscreen-md-down .modal-footer{border-radius:0}}@media (max-width:991.98px){.modal-fullscreen-lg-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-lg-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-lg-down .modal-header{border-radius:0}.modal-fullscreen-lg-down .modal-body{overflow-y:auto}.modal-fullscreen-lg-down .modal-footer{border-radius:0}}@media (max-width:1199.98px){.modal-fullscreen-xl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xl-down .modal-header{border-radius:0}.modal-fullscreen-xl-down .modal-body{overflow-y:auto}.modal-fullscreen-xl-down .modal-footer{border-radius:0}}@media (max-width:1399.98px){.modal-fullscreen-xxl-down{width:100vw;max-width:none;height:100%;margin:0}.modal-fullscreen-xxl-down .modal-content{height:100%;border:0;border-radius:0}.modal-fullscreen-xxl-down .modal-header{border-radius:0}.modal-fullscreen-xxl-down .modal-body{overflow-y:auto}.modal-fullscreen-xxl-down .modal-footer{border-radius:0}}.tooltip{position:absolute;z-index:1080;display:block;margin:0;font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;opacity:0}.tooltip.show{opacity:.9}.tooltip .tooltip-arrow{position:absolute;display:block;width:.8rem;height:.4rem}.tooltip .tooltip-arrow::before{position:absolute;content:"";border-color:transparent;border-style:solid}.bs-tooltip-auto[data-popper-placement^=top],.bs-tooltip-top{padding:.4rem 0}.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow,.bs-tooltip-top .tooltip-arrow{bottom:0}.bs-tooltip-auto[data-popper-placement^=top] .tooltip-arrow::before,.bs-tooltip-top .tooltip-arrow::before{top:-1px;border-width:.4rem .4rem 0;border-top-color:#000}.bs-tooltip-auto[data-popper-placement^=right],.bs-tooltip-end{padding:0 .4rem}.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow,.bs-tooltip-end .tooltip-arrow{left:0;width:.4rem;height:.8rem}.bs-tooltip-auto[data-popper-placement^=right] .tooltip-arrow::before,.bs-tooltip-end .tooltip-arrow::before{right:-1px;border-width:.4rem .4rem .4rem 0;border-right-color:#000}.bs-tooltip-auto[data-popper-placement^=bottom],.bs-tooltip-bottom{padding:.4rem 0}.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow,.bs-tooltip-bottom .tooltip-arrow{top:0}.bs-tooltip-auto[data-popper-placement^=bottom] .tooltip-arrow::before,.bs-tooltip-bottom .tooltip-arrow::before{bottom:-1px;border-width:0 .4rem .4rem;border-bottom-color:#000}.bs-tooltip-auto[data-popper-placement^=left],.bs-tooltip-start{padding:0 .4rem}.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow,.bs-tooltip-start .tooltip-arrow{right:0;width:.4rem;height:.8rem}.bs-tooltip-auto[data-popper-placement^=left] .tooltip-arrow::before,.bs-tooltip-start .tooltip-arrow::before{left:-1px;border-width:.4rem 0 .4rem .4rem;border-left-color:#000}.tooltip-inner{max-width:200px;padding:.25rem .5rem;color:#fff;text-align:center;background-color:#000;border-radius:.25rem}.popover{position:absolute;top:0;left:0;z-index:1070;display:block;max-width:276px;font-family:var(--bs-font-sans-serif);font-style:normal;font-weight:400;line-height:1.5;text-align:left;text-align:start;text-decoration:none;text-shadow:none;text-transform:none;letter-spacing:normal;word-break:normal;word-spacing:normal;white-space:normal;line-break:auto;font-size:.875rem;word-wrap:break-word;background-color:#fff;background-clip:padding-box;border:1px solid rgba(0,0,0,.2);border-radius:.3rem}.popover .popover-arrow{position:absolute;display:block;width:1rem;height:.5rem}.popover .popover-arrow::after,.popover .popover-arrow::before{position:absolute;display:block;content:"";border-color:transparent;border-style:solid}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow,.bs-popover-top>.popover-arrow{bottom:calc(-.5rem - 1px)}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::before,.bs-popover-top>.popover-arrow::before{bottom:0;border-width:.5rem .5rem 0;border-top-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=top]>.popover-arrow::after,.bs-popover-top>.popover-arrow::after{bottom:1px;border-width:.5rem .5rem 0;border-top-color:#fff}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow,.bs-popover-end>.popover-arrow{left:calc(-.5rem - 1px);width:.5rem;height:1rem}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::before,.bs-popover-end>.popover-arrow::before{left:0;border-width:.5rem .5rem .5rem 0;border-right-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=right]>.popover-arrow::after,.bs-popover-end>.popover-arrow::after{left:1px;border-width:.5rem .5rem .5rem 0;border-right-color:#fff}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow,.bs-popover-bottom>.popover-arrow{top:calc(-.5rem - 1px)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::before,.bs-popover-bottom>.popover-arrow::before{top:0;border-width:0 .5rem .5rem .5rem;border-bottom-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=bottom]>.popover-arrow::after,.bs-popover-bottom>.popover-arrow::after{top:1px;border-width:0 .5rem .5rem .5rem;border-bottom-color:#fff}.bs-popover-auto[data-popper-placement^=bottom] .popover-header::before,.bs-popover-bottom .popover-header::before{position:absolute;top:0;left:50%;display:block;width:1rem;margin-left:-.5rem;content:"";border-bottom:1px solid #f0f0f0}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow,.bs-popover-start>.popover-arrow{right:calc(-.5rem - 1px);width:.5rem;height:1rem}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::before,.bs-popover-start>.popover-arrow::before{right:0;border-width:.5rem 0 .5rem .5rem;border-left-color:rgba(0,0,0,.25)}.bs-popover-auto[data-popper-placement^=left]>.popover-arrow::after,.bs-popover-start>.popover-arrow::after{right:1px;border-width:.5rem 0 .5rem .5rem;border-left-color:#fff}.popover-header{padding:.5rem 1rem;margin-bottom:0;font-size:1rem;background-color:#f0f0f0;border-bottom:1px solid rgba(0,0,0,.2);border-top-left-radius:calc(.3rem - 1px);border-top-right-radius:calc(.3rem - 1px)}.popover-header:empty{display:none}.popover-body{padding:1rem 1rem;color:#212529}.carousel{position:relative}.carousel.pointer-event{touch-action:pan-y}.carousel-inner{position:relative;width:100%;overflow:hidden}.carousel-inner::after{display:block;clear:both;content:""}.carousel-item{position:relative;display:none;float:left;width:100%;margin-right:-100%;-webkit-backface-visibility:hidden;backface-visibility:hidden;transition:transform .6s ease-in-out}@media (prefers-reduced-motion:reduce){.carousel-item{transition:none}}.carousel-item-next,.carousel-item-prev,.carousel-item.active{display:block}.active.carousel-item-end,.carousel-item-next:not(.carousel-item-start){transform:translateX(100%)}.active.carousel-item-start,.carousel-item-prev:not(.carousel-item-end){transform:translateX(-100%)}.carousel-fade .carousel-item{opacity:0;transition-property:opacity;transform:none}.carousel-fade .carousel-item-next.carousel-item-start,.carousel-fade .carousel-item-prev.carousel-item-end,.carousel-fade .carousel-item.active{z-index:1;opacity:1}.carousel-fade .active.carousel-item-end,.carousel-fade .active.carousel-item-start{z-index:0;opacity:0;transition:opacity 0s .6s}@media (prefers-reduced-motion:reduce){.carousel-fade .active.carousel-item-end,.carousel-fade .active.carousel-item-start{transition:none}}.carousel-control-next,.carousel-control-prev{position:absolute;top:0;bottom:0;z-index:1;display:flex;align-items:center;justify-content:center;width:15%;padding:0;color:#fff;text-align:center;background:0 0;border:0;opacity:.5;transition:opacity .15s ease}@media (prefers-reduced-motion:reduce){.carousel-control-next,.carousel-control-prev{transition:none}}.carousel-control-next:focus,.carousel-control-next:hover,.carousel-control-prev:focus,.carousel-control-prev:hover{color:#fff;text-decoration:none;outline:0;opacity:.9}.carousel-control-prev{left:0}.carousel-control-next{right:0}.carousel-control-next-icon,.carousel-control-prev-icon{display:inline-block;width:2rem;height:2rem;background-repeat:no-repeat;background-position:50%;background-size:100% 100%}.carousel-control-prev-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M11.354 1.646a.5.5 0 0 1 0 .708L5.707 8l5.647 5.646a.5.5 0 0 1-.708.708l-6-6a.5.5 0 0 1 0-.708l6-6a.5.5 0 0 1 .708 0z'/%3e%3c/svg%3e")}.carousel-control-next-icon{background-image:url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16' fill='%23fff'%3e%3cpath d='M4.646 1.646a.5.5 0 0 1 .708 0l6 6a.5.5 0 0 1 0 .708l-6 6a.5.5 0 0 1-.708-.708L10.293 8 4.646 2.354a.5.5 0 0 1 0-.708z'/%3e%3c/svg%3e")}.carousel-indicators{position:absolute;right:0;bottom:0;left:0;z-index:2;display:flex;justify-content:center;padding:0;margin-right:15%;margin-bottom:1rem;margin-left:15%;list-style:none}.carousel-indicators [data-bs-target]{box-sizing:content-box;flex:0 1 auto;width:30px;height:3px;padding:0;margin-right:3px;margin-left:3px;text-indent:-999px;cursor:pointer;background-color:#fff;background-clip:padding-box;border:0;border-top:10px solid transparent;border-bottom:10px solid transparent;opacity:.5;transition:opacity .6s ease}@media (prefers-reduced-motion:reduce){.carousel-indicators [data-bs-target]{transition:none}}.carousel-indicators .active{opacity:1}.carousel-caption{position:absolute;right:15%;bottom:1.25rem;left:15%;padding-top:1.25rem;padding-bottom:1.25rem;color:#fff;text-align:center}.carousel-dark .carousel-control-next-icon,.carousel-dark .carousel-control-prev-icon{filter:invert(1) grayscale(100)}.carousel-dark .carousel-indicators [data-bs-target]{background-color:#000}.carousel-dark .carousel-caption{color:#000}@-webkit-keyframes spinner-border{to{transform:rotate(360deg)}}@keyframes spinner-border{to{transform:rotate(360deg)}}.spinner-border{display:inline-block;width:2rem;height:2rem;vertical-align:-.125em;border:.25em solid currentColor;border-right-color:transparent;border-radius:50%;-webkit-animation:.75s linear infinite spinner-border;animation:.75s linear infinite spinner-border}.spinner-border-sm{width:1rem;height:1rem;border-width:.2em}@-webkit-keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}@keyframes spinner-grow{0%{transform:scale(0)}50%{opacity:1;transform:none}}.spinner-grow{display:inline-block;width:2rem;height:2rem;vertical-align:-.125em;background-color:currentColor;border-radius:50%;opacity:0;-webkit-animation:.75s linear infinite spinner-grow;animation:.75s linear infinite spinner-grow}.spinner-grow-sm{width:1rem;height:1rem}@media (prefers-reduced-motion:reduce){.spinner-border,.spinner-grow{-webkit-animation-duration:1.5s;animation-duration:1.5s}}.offcanvas{position:fixed;bottom:0;z-index:1045;display:flex;flex-direction:column;max-width:100%;visibility:hidden;background-color:#fff;background-clip:padding-box;outline:0;transition:transform .3s ease-in-out}@media (prefers-reduced-motion:reduce){.offcanvas{transition:none}}.offcanvas-backdrop{position:fixed;top:0;left:0;z-index:1040;width:100vw;height:100vh;background-color:#000}.offcanvas-backdrop.fade{opacity:0}.offcanvas-backdrop.show{opacity:.5}.offcanvas-header{display:flex;align-items:center;justify-content:space-between;padding:1rem 1rem}.offcanvas-header .btn-close{padding:.5rem .5rem;margin-top:-.5rem;margin-right:-.5rem;margin-bottom:-.5rem}.offcanvas-title{margin-bottom:0;line-height:1.5}.offcanvas-body{flex-grow:1;padding:1rem 1rem;overflow-y:auto}.offcanvas-start{top:0;left:0;width:400px;border-right:1px solid rgba(0,0,0,.2);transform:translateX(-100%)}.offcanvas-end{top:0;right:0;width:400px;border-left:1px solid rgba(0,0,0,.2);transform:translateX(100%)}.offcanvas-top{top:0;right:0;left:0;height:30vh;max-height:100%;border-bottom:1px solid rgba(0,0,0,.2);transform:translateY(-100%)}.offcanvas-bottom{right:0;left:0;height:30vh;max-height:100%;border-top:1px solid rgba(0,0,0,.2);transform:translateY(100%)}.offcanvas.show{transform:none}.placeholder{display:inline-block;min-height:1em;vertical-align:middle;cursor:wait;background-color:currentColor;opacity:.5}.placeholder.btn::before{display:inline-block;content:""}.placeholder-xs{min-height:.6em}.placeholder-sm{min-height:.8em}.placeholder-lg{min-height:1.2em}.placeholder-glow .placeholder{-webkit-animation:placeholder-glow 2s ease-in-out infinite;animation:placeholder-glow 2s ease-in-out infinite}@-webkit-keyframes placeholder-glow{50%{opacity:.2}}@keyframes placeholder-glow{50%{opacity:.2}}.placeholder-wave{-webkit-mask-image:linear-gradient(130deg,#000 55%,rgba(0,0,0,0.8) 75%,#000 95%);mask-image:linear-gradient(130deg,#000 55%,rgba(0,0,0,0.8) 75%,#000 95%);-webkit-mask-size:200% 100%;mask-size:200% 100%;-webkit-animation:placeholder-wave 2s linear infinite;animation:placeholder-wave 2s linear infinite}@-webkit-keyframes placeholder-wave{100%{-webkit-mask-position:-200% 0%;mask-position:-200% 0%}}@keyframes placeholder-wave{100%{-webkit-mask-position:-200% 0%;mask-position:-200% 0%}}.clearfix::after{display:block;clear:both;content:""}.link-primary{color:#0d6efd}.link-primary:focus,.link-primary:hover{color:#0a58ca}.link-secondary{color:#6c757d}.link-secondary:focus,.link-secondary:hover{color:#565e64}.link-success{color:#198754}.link-success:focus,.link-success:hover{color:#146c43}.link-info{color:#0dcaf0}.link-info:focus,.link-info:hover{color:#3dd5f3}.link-warning{color:#ffc107}.link-warning:focus,.link-warning:hover{color:#ffcd39}.link-danger{color:#dc3545}.link-danger:focus,.link-danger:hover{color:#b02a37}.link-light{color:#f8f9fa}.link-light:focus,.link-light:hover{color:#f9fafb}.link-dark{color:#212529}.link-dark:focus,.link-dark:hover{color:#1a1e21}.ratio{position:relative;width:100%}.ratio::before{display:block;padding-top:var(--bs-aspect-ratio);content:""}.ratio>*{position:absolute;top:0;left:0;width:100%;height:100%}.ratio-1x1{--bs-aspect-ratio:100%}.ratio-4x3{--bs-aspect-ratio:75%}.ratio-16x9{--bs-aspect-ratio:56.25%}.ratio-21x9{--bs-aspect-ratio:42.8571428571%}.fixed-top{position:fixed;top:0;right:0;left:0;z-index:1030}.fixed-bottom{position:fixed;right:0;bottom:0;left:0;z-index:1030}.sticky-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}@media (min-width:576px){.sticky-sm-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:768px){.sticky-md-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:992px){.sticky-lg-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:1200px){.sticky-xl-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}@media (min-width:1400px){.sticky-xxl-top{position:-webkit-sticky;position:sticky;top:0;z-index:1020}}.hstack{display:flex;flex-direction:row;align-items:center;align-self:stretch}.vstack{display:flex;flex:1 1 auto;flex-direction:column;align-self:stretch}.visually-hidden,.visually-hidden-focusable:not(:focus):not(:focus-within){position:absolute!important;width:1px!important;height:1px!important;padding:0!important;margin:-1px!important;overflow:hidden!important;clip:rect(0,0,0,0)!important;white-space:nowrap!important;border:0!important}.stretched-link::after{position:absolute;top:0;right:0;bottom:0;left:0;z-index:1;content:""}.text-truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.vr{display:inline-block;align-self:stretch;width:1px;min-height:1em;background-color:currentColor;opacity:.25}.align-baseline{vertical-align:baseline!important}.align-top{vertical-align:top!important}.align-middle{vertical-align:middle!important}.align-bottom{vertical-align:bottom!important}.align-text-bottom{vertical-align:text-bottom!important}.align-text-top{vertical-align:text-top!important}.float-start{float:left!important}.float-end{float:right!important}.float-none{float:none!important}.opacity-0{opacity:0!important}.opacity-25{opacity:.25!important}.opacity-50{opacity:.5!important}.opacity-75{opacity:.75!important}.opacity-100{opacity:1!important}.overflow-auto{overflow:auto!important}.overflow-hidden{overflow:hidden!important}.overflow-visible{overflow:visible!important}.overflow-scroll{overflow:scroll!important}.d-inline{display:inline!important}.d-inline-block{display:inline-block!important}.d-block{display:block!important}.d-grid{display:grid!important}.d-table{display:table!important}.d-table-row{display:table-row!important}.d-table-cell{display:table-cell!important}.d-flex{display:flex!important}.d-inline-flex{display:inline-flex!important}.d-none{display:none!important}.shadow{box-shadow:0 .5rem 1rem rgba(0,0,0,.15)!important}.shadow-sm{box-shadow:0 .125rem .25rem rgba(0,0,0,.075)!important}.shadow-lg{box-shadow:0 1rem 3rem rgba(0,0,0,.175)!important}.shadow-none{box-shadow:none!important}.position-static{position:static!important}.position-relative{position:relative!important}.position-absolute{position:absolute!important}.position-fixed{position:fixed!important}.position-sticky{position:-webkit-sticky!important;position:sticky!important}.top-0{top:0!important}.top-50{top:50%!important}.top-100{top:100%!important}.bottom-0{bottom:0!important}.bottom-50{bottom:50%!important}.bottom-100{bottom:100%!important}.start-0{left:0!important}.start-50{left:50%!important}.start-100{left:100%!important}.end-0{right:0!important}.end-50{right:50%!important}.end-100{right:100%!important}.translate-middle{transform:translate(-50%,-50%)!important}.translate-middle-x{transform:translateX(-50%)!important}.translate-middle-y{transform:translateY(-50%)!important}.border{border:1px solid #dee2e6!important}.border-0{border:0!important}.border-top{border-top:1px solid #dee2e6!important}.border-top-0{border-top:0!important}.border-end{border-right:1px solid #dee2e6!important}.border-end-0{border-right:0!important}.border-bottom{border-bottom:1px solid #dee2e6!important}.border-bottom-0{border-bottom:0!important}.border-start{border-left:1px solid #dee2e6!important}.border-start-0{border-left:0!important}.border-primary{border-color:#0d6efd!important}.border-secondary{border-color:#6c757d!important}.border-success{border-color:#198754!important}.border-info{border-color:#0dcaf0!important}.border-warning{border-color:#ffc107!important}.border-danger{border-color:#dc3545!important}.border-light{border-color:#f8f9fa!important}.border-dark{border-color:#212529!important}.border-white{border-color:#fff!important}.border-1{border-width:1px!important}.border-2{border-width:2px!important}.border-3{border-width:3px!important}.border-4{border-width:4px!important}.border-5{border-width:5px!important}.w-25{width:25%!important}.w-50{width:50%!important}.w-75{width:75%!important}.w-100{width:100%!important}.w-auto{width:auto!important}.mw-100{max-width:100%!important}.vw-100{width:100vw!important}.min-vw-100{min-width:100vw!important}.h-25{height:25%!important}.h-50{height:50%!important}.h-75{height:75%!important}.h-100{height:100%!important}.h-auto{height:auto!important}.mh-100{max-height:100%!important}.vh-100{height:100vh!important}.min-vh-100{min-height:100vh!important}.flex-fill{flex:1 1 auto!important}.flex-row{flex-direction:row!important}.flex-column{flex-direction:column!important}.flex-row-reverse{flex-direction:row-reverse!important}.flex-column-reverse{flex-direction:column-reverse!important}.flex-grow-0{flex-grow:0!important}.flex-grow-1{flex-grow:1!important}.flex-shrink-0{flex-shrink:0!important}.flex-shrink-1{flex-shrink:1!important}.flex-wrap{flex-wrap:wrap!important}.flex-nowrap{flex-wrap:nowrap!important}.flex-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-0{gap:0!important}.gap-1{gap:.25rem!important}.gap-2{gap:.5rem!important}.gap-3{gap:1rem!important}.gap-4{gap:1.5rem!important}.gap-5{gap:3rem!important}.justify-content-start{justify-content:flex-start!important}.justify-content-end{justify-content:flex-end!important}.justify-content-center{justify-content:center!important}.justify-content-between{justify-content:space-between!important}.justify-content-around{justify-content:space-around!important}.justify-content-evenly{justify-content:space-evenly!important}.align-items-start{align-items:flex-start!important}.align-items-end{align-items:flex-end!important}.align-items-center{align-items:center!important}.align-items-baseline{align-items:baseline!important}.align-items-stretch{align-items:stretch!important}.align-content-start{align-content:flex-start!important}.align-content-end{align-content:flex-end!important}.align-content-center{align-content:center!important}.align-content-between{align-content:space-between!important}.align-content-around{align-content:space-around!important}.align-content-stretch{align-content:stretch!important}.align-self-auto{align-self:auto!important}.align-self-start{align-self:flex-start!important}.align-self-end{align-self:flex-end!important}.align-self-center{align-self:center!important}.align-self-baseline{align-self:baseline!important}.align-self-stretch{align-self:stretch!important}.order-first{order:-1!important}.order-0{order:0!important}.order-1{order:1!important}.order-2{order:2!important}.order-3{order:3!important}.order-4{order:4!important}.order-5{order:5!important}.order-last{order:6!important}.m-0{margin:0!important}.m-1{margin:.25rem!important}.m-2{margin:.5rem!important}.m-3{margin:1rem!important}.m-4{margin:1.5rem!important}.m-5{margin:3rem!important}.m-auto{margin:auto!important}.mx-0{margin-right:0!important;margin-left:0!important}.mx-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-3{margin-right:1rem!important;margin-left:1rem!important}.mx-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-5{margin-right:3rem!important;margin-left:3rem!important}.mx-auto{margin-right:auto!important;margin-left:auto!important}.my-0{margin-top:0!important;margin-bottom:0!important}.my-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-0{margin-top:0!important}.mt-1{margin-top:.25rem!important}.mt-2{margin-top:.5rem!important}.mt-3{margin-top:1rem!important}.mt-4{margin-top:1.5rem!important}.mt-5{margin-top:3rem!important}.mt-auto{margin-top:auto!important}.me-0{margin-right:0!important}.me-1{margin-right:.25rem!important}.me-2{margin-right:.5rem!important}.me-3{margin-right:1rem!important}.me-4{margin-right:1.5rem!important}.me-5{margin-right:3rem!important}.me-auto{margin-right:auto!important}.mb-0{margin-bottom:0!important}.mb-1{margin-bottom:.25rem!important}.mb-2{margin-bottom:.5rem!important}.mb-3{margin-bottom:1rem!important}.mb-4{margin-bottom:1.5rem!important}.mb-5{margin-bottom:3rem!important}.mb-auto{margin-bottom:auto!important}.ms-0{margin-left:0!important}.ms-1{margin-left:.25rem!important}.ms-2{margin-left:.5rem!important}.ms-3{margin-left:1rem!important}.ms-4{margin-left:1.5rem!important}.ms-5{margin-left:3rem!important}.ms-auto{margin-left:auto!important}.p-0{padding:0!important}.p-1{padding:.25rem!important}.p-2{padding:.5rem!important}.p-3{padding:1rem!important}.p-4{padding:1.5rem!important}.p-5{padding:3rem!important}.px-0{padding-right:0!important;padding-left:0!important}.px-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-3{padding-right:1rem!important;padding-left:1rem!important}.px-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-5{padding-right:3rem!important;padding-left:3rem!important}.py-0{padding-top:0!important;padding-bottom:0!important}.py-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-0{padding-top:0!important}.pt-1{padding-top:.25rem!important}.pt-2{padding-top:.5rem!important}.pt-3{padding-top:1rem!important}.pt-4{padding-top:1.5rem!important}.pt-5{padding-top:3rem!important}.pe-0{padding-right:0!important}.pe-1{padding-right:.25rem!important}.pe-2{padding-right:.5rem!important}.pe-3{padding-right:1rem!important}.pe-4{padding-right:1.5rem!important}.pe-5{padding-right:3rem!important}.pb-0{padding-bottom:0!important}.pb-1{padding-bottom:.25rem!important}.pb-2{padding-bottom:.5rem!important}.pb-3{padding-bottom:1rem!important}.pb-4{padding-bottom:1.5rem!important}.pb-5{padding-bottom:3rem!important}.ps-0{padding-left:0!important}.ps-1{padding-left:.25rem!important}.ps-2{padding-left:.5rem!important}.ps-3{padding-left:1rem!important}.ps-4{padding-left:1.5rem!important}.ps-5{padding-left:3rem!important}.font-monospace{font-family:var(--bs-font-monospace)!important}.fs-1{font-size:calc(1.375rem + 1.5vw)!important}.fs-2{font-size:calc(1.325rem + .9vw)!important}.fs-3{font-size:calc(1.3rem + .6vw)!important}.fs-4{font-size:calc(1.275rem + .3vw)!important}.fs-5{font-size:1.25rem!important}.fs-6{font-size:1rem!important}.fst-italic{font-style:italic!important}.fst-normal{font-style:normal!important}.fw-light{font-weight:300!important}.fw-lighter{font-weight:lighter!important}.fw-normal{font-weight:400!important}.fw-bold{font-weight:700!important}.fw-bolder{font-weight:bolder!important}.lh-1{line-height:1!important}.lh-sm{line-height:1.25!important}.lh-base{line-height:1.5!important}.lh-lg{line-height:2!important}.text-start{text-align:left!important}.text-end{text-align:right!important}.text-center{text-align:center!important}.text-decoration-none{text-decoration:none!important}.text-decoration-underline{text-decoration:underline!important}.text-decoration-line-through{text-decoration:line-through!important}.text-lowercase{text-transform:lowercase!important}.text-uppercase{text-transform:uppercase!important}.text-capitalize{text-transform:capitalize!important}.text-wrap{white-space:normal!important}.text-nowrap{white-space:nowrap!important}.text-break{word-wrap:break-word!important;word-break:break-word!important}.text-primary{--bs-text-opacity:1;color:rgba(var(--bs-primary-rgb),var(--bs-text-opacity))!important}.text-secondary{--bs-text-opacity:1;color:rgba(var(--bs-secondary-rgb),var(--bs-text-opacity))!important}.text-success{--bs-text-opacity:1;color:rgba(var(--bs-success-rgb),var(--bs-text-opacity))!important}.text-info{--bs-text-opacity:1;color:rgba(var(--bs-info-rgb),var(--bs-text-opacity))!important}.text-warning{--bs-text-opacity:1;color:rgba(var(--bs-warning-rgb),var(--bs-text-opacity))!important}.text-danger{--bs-text-opacity:1;color:rgba(var(--bs-danger-rgb),var(--bs-text-opacity))!important}.text-light{--bs-text-opacity:1;color:rgba(var(--bs-light-rgb),var(--bs-text-opacity))!important}.text-dark{--bs-text-opacity:1;color:rgba(var(--bs-dark-rgb),var(--bs-text-opacity))!important}.text-black{--bs-text-opacity:1;color:rgba(var(--bs-black-rgb),var(--bs-text-opacity))!important}.text-white{--bs-text-opacity:1;color:rgba(var(--bs-white-rgb),var(--bs-text-opacity))!important}.text-body{--bs-text-opacity:1;color:rgba(var(--bs-body-color-rgb),var(--bs-text-opacity))!important}.text-muted{--bs-text-opacity:1;color:#6c757d!important}.text-black-50{--bs-text-opacity:1;color:rgba(0,0,0,.5)!important}.text-white-50{--bs-text-opacity:1;color:rgba(255,255,255,.5)!important}.text-reset{--bs-text-opacity:1;color:inherit!important}.text-opacity-25{--bs-text-opacity:0.25}.text-opacity-50{--bs-text-opacity:0.5}.text-opacity-75{--bs-text-opacity:0.75}.text-opacity-100{--bs-text-opacity:1}.bg-primary{--bs-bg-opacity:1;background-color:rgba(var(--bs-primary-rgb),var(--bs-bg-opacity))!important}.bg-secondary{--bs-bg-opacity:1;background-color:rgba(var(--bs-secondary-rgb),var(--bs-bg-opacity))!important}.bg-success{--bs-bg-opacity:1;background-color:rgba(var(--bs-success-rgb),var(--bs-bg-opacity))!important}.bg-info{--bs-bg-opacity:1;background-color:rgba(var(--bs-info-rgb),var(--bs-bg-opacity))!important}.bg-warning{--bs-bg-opacity:1;background-color:rgba(var(--bs-warning-rgb),var(--bs-bg-opacity))!important}.bg-danger{--bs-bg-opacity:1;background-color:rgba(var(--bs-danger-rgb),var(--bs-bg-opacity))!important}.bg-light{--bs-bg-opacity:1;background-color:rgba(var(--bs-light-rgb),var(--bs-bg-opacity))!important}.bg-dark{--bs-bg-opacity:1;background-color:rgba(var(--bs-dark-rgb),var(--bs-bg-opacity))!important}.bg-black{--bs-bg-opacity:1;background-color:rgba(var(--bs-black-rgb),var(--bs-bg-opacity))!important}.bg-white{--bs-bg-opacity:1;background-color:rgba(var(--bs-white-rgb),var(--bs-bg-opacity))!important}.bg-body{--bs-bg-opacity:1;background-color:rgba(var(--bs-body-bg-rgb),var(--bs-bg-opacity))!important}.bg-transparent{--bs-bg-opacity:1;background-color:transparent!important}.bg-opacity-10{--bs-bg-opacity:0.1}.bg-opacity-25{--bs-bg-opacity:0.25}.bg-opacity-50{--bs-bg-opacity:0.5}.bg-opacity-75{--bs-bg-opacity:0.75}.bg-opacity-100{--bs-bg-opacity:1}.bg-gradient{background-image:var(--bs-gradient)!important}.user-select-all{-webkit-user-select:all!important;-moz-user-select:all!important;user-select:all!important}.user-select-auto{-webkit-user-select:auto!important;-moz-user-select:auto!important;user-select:auto!important}.user-select-none{-webkit-user-select:none!important;-moz-user-select:none!important;user-select:none!important}.pe-none{pointer-events:none!important}.pe-auto{pointer-events:auto!important}.rounded{border-radius:.25rem!important}.rounded-0{border-radius:0!important}.rounded-1{border-radius:.2rem!important}.rounded-2{border-radius:.25rem!important}.rounded-3{border-radius:.3rem!important}.rounded-circle{border-radius:50%!important}.rounded-pill{border-radius:50rem!important}.rounded-top{border-top-left-radius:.25rem!important;border-top-right-radius:.25rem!important}.rounded-end{border-top-right-radius:.25rem!important;border-bottom-right-radius:.25rem!important}.rounded-bottom{border-bottom-right-radius:.25rem!important;border-bottom-left-radius:.25rem!important}.rounded-start{border-bottom-left-radius:.25rem!important;border-top-left-radius:.25rem!important}.visible{visibility:visible!important}.invisible{visibility:hidden!important}@media (min-width:576px){.float-sm-start{float:left!important}.float-sm-end{float:right!important}.float-sm-none{float:none!important}.d-sm-inline{display:inline!important}.d-sm-inline-block{display:inline-block!important}.d-sm-block{display:block!important}.d-sm-grid{display:grid!important}.d-sm-table{display:table!important}.d-sm-table-row{display:table-row!important}.d-sm-table-cell{display:table-cell!important}.d-sm-flex{display:flex!important}.d-sm-inline-flex{display:inline-flex!important}.d-sm-none{display:none!important}.flex-sm-fill{flex:1 1 auto!important}.flex-sm-row{flex-direction:row!important}.flex-sm-column{flex-direction:column!important}.flex-sm-row-reverse{flex-direction:row-reverse!important}.flex-sm-column-reverse{flex-direction:column-reverse!important}.flex-sm-grow-0{flex-grow:0!important}.flex-sm-grow-1{flex-grow:1!important}.flex-sm-shrink-0{flex-shrink:0!important}.flex-sm-shrink-1{flex-shrink:1!important}.flex-sm-wrap{flex-wrap:wrap!important}.flex-sm-nowrap{flex-wrap:nowrap!important}.flex-sm-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-sm-0{gap:0!important}.gap-sm-1{gap:.25rem!important}.gap-sm-2{gap:.5rem!important}.gap-sm-3{gap:1rem!important}.gap-sm-4{gap:1.5rem!important}.gap-sm-5{gap:3rem!important}.justify-content-sm-start{justify-content:flex-start!important}.justify-content-sm-end{justify-content:flex-end!important}.justify-content-sm-center{justify-content:center!important}.justify-content-sm-between{justify-content:space-between!important}.justify-content-sm-around{justify-content:space-around!important}.justify-content-sm-evenly{justify-content:space-evenly!important}.align-items-sm-start{align-items:flex-start!important}.align-items-sm-end{align-items:flex-end!important}.align-items-sm-center{align-items:center!important}.align-items-sm-baseline{align-items:baseline!important}.align-items-sm-stretch{align-items:stretch!important}.align-content-sm-start{align-content:flex-start!important}.align-content-sm-end{align-content:flex-end!important}.align-content-sm-center{align-content:center!important}.align-content-sm-between{align-content:space-between!important}.align-content-sm-around{align-content:space-around!important}.align-content-sm-stretch{align-content:stretch!important}.align-self-sm-auto{align-self:auto!important}.align-self-sm-start{align-self:flex-start!important}.align-self-sm-end{align-self:flex-end!important}.align-self-sm-center{align-self:center!important}.align-self-sm-baseline{align-self:baseline!important}.align-self-sm-stretch{align-self:stretch!important}.order-sm-first{order:-1!important}.order-sm-0{order:0!important}.order-sm-1{order:1!important}.order-sm-2{order:2!important}.order-sm-3{order:3!important}.order-sm-4{order:4!important}.order-sm-5{order:5!important}.order-sm-last{order:6!important}.m-sm-0{margin:0!important}.m-sm-1{margin:.25rem!important}.m-sm-2{margin:.5rem!important}.m-sm-3{margin:1rem!important}.m-sm-4{margin:1.5rem!important}.m-sm-5{margin:3rem!important}.m-sm-auto{margin:auto!important}.mx-sm-0{margin-right:0!important;margin-left:0!important}.mx-sm-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-sm-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-sm-3{margin-right:1rem!important;margin-left:1rem!important}.mx-sm-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-sm-5{margin-right:3rem!important;margin-left:3rem!important}.mx-sm-auto{margin-right:auto!important;margin-left:auto!important}.my-sm-0{margin-top:0!important;margin-bottom:0!important}.my-sm-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-sm-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-sm-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-sm-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-sm-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-sm-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-sm-0{margin-top:0!important}.mt-sm-1{margin-top:.25rem!important}.mt-sm-2{margin-top:.5rem!important}.mt-sm-3{margin-top:1rem!important}.mt-sm-4{margin-top:1.5rem!important}.mt-sm-5{margin-top:3rem!important}.mt-sm-auto{margin-top:auto!important}.me-sm-0{margin-right:0!important}.me-sm-1{margin-right:.25rem!important}.me-sm-2{margin-right:.5rem!important}.me-sm-3{margin-right:1rem!important}.me-sm-4{margin-right:1.5rem!important}.me-sm-5{margin-right:3rem!important}.me-sm-auto{margin-right:auto!important}.mb-sm-0{margin-bottom:0!important}.mb-sm-1{margin-bottom:.25rem!important}.mb-sm-2{margin-bottom:.5rem!important}.mb-sm-3{margin-bottom:1rem!important}.mb-sm-4{margin-bottom:1.5rem!important}.mb-sm-5{margin-bottom:3rem!important}.mb-sm-auto{margin-bottom:auto!important}.ms-sm-0{margin-left:0!important}.ms-sm-1{margin-left:.25rem!important}.ms-sm-2{margin-left:.5rem!important}.ms-sm-3{margin-left:1rem!important}.ms-sm-4{margin-left:1.5rem!important}.ms-sm-5{margin-left:3rem!important}.ms-sm-auto{margin-left:auto!important}.p-sm-0{padding:0!important}.p-sm-1{padding:.25rem!important}.p-sm-2{padding:.5rem!important}.p-sm-3{padding:1rem!important}.p-sm-4{padding:1.5rem!important}.p-sm-5{padding:3rem!important}.px-sm-0{padding-right:0!important;padding-left:0!important}.px-sm-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-sm-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-sm-3{padding-right:1rem!important;padding-left:1rem!important}.px-sm-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-sm-5{padding-right:3rem!important;padding-left:3rem!important}.py-sm-0{padding-top:0!important;padding-bottom:0!important}.py-sm-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-sm-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-sm-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-sm-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-sm-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-sm-0{padding-top:0!important}.pt-sm-1{padding-top:.25rem!important}.pt-sm-2{padding-top:.5rem!important}.pt-sm-3{padding-top:1rem!important}.pt-sm-4{padding-top:1.5rem!important}.pt-sm-5{padding-top:3rem!important}.pe-sm-0{padding-right:0!important}.pe-sm-1{padding-right:.25rem!important}.pe-sm-2{padding-right:.5rem!important}.pe-sm-3{padding-right:1rem!important}.pe-sm-4{padding-right:1.5rem!important}.pe-sm-5{padding-right:3rem!important}.pb-sm-0{padding-bottom:0!important}.pb-sm-1{padding-bottom:.25rem!important}.pb-sm-2{padding-bottom:.5rem!important}.pb-sm-3{padding-bottom:1rem!important}.pb-sm-4{padding-bottom:1.5rem!important}.pb-sm-5{padding-bottom:3rem!important}.ps-sm-0{padding-left:0!important}.ps-sm-1{padding-left:.25rem!important}.ps-sm-2{padding-left:.5rem!important}.ps-sm-3{padding-left:1rem!important}.ps-sm-4{padding-left:1.5rem!important}.ps-sm-5{padding-left:3rem!important}.text-sm-start{text-align:left!important}.text-sm-end{text-align:right!important}.text-sm-center{text-align:center!important}}@media (min-width:768px){.float-md-start{float:left!important}.float-md-end{float:right!important}.float-md-none{float:none!important}.d-md-inline{display:inline!important}.d-md-inline-block{display:inline-block!important}.d-md-block{display:block!important}.d-md-grid{display:grid!important}.d-md-table{display:table!important}.d-md-table-row{display:table-row!important}.d-md-table-cell{display:table-cell!important}.d-md-flex{display:flex!important}.d-md-inline-flex{display:inline-flex!important}.d-md-none{display:none!important}.flex-md-fill{flex:1 1 auto!important}.flex-md-row{flex-direction:row!important}.flex-md-column{flex-direction:column!important}.flex-md-row-reverse{flex-direction:row-reverse!important}.flex-md-column-reverse{flex-direction:column-reverse!important}.flex-md-grow-0{flex-grow:0!important}.flex-md-grow-1{flex-grow:1!important}.flex-md-shrink-0{flex-shrink:0!important}.flex-md-shrink-1{flex-shrink:1!important}.flex-md-wrap{flex-wrap:wrap!important}.flex-md-nowrap{flex-wrap:nowrap!important}.flex-md-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-md-0{gap:0!important}.gap-md-1{gap:.25rem!important}.gap-md-2{gap:.5rem!important}.gap-md-3{gap:1rem!important}.gap-md-4{gap:1.5rem!important}.gap-md-5{gap:3rem!important}.justify-content-md-start{justify-content:flex-start!important}.justify-content-md-end{justify-content:flex-end!important}.justify-content-md-center{justify-content:center!important}.justify-content-md-between{justify-content:space-between!important}.justify-content-md-around{justify-content:space-around!important}.justify-content-md-evenly{justify-content:space-evenly!important}.align-items-md-start{align-items:flex-start!important}.align-items-md-end{align-items:flex-end!important}.align-items-md-center{align-items:center!important}.align-items-md-baseline{align-items:baseline!important}.align-items-md-stretch{align-items:stretch!important}.align-content-md-start{align-content:flex-start!important}.align-content-md-end{align-content:flex-end!important}.align-content-md-center{align-content:center!important}.align-content-md-between{align-content:space-between!important}.align-content-md-around{align-content:space-around!important}.align-content-md-stretch{align-content:stretch!important}.align-self-md-auto{align-self:auto!important}.align-self-md-start{align-self:flex-start!important}.align-self-md-end{align-self:flex-end!important}.align-self-md-center{align-self:center!important}.align-self-md-baseline{align-self:baseline!important}.align-self-md-stretch{align-self:stretch!important}.order-md-first{order:-1!important}.order-md-0{order:0!important}.order-md-1{order:1!important}.order-md-2{order:2!important}.order-md-3{order:3!important}.order-md-4{order:4!important}.order-md-5{order:5!important}.order-md-last{order:6!important}.m-md-0{margin:0!important}.m-md-1{margin:.25rem!important}.m-md-2{margin:.5rem!important}.m-md-3{margin:1rem!important}.m-md-4{margin:1.5rem!important}.m-md-5{margin:3rem!important}.m-md-auto{margin:auto!important}.mx-md-0{margin-right:0!important;margin-left:0!important}.mx-md-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-md-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-md-3{margin-right:1rem!important;margin-left:1rem!important}.mx-md-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-md-5{margin-right:3rem!important;margin-left:3rem!important}.mx-md-auto{margin-right:auto!important;margin-left:auto!important}.my-md-0{margin-top:0!important;margin-bottom:0!important}.my-md-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-md-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-md-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-md-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-md-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-md-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-md-0{margin-top:0!important}.mt-md-1{margin-top:.25rem!important}.mt-md-2{margin-top:.5rem!important}.mt-md-3{margin-top:1rem!important}.mt-md-4{margin-top:1.5rem!important}.mt-md-5{margin-top:3rem!important}.mt-md-auto{margin-top:auto!important}.me-md-0{margin-right:0!important}.me-md-1{margin-right:.25rem!important}.me-md-2{margin-right:.5rem!important}.me-md-3{margin-right:1rem!important}.me-md-4{margin-right:1.5rem!important}.me-md-5{margin-right:3rem!important}.me-md-auto{margin-right:auto!important}.mb-md-0{margin-bottom:0!important}.mb-md-1{margin-bottom:.25rem!important}.mb-md-2{margin-bottom:.5rem!important}.mb-md-3{margin-bottom:1rem!important}.mb-md-4{margin-bottom:1.5rem!important}.mb-md-5{margin-bottom:3rem!important}.mb-md-auto{margin-bottom:auto!important}.ms-md-0{margin-left:0!important}.ms-md-1{margin-left:.25rem!important}.ms-md-2{margin-left:.5rem!important}.ms-md-3{margin-left:1rem!important}.ms-md-4{margin-left:1.5rem!important}.ms-md-5{margin-left:3rem!important}.ms-md-auto{margin-left:auto!important}.p-md-0{padding:0!important}.p-md-1{padding:.25rem!important}.p-md-2{padding:.5rem!important}.p-md-3{padding:1rem!important}.p-md-4{padding:1.5rem!important}.p-md-5{padding:3rem!important}.px-md-0{padding-right:0!important;padding-left:0!important}.px-md-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-md-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-md-3{padding-right:1rem!important;padding-left:1rem!important}.px-md-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-md-5{padding-right:3rem!important;padding-left:3rem!important}.py-md-0{padding-top:0!important;padding-bottom:0!important}.py-md-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-md-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-md-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-md-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-md-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-md-0{padding-top:0!important}.pt-md-1{padding-top:.25rem!important}.pt-md-2{padding-top:.5rem!important}.pt-md-3{padding-top:1rem!important}.pt-md-4{padding-top:1.5rem!important}.pt-md-5{padding-top:3rem!important}.pe-md-0{padding-right:0!important}.pe-md-1{padding-right:.25rem!important}.pe-md-2{padding-right:.5rem!important}.pe-md-3{padding-right:1rem!important}.pe-md-4{padding-right:1.5rem!important}.pe-md-5{padding-right:3rem!important}.pb-md-0{padding-bottom:0!important}.pb-md-1{padding-bottom:.25rem!important}.pb-md-2{padding-bottom:.5rem!important}.pb-md-3{padding-bottom:1rem!important}.pb-md-4{padding-bottom:1.5rem!important}.pb-md-5{padding-bottom:3rem!important}.ps-md-0{padding-left:0!important}.ps-md-1{padding-left:.25rem!important}.ps-md-2{padding-left:.5rem!important}.ps-md-3{padding-left:1rem!important}.ps-md-4{padding-left:1.5rem!important}.ps-md-5{padding-left:3rem!important}.text-md-start{text-align:left!important}.text-md-end{text-align:right!important}.text-md-center{text-align:center!important}}@media (min-width:992px){.float-lg-start{float:left!important}.float-lg-end{float:right!important}.float-lg-none{float:none!important}.d-lg-inline{display:inline!important}.d-lg-inline-block{display:inline-block!important}.d-lg-block{display:block!important}.d-lg-grid{display:grid!important}.d-lg-table{display:table!important}.d-lg-table-row{display:table-row!important}.d-lg-table-cell{display:table-cell!important}.d-lg-flex{display:flex!important}.d-lg-inline-flex{display:inline-flex!important}.d-lg-none{display:none!important}.flex-lg-fill{flex:1 1 auto!important}.flex-lg-row{flex-direction:row!important}.flex-lg-column{flex-direction:column!important}.flex-lg-row-reverse{flex-direction:row-reverse!important}.flex-lg-column-reverse{flex-direction:column-reverse!important}.flex-lg-grow-0{flex-grow:0!important}.flex-lg-grow-1{flex-grow:1!important}.flex-lg-shrink-0{flex-shrink:0!important}.flex-lg-shrink-1{flex-shrink:1!important}.flex-lg-wrap{flex-wrap:wrap!important}.flex-lg-nowrap{flex-wrap:nowrap!important}.flex-lg-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-lg-0{gap:0!important}.gap-lg-1{gap:.25rem!important}.gap-lg-2{gap:.5rem!important}.gap-lg-3{gap:1rem!important}.gap-lg-4{gap:1.5rem!important}.gap-lg-5{gap:3rem!important}.justify-content-lg-start{justify-content:flex-start!important}.justify-content-lg-end{justify-content:flex-end!important}.justify-content-lg-center{justify-content:center!important}.justify-content-lg-between{justify-content:space-between!important}.justify-content-lg-around{justify-content:space-around!important}.justify-content-lg-evenly{justify-content:space-evenly!important}.align-items-lg-start{align-items:flex-start!important}.align-items-lg-end{align-items:flex-end!important}.align-items-lg-center{align-items:center!important}.align-items-lg-baseline{align-items:baseline!important}.align-items-lg-stretch{align-items:stretch!important}.align-content-lg-start{align-content:flex-start!important}.align-content-lg-end{align-content:flex-end!important}.align-content-lg-center{align-content:center!important}.align-content-lg-between{align-content:space-between!important}.align-content-lg-around{align-content:space-around!important}.align-content-lg-stretch{align-content:stretch!important}.align-self-lg-auto{align-self:auto!important}.align-self-lg-start{align-self:flex-start!important}.align-self-lg-end{align-self:flex-end!important}.align-self-lg-center{align-self:center!important}.align-self-lg-baseline{align-self:baseline!important}.align-self-lg-stretch{align-self:stretch!important}.order-lg-first{order:-1!important}.order-lg-0{order:0!important}.order-lg-1{order:1!important}.order-lg-2{order:2!important}.order-lg-3{order:3!important}.order-lg-4{order:4!important}.order-lg-5{order:5!important}.order-lg-last{order:6!important}.m-lg-0{margin:0!important}.m-lg-1{margin:.25rem!important}.m-lg-2{margin:.5rem!important}.m-lg-3{margin:1rem!important}.m-lg-4{margin:1.5rem!important}.m-lg-5{margin:3rem!important}.m-lg-auto{margin:auto!important}.mx-lg-0{margin-right:0!important;margin-left:0!important}.mx-lg-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-lg-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-lg-3{margin-right:1rem!important;margin-left:1rem!important}.mx-lg-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-lg-5{margin-right:3rem!important;margin-left:3rem!important}.mx-lg-auto{margin-right:auto!important;margin-left:auto!important}.my-lg-0{margin-top:0!important;margin-bottom:0!important}.my-lg-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-lg-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-lg-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-lg-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-lg-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-lg-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-lg-0{margin-top:0!important}.mt-lg-1{margin-top:.25rem!important}.mt-lg-2{margin-top:.5rem!important}.mt-lg-3{margin-top:1rem!important}.mt-lg-4{margin-top:1.5rem!important}.mt-lg-5{margin-top:3rem!important}.mt-lg-auto{margin-top:auto!important}.me-lg-0{margin-right:0!important}.me-lg-1{margin-right:.25rem!important}.me-lg-2{margin-right:.5rem!important}.me-lg-3{margin-right:1rem!important}.me-lg-4{margin-right:1.5rem!important}.me-lg-5{margin-right:3rem!important}.me-lg-auto{margin-right:auto!important}.mb-lg-0{margin-bottom:0!important}.mb-lg-1{margin-bottom:.25rem!important}.mb-lg-2{margin-bottom:.5rem!important}.mb-lg-3{margin-bottom:1rem!important}.mb-lg-4{margin-bottom:1.5rem!important}.mb-lg-5{margin-bottom:3rem!important}.mb-lg-auto{margin-bottom:auto!important}.ms-lg-0{margin-left:0!important}.ms-lg-1{margin-left:.25rem!important}.ms-lg-2{margin-left:.5rem!important}.ms-lg-3{margin-left:1rem!important}.ms-lg-4{margin-left:1.5rem!important}.ms-lg-5{margin-left:3rem!important}.ms-lg-auto{margin-left:auto!important}.p-lg-0{padding:0!important}.p-lg-1{padding:.25rem!important}.p-lg-2{padding:.5rem!important}.p-lg-3{padding:1rem!important}.p-lg-4{padding:1.5rem!important}.p-lg-5{padding:3rem!important}.px-lg-0{padding-right:0!important;padding-left:0!important}.px-lg-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-lg-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-lg-3{padding-right:1rem!important;padding-left:1rem!important}.px-lg-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-lg-5{padding-right:3rem!important;padding-left:3rem!important}.py-lg-0{padding-top:0!important;padding-bottom:0!important}.py-lg-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-lg-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-lg-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-lg-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-lg-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-lg-0{padding-top:0!important}.pt-lg-1{padding-top:.25rem!important}.pt-lg-2{padding-top:.5rem!important}.pt-lg-3{padding-top:1rem!important}.pt-lg-4{padding-top:1.5rem!important}.pt-lg-5{padding-top:3rem!important}.pe-lg-0{padding-right:0!important}.pe-lg-1{padding-right:.25rem!important}.pe-lg-2{padding-right:.5rem!important}.pe-lg-3{padding-right:1rem!important}.pe-lg-4{padding-right:1.5rem!important}.pe-lg-5{padding-right:3rem!important}.pb-lg-0{padding-bottom:0!important}.pb-lg-1{padding-bottom:.25rem!important}.pb-lg-2{padding-bottom:.5rem!important}.pb-lg-3{padding-bottom:1rem!important}.pb-lg-4{padding-bottom:1.5rem!important}.pb-lg-5{padding-bottom:3rem!important}.ps-lg-0{padding-left:0!important}.ps-lg-1{padding-left:.25rem!important}.ps-lg-2{padding-left:.5rem!important}.ps-lg-3{padding-left:1rem!important}.ps-lg-4{padding-left:1.5rem!important}.ps-lg-5{padding-left:3rem!important}.text-lg-start{text-align:left!important}.text-lg-end{text-align:right!important}.text-lg-center{text-align:center!important}}@media (min-width:1200px){.float-xl-start{float:left!important}.float-xl-end{float:right!important}.float-xl-none{float:none!important}.d-xl-inline{display:inline!important}.d-xl-inline-block{display:inline-block!important}.d-xl-block{display:block!important}.d-xl-grid{display:grid!important}.d-xl-table{display:table!important}.d-xl-table-row{display:table-row!important}.d-xl-table-cell{display:table-cell!important}.d-xl-flex{display:flex!important}.d-xl-inline-flex{display:inline-flex!important}.d-xl-none{display:none!important}.flex-xl-fill{flex:1 1 auto!important}.flex-xl-row{flex-direction:row!important}.flex-xl-column{flex-direction:column!important}.flex-xl-row-reverse{flex-direction:row-reverse!important}.flex-xl-column-reverse{flex-direction:column-reverse!important}.flex-xl-grow-0{flex-grow:0!important}.flex-xl-grow-1{flex-grow:1!important}.flex-xl-shrink-0{flex-shrink:0!important}.flex-xl-shrink-1{flex-shrink:1!important}.flex-xl-wrap{flex-wrap:wrap!important}.flex-xl-nowrap{flex-wrap:nowrap!important}.flex-xl-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-xl-0{gap:0!important}.gap-xl-1{gap:.25rem!important}.gap-xl-2{gap:.5rem!important}.gap-xl-3{gap:1rem!important}.gap-xl-4{gap:1.5rem!important}.gap-xl-5{gap:3rem!important}.justify-content-xl-start{justify-content:flex-start!important}.justify-content-xl-end{justify-content:flex-end!important}.justify-content-xl-center{justify-content:center!important}.justify-content-xl-between{justify-content:space-between!important}.justify-content-xl-around{justify-content:space-around!important}.justify-content-xl-evenly{justify-content:space-evenly!important}.align-items-xl-start{align-items:flex-start!important}.align-items-xl-end{align-items:flex-end!important}.align-items-xl-center{align-items:center!important}.align-items-xl-baseline{align-items:baseline!important}.align-items-xl-stretch{align-items:stretch!important}.align-content-xl-start{align-content:flex-start!important}.align-content-xl-end{align-content:flex-end!important}.align-content-xl-center{align-content:center!important}.align-content-xl-between{align-content:space-between!important}.align-content-xl-around{align-content:space-around!important}.align-content-xl-stretch{align-content:stretch!important}.align-self-xl-auto{align-self:auto!important}.align-self-xl-start{align-self:flex-start!important}.align-self-xl-end{align-self:flex-end!important}.align-self-xl-center{align-self:center!important}.align-self-xl-baseline{align-self:baseline!important}.align-self-xl-stretch{align-self:stretch!important}.order-xl-first{order:-1!important}.order-xl-0{order:0!important}.order-xl-1{order:1!important}.order-xl-2{order:2!important}.order-xl-3{order:3!important}.order-xl-4{order:4!important}.order-xl-5{order:5!important}.order-xl-last{order:6!important}.m-xl-0{margin:0!important}.m-xl-1{margin:.25rem!important}.m-xl-2{margin:.5rem!important}.m-xl-3{margin:1rem!important}.m-xl-4{margin:1.5rem!important}.m-xl-5{margin:3rem!important}.m-xl-auto{margin:auto!important}.mx-xl-0{margin-right:0!important;margin-left:0!important}.mx-xl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xl-auto{margin-right:auto!important;margin-left:auto!important}.my-xl-0{margin-top:0!important;margin-bottom:0!important}.my-xl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xl-0{margin-top:0!important}.mt-xl-1{margin-top:.25rem!important}.mt-xl-2{margin-top:.5rem!important}.mt-xl-3{margin-top:1rem!important}.mt-xl-4{margin-top:1.5rem!important}.mt-xl-5{margin-top:3rem!important}.mt-xl-auto{margin-top:auto!important}.me-xl-0{margin-right:0!important}.me-xl-1{margin-right:.25rem!important}.me-xl-2{margin-right:.5rem!important}.me-xl-3{margin-right:1rem!important}.me-xl-4{margin-right:1.5rem!important}.me-xl-5{margin-right:3rem!important}.me-xl-auto{margin-right:auto!important}.mb-xl-0{margin-bottom:0!important}.mb-xl-1{margin-bottom:.25rem!important}.mb-xl-2{margin-bottom:.5rem!important}.mb-xl-3{margin-bottom:1rem!important}.mb-xl-4{margin-bottom:1.5rem!important}.mb-xl-5{margin-bottom:3rem!important}.mb-xl-auto{margin-bottom:auto!important}.ms-xl-0{margin-left:0!important}.ms-xl-1{margin-left:.25rem!important}.ms-xl-2{margin-left:.5rem!important}.ms-xl-3{margin-left:1rem!important}.ms-xl-4{margin-left:1.5rem!important}.ms-xl-5{margin-left:3rem!important}.ms-xl-auto{margin-left:auto!important}.p-xl-0{padding:0!important}.p-xl-1{padding:.25rem!important}.p-xl-2{padding:.5rem!important}.p-xl-3{padding:1rem!important}.p-xl-4{padding:1.5rem!important}.p-xl-5{padding:3rem!important}.px-xl-0{padding-right:0!important;padding-left:0!important}.px-xl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xl-0{padding-top:0!important;padding-bottom:0!important}.py-xl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xl-0{padding-top:0!important}.pt-xl-1{padding-top:.25rem!important}.pt-xl-2{padding-top:.5rem!important}.pt-xl-3{padding-top:1rem!important}.pt-xl-4{padding-top:1.5rem!important}.pt-xl-5{padding-top:3rem!important}.pe-xl-0{padding-right:0!important}.pe-xl-1{padding-right:.25rem!important}.pe-xl-2{padding-right:.5rem!important}.pe-xl-3{padding-right:1rem!important}.pe-xl-4{padding-right:1.5rem!important}.pe-xl-5{padding-right:3rem!important}.pb-xl-0{padding-bottom:0!important}.pb-xl-1{padding-bottom:.25rem!important}.pb-xl-2{padding-bottom:.5rem!important}.pb-xl-3{padding-bottom:1rem!important}.pb-xl-4{padding-bottom:1.5rem!important}.pb-xl-5{padding-bottom:3rem!important}.ps-xl-0{padding-left:0!important}.ps-xl-1{padding-left:.25rem!important}.ps-xl-2{padding-left:.5rem!important}.ps-xl-3{padding-left:1rem!important}.ps-xl-4{padding-left:1.5rem!important}.ps-xl-5{padding-left:3rem!important}.text-xl-start{text-align:left!important}.text-xl-end{text-align:right!important}.text-xl-center{text-align:center!important}}@media (min-width:1400px){.float-xxl-start{float:left!important}.float-xxl-end{float:right!important}.float-xxl-none{float:none!important}.d-xxl-inline{display:inline!important}.d-xxl-inline-block{display:inline-block!important}.d-xxl-block{display:block!important}.d-xxl-grid{display:grid!important}.d-xxl-table{display:table!important}.d-xxl-table-row{display:table-row!important}.d-xxl-table-cell{display:table-cell!important}.d-xxl-flex{display:flex!important}.d-xxl-inline-flex{display:inline-flex!important}.d-xxl-none{display:none!important}.flex-xxl-fill{flex:1 1 auto!important}.flex-xxl-row{flex-direction:row!important}.flex-xxl-column{flex-direction:column!important}.flex-xxl-row-reverse{flex-direction:row-reverse!important}.flex-xxl-column-reverse{flex-direction:column-reverse!important}.flex-xxl-grow-0{flex-grow:0!important}.flex-xxl-grow-1{flex-grow:1!important}.flex-xxl-shrink-0{flex-shrink:0!important}.flex-xxl-shrink-1{flex-shrink:1!important}.flex-xxl-wrap{flex-wrap:wrap!important}.flex-xxl-nowrap{flex-wrap:nowrap!important}.flex-xxl-wrap-reverse{flex-wrap:wrap-reverse!important}.gap-xxl-0{gap:0!important}.gap-xxl-1{gap:.25rem!important}.gap-xxl-2{gap:.5rem!important}.gap-xxl-3{gap:1rem!important}.gap-xxl-4{gap:1.5rem!important}.gap-xxl-5{gap:3rem!important}.justify-content-xxl-start{justify-content:flex-start!important}.justify-content-xxl-end{justify-content:flex-end!important}.justify-content-xxl-center{justify-content:center!important}.justify-content-xxl-between{justify-content:space-between!important}.justify-content-xxl-around{justify-content:space-around!important}.justify-content-xxl-evenly{justify-content:space-evenly!important}.align-items-xxl-start{align-items:flex-start!important}.align-items-xxl-end{align-items:flex-end!important}.align-items-xxl-center{align-items:center!important}.align-items-xxl-baseline{align-items:baseline!important}.align-items-xxl-stretch{align-items:stretch!important}.align-content-xxl-start{align-content:flex-start!important}.align-content-xxl-end{align-content:flex-end!important}.align-content-xxl-center{align-content:center!important}.align-content-xxl-between{align-content:space-between!important}.align-content-xxl-around{align-content:space-around!important}.align-content-xxl-stretch{align-content:stretch!important}.align-self-xxl-auto{align-self:auto!important}.align-self-xxl-start{align-self:flex-start!important}.align-self-xxl-end{align-self:flex-end!important}.align-self-xxl-center{align-self:center!important}.align-self-xxl-baseline{align-self:baseline!important}.align-self-xxl-stretch{align-self:stretch!important}.order-xxl-first{order:-1!important}.order-xxl-0{order:0!important}.order-xxl-1{order:1!important}.order-xxl-2{order:2!important}.order-xxl-3{order:3!important}.order-xxl-4{order:4!important}.order-xxl-5{order:5!important}.order-xxl-last{order:6!important}.m-xxl-0{margin:0!important}.m-xxl-1{margin:.25rem!important}.m-xxl-2{margin:.5rem!important}.m-xxl-3{margin:1rem!important}.m-xxl-4{margin:1.5rem!important}.m-xxl-5{margin:3rem!important}.m-xxl-auto{margin:auto!important}.mx-xxl-0{margin-right:0!important;margin-left:0!important}.mx-xxl-1{margin-right:.25rem!important;margin-left:.25rem!important}.mx-xxl-2{margin-right:.5rem!important;margin-left:.5rem!important}.mx-xxl-3{margin-right:1rem!important;margin-left:1rem!important}.mx-xxl-4{margin-right:1.5rem!important;margin-left:1.5rem!important}.mx-xxl-5{margin-right:3rem!important;margin-left:3rem!important}.mx-xxl-auto{margin-right:auto!important;margin-left:auto!important}.my-xxl-0{margin-top:0!important;margin-bottom:0!important}.my-xxl-1{margin-top:.25rem!important;margin-bottom:.25rem!important}.my-xxl-2{margin-top:.5rem!important;margin-bottom:.5rem!important}.my-xxl-3{margin-top:1rem!important;margin-bottom:1rem!important}.my-xxl-4{margin-top:1.5rem!important;margin-bottom:1.5rem!important}.my-xxl-5{margin-top:3rem!important;margin-bottom:3rem!important}.my-xxl-auto{margin-top:auto!important;margin-bottom:auto!important}.mt-xxl-0{margin-top:0!important}.mt-xxl-1{margin-top:.25rem!important}.mt-xxl-2{margin-top:.5rem!important}.mt-xxl-3{margin-top:1rem!important}.mt-xxl-4{margin-top:1.5rem!important}.mt-xxl-5{margin-top:3rem!important}.mt-xxl-auto{margin-top:auto!important}.me-xxl-0{margin-right:0!important}.me-xxl-1{margin-right:.25rem!important}.me-xxl-2{margin-right:.5rem!important}.me-xxl-3{margin-right:1rem!important}.me-xxl-4{margin-right:1.5rem!important}.me-xxl-5{margin-right:3rem!important}.me-xxl-auto{margin-right:auto!important}.mb-xxl-0{margin-bottom:0!important}.mb-xxl-1{margin-bottom:.25rem!important}.mb-xxl-2{margin-bottom:.5rem!important}.mb-xxl-3{margin-bottom:1rem!important}.mb-xxl-4{margin-bottom:1.5rem!important}.mb-xxl-5{margin-bottom:3rem!important}.mb-xxl-auto{margin-bottom:auto!important}.ms-xxl-0{margin-left:0!important}.ms-xxl-1{margin-left:.25rem!important}.ms-xxl-2{margin-left:.5rem!important}.ms-xxl-3{margin-left:1rem!important}.ms-xxl-4{margin-left:1.5rem!important}.ms-xxl-5{margin-left:3rem!important}.ms-xxl-auto{margin-left:auto!important}.p-xxl-0{padding:0!important}.p-xxl-1{padding:.25rem!important}.p-xxl-2{padding:.5rem!important}.p-xxl-3{padding:1rem!important}.p-xxl-4{padding:1.5rem!important}.p-xxl-5{padding:3rem!important}.px-xxl-0{padding-right:0!important;padding-left:0!important}.px-xxl-1{padding-right:.25rem!important;padding-left:.25rem!important}.px-xxl-2{padding-right:.5rem!important;padding-left:.5rem!important}.px-xxl-3{padding-right:1rem!important;padding-left:1rem!important}.px-xxl-4{padding-right:1.5rem!important;padding-left:1.5rem!important}.px-xxl-5{padding-right:3rem!important;padding-left:3rem!important}.py-xxl-0{padding-top:0!important;padding-bottom:0!important}.py-xxl-1{padding-top:.25rem!important;padding-bottom:.25rem!important}.py-xxl-2{padding-top:.5rem!important;padding-bottom:.5rem!important}.py-xxl-3{padding-top:1rem!important;padding-bottom:1rem!important}.py-xxl-4{padding-top:1.5rem!important;padding-bottom:1.5rem!important}.py-xxl-5{padding-top:3rem!important;padding-bottom:3rem!important}.pt-xxl-0{padding-top:0!important}.pt-xxl-1{padding-top:.25rem!important}.pt-xxl-2{padding-top:.5rem!important}.pt-xxl-3{padding-top:1rem!important}.pt-xxl-4{padding-top:1.5rem!important}.pt-xxl-5{padding-top:3rem!important}.pe-xxl-0{padding-right:0!important}.pe-xxl-1{padding-right:.25rem!important}.pe-xxl-2{padding-right:.5rem!important}.pe-xxl-3{padding-right:1rem!important}.pe-xxl-4{padding-right:1.5rem!important}.pe-xxl-5{padding-right:3rem!important}.pb-xxl-0{padding-bottom:0!important}.pb-xxl-1{padding-bottom:.25rem!important}.pb-xxl-2{padding-bottom:.5rem!important}.pb-xxl-3{padding-bottom:1rem!important}.pb-xxl-4{padding-bottom:1.5rem!important}.pb-xxl-5{padding-bottom:3rem!important}.ps-xxl-0{padding-left:0!important}.ps-xxl-1{padding-left:.25rem!important}.ps-xxl-2{padding-left:.5rem!important}.ps-xxl-3{padding-left:1rem!important}.ps-xxl-4{padding-left:1.5rem!important}.ps-xxl-5{padding-left:3rem!important}.text-xxl-start{text-align:left!important}.text-xxl-end{text-align:right!important}.text-xxl-center{text-align:center!important}}@media (min-width:1200px){.fs-1{font-size:2.5rem!important}.fs-2{font-size:2rem!important}.fs-3{font-size:1.75rem!important}.fs-4{font-size:1.5rem!important}}@media print{.d-print-inline{display:inline!important}.d-print-inline-block{display:inline-block!important}.d-print-block{display:block!important}.d-print-grid{display:grid!important}.d-print-table{display:table!important}.d-print-table-row{display:table-row!important}.d-print-table-cell{display:table-cell!important}.d-print-flex{display:flex!important}.d-print-inline-flex{display:inline-flex!important}.d-print-none{display:none!important}} +/*# sourceMappingURL=bootstrap.min.css.map */ \ No newline at end of file diff --git a/pkg/api/static/bootstrap-icons-1.8.1.css b/pkg/api/static/bootstrap-icons-1.8.1.css new file mode 100644 index 00000000000..5712315ebab --- /dev/null +++ b/pkg/api/static/bootstrap-icons-1.8.1.css @@ -0,0 +1,1704 @@ +@font-face { + font-family: "bootstrap-icons"; + src: url("./fonts/bootstrap-icons.woff2?524846017b983fc8ded9325d94ed40f3") format("woff2"), +url("./fonts/bootstrap-icons.woff?524846017b983fc8ded9325d94ed40f3") format("woff"); +} + +.bi::before, +[class^="bi-"]::before, +[class*=" bi-"]::before { + display: inline-block; + font-family: bootstrap-icons !important; + font-style: normal; + font-weight: normal !important; + font-variant: normal; + text-transform: none; + line-height: 1; + vertical-align: -.125em; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.bi-123::before { content: "\f67f"; } +.bi-alarm-fill::before { content: "\f101"; } +.bi-alarm::before { content: "\f102"; } +.bi-align-bottom::before { content: "\f103"; } +.bi-align-center::before { content: "\f104"; } +.bi-align-end::before { content: "\f105"; } +.bi-align-middle::before { content: "\f106"; } +.bi-align-start::before { content: "\f107"; } +.bi-align-top::before { content: "\f108"; } +.bi-alt::before { content: "\f109"; } +.bi-app-indicator::before { content: "\f10a"; } +.bi-app::before { content: "\f10b"; } +.bi-archive-fill::before { content: "\f10c"; } +.bi-archive::before { content: "\f10d"; } +.bi-arrow-90deg-down::before { content: "\f10e"; } +.bi-arrow-90deg-left::before { content: "\f10f"; } +.bi-arrow-90deg-right::before { content: "\f110"; } +.bi-arrow-90deg-up::before { content: "\f111"; } +.bi-arrow-bar-down::before { content: "\f112"; } +.bi-arrow-bar-left::before { content: "\f113"; } +.bi-arrow-bar-right::before { content: "\f114"; } +.bi-arrow-bar-up::before { content: "\f115"; } +.bi-arrow-clockwise::before { content: "\f116"; } +.bi-arrow-counterclockwise::before { content: "\f117"; } +.bi-arrow-down-circle-fill::before { content: "\f118"; } +.bi-arrow-down-circle::before { content: "\f119"; } +.bi-arrow-down-left-circle-fill::before { content: "\f11a"; } +.bi-arrow-down-left-circle::before { content: "\f11b"; } +.bi-arrow-down-left-square-fill::before { content: "\f11c"; } +.bi-arrow-down-left-square::before { content: "\f11d"; } +.bi-arrow-down-left::before { content: "\f11e"; } +.bi-arrow-down-right-circle-fill::before { content: "\f11f"; } +.bi-arrow-down-right-circle::before { content: "\f120"; } +.bi-arrow-down-right-square-fill::before { content: "\f121"; } +.bi-arrow-down-right-square::before { content: "\f122"; } +.bi-arrow-down-right::before { content: "\f123"; } +.bi-arrow-down-short::before { content: "\f124"; } +.bi-arrow-down-square-fill::before { content: "\f125"; } +.bi-arrow-down-square::before { content: "\f126"; } +.bi-arrow-down-up::before { content: "\f127"; } +.bi-arrow-down::before { content: "\f128"; } +.bi-arrow-left-circle-fill::before { content: "\f129"; } +.bi-arrow-left-circle::before { content: "\f12a"; } +.bi-arrow-left-right::before { content: "\f12b"; } +.bi-arrow-left-short::before { content: "\f12c"; } +.bi-arrow-left-square-fill::before { content: "\f12d"; } +.bi-arrow-left-square::before { content: "\f12e"; } +.bi-arrow-left::before { content: "\f12f"; } +.bi-arrow-repeat::before { content: "\f130"; } +.bi-arrow-return-left::before { content: "\f131"; } +.bi-arrow-return-right::before { content: "\f132"; } +.bi-arrow-right-circle-fill::before { content: "\f133"; } +.bi-arrow-right-circle::before { content: "\f134"; } +.bi-arrow-right-short::before { content: "\f135"; } +.bi-arrow-right-square-fill::before { content: "\f136"; } +.bi-arrow-right-square::before { content: "\f137"; } +.bi-arrow-right::before { content: "\f138"; } +.bi-arrow-up-circle-fill::before { content: "\f139"; } +.bi-arrow-up-circle::before { content: "\f13a"; } +.bi-arrow-up-left-circle-fill::before { content: "\f13b"; } +.bi-arrow-up-left-circle::before { content: "\f13c"; } +.bi-arrow-up-left-square-fill::before { content: "\f13d"; } +.bi-arrow-up-left-square::before { content: "\f13e"; } +.bi-arrow-up-left::before { content: "\f13f"; } +.bi-arrow-up-right-circle-fill::before { content: "\f140"; } +.bi-arrow-up-right-circle::before { content: "\f141"; } +.bi-arrow-up-right-square-fill::before { content: "\f142"; } +.bi-arrow-up-right-square::before { content: "\f143"; } +.bi-arrow-up-right::before { content: "\f144"; } +.bi-arrow-up-short::before { content: "\f145"; } +.bi-arrow-up-square-fill::before { content: "\f146"; } +.bi-arrow-up-square::before { content: "\f147"; } +.bi-arrow-up::before { content: "\f148"; } +.bi-arrows-angle-contract::before { content: "\f149"; } +.bi-arrows-angle-expand::before { content: "\f14a"; } +.bi-arrows-collapse::before { content: "\f14b"; } +.bi-arrows-expand::before { content: "\f14c"; } +.bi-arrows-fullscreen::before { content: "\f14d"; } +.bi-arrows-move::before { content: "\f14e"; } +.bi-aspect-ratio-fill::before { content: "\f14f"; } +.bi-aspect-ratio::before { content: "\f150"; } +.bi-asterisk::before { content: "\f151"; } +.bi-at::before { content: "\f152"; } +.bi-award-fill::before { content: "\f153"; } +.bi-award::before { content: "\f154"; } +.bi-back::before { content: "\f155"; } +.bi-backspace-fill::before { content: "\f156"; } +.bi-backspace-reverse-fill::before { content: "\f157"; } +.bi-backspace-reverse::before { content: "\f158"; } +.bi-backspace::before { content: "\f159"; } +.bi-badge-3d-fill::before { content: "\f15a"; } +.bi-badge-3d::before { content: "\f15b"; } +.bi-badge-4k-fill::before { content: "\f15c"; } +.bi-badge-4k::before { content: "\f15d"; } +.bi-badge-8k-fill::before { content: "\f15e"; } +.bi-badge-8k::before { content: "\f15f"; } +.bi-badge-ad-fill::before { content: "\f160"; } +.bi-badge-ad::before { content: "\f161"; } +.bi-badge-ar-fill::before { content: "\f162"; } +.bi-badge-ar::before { content: "\f163"; } +.bi-badge-cc-fill::before { content: "\f164"; } +.bi-badge-cc::before { content: "\f165"; } +.bi-badge-hd-fill::before { content: "\f166"; } +.bi-badge-hd::before { content: "\f167"; } +.bi-badge-tm-fill::before { content: "\f168"; } +.bi-badge-tm::before { content: "\f169"; } +.bi-badge-vo-fill::before { content: "\f16a"; } +.bi-badge-vo::before { content: "\f16b"; } +.bi-badge-vr-fill::before { content: "\f16c"; } +.bi-badge-vr::before { content: "\f16d"; } +.bi-badge-wc-fill::before { content: "\f16e"; } +.bi-badge-wc::before { content: "\f16f"; } +.bi-bag-check-fill::before { content: "\f170"; } +.bi-bag-check::before { content: "\f171"; } +.bi-bag-dash-fill::before { content: "\f172"; } +.bi-bag-dash::before { content: "\f173"; } +.bi-bag-fill::before { content: "\f174"; } +.bi-bag-plus-fill::before { content: "\f175"; } +.bi-bag-plus::before { content: "\f176"; } +.bi-bag-x-fill::before { content: "\f177"; } +.bi-bag-x::before { content: "\f178"; } +.bi-bag::before { content: "\f179"; } +.bi-bar-chart-fill::before { content: "\f17a"; } +.bi-bar-chart-line-fill::before { content: "\f17b"; } +.bi-bar-chart-line::before { content: "\f17c"; } +.bi-bar-chart-steps::before { content: "\f17d"; } +.bi-bar-chart::before { content: "\f17e"; } +.bi-basket-fill::before { content: "\f17f"; } +.bi-basket::before { content: "\f180"; } +.bi-basket2-fill::before { content: "\f181"; } +.bi-basket2::before { content: "\f182"; } +.bi-basket3-fill::before { content: "\f183"; } +.bi-basket3::before { content: "\f184"; } +.bi-battery-charging::before { content: "\f185"; } +.bi-battery-full::before { content: "\f186"; } +.bi-battery-half::before { content: "\f187"; } +.bi-battery::before { content: "\f188"; } +.bi-bell-fill::before { content: "\f189"; } +.bi-bell::before { content: "\f18a"; } +.bi-bezier::before { content: "\f18b"; } +.bi-bezier2::before { content: "\f18c"; } +.bi-bicycle::before { content: "\f18d"; } +.bi-binoculars-fill::before { content: "\f18e"; } +.bi-binoculars::before { content: "\f18f"; } +.bi-blockquote-left::before { content: "\f190"; } +.bi-blockquote-right::before { content: "\f191"; } +.bi-book-fill::before { content: "\f192"; } +.bi-book-half::before { content: "\f193"; } +.bi-book::before { content: "\f194"; } +.bi-bookmark-check-fill::before { content: "\f195"; } +.bi-bookmark-check::before { content: "\f196"; } +.bi-bookmark-dash-fill::before { content: "\f197"; } +.bi-bookmark-dash::before { content: "\f198"; } +.bi-bookmark-fill::before { content: "\f199"; } +.bi-bookmark-heart-fill::before { content: "\f19a"; } +.bi-bookmark-heart::before { content: "\f19b"; } +.bi-bookmark-plus-fill::before { content: "\f19c"; } +.bi-bookmark-plus::before { content: "\f19d"; } +.bi-bookmark-star-fill::before { content: "\f19e"; } +.bi-bookmark-star::before { content: "\f19f"; } +.bi-bookmark-x-fill::before { content: "\f1a0"; } +.bi-bookmark-x::before { content: "\f1a1"; } +.bi-bookmark::before { content: "\f1a2"; } +.bi-bookmarks-fill::before { content: "\f1a3"; } +.bi-bookmarks::before { content: "\f1a4"; } +.bi-bookshelf::before { content: "\f1a5"; } +.bi-bootstrap-fill::before { content: "\f1a6"; } +.bi-bootstrap-reboot::before { content: "\f1a7"; } +.bi-bootstrap::before { content: "\f1a8"; } +.bi-border-all::before { content: "\f1a9"; } +.bi-border-bottom::before { content: "\f1aa"; } +.bi-border-center::before { content: "\f1ab"; } +.bi-border-inner::before { content: "\f1ac"; } +.bi-border-left::before { content: "\f1ad"; } +.bi-border-middle::before { content: "\f1ae"; } +.bi-border-outer::before { content: "\f1af"; } +.bi-border-right::before { content: "\f1b0"; } +.bi-border-style::before { content: "\f1b1"; } +.bi-border-top::before { content: "\f1b2"; } +.bi-border-width::before { content: "\f1b3"; } +.bi-border::before { content: "\f1b4"; } +.bi-bounding-box-circles::before { content: "\f1b5"; } +.bi-bounding-box::before { content: "\f1b6"; } +.bi-box-arrow-down-left::before { content: "\f1b7"; } +.bi-box-arrow-down-right::before { content: "\f1b8"; } +.bi-box-arrow-down::before { content: "\f1b9"; } +.bi-box-arrow-in-down-left::before { content: "\f1ba"; } +.bi-box-arrow-in-down-right::before { content: "\f1bb"; } +.bi-box-arrow-in-down::before { content: "\f1bc"; } +.bi-box-arrow-in-left::before { content: "\f1bd"; } +.bi-box-arrow-in-right::before { content: "\f1be"; } +.bi-box-arrow-in-up-left::before { content: "\f1bf"; } +.bi-box-arrow-in-up-right::before { content: "\f1c0"; } +.bi-box-arrow-in-up::before { content: "\f1c1"; } +.bi-box-arrow-left::before { content: "\f1c2"; } +.bi-box-arrow-right::before { content: "\f1c3"; } +.bi-box-arrow-up-left::before { content: "\f1c4"; } +.bi-box-arrow-up-right::before { content: "\f1c5"; } +.bi-box-arrow-up::before { content: "\f1c6"; } +.bi-box-seam::before { content: "\f1c7"; } +.bi-box::before { content: "\f1c8"; } +.bi-braces::before { content: "\f1c9"; } +.bi-bricks::before { content: "\f1ca"; } +.bi-briefcase-fill::before { content: "\f1cb"; } +.bi-briefcase::before { content: "\f1cc"; } +.bi-brightness-alt-high-fill::before { content: "\f1cd"; } +.bi-brightness-alt-high::before { content: "\f1ce"; } +.bi-brightness-alt-low-fill::before { content: "\f1cf"; } +.bi-brightness-alt-low::before { content: "\f1d0"; } +.bi-brightness-high-fill::before { content: "\f1d1"; } +.bi-brightness-high::before { content: "\f1d2"; } +.bi-brightness-low-fill::before { content: "\f1d3"; } +.bi-brightness-low::before { content: "\f1d4"; } +.bi-broadcast-pin::before { content: "\f1d5"; } +.bi-broadcast::before { content: "\f1d6"; } +.bi-brush-fill::before { content: "\f1d7"; } +.bi-brush::before { content: "\f1d8"; } +.bi-bucket-fill::before { content: "\f1d9"; } +.bi-bucket::before { content: "\f1da"; } +.bi-bug-fill::before { content: "\f1db"; } +.bi-bug::before { content: "\f1dc"; } +.bi-building::before { content: "\f1dd"; } +.bi-bullseye::before { content: "\f1de"; } +.bi-calculator-fill::before { content: "\f1df"; } +.bi-calculator::before { content: "\f1e0"; } +.bi-calendar-check-fill::before { content: "\f1e1"; } +.bi-calendar-check::before { content: "\f1e2"; } +.bi-calendar-date-fill::before { content: "\f1e3"; } +.bi-calendar-date::before { content: "\f1e4"; } +.bi-calendar-day-fill::before { content: "\f1e5"; } +.bi-calendar-day::before { content: "\f1e6"; } +.bi-calendar-event-fill::before { content: "\f1e7"; } +.bi-calendar-event::before { content: "\f1e8"; } +.bi-calendar-fill::before { content: "\f1e9"; } +.bi-calendar-minus-fill::before { content: "\f1ea"; } +.bi-calendar-minus::before { content: "\f1eb"; } +.bi-calendar-month-fill::before { content: "\f1ec"; } +.bi-calendar-month::before { content: "\f1ed"; } +.bi-calendar-plus-fill::before { content: "\f1ee"; } +.bi-calendar-plus::before { content: "\f1ef"; } +.bi-calendar-range-fill::before { content: "\f1f0"; } +.bi-calendar-range::before { content: "\f1f1"; } +.bi-calendar-week-fill::before { content: "\f1f2"; } +.bi-calendar-week::before { content: "\f1f3"; } +.bi-calendar-x-fill::before { content: "\f1f4"; } +.bi-calendar-x::before { content: "\f1f5"; } +.bi-calendar::before { content: "\f1f6"; } +.bi-calendar2-check-fill::before { content: "\f1f7"; } +.bi-calendar2-check::before { content: "\f1f8"; } +.bi-calendar2-date-fill::before { content: "\f1f9"; } +.bi-calendar2-date::before { content: "\f1fa"; } +.bi-calendar2-day-fill::before { content: "\f1fb"; } +.bi-calendar2-day::before { content: "\f1fc"; } +.bi-calendar2-event-fill::before { content: "\f1fd"; } +.bi-calendar2-event::before { content: "\f1fe"; } +.bi-calendar2-fill::before { content: "\f1ff"; } +.bi-calendar2-minus-fill::before { content: "\f200"; } +.bi-calendar2-minus::before { content: "\f201"; } +.bi-calendar2-month-fill::before { content: "\f202"; } +.bi-calendar2-month::before { content: "\f203"; } +.bi-calendar2-plus-fill::before { content: "\f204"; } +.bi-calendar2-plus::before { content: "\f205"; } +.bi-calendar2-range-fill::before { content: "\f206"; } +.bi-calendar2-range::before { content: "\f207"; } +.bi-calendar2-week-fill::before { content: "\f208"; } +.bi-calendar2-week::before { content: "\f209"; } +.bi-calendar2-x-fill::before { content: "\f20a"; } +.bi-calendar2-x::before { content: "\f20b"; } +.bi-calendar2::before { content: "\f20c"; } +.bi-calendar3-event-fill::before { content: "\f20d"; } +.bi-calendar3-event::before { content: "\f20e"; } +.bi-calendar3-fill::before { content: "\f20f"; } +.bi-calendar3-range-fill::before { content: "\f210"; } +.bi-calendar3-range::before { content: "\f211"; } +.bi-calendar3-week-fill::before { content: "\f212"; } +.bi-calendar3-week::before { content: "\f213"; } +.bi-calendar3::before { content: "\f214"; } +.bi-calendar4-event::before { content: "\f215"; } +.bi-calendar4-range::before { content: "\f216"; } +.bi-calendar4-week::before { content: "\f217"; } +.bi-calendar4::before { content: "\f218"; } +.bi-camera-fill::before { content: "\f219"; } +.bi-camera-reels-fill::before { content: "\f21a"; } +.bi-camera-reels::before { content: "\f21b"; } +.bi-camera-video-fill::before { content: "\f21c"; } +.bi-camera-video-off-fill::before { content: "\f21d"; } +.bi-camera-video-off::before { content: "\f21e"; } +.bi-camera-video::before { content: "\f21f"; } +.bi-camera::before { content: "\f220"; } +.bi-camera2::before { content: "\f221"; } +.bi-capslock-fill::before { content: "\f222"; } +.bi-capslock::before { content: "\f223"; } +.bi-card-checklist::before { content: "\f224"; } +.bi-card-heading::before { content: "\f225"; } +.bi-card-image::before { content: "\f226"; } +.bi-card-list::before { content: "\f227"; } +.bi-card-text::before { content: "\f228"; } +.bi-caret-down-fill::before { content: "\f229"; } +.bi-caret-down-square-fill::before { content: "\f22a"; } +.bi-caret-down-square::before { content: "\f22b"; } +.bi-caret-down::before { content: "\f22c"; } +.bi-caret-left-fill::before { content: "\f22d"; } +.bi-caret-left-square-fill::before { content: "\f22e"; } +.bi-caret-left-square::before { content: "\f22f"; } +.bi-caret-left::before { content: "\f230"; } +.bi-caret-right-fill::before { content: "\f231"; } +.bi-caret-right-square-fill::before { content: "\f232"; } +.bi-caret-right-square::before { content: "\f233"; } +.bi-caret-right::before { content: "\f234"; } +.bi-caret-up-fill::before { content: "\f235"; } +.bi-caret-up-square-fill::before { content: "\f236"; } +.bi-caret-up-square::before { content: "\f237"; } +.bi-caret-up::before { content: "\f238"; } +.bi-cart-check-fill::before { content: "\f239"; } +.bi-cart-check::before { content: "\f23a"; } +.bi-cart-dash-fill::before { content: "\f23b"; } +.bi-cart-dash::before { content: "\f23c"; } +.bi-cart-fill::before { content: "\f23d"; } +.bi-cart-plus-fill::before { content: "\f23e"; } +.bi-cart-plus::before { content: "\f23f"; } +.bi-cart-x-fill::before { content: "\f240"; } +.bi-cart-x::before { content: "\f241"; } +.bi-cart::before { content: "\f242"; } +.bi-cart2::before { content: "\f243"; } +.bi-cart3::before { content: "\f244"; } +.bi-cart4::before { content: "\f245"; } +.bi-cash-stack::before { content: "\f246"; } +.bi-cash::before { content: "\f247"; } +.bi-cast::before { content: "\f248"; } +.bi-chat-dots-fill::before { content: "\f249"; } +.bi-chat-dots::before { content: "\f24a"; } +.bi-chat-fill::before { content: "\f24b"; } +.bi-chat-left-dots-fill::before { content: "\f24c"; } +.bi-chat-left-dots::before { content: "\f24d"; } +.bi-chat-left-fill::before { content: "\f24e"; } +.bi-chat-left-quote-fill::before { content: "\f24f"; } +.bi-chat-left-quote::before { content: "\f250"; } +.bi-chat-left-text-fill::before { content: "\f251"; } +.bi-chat-left-text::before { content: "\f252"; } +.bi-chat-left::before { content: "\f253"; } +.bi-chat-quote-fill::before { content: "\f254"; } +.bi-chat-quote::before { content: "\f255"; } +.bi-chat-right-dots-fill::before { content: "\f256"; } +.bi-chat-right-dots::before { content: "\f257"; } +.bi-chat-right-fill::before { content: "\f258"; } +.bi-chat-right-quote-fill::before { content: "\f259"; } +.bi-chat-right-quote::before { content: "\f25a"; } +.bi-chat-right-text-fill::before { content: "\f25b"; } +.bi-chat-right-text::before { content: "\f25c"; } +.bi-chat-right::before { content: "\f25d"; } +.bi-chat-square-dots-fill::before { content: "\f25e"; } +.bi-chat-square-dots::before { content: "\f25f"; } +.bi-chat-square-fill::before { content: "\f260"; } +.bi-chat-square-quote-fill::before { content: "\f261"; } +.bi-chat-square-quote::before { content: "\f262"; } +.bi-chat-square-text-fill::before { content: "\f263"; } +.bi-chat-square-text::before { content: "\f264"; } +.bi-chat-square::before { content: "\f265"; } +.bi-chat-text-fill::before { content: "\f266"; } +.bi-chat-text::before { content: "\f267"; } +.bi-chat::before { content: "\f268"; } +.bi-check-all::before { content: "\f269"; } +.bi-check-circle-fill::before { content: "\f26a"; } +.bi-check-circle::before { content: "\f26b"; } +.bi-check-square-fill::before { content: "\f26c"; } +.bi-check-square::before { content: "\f26d"; } +.bi-check::before { content: "\f26e"; } +.bi-check2-all::before { content: "\f26f"; } +.bi-check2-circle::before { content: "\f270"; } +.bi-check2-square::before { content: "\f271"; } +.bi-check2::before { content: "\f272"; } +.bi-chevron-bar-contract::before { content: "\f273"; } +.bi-chevron-bar-down::before { content: "\f274"; } +.bi-chevron-bar-expand::before { content: "\f275"; } +.bi-chevron-bar-left::before { content: "\f276"; } +.bi-chevron-bar-right::before { content: "\f277"; } +.bi-chevron-bar-up::before { content: "\f278"; } +.bi-chevron-compact-down::before { content: "\f279"; } +.bi-chevron-compact-left::before { content: "\f27a"; } +.bi-chevron-compact-right::before { content: "\f27b"; } +.bi-chevron-compact-up::before { content: "\f27c"; } +.bi-chevron-contract::before { content: "\f27d"; } +.bi-chevron-double-down::before { content: "\f27e"; } +.bi-chevron-double-left::before { content: "\f27f"; } +.bi-chevron-double-right::before { content: "\f280"; } +.bi-chevron-double-up::before { content: "\f281"; } +.bi-chevron-down::before { content: "\f282"; } +.bi-chevron-expand::before { content: "\f283"; } +.bi-chevron-left::before { content: "\f284"; } +.bi-chevron-right::before { content: "\f285"; } +.bi-chevron-up::before { content: "\f286"; } +.bi-circle-fill::before { content: "\f287"; } +.bi-circle-half::before { content: "\f288"; } +.bi-circle-square::before { content: "\f289"; } +.bi-circle::before { content: "\f28a"; } +.bi-clipboard-check::before { content: "\f28b"; } +.bi-clipboard-data::before { content: "\f28c"; } +.bi-clipboard-minus::before { content: "\f28d"; } +.bi-clipboard-plus::before { content: "\f28e"; } +.bi-clipboard-x::before { content: "\f28f"; } +.bi-clipboard::before { content: "\f290"; } +.bi-clock-fill::before { content: "\f291"; } +.bi-clock-history::before { content: "\f292"; } +.bi-clock::before { content: "\f293"; } +.bi-cloud-arrow-down-fill::before { content: "\f294"; } +.bi-cloud-arrow-down::before { content: "\f295"; } +.bi-cloud-arrow-up-fill::before { content: "\f296"; } +.bi-cloud-arrow-up::before { content: "\f297"; } +.bi-cloud-check-fill::before { content: "\f298"; } +.bi-cloud-check::before { content: "\f299"; } +.bi-cloud-download-fill::before { content: "\f29a"; } +.bi-cloud-download::before { content: "\f29b"; } +.bi-cloud-drizzle-fill::before { content: "\f29c"; } +.bi-cloud-drizzle::before { content: "\f29d"; } +.bi-cloud-fill::before { content: "\f29e"; } +.bi-cloud-fog-fill::before { content: "\f29f"; } +.bi-cloud-fog::before { content: "\f2a0"; } +.bi-cloud-fog2-fill::before { content: "\f2a1"; } +.bi-cloud-fog2::before { content: "\f2a2"; } +.bi-cloud-hail-fill::before { content: "\f2a3"; } +.bi-cloud-hail::before { content: "\f2a4"; } +.bi-cloud-haze-1::before { content: "\f2a5"; } +.bi-cloud-haze-fill::before { content: "\f2a6"; } +.bi-cloud-haze::before { content: "\f2a7"; } +.bi-cloud-haze2-fill::before { content: "\f2a8"; } +.bi-cloud-lightning-fill::before { content: "\f2a9"; } +.bi-cloud-lightning-rain-fill::before { content: "\f2aa"; } +.bi-cloud-lightning-rain::before { content: "\f2ab"; } +.bi-cloud-lightning::before { content: "\f2ac"; } +.bi-cloud-minus-fill::before { content: "\f2ad"; } +.bi-cloud-minus::before { content: "\f2ae"; } +.bi-cloud-moon-fill::before { content: "\f2af"; } +.bi-cloud-moon::before { content: "\f2b0"; } +.bi-cloud-plus-fill::before { content: "\f2b1"; } +.bi-cloud-plus::before { content: "\f2b2"; } +.bi-cloud-rain-fill::before { content: "\f2b3"; } +.bi-cloud-rain-heavy-fill::before { content: "\f2b4"; } +.bi-cloud-rain-heavy::before { content: "\f2b5"; } +.bi-cloud-rain::before { content: "\f2b6"; } +.bi-cloud-slash-fill::before { content: "\f2b7"; } +.bi-cloud-slash::before { content: "\f2b8"; } +.bi-cloud-sleet-fill::before { content: "\f2b9"; } +.bi-cloud-sleet::before { content: "\f2ba"; } +.bi-cloud-snow-fill::before { content: "\f2bb"; } +.bi-cloud-snow::before { content: "\f2bc"; } +.bi-cloud-sun-fill::before { content: "\f2bd"; } +.bi-cloud-sun::before { content: "\f2be"; } +.bi-cloud-upload-fill::before { content: "\f2bf"; } +.bi-cloud-upload::before { content: "\f2c0"; } +.bi-cloud::before { content: "\f2c1"; } +.bi-clouds-fill::before { content: "\f2c2"; } +.bi-clouds::before { content: "\f2c3"; } +.bi-cloudy-fill::before { content: "\f2c4"; } +.bi-cloudy::before { content: "\f2c5"; } +.bi-code-slash::before { content: "\f2c6"; } +.bi-code-square::before { content: "\f2c7"; } +.bi-code::before { content: "\f2c8"; } +.bi-collection-fill::before { content: "\f2c9"; } +.bi-collection-play-fill::before { content: "\f2ca"; } +.bi-collection-play::before { content: "\f2cb"; } +.bi-collection::before { content: "\f2cc"; } +.bi-columns-gap::before { content: "\f2cd"; } +.bi-columns::before { content: "\f2ce"; } +.bi-command::before { content: "\f2cf"; } +.bi-compass-fill::before { content: "\f2d0"; } +.bi-compass::before { content: "\f2d1"; } +.bi-cone-striped::before { content: "\f2d2"; } +.bi-cone::before { content: "\f2d3"; } +.bi-controller::before { content: "\f2d4"; } +.bi-cpu-fill::before { content: "\f2d5"; } +.bi-cpu::before { content: "\f2d6"; } +.bi-credit-card-2-back-fill::before { content: "\f2d7"; } +.bi-credit-card-2-back::before { content: "\f2d8"; } +.bi-credit-card-2-front-fill::before { content: "\f2d9"; } +.bi-credit-card-2-front::before { content: "\f2da"; } +.bi-credit-card-fill::before { content: "\f2db"; } +.bi-credit-card::before { content: "\f2dc"; } +.bi-crop::before { content: "\f2dd"; } +.bi-cup-fill::before { content: "\f2de"; } +.bi-cup-straw::before { content: "\f2df"; } +.bi-cup::before { content: "\f2e0"; } +.bi-cursor-fill::before { content: "\f2e1"; } +.bi-cursor-text::before { content: "\f2e2"; } +.bi-cursor::before { content: "\f2e3"; } +.bi-dash-circle-dotted::before { content: "\f2e4"; } +.bi-dash-circle-fill::before { content: "\f2e5"; } +.bi-dash-circle::before { content: "\f2e6"; } +.bi-dash-square-dotted::before { content: "\f2e7"; } +.bi-dash-square-fill::before { content: "\f2e8"; } +.bi-dash-square::before { content: "\f2e9"; } +.bi-dash::before { content: "\f2ea"; } +.bi-diagram-2-fill::before { content: "\f2eb"; } +.bi-diagram-2::before { content: "\f2ec"; } +.bi-diagram-3-fill::before { content: "\f2ed"; } +.bi-diagram-3::before { content: "\f2ee"; } +.bi-diamond-fill::before { content: "\f2ef"; } +.bi-diamond-half::before { content: "\f2f0"; } +.bi-diamond::before { content: "\f2f1"; } +.bi-dice-1-fill::before { content: "\f2f2"; } +.bi-dice-1::before { content: "\f2f3"; } +.bi-dice-2-fill::before { content: "\f2f4"; } +.bi-dice-2::before { content: "\f2f5"; } +.bi-dice-3-fill::before { content: "\f2f6"; } +.bi-dice-3::before { content: "\f2f7"; } +.bi-dice-4-fill::before { content: "\f2f8"; } +.bi-dice-4::before { content: "\f2f9"; } +.bi-dice-5-fill::before { content: "\f2fa"; } +.bi-dice-5::before { content: "\f2fb"; } +.bi-dice-6-fill::before { content: "\f2fc"; } +.bi-dice-6::before { content: "\f2fd"; } +.bi-disc-fill::before { content: "\f2fe"; } +.bi-disc::before { content: "\f2ff"; } +.bi-discord::before { content: "\f300"; } +.bi-display-fill::before { content: "\f301"; } +.bi-display::before { content: "\f302"; } +.bi-distribute-horizontal::before { content: "\f303"; } +.bi-distribute-vertical::before { content: "\f304"; } +.bi-door-closed-fill::before { content: "\f305"; } +.bi-door-closed::before { content: "\f306"; } +.bi-door-open-fill::before { content: "\f307"; } +.bi-door-open::before { content: "\f308"; } +.bi-dot::before { content: "\f309"; } +.bi-download::before { content: "\f30a"; } +.bi-droplet-fill::before { content: "\f30b"; } +.bi-droplet-half::before { content: "\f30c"; } +.bi-droplet::before { content: "\f30d"; } +.bi-earbuds::before { content: "\f30e"; } +.bi-easel-fill::before { content: "\f30f"; } +.bi-easel::before { content: "\f310"; } +.bi-egg-fill::before { content: "\f311"; } +.bi-egg-fried::before { content: "\f312"; } +.bi-egg::before { content: "\f313"; } +.bi-eject-fill::before { content: "\f314"; } +.bi-eject::before { content: "\f315"; } +.bi-emoji-angry-fill::before { content: "\f316"; } +.bi-emoji-angry::before { content: "\f317"; } +.bi-emoji-dizzy-fill::before { content: "\f318"; } +.bi-emoji-dizzy::before { content: "\f319"; } +.bi-emoji-expressionless-fill::before { content: "\f31a"; } +.bi-emoji-expressionless::before { content: "\f31b"; } +.bi-emoji-frown-fill::before { content: "\f31c"; } +.bi-emoji-frown::before { content: "\f31d"; } +.bi-emoji-heart-eyes-fill::before { content: "\f31e"; } +.bi-emoji-heart-eyes::before { content: "\f31f"; } +.bi-emoji-laughing-fill::before { content: "\f320"; } +.bi-emoji-laughing::before { content: "\f321"; } +.bi-emoji-neutral-fill::before { content: "\f322"; } +.bi-emoji-neutral::before { content: "\f323"; } +.bi-emoji-smile-fill::before { content: "\f324"; } +.bi-emoji-smile-upside-down-fill::before { content: "\f325"; } +.bi-emoji-smile-upside-down::before { content: "\f326"; } +.bi-emoji-smile::before { content: "\f327"; } +.bi-emoji-sunglasses-fill::before { content: "\f328"; } +.bi-emoji-sunglasses::before { content: "\f329"; } +.bi-emoji-wink-fill::before { content: "\f32a"; } +.bi-emoji-wink::before { content: "\f32b"; } +.bi-envelope-fill::before { content: "\f32c"; } +.bi-envelope-open-fill::before { content: "\f32d"; } +.bi-envelope-open::before { content: "\f32e"; } +.bi-envelope::before { content: "\f32f"; } +.bi-eraser-fill::before { content: "\f330"; } +.bi-eraser::before { content: "\f331"; } +.bi-exclamation-circle-fill::before { content: "\f332"; } +.bi-exclamation-circle::before { content: "\f333"; } +.bi-exclamation-diamond-fill::before { content: "\f334"; } +.bi-exclamation-diamond::before { content: "\f335"; } +.bi-exclamation-octagon-fill::before { content: "\f336"; } +.bi-exclamation-octagon::before { content: "\f337"; } +.bi-exclamation-square-fill::before { content: "\f338"; } +.bi-exclamation-square::before { content: "\f339"; } +.bi-exclamation-triangle-fill::before { content: "\f33a"; } +.bi-exclamation-triangle::before { content: "\f33b"; } +.bi-exclamation::before { content: "\f33c"; } +.bi-exclude::before { content: "\f33d"; } +.bi-eye-fill::before { content: "\f33e"; } +.bi-eye-slash-fill::before { content: "\f33f"; } +.bi-eye-slash::before { content: "\f340"; } +.bi-eye::before { content: "\f341"; } +.bi-eyedropper::before { content: "\f342"; } +.bi-eyeglasses::before { content: "\f343"; } +.bi-facebook::before { content: "\f344"; } +.bi-file-arrow-down-fill::before { content: "\f345"; } +.bi-file-arrow-down::before { content: "\f346"; } +.bi-file-arrow-up-fill::before { content: "\f347"; } +.bi-file-arrow-up::before { content: "\f348"; } +.bi-file-bar-graph-fill::before { content: "\f349"; } +.bi-file-bar-graph::before { content: "\f34a"; } +.bi-file-binary-fill::before { content: "\f34b"; } +.bi-file-binary::before { content: "\f34c"; } +.bi-file-break-fill::before { content: "\f34d"; } +.bi-file-break::before { content: "\f34e"; } +.bi-file-check-fill::before { content: "\f34f"; } +.bi-file-check::before { content: "\f350"; } +.bi-file-code-fill::before { content: "\f351"; } +.bi-file-code::before { content: "\f352"; } +.bi-file-diff-fill::before { content: "\f353"; } +.bi-file-diff::before { content: "\f354"; } +.bi-file-earmark-arrow-down-fill::before { content: "\f355"; } +.bi-file-earmark-arrow-down::before { content: "\f356"; } +.bi-file-earmark-arrow-up-fill::before { content: "\f357"; } +.bi-file-earmark-arrow-up::before { content: "\f358"; } +.bi-file-earmark-bar-graph-fill::before { content: "\f359"; } +.bi-file-earmark-bar-graph::before { content: "\f35a"; } +.bi-file-earmark-binary-fill::before { content: "\f35b"; } +.bi-file-earmark-binary::before { content: "\f35c"; } +.bi-file-earmark-break-fill::before { content: "\f35d"; } +.bi-file-earmark-break::before { content: "\f35e"; } +.bi-file-earmark-check-fill::before { content: "\f35f"; } +.bi-file-earmark-check::before { content: "\f360"; } +.bi-file-earmark-code-fill::before { content: "\f361"; } +.bi-file-earmark-code::before { content: "\f362"; } +.bi-file-earmark-diff-fill::before { content: "\f363"; } +.bi-file-earmark-diff::before { content: "\f364"; } +.bi-file-earmark-easel-fill::before { content: "\f365"; } +.bi-file-earmark-easel::before { content: "\f366"; } +.bi-file-earmark-excel-fill::before { content: "\f367"; } +.bi-file-earmark-excel::before { content: "\f368"; } +.bi-file-earmark-fill::before { content: "\f369"; } +.bi-file-earmark-font-fill::before { content: "\f36a"; } +.bi-file-earmark-font::before { content: "\f36b"; } +.bi-file-earmark-image-fill::before { content: "\f36c"; } +.bi-file-earmark-image::before { content: "\f36d"; } +.bi-file-earmark-lock-fill::before { content: "\f36e"; } +.bi-file-earmark-lock::before { content: "\f36f"; } +.bi-file-earmark-lock2-fill::before { content: "\f370"; } +.bi-file-earmark-lock2::before { content: "\f371"; } +.bi-file-earmark-medical-fill::before { content: "\f372"; } +.bi-file-earmark-medical::before { content: "\f373"; } +.bi-file-earmark-minus-fill::before { content: "\f374"; } +.bi-file-earmark-minus::before { content: "\f375"; } +.bi-file-earmark-music-fill::before { content: "\f376"; } +.bi-file-earmark-music::before { content: "\f377"; } +.bi-file-earmark-person-fill::before { content: "\f378"; } +.bi-file-earmark-person::before { content: "\f379"; } +.bi-file-earmark-play-fill::before { content: "\f37a"; } +.bi-file-earmark-play::before { content: "\f37b"; } +.bi-file-earmark-plus-fill::before { content: "\f37c"; } +.bi-file-earmark-plus::before { content: "\f37d"; } +.bi-file-earmark-post-fill::before { content: "\f37e"; } +.bi-file-earmark-post::before { content: "\f37f"; } +.bi-file-earmark-ppt-fill::before { content: "\f380"; } +.bi-file-earmark-ppt::before { content: "\f381"; } +.bi-file-earmark-richtext-fill::before { content: "\f382"; } +.bi-file-earmark-richtext::before { content: "\f383"; } +.bi-file-earmark-ruled-fill::before { content: "\f384"; } +.bi-file-earmark-ruled::before { content: "\f385"; } +.bi-file-earmark-slides-fill::before { content: "\f386"; } +.bi-file-earmark-slides::before { content: "\f387"; } +.bi-file-earmark-spreadsheet-fill::before { content: "\f388"; } +.bi-file-earmark-spreadsheet::before { content: "\f389"; } +.bi-file-earmark-text-fill::before { content: "\f38a"; } +.bi-file-earmark-text::before { content: "\f38b"; } +.bi-file-earmark-word-fill::before { content: "\f38c"; } +.bi-file-earmark-word::before { content: "\f38d"; } +.bi-file-earmark-x-fill::before { content: "\f38e"; } +.bi-file-earmark-x::before { content: "\f38f"; } +.bi-file-earmark-zip-fill::before { content: "\f390"; } +.bi-file-earmark-zip::before { content: "\f391"; } +.bi-file-earmark::before { content: "\f392"; } +.bi-file-easel-fill::before { content: "\f393"; } +.bi-file-easel::before { content: "\f394"; } +.bi-file-excel-fill::before { content: "\f395"; } +.bi-file-excel::before { content: "\f396"; } +.bi-file-fill::before { content: "\f397"; } +.bi-file-font-fill::before { content: "\f398"; } +.bi-file-font::before { content: "\f399"; } +.bi-file-image-fill::before { content: "\f39a"; } +.bi-file-image::before { content: "\f39b"; } +.bi-file-lock-fill::before { content: "\f39c"; } +.bi-file-lock::before { content: "\f39d"; } +.bi-file-lock2-fill::before { content: "\f39e"; } +.bi-file-lock2::before { content: "\f39f"; } +.bi-file-medical-fill::before { content: "\f3a0"; } +.bi-file-medical::before { content: "\f3a1"; } +.bi-file-minus-fill::before { content: "\f3a2"; } +.bi-file-minus::before { content: "\f3a3"; } +.bi-file-music-fill::before { content: "\f3a4"; } +.bi-file-music::before { content: "\f3a5"; } +.bi-file-person-fill::before { content: "\f3a6"; } +.bi-file-person::before { content: "\f3a7"; } +.bi-file-play-fill::before { content: "\f3a8"; } +.bi-file-play::before { content: "\f3a9"; } +.bi-file-plus-fill::before { content: "\f3aa"; } +.bi-file-plus::before { content: "\f3ab"; } +.bi-file-post-fill::before { content: "\f3ac"; } +.bi-file-post::before { content: "\f3ad"; } +.bi-file-ppt-fill::before { content: "\f3ae"; } +.bi-file-ppt::before { content: "\f3af"; } +.bi-file-richtext-fill::before { content: "\f3b0"; } +.bi-file-richtext::before { content: "\f3b1"; } +.bi-file-ruled-fill::before { content: "\f3b2"; } +.bi-file-ruled::before { content: "\f3b3"; } +.bi-file-slides-fill::before { content: "\f3b4"; } +.bi-file-slides::before { content: "\f3b5"; } +.bi-file-spreadsheet-fill::before { content: "\f3b6"; } +.bi-file-spreadsheet::before { content: "\f3b7"; } +.bi-file-text-fill::before { content: "\f3b8"; } +.bi-file-text::before { content: "\f3b9"; } +.bi-file-word-fill::before { content: "\f3ba"; } +.bi-file-word::before { content: "\f3bb"; } +.bi-file-x-fill::before { content: "\f3bc"; } +.bi-file-x::before { content: "\f3bd"; } +.bi-file-zip-fill::before { content: "\f3be"; } +.bi-file-zip::before { content: "\f3bf"; } +.bi-file::before { content: "\f3c0"; } +.bi-files-alt::before { content: "\f3c1"; } +.bi-files::before { content: "\f3c2"; } +.bi-film::before { content: "\f3c3"; } +.bi-filter-circle-fill::before { content: "\f3c4"; } +.bi-filter-circle::before { content: "\f3c5"; } +.bi-filter-left::before { content: "\f3c6"; } +.bi-filter-right::before { content: "\f3c7"; } +.bi-filter-square-fill::before { content: "\f3c8"; } +.bi-filter-square::before { content: "\f3c9"; } +.bi-filter::before { content: "\f3ca"; } +.bi-flag-fill::before { content: "\f3cb"; } +.bi-flag::before { content: "\f3cc"; } +.bi-flower1::before { content: "\f3cd"; } +.bi-flower2::before { content: "\f3ce"; } +.bi-flower3::before { content: "\f3cf"; } +.bi-folder-check::before { content: "\f3d0"; } +.bi-folder-fill::before { content: "\f3d1"; } +.bi-folder-minus::before { content: "\f3d2"; } +.bi-folder-plus::before { content: "\f3d3"; } +.bi-folder-symlink-fill::before { content: "\f3d4"; } +.bi-folder-symlink::before { content: "\f3d5"; } +.bi-folder-x::before { content: "\f3d6"; } +.bi-folder::before { content: "\f3d7"; } +.bi-folder2-open::before { content: "\f3d8"; } +.bi-folder2::before { content: "\f3d9"; } +.bi-fonts::before { content: "\f3da"; } +.bi-forward-fill::before { content: "\f3db"; } +.bi-forward::before { content: "\f3dc"; } +.bi-front::before { content: "\f3dd"; } +.bi-fullscreen-exit::before { content: "\f3de"; } +.bi-fullscreen::before { content: "\f3df"; } +.bi-funnel-fill::before { content: "\f3e0"; } +.bi-funnel::before { content: "\f3e1"; } +.bi-gear-fill::before { content: "\f3e2"; } +.bi-gear-wide-connected::before { content: "\f3e3"; } +.bi-gear-wide::before { content: "\f3e4"; } +.bi-gear::before { content: "\f3e5"; } +.bi-gem::before { content: "\f3e6"; } +.bi-geo-alt-fill::before { content: "\f3e7"; } +.bi-geo-alt::before { content: "\f3e8"; } +.bi-geo-fill::before { content: "\f3e9"; } +.bi-geo::before { content: "\f3ea"; } +.bi-gift-fill::before { content: "\f3eb"; } +.bi-gift::before { content: "\f3ec"; } +.bi-github::before { content: "\f3ed"; } +.bi-globe::before { content: "\f3ee"; } +.bi-globe2::before { content: "\f3ef"; } +.bi-google::before { content: "\f3f0"; } +.bi-graph-down::before { content: "\f3f1"; } +.bi-graph-up::before { content: "\f3f2"; } +.bi-grid-1x2-fill::before { content: "\f3f3"; } +.bi-grid-1x2::before { content: "\f3f4"; } +.bi-grid-3x2-gap-fill::before { content: "\f3f5"; } +.bi-grid-3x2-gap::before { content: "\f3f6"; } +.bi-grid-3x2::before { content: "\f3f7"; } +.bi-grid-3x3-gap-fill::before { content: "\f3f8"; } +.bi-grid-3x3-gap::before { content: "\f3f9"; } +.bi-grid-3x3::before { content: "\f3fa"; } +.bi-grid-fill::before { content: "\f3fb"; } +.bi-grid::before { content: "\f3fc"; } +.bi-grip-horizontal::before { content: "\f3fd"; } +.bi-grip-vertical::before { content: "\f3fe"; } +.bi-hammer::before { content: "\f3ff"; } +.bi-hand-index-fill::before { content: "\f400"; } +.bi-hand-index-thumb-fill::before { content: "\f401"; } +.bi-hand-index-thumb::before { content: "\f402"; } +.bi-hand-index::before { content: "\f403"; } +.bi-hand-thumbs-down-fill::before { content: "\f404"; } +.bi-hand-thumbs-down::before { content: "\f405"; } +.bi-hand-thumbs-up-fill::before { content: "\f406"; } +.bi-hand-thumbs-up::before { content: "\f407"; } +.bi-handbag-fill::before { content: "\f408"; } +.bi-handbag::before { content: "\f409"; } +.bi-hash::before { content: "\f40a"; } +.bi-hdd-fill::before { content: "\f40b"; } +.bi-hdd-network-fill::before { content: "\f40c"; } +.bi-hdd-network::before { content: "\f40d"; } +.bi-hdd-rack-fill::before { content: "\f40e"; } +.bi-hdd-rack::before { content: "\f40f"; } +.bi-hdd-stack-fill::before { content: "\f410"; } +.bi-hdd-stack::before { content: "\f411"; } +.bi-hdd::before { content: "\f412"; } +.bi-headphones::before { content: "\f413"; } +.bi-headset::before { content: "\f414"; } +.bi-heart-fill::before { content: "\f415"; } +.bi-heart-half::before { content: "\f416"; } +.bi-heart::before { content: "\f417"; } +.bi-heptagon-fill::before { content: "\f418"; } +.bi-heptagon-half::before { content: "\f419"; } +.bi-heptagon::before { content: "\f41a"; } +.bi-hexagon-fill::before { content: "\f41b"; } +.bi-hexagon-half::before { content: "\f41c"; } +.bi-hexagon::before { content: "\f41d"; } +.bi-hourglass-bottom::before { content: "\f41e"; } +.bi-hourglass-split::before { content: "\f41f"; } +.bi-hourglass-top::before { content: "\f420"; } +.bi-hourglass::before { content: "\f421"; } +.bi-house-door-fill::before { content: "\f422"; } +.bi-house-door::before { content: "\f423"; } +.bi-house-fill::before { content: "\f424"; } +.bi-house::before { content: "\f425"; } +.bi-hr::before { content: "\f426"; } +.bi-hurricane::before { content: "\f427"; } +.bi-image-alt::before { content: "\f428"; } +.bi-image-fill::before { content: "\f429"; } +.bi-image::before { content: "\f42a"; } +.bi-images::before { content: "\f42b"; } +.bi-inbox-fill::before { content: "\f42c"; } +.bi-inbox::before { content: "\f42d"; } +.bi-inboxes-fill::before { content: "\f42e"; } +.bi-inboxes::before { content: "\f42f"; } +.bi-info-circle-fill::before { content: "\f430"; } +.bi-info-circle::before { content: "\f431"; } +.bi-info-square-fill::before { content: "\f432"; } +.bi-info-square::before { content: "\f433"; } +.bi-info::before { content: "\f434"; } +.bi-input-cursor-text::before { content: "\f435"; } +.bi-input-cursor::before { content: "\f436"; } +.bi-instagram::before { content: "\f437"; } +.bi-intersect::before { content: "\f438"; } +.bi-journal-album::before { content: "\f439"; } +.bi-journal-arrow-down::before { content: "\f43a"; } +.bi-journal-arrow-up::before { content: "\f43b"; } +.bi-journal-bookmark-fill::before { content: "\f43c"; } +.bi-journal-bookmark::before { content: "\f43d"; } +.bi-journal-check::before { content: "\f43e"; } +.bi-journal-code::before { content: "\f43f"; } +.bi-journal-medical::before { content: "\f440"; } +.bi-journal-minus::before { content: "\f441"; } +.bi-journal-plus::before { content: "\f442"; } +.bi-journal-richtext::before { content: "\f443"; } +.bi-journal-text::before { content: "\f444"; } +.bi-journal-x::before { content: "\f445"; } +.bi-journal::before { content: "\f446"; } +.bi-journals::before { content: "\f447"; } +.bi-joystick::before { content: "\f448"; } +.bi-justify-left::before { content: "\f449"; } +.bi-justify-right::before { content: "\f44a"; } +.bi-justify::before { content: "\f44b"; } +.bi-kanban-fill::before { content: "\f44c"; } +.bi-kanban::before { content: "\f44d"; } +.bi-key-fill::before { content: "\f44e"; } +.bi-key::before { content: "\f44f"; } +.bi-keyboard-fill::before { content: "\f450"; } +.bi-keyboard::before { content: "\f451"; } +.bi-ladder::before { content: "\f452"; } +.bi-lamp-fill::before { content: "\f453"; } +.bi-lamp::before { content: "\f454"; } +.bi-laptop-fill::before { content: "\f455"; } +.bi-laptop::before { content: "\f456"; } +.bi-layer-backward::before { content: "\f457"; } +.bi-layer-forward::before { content: "\f458"; } +.bi-layers-fill::before { content: "\f459"; } +.bi-layers-half::before { content: "\f45a"; } +.bi-layers::before { content: "\f45b"; } +.bi-layout-sidebar-inset-reverse::before { content: "\f45c"; } +.bi-layout-sidebar-inset::before { content: "\f45d"; } +.bi-layout-sidebar-reverse::before { content: "\f45e"; } +.bi-layout-sidebar::before { content: "\f45f"; } +.bi-layout-split::before { content: "\f460"; } +.bi-layout-text-sidebar-reverse::before { content: "\f461"; } +.bi-layout-text-sidebar::before { content: "\f462"; } +.bi-layout-text-window-reverse::before { content: "\f463"; } +.bi-layout-text-window::before { content: "\f464"; } +.bi-layout-three-columns::before { content: "\f465"; } +.bi-layout-wtf::before { content: "\f466"; } +.bi-life-preserver::before { content: "\f467"; } +.bi-lightbulb-fill::before { content: "\f468"; } +.bi-lightbulb-off-fill::before { content: "\f469"; } +.bi-lightbulb-off::before { content: "\f46a"; } +.bi-lightbulb::before { content: "\f46b"; } +.bi-lightning-charge-fill::before { content: "\f46c"; } +.bi-lightning-charge::before { content: "\f46d"; } +.bi-lightning-fill::before { content: "\f46e"; } +.bi-lightning::before { content: "\f46f"; } +.bi-link-45deg::before { content: "\f470"; } +.bi-link::before { content: "\f471"; } +.bi-linkedin::before { content: "\f472"; } +.bi-list-check::before { content: "\f473"; } +.bi-list-nested::before { content: "\f474"; } +.bi-list-ol::before { content: "\f475"; } +.bi-list-stars::before { content: "\f476"; } +.bi-list-task::before { content: "\f477"; } +.bi-list-ul::before { content: "\f478"; } +.bi-list::before { content: "\f479"; } +.bi-lock-fill::before { content: "\f47a"; } +.bi-lock::before { content: "\f47b"; } +.bi-mailbox::before { content: "\f47c"; } +.bi-mailbox2::before { content: "\f47d"; } +.bi-map-fill::before { content: "\f47e"; } +.bi-map::before { content: "\f47f"; } +.bi-markdown-fill::before { content: "\f480"; } +.bi-markdown::before { content: "\f481"; } +.bi-mask::before { content: "\f482"; } +.bi-megaphone-fill::before { content: "\f483"; } +.bi-megaphone::before { content: "\f484"; } +.bi-menu-app-fill::before { content: "\f485"; } +.bi-menu-app::before { content: "\f486"; } +.bi-menu-button-fill::before { content: "\f487"; } +.bi-menu-button-wide-fill::before { content: "\f488"; } +.bi-menu-button-wide::before { content: "\f489"; } +.bi-menu-button::before { content: "\f48a"; } +.bi-menu-down::before { content: "\f48b"; } +.bi-menu-up::before { content: "\f48c"; } +.bi-mic-fill::before { content: "\f48d"; } +.bi-mic-mute-fill::before { content: "\f48e"; } +.bi-mic-mute::before { content: "\f48f"; } +.bi-mic::before { content: "\f490"; } +.bi-minecart-loaded::before { content: "\f491"; } +.bi-minecart::before { content: "\f492"; } +.bi-moisture::before { content: "\f493"; } +.bi-moon-fill::before { content: "\f494"; } +.bi-moon-stars-fill::before { content: "\f495"; } +.bi-moon-stars::before { content: "\f496"; } +.bi-moon::before { content: "\f497"; } +.bi-mouse-fill::before { content: "\f498"; } +.bi-mouse::before { content: "\f499"; } +.bi-mouse2-fill::before { content: "\f49a"; } +.bi-mouse2::before { content: "\f49b"; } +.bi-mouse3-fill::before { content: "\f49c"; } +.bi-mouse3::before { content: "\f49d"; } +.bi-music-note-beamed::before { content: "\f49e"; } +.bi-music-note-list::before { content: "\f49f"; } +.bi-music-note::before { content: "\f4a0"; } +.bi-music-player-fill::before { content: "\f4a1"; } +.bi-music-player::before { content: "\f4a2"; } +.bi-newspaper::before { content: "\f4a3"; } +.bi-node-minus-fill::before { content: "\f4a4"; } +.bi-node-minus::before { content: "\f4a5"; } +.bi-node-plus-fill::before { content: "\f4a6"; } +.bi-node-plus::before { content: "\f4a7"; } +.bi-nut-fill::before { content: "\f4a8"; } +.bi-nut::before { content: "\f4a9"; } +.bi-octagon-fill::before { content: "\f4aa"; } +.bi-octagon-half::before { content: "\f4ab"; } +.bi-octagon::before { content: "\f4ac"; } +.bi-option::before { content: "\f4ad"; } +.bi-outlet::before { content: "\f4ae"; } +.bi-paint-bucket::before { content: "\f4af"; } +.bi-palette-fill::before { content: "\f4b0"; } +.bi-palette::before { content: "\f4b1"; } +.bi-palette2::before { content: "\f4b2"; } +.bi-paperclip::before { content: "\f4b3"; } +.bi-paragraph::before { content: "\f4b4"; } +.bi-patch-check-fill::before { content: "\f4b5"; } +.bi-patch-check::before { content: "\f4b6"; } +.bi-patch-exclamation-fill::before { content: "\f4b7"; } +.bi-patch-exclamation::before { content: "\f4b8"; } +.bi-patch-minus-fill::before { content: "\f4b9"; } +.bi-patch-minus::before { content: "\f4ba"; } +.bi-patch-plus-fill::before { content: "\f4bb"; } +.bi-patch-plus::before { content: "\f4bc"; } +.bi-patch-question-fill::before { content: "\f4bd"; } +.bi-patch-question::before { content: "\f4be"; } +.bi-pause-btn-fill::before { content: "\f4bf"; } +.bi-pause-btn::before { content: "\f4c0"; } +.bi-pause-circle-fill::before { content: "\f4c1"; } +.bi-pause-circle::before { content: "\f4c2"; } +.bi-pause-fill::before { content: "\f4c3"; } +.bi-pause::before { content: "\f4c4"; } +.bi-peace-fill::before { content: "\f4c5"; } +.bi-peace::before { content: "\f4c6"; } +.bi-pen-fill::before { content: "\f4c7"; } +.bi-pen::before { content: "\f4c8"; } +.bi-pencil-fill::before { content: "\f4c9"; } +.bi-pencil-square::before { content: "\f4ca"; } +.bi-pencil::before { content: "\f4cb"; } +.bi-pentagon-fill::before { content: "\f4cc"; } +.bi-pentagon-half::before { content: "\f4cd"; } +.bi-pentagon::before { content: "\f4ce"; } +.bi-people-fill::before { content: "\f4cf"; } +.bi-people::before { content: "\f4d0"; } +.bi-percent::before { content: "\f4d1"; } +.bi-person-badge-fill::before { content: "\f4d2"; } +.bi-person-badge::before { content: "\f4d3"; } +.bi-person-bounding-box::before { content: "\f4d4"; } +.bi-person-check-fill::before { content: "\f4d5"; } +.bi-person-check::before { content: "\f4d6"; } +.bi-person-circle::before { content: "\f4d7"; } +.bi-person-dash-fill::before { content: "\f4d8"; } +.bi-person-dash::before { content: "\f4d9"; } +.bi-person-fill::before { content: "\f4da"; } +.bi-person-lines-fill::before { content: "\f4db"; } +.bi-person-plus-fill::before { content: "\f4dc"; } +.bi-person-plus::before { content: "\f4dd"; } +.bi-person-square::before { content: "\f4de"; } +.bi-person-x-fill::before { content: "\f4df"; } +.bi-person-x::before { content: "\f4e0"; } +.bi-person::before { content: "\f4e1"; } +.bi-phone-fill::before { content: "\f4e2"; } +.bi-phone-landscape-fill::before { content: "\f4e3"; } +.bi-phone-landscape::before { content: "\f4e4"; } +.bi-phone-vibrate-fill::before { content: "\f4e5"; } +.bi-phone-vibrate::before { content: "\f4e6"; } +.bi-phone::before { content: "\f4e7"; } +.bi-pie-chart-fill::before { content: "\f4e8"; } +.bi-pie-chart::before { content: "\f4e9"; } +.bi-pin-angle-fill::before { content: "\f4ea"; } +.bi-pin-angle::before { content: "\f4eb"; } +.bi-pin-fill::before { content: "\f4ec"; } +.bi-pin::before { content: "\f4ed"; } +.bi-pip-fill::before { content: "\f4ee"; } +.bi-pip::before { content: "\f4ef"; } +.bi-play-btn-fill::before { content: "\f4f0"; } +.bi-play-btn::before { content: "\f4f1"; } +.bi-play-circle-fill::before { content: "\f4f2"; } +.bi-play-circle::before { content: "\f4f3"; } +.bi-play-fill::before { content: "\f4f4"; } +.bi-play::before { content: "\f4f5"; } +.bi-plug-fill::before { content: "\f4f6"; } +.bi-plug::before { content: "\f4f7"; } +.bi-plus-circle-dotted::before { content: "\f4f8"; } +.bi-plus-circle-fill::before { content: "\f4f9"; } +.bi-plus-circle::before { content: "\f4fa"; } +.bi-plus-square-dotted::before { content: "\f4fb"; } +.bi-plus-square-fill::before { content: "\f4fc"; } +.bi-plus-square::before { content: "\f4fd"; } +.bi-plus::before { content: "\f4fe"; } +.bi-power::before { content: "\f4ff"; } +.bi-printer-fill::before { content: "\f500"; } +.bi-printer::before { content: "\f501"; } +.bi-puzzle-fill::before { content: "\f502"; } +.bi-puzzle::before { content: "\f503"; } +.bi-question-circle-fill::before { content: "\f504"; } +.bi-question-circle::before { content: "\f505"; } +.bi-question-diamond-fill::before { content: "\f506"; } +.bi-question-diamond::before { content: "\f507"; } +.bi-question-octagon-fill::before { content: "\f508"; } +.bi-question-octagon::before { content: "\f509"; } +.bi-question-square-fill::before { content: "\f50a"; } +.bi-question-square::before { content: "\f50b"; } +.bi-question::before { content: "\f50c"; } +.bi-rainbow::before { content: "\f50d"; } +.bi-receipt-cutoff::before { content: "\f50e"; } +.bi-receipt::before { content: "\f50f"; } +.bi-reception-0::before { content: "\f510"; } +.bi-reception-1::before { content: "\f511"; } +.bi-reception-2::before { content: "\f512"; } +.bi-reception-3::before { content: "\f513"; } +.bi-reception-4::before { content: "\f514"; } +.bi-record-btn-fill::before { content: "\f515"; } +.bi-record-btn::before { content: "\f516"; } +.bi-record-circle-fill::before { content: "\f517"; } +.bi-record-circle::before { content: "\f518"; } +.bi-record-fill::before { content: "\f519"; } +.bi-record::before { content: "\f51a"; } +.bi-record2-fill::before { content: "\f51b"; } +.bi-record2::before { content: "\f51c"; } +.bi-reply-all-fill::before { content: "\f51d"; } +.bi-reply-all::before { content: "\f51e"; } +.bi-reply-fill::before { content: "\f51f"; } +.bi-reply::before { content: "\f520"; } +.bi-rss-fill::before { content: "\f521"; } +.bi-rss::before { content: "\f522"; } +.bi-rulers::before { content: "\f523"; } +.bi-save-fill::before { content: "\f524"; } +.bi-save::before { content: "\f525"; } +.bi-save2-fill::before { content: "\f526"; } +.bi-save2::before { content: "\f527"; } +.bi-scissors::before { content: "\f528"; } +.bi-screwdriver::before { content: "\f529"; } +.bi-search::before { content: "\f52a"; } +.bi-segmented-nav::before { content: "\f52b"; } +.bi-server::before { content: "\f52c"; } +.bi-share-fill::before { content: "\f52d"; } +.bi-share::before { content: "\f52e"; } +.bi-shield-check::before { content: "\f52f"; } +.bi-shield-exclamation::before { content: "\f530"; } +.bi-shield-fill-check::before { content: "\f531"; } +.bi-shield-fill-exclamation::before { content: "\f532"; } +.bi-shield-fill-minus::before { content: "\f533"; } +.bi-shield-fill-plus::before { content: "\f534"; } +.bi-shield-fill-x::before { content: "\f535"; } +.bi-shield-fill::before { content: "\f536"; } +.bi-shield-lock-fill::before { content: "\f537"; } +.bi-shield-lock::before { content: "\f538"; } +.bi-shield-minus::before { content: "\f539"; } +.bi-shield-plus::before { content: "\f53a"; } +.bi-shield-shaded::before { content: "\f53b"; } +.bi-shield-slash-fill::before { content: "\f53c"; } +.bi-shield-slash::before { content: "\f53d"; } +.bi-shield-x::before { content: "\f53e"; } +.bi-shield::before { content: "\f53f"; } +.bi-shift-fill::before { content: "\f540"; } +.bi-shift::before { content: "\f541"; } +.bi-shop-window::before { content: "\f542"; } +.bi-shop::before { content: "\f543"; } +.bi-shuffle::before { content: "\f544"; } +.bi-signpost-2-fill::before { content: "\f545"; } +.bi-signpost-2::before { content: "\f546"; } +.bi-signpost-fill::before { content: "\f547"; } +.bi-signpost-split-fill::before { content: "\f548"; } +.bi-signpost-split::before { content: "\f549"; } +.bi-signpost::before { content: "\f54a"; } +.bi-sim-fill::before { content: "\f54b"; } +.bi-sim::before { content: "\f54c"; } +.bi-skip-backward-btn-fill::before { content: "\f54d"; } +.bi-skip-backward-btn::before { content: "\f54e"; } +.bi-skip-backward-circle-fill::before { content: "\f54f"; } +.bi-skip-backward-circle::before { content: "\f550"; } +.bi-skip-backward-fill::before { content: "\f551"; } +.bi-skip-backward::before { content: "\f552"; } +.bi-skip-end-btn-fill::before { content: "\f553"; } +.bi-skip-end-btn::before { content: "\f554"; } +.bi-skip-end-circle-fill::before { content: "\f555"; } +.bi-skip-end-circle::before { content: "\f556"; } +.bi-skip-end-fill::before { content: "\f557"; } +.bi-skip-end::before { content: "\f558"; } +.bi-skip-forward-btn-fill::before { content: "\f559"; } +.bi-skip-forward-btn::before { content: "\f55a"; } +.bi-skip-forward-circle-fill::before { content: "\f55b"; } +.bi-skip-forward-circle::before { content: "\f55c"; } +.bi-skip-forward-fill::before { content: "\f55d"; } +.bi-skip-forward::before { content: "\f55e"; } +.bi-skip-start-btn-fill::before { content: "\f55f"; } +.bi-skip-start-btn::before { content: "\f560"; } +.bi-skip-start-circle-fill::before { content: "\f561"; } +.bi-skip-start-circle::before { content: "\f562"; } +.bi-skip-start-fill::before { content: "\f563"; } +.bi-skip-start::before { content: "\f564"; } +.bi-slack::before { content: "\f565"; } +.bi-slash-circle-fill::before { content: "\f566"; } +.bi-slash-circle::before { content: "\f567"; } +.bi-slash-square-fill::before { content: "\f568"; } +.bi-slash-square::before { content: "\f569"; } +.bi-slash::before { content: "\f56a"; } +.bi-sliders::before { content: "\f56b"; } +.bi-smartwatch::before { content: "\f56c"; } +.bi-snow::before { content: "\f56d"; } +.bi-snow2::before { content: "\f56e"; } +.bi-snow3::before { content: "\f56f"; } +.bi-sort-alpha-down-alt::before { content: "\f570"; } +.bi-sort-alpha-down::before { content: "\f571"; } +.bi-sort-alpha-up-alt::before { content: "\f572"; } +.bi-sort-alpha-up::before { content: "\f573"; } +.bi-sort-down-alt::before { content: "\f574"; } +.bi-sort-down::before { content: "\f575"; } +.bi-sort-numeric-down-alt::before { content: "\f576"; } +.bi-sort-numeric-down::before { content: "\f577"; } +.bi-sort-numeric-up-alt::before { content: "\f578"; } +.bi-sort-numeric-up::before { content: "\f579"; } +.bi-sort-up-alt::before { content: "\f57a"; } +.bi-sort-up::before { content: "\f57b"; } +.bi-soundwave::before { content: "\f57c"; } +.bi-speaker-fill::before { content: "\f57d"; } +.bi-speaker::before { content: "\f57e"; } +.bi-speedometer::before { content: "\f57f"; } +.bi-speedometer2::before { content: "\f580"; } +.bi-spellcheck::before { content: "\f581"; } +.bi-square-fill::before { content: "\f582"; } +.bi-square-half::before { content: "\f583"; } +.bi-square::before { content: "\f584"; } +.bi-stack::before { content: "\f585"; } +.bi-star-fill::before { content: "\f586"; } +.bi-star-half::before { content: "\f587"; } +.bi-star::before { content: "\f588"; } +.bi-stars::before { content: "\f589"; } +.bi-stickies-fill::before { content: "\f58a"; } +.bi-stickies::before { content: "\f58b"; } +.bi-sticky-fill::before { content: "\f58c"; } +.bi-sticky::before { content: "\f58d"; } +.bi-stop-btn-fill::before { content: "\f58e"; } +.bi-stop-btn::before { content: "\f58f"; } +.bi-stop-circle-fill::before { content: "\f590"; } +.bi-stop-circle::before { content: "\f591"; } +.bi-stop-fill::before { content: "\f592"; } +.bi-stop::before { content: "\f593"; } +.bi-stoplights-fill::before { content: "\f594"; } +.bi-stoplights::before { content: "\f595"; } +.bi-stopwatch-fill::before { content: "\f596"; } +.bi-stopwatch::before { content: "\f597"; } +.bi-subtract::before { content: "\f598"; } +.bi-suit-club-fill::before { content: "\f599"; } +.bi-suit-club::before { content: "\f59a"; } +.bi-suit-diamond-fill::before { content: "\f59b"; } +.bi-suit-diamond::before { content: "\f59c"; } +.bi-suit-heart-fill::before { content: "\f59d"; } +.bi-suit-heart::before { content: "\f59e"; } +.bi-suit-spade-fill::before { content: "\f59f"; } +.bi-suit-spade::before { content: "\f5a0"; } +.bi-sun-fill::before { content: "\f5a1"; } +.bi-sun::before { content: "\f5a2"; } +.bi-sunglasses::before { content: "\f5a3"; } +.bi-sunrise-fill::before { content: "\f5a4"; } +.bi-sunrise::before { content: "\f5a5"; } +.bi-sunset-fill::before { content: "\f5a6"; } +.bi-sunset::before { content: "\f5a7"; } +.bi-symmetry-horizontal::before { content: "\f5a8"; } +.bi-symmetry-vertical::before { content: "\f5a9"; } +.bi-table::before { content: "\f5aa"; } +.bi-tablet-fill::before { content: "\f5ab"; } +.bi-tablet-landscape-fill::before { content: "\f5ac"; } +.bi-tablet-landscape::before { content: "\f5ad"; } +.bi-tablet::before { content: "\f5ae"; } +.bi-tag-fill::before { content: "\f5af"; } +.bi-tag::before { content: "\f5b0"; } +.bi-tags-fill::before { content: "\f5b1"; } +.bi-tags::before { content: "\f5b2"; } +.bi-telegram::before { content: "\f5b3"; } +.bi-telephone-fill::before { content: "\f5b4"; } +.bi-telephone-forward-fill::before { content: "\f5b5"; } +.bi-telephone-forward::before { content: "\f5b6"; } +.bi-telephone-inbound-fill::before { content: "\f5b7"; } +.bi-telephone-inbound::before { content: "\f5b8"; } +.bi-telephone-minus-fill::before { content: "\f5b9"; } +.bi-telephone-minus::before { content: "\f5ba"; } +.bi-telephone-outbound-fill::before { content: "\f5bb"; } +.bi-telephone-outbound::before { content: "\f5bc"; } +.bi-telephone-plus-fill::before { content: "\f5bd"; } +.bi-telephone-plus::before { content: "\f5be"; } +.bi-telephone-x-fill::before { content: "\f5bf"; } +.bi-telephone-x::before { content: "\f5c0"; } +.bi-telephone::before { content: "\f5c1"; } +.bi-terminal-fill::before { content: "\f5c2"; } +.bi-terminal::before { content: "\f5c3"; } +.bi-text-center::before { content: "\f5c4"; } +.bi-text-indent-left::before { content: "\f5c5"; } +.bi-text-indent-right::before { content: "\f5c6"; } +.bi-text-left::before { content: "\f5c7"; } +.bi-text-paragraph::before { content: "\f5c8"; } +.bi-text-right::before { content: "\f5c9"; } +.bi-textarea-resize::before { content: "\f5ca"; } +.bi-textarea-t::before { content: "\f5cb"; } +.bi-textarea::before { content: "\f5cc"; } +.bi-thermometer-half::before { content: "\f5cd"; } +.bi-thermometer-high::before { content: "\f5ce"; } +.bi-thermometer-low::before { content: "\f5cf"; } +.bi-thermometer-snow::before { content: "\f5d0"; } +.bi-thermometer-sun::before { content: "\f5d1"; } +.bi-thermometer::before { content: "\f5d2"; } +.bi-three-dots-vertical::before { content: "\f5d3"; } +.bi-three-dots::before { content: "\f5d4"; } +.bi-toggle-off::before { content: "\f5d5"; } +.bi-toggle-on::before { content: "\f5d6"; } +.bi-toggle2-off::before { content: "\f5d7"; } +.bi-toggle2-on::before { content: "\f5d8"; } +.bi-toggles::before { content: "\f5d9"; } +.bi-toggles2::before { content: "\f5da"; } +.bi-tools::before { content: "\f5db"; } +.bi-tornado::before { content: "\f5dc"; } +.bi-trash-fill::before { content: "\f5dd"; } +.bi-trash::before { content: "\f5de"; } +.bi-trash2-fill::before { content: "\f5df"; } +.bi-trash2::before { content: "\f5e0"; } +.bi-tree-fill::before { content: "\f5e1"; } +.bi-tree::before { content: "\f5e2"; } +.bi-triangle-fill::before { content: "\f5e3"; } +.bi-triangle-half::before { content: "\f5e4"; } +.bi-triangle::before { content: "\f5e5"; } +.bi-trophy-fill::before { content: "\f5e6"; } +.bi-trophy::before { content: "\f5e7"; } +.bi-tropical-storm::before { content: "\f5e8"; } +.bi-truck-flatbed::before { content: "\f5e9"; } +.bi-truck::before { content: "\f5ea"; } +.bi-tsunami::before { content: "\f5eb"; } +.bi-tv-fill::before { content: "\f5ec"; } +.bi-tv::before { content: "\f5ed"; } +.bi-twitch::before { content: "\f5ee"; } +.bi-twitter::before { content: "\f5ef"; } +.bi-type-bold::before { content: "\f5f0"; } +.bi-type-h1::before { content: "\f5f1"; } +.bi-type-h2::before { content: "\f5f2"; } +.bi-type-h3::before { content: "\f5f3"; } +.bi-type-italic::before { content: "\f5f4"; } +.bi-type-strikethrough::before { content: "\f5f5"; } +.bi-type-underline::before { content: "\f5f6"; } +.bi-type::before { content: "\f5f7"; } +.bi-ui-checks-grid::before { content: "\f5f8"; } +.bi-ui-checks::before { content: "\f5f9"; } +.bi-ui-radios-grid::before { content: "\f5fa"; } +.bi-ui-radios::before { content: "\f5fb"; } +.bi-umbrella-fill::before { content: "\f5fc"; } +.bi-umbrella::before { content: "\f5fd"; } +.bi-union::before { content: "\f5fe"; } +.bi-unlock-fill::before { content: "\f5ff"; } +.bi-unlock::before { content: "\f600"; } +.bi-upc-scan::before { content: "\f601"; } +.bi-upc::before { content: "\f602"; } +.bi-upload::before { content: "\f603"; } +.bi-vector-pen::before { content: "\f604"; } +.bi-view-list::before { content: "\f605"; } +.bi-view-stacked::before { content: "\f606"; } +.bi-vinyl-fill::before { content: "\f607"; } +.bi-vinyl::before { content: "\f608"; } +.bi-voicemail::before { content: "\f609"; } +.bi-volume-down-fill::before { content: "\f60a"; } +.bi-volume-down::before { content: "\f60b"; } +.bi-volume-mute-fill::before { content: "\f60c"; } +.bi-volume-mute::before { content: "\f60d"; } +.bi-volume-off-fill::before { content: "\f60e"; } +.bi-volume-off::before { content: "\f60f"; } +.bi-volume-up-fill::before { content: "\f610"; } +.bi-volume-up::before { content: "\f611"; } +.bi-vr::before { content: "\f612"; } +.bi-wallet-fill::before { content: "\f613"; } +.bi-wallet::before { content: "\f614"; } +.bi-wallet2::before { content: "\f615"; } +.bi-watch::before { content: "\f616"; } +.bi-water::before { content: "\f617"; } +.bi-whatsapp::before { content: "\f618"; } +.bi-wifi-1::before { content: "\f619"; } +.bi-wifi-2::before { content: "\f61a"; } +.bi-wifi-off::before { content: "\f61b"; } +.bi-wifi::before { content: "\f61c"; } +.bi-wind::before { content: "\f61d"; } +.bi-window-dock::before { content: "\f61e"; } +.bi-window-sidebar::before { content: "\f61f"; } +.bi-window::before { content: "\f620"; } +.bi-wrench::before { content: "\f621"; } +.bi-x-circle-fill::before { content: "\f622"; } +.bi-x-circle::before { content: "\f623"; } +.bi-x-diamond-fill::before { content: "\f624"; } +.bi-x-diamond::before { content: "\f625"; } +.bi-x-octagon-fill::before { content: "\f626"; } +.bi-x-octagon::before { content: "\f627"; } +.bi-x-square-fill::before { content: "\f628"; } +.bi-x-square::before { content: "\f629"; } +.bi-x::before { content: "\f62a"; } +.bi-youtube::before { content: "\f62b"; } +.bi-zoom-in::before { content: "\f62c"; } +.bi-zoom-out::before { content: "\f62d"; } +.bi-bank::before { content: "\f62e"; } +.bi-bank2::before { content: "\f62f"; } +.bi-bell-slash-fill::before { content: "\f630"; } +.bi-bell-slash::before { content: "\f631"; } +.bi-cash-coin::before { content: "\f632"; } +.bi-check-lg::before { content: "\f633"; } +.bi-coin::before { content: "\f634"; } +.bi-currency-bitcoin::before { content: "\f635"; } +.bi-currency-dollar::before { content: "\f636"; } +.bi-currency-euro::before { content: "\f637"; } +.bi-currency-exchange::before { content: "\f638"; } +.bi-currency-pound::before { content: "\f639"; } +.bi-currency-yen::before { content: "\f63a"; } +.bi-dash-lg::before { content: "\f63b"; } +.bi-exclamation-lg::before { content: "\f63c"; } +.bi-file-earmark-pdf-fill::before { content: "\f63d"; } +.bi-file-earmark-pdf::before { content: "\f63e"; } +.bi-file-pdf-fill::before { content: "\f63f"; } +.bi-file-pdf::before { content: "\f640"; } +.bi-gender-ambiguous::before { content: "\f641"; } +.bi-gender-female::before { content: "\f642"; } +.bi-gender-male::before { content: "\f643"; } +.bi-gender-trans::before { content: "\f644"; } +.bi-headset-vr::before { content: "\f645"; } +.bi-info-lg::before { content: "\f646"; } +.bi-mastodon::before { content: "\f647"; } +.bi-messenger::before { content: "\f648"; } +.bi-piggy-bank-fill::before { content: "\f649"; } +.bi-piggy-bank::before { content: "\f64a"; } +.bi-pin-map-fill::before { content: "\f64b"; } +.bi-pin-map::before { content: "\f64c"; } +.bi-plus-lg::before { content: "\f64d"; } +.bi-question-lg::before { content: "\f64e"; } +.bi-recycle::before { content: "\f64f"; } +.bi-reddit::before { content: "\f650"; } +.bi-safe-fill::before { content: "\f651"; } +.bi-safe2-fill::before { content: "\f652"; } +.bi-safe2::before { content: "\f653"; } +.bi-sd-card-fill::before { content: "\f654"; } +.bi-sd-card::before { content: "\f655"; } +.bi-skype::before { content: "\f656"; } +.bi-slash-lg::before { content: "\f657"; } +.bi-translate::before { content: "\f658"; } +.bi-x-lg::before { content: "\f659"; } +.bi-safe::before { content: "\f65a"; } +.bi-apple::before { content: "\f65b"; } +.bi-microsoft::before { content: "\f65d"; } +.bi-windows::before { content: "\f65e"; } +.bi-behance::before { content: "\f65c"; } +.bi-dribbble::before { content: "\f65f"; } +.bi-line::before { content: "\f660"; } +.bi-medium::before { content: "\f661"; } +.bi-paypal::before { content: "\f662"; } +.bi-pinterest::before { content: "\f663"; } +.bi-signal::before { content: "\f664"; } +.bi-snapchat::before { content: "\f665"; } +.bi-spotify::before { content: "\f666"; } +.bi-stack-overflow::before { content: "\f667"; } +.bi-strava::before { content: "\f668"; } +.bi-wordpress::before { content: "\f669"; } +.bi-vimeo::before { content: "\f66a"; } +.bi-activity::before { content: "\f66b"; } +.bi-easel2-fill::before { content: "\f66c"; } +.bi-easel2::before { content: "\f66d"; } +.bi-easel3-fill::before { content: "\f66e"; } +.bi-easel3::before { content: "\f66f"; } +.bi-fan::before { content: "\f670"; } +.bi-fingerprint::before { content: "\f671"; } +.bi-graph-down-arrow::before { content: "\f672"; } +.bi-graph-up-arrow::before { content: "\f673"; } +.bi-hypnotize::before { content: "\f674"; } +.bi-magic::before { content: "\f675"; } +.bi-person-rolodex::before { content: "\f676"; } +.bi-person-video::before { content: "\f677"; } +.bi-person-video2::before { content: "\f678"; } +.bi-person-video3::before { content: "\f679"; } +.bi-person-workspace::before { content: "\f67a"; } +.bi-radioactive::before { content: "\f67b"; } +.bi-webcam-fill::before { content: "\f67c"; } +.bi-webcam::before { content: "\f67d"; } +.bi-yin-yang::before { content: "\f67e"; } +.bi-bandaid-fill::before { content: "\f680"; } +.bi-bandaid::before { content: "\f681"; } +.bi-bluetooth::before { content: "\f682"; } +.bi-body-text::before { content: "\f683"; } +.bi-boombox::before { content: "\f684"; } +.bi-boxes::before { content: "\f685"; } +.bi-dpad-fill::before { content: "\f686"; } +.bi-dpad::before { content: "\f687"; } +.bi-ear-fill::before { content: "\f688"; } +.bi-ear::before { content: "\f689"; } +.bi-envelope-check-1::before { content: "\f68a"; } +.bi-envelope-check-fill::before { content: "\f68b"; } +.bi-envelope-check::before { content: "\f68c"; } +.bi-envelope-dash-1::before { content: "\f68d"; } +.bi-envelope-dash-fill::before { content: "\f68e"; } +.bi-envelope-dash::before { content: "\f68f"; } +.bi-envelope-exclamation-1::before { content: "\f690"; } +.bi-envelope-exclamation-fill::before { content: "\f691"; } +.bi-envelope-exclamation::before { content: "\f692"; } +.bi-envelope-plus-fill::before { content: "\f693"; } +.bi-envelope-plus::before { content: "\f694"; } +.bi-envelope-slash-1::before { content: "\f695"; } +.bi-envelope-slash-fill::before { content: "\f696"; } +.bi-envelope-slash::before { content: "\f697"; } +.bi-envelope-x-1::before { content: "\f698"; } +.bi-envelope-x-fill::before { content: "\f699"; } +.bi-envelope-x::before { content: "\f69a"; } +.bi-explicit-fill::before { content: "\f69b"; } +.bi-explicit::before { content: "\f69c"; } +.bi-git::before { content: "\f69d"; } +.bi-infinity::before { content: "\f69e"; } +.bi-list-columns-reverse::before { content: "\f69f"; } +.bi-list-columns::before { content: "\f6a0"; } +.bi-meta::before { content: "\f6a1"; } +.bi-mortorboard-fill::before { content: "\f6a2"; } +.bi-mortorboard::before { content: "\f6a3"; } +.bi-nintendo-switch::before { content: "\f6a4"; } +.bi-pc-display-horizontal::before { content: "\f6a5"; } +.bi-pc-display::before { content: "\f6a6"; } +.bi-pc-horizontal::before { content: "\f6a7"; } +.bi-pc::before { content: "\f6a8"; } +.bi-playstation::before { content: "\f6a9"; } +.bi-plus-slash-minus::before { content: "\f6aa"; } +.bi-projector-fill::before { content: "\f6ab"; } +.bi-projector::before { content: "\f6ac"; } +.bi-qr-code-scan::before { content: "\f6ad"; } +.bi-qr-code::before { content: "\f6ae"; } +.bi-quora::before { content: "\f6af"; } +.bi-quote::before { content: "\f6b0"; } +.bi-robot::before { content: "\f6b1"; } +.bi-send-check-fill::before { content: "\f6b2"; } +.bi-send-check::before { content: "\f6b3"; } +.bi-send-dash-fill::before { content: "\f6b4"; } +.bi-send-dash::before { content: "\f6b5"; } +.bi-send-exclamation-1::before { content: "\f6b6"; } +.bi-send-exclamation-fill::before { content: "\f6b7"; } +.bi-send-exclamation::before { content: "\f6b8"; } +.bi-send-fill::before { content: "\f6b9"; } +.bi-send-plus-fill::before { content: "\f6ba"; } +.bi-send-plus::before { content: "\f6bb"; } +.bi-send-slash-fill::before { content: "\f6bc"; } +.bi-send-slash::before { content: "\f6bd"; } +.bi-send-x-fill::before { content: "\f6be"; } +.bi-send-x::before { content: "\f6bf"; } +.bi-send::before { content: "\f6c0"; } +.bi-steam::before { content: "\f6c1"; } +.bi-terminal-dash-1::before { content: "\f6c2"; } +.bi-terminal-dash::before { content: "\f6c3"; } +.bi-terminal-plus::before { content: "\f6c4"; } +.bi-terminal-split::before { content: "\f6c5"; } +.bi-ticket-detailed-fill::before { content: "\f6c6"; } +.bi-ticket-detailed::before { content: "\f6c7"; } +.bi-ticket-fill::before { content: "\f6c8"; } +.bi-ticket-perforated-fill::before { content: "\f6c9"; } +.bi-ticket-perforated::before { content: "\f6ca"; } +.bi-ticket::before { content: "\f6cb"; } +.bi-tiktok::before { content: "\f6cc"; } +.bi-window-dash::before { content: "\f6cd"; } +.bi-window-desktop::before { content: "\f6ce"; } +.bi-window-fullscreen::before { content: "\f6cf"; } +.bi-window-plus::before { content: "\f6d0"; } +.bi-window-split::before { content: "\f6d1"; } +.bi-window-stack::before { content: "\f6d2"; } +.bi-window-x::before { content: "\f6d3"; } +.bi-xbox::before { content: "\f6d4"; } +.bi-ethernet::before { content: "\f6d5"; } +.bi-hdmi-fill::before { content: "\f6d6"; } +.bi-hdmi::before { content: "\f6d7"; } +.bi-usb-c-fill::before { content: "\f6d8"; } +.bi-usb-c::before { content: "\f6d9"; } +.bi-usb-fill::before { content: "\f6da"; } +.bi-usb-plug-fill::before { content: "\f6db"; } +.bi-usb-plug::before { content: "\f6dc"; } +.bi-usb-symbol::before { content: "\f6dd"; } +.bi-usb::before { content: "\f6de"; } +.bi-boombox-fill::before { content: "\f6df"; } +.bi-displayport-1::before { content: "\f6e0"; } +.bi-displayport::before { content: "\f6e1"; } +.bi-gpu-card::before { content: "\f6e2"; } +.bi-memory::before { content: "\f6e3"; } +.bi-modem-fill::before { content: "\f6e4"; } +.bi-modem::before { content: "\f6e5"; } +.bi-motherboard-fill::before { content: "\f6e6"; } +.bi-motherboard::before { content: "\f6e7"; } +.bi-optical-audio-fill::before { content: "\f6e8"; } +.bi-optical-audio::before { content: "\f6e9"; } +.bi-pci-card::before { content: "\f6ea"; } +.bi-router-fill::before { content: "\f6eb"; } +.bi-router::before { content: "\f6ec"; } +.bi-ssd-fill::before { content: "\f6ed"; } +.bi-ssd::before { content: "\f6ee"; } +.bi-thunderbolt-fill::before { content: "\f6ef"; } +.bi-thunderbolt::before { content: "\f6f0"; } +.bi-usb-drive-fill::before { content: "\f6f1"; } +.bi-usb-drive::before { content: "\f6f2"; } +.bi-usb-micro-fill::before { content: "\f6f3"; } +.bi-usb-micro::before { content: "\f6f4"; } +.bi-usb-mini-fill::before { content: "\f6f5"; } +.bi-usb-mini::before { content: "\f6f6"; } +.bi-cloud-haze2::before { content: "\f6f7"; } +.bi-device-hdd-fill::before { content: "\f6f8"; } +.bi-device-hdd::before { content: "\f6f9"; } +.bi-device-ssd-fill::before { content: "\f6fa"; } +.bi-device-ssd::before { content: "\f6fb"; } +.bi-displayport-fill::before { content: "\f6fc"; } +.bi-mortarboard-fill::before { content: "\f6fd"; } +.bi-mortarboard::before { content: "\f6fe"; } +.bi-terminal-x::before { content: "\f6ff"; } +.bi-arrow-through-heart-fill::before { content: "\f700"; } +.bi-arrow-through-heart::before { content: "\f701"; } +.bi-badge-sd-fill::before { content: "\f702"; } +.bi-badge-sd::before { content: "\f703"; } +.bi-bag-heart-fill::before { content: "\f704"; } +.bi-bag-heart::before { content: "\f705"; } +.bi-balloon-fill::before { content: "\f706"; } +.bi-balloon-heart-fill::before { content: "\f707"; } +.bi-balloon-heart::before { content: "\f708"; } +.bi-balloon::before { content: "\f709"; } +.bi-box2-fill::before { content: "\f70a"; } +.bi-box2-heart-fill::before { content: "\f70b"; } +.bi-box2-heart::before { content: "\f70c"; } +.bi-box2::before { content: "\f70d"; } +.bi-braces-asterisk::before { content: "\f70e"; } +.bi-calendar-heart-fill::before { content: "\f70f"; } +.bi-calendar-heart::before { content: "\f710"; } +.bi-calendar2-heart-fill::before { content: "\f711"; } +.bi-calendar2-heart::before { content: "\f712"; } +.bi-chat-heart-fill::before { content: "\f713"; } +.bi-chat-heart::before { content: "\f714"; } +.bi-chat-left-heart-fill::before { content: "\f715"; } +.bi-chat-left-heart::before { content: "\f716"; } +.bi-chat-right-heart-fill::before { content: "\f717"; } +.bi-chat-right-heart::before { content: "\f718"; } +.bi-chat-square-heart-fill::before { content: "\f719"; } +.bi-chat-square-heart::before { content: "\f71a"; } +.bi-clipboard-check-fill::before { content: "\f71b"; } +.bi-clipboard-data-fill::before { content: "\f71c"; } +.bi-clipboard-fill::before { content: "\f71d"; } +.bi-clipboard-heart-fill::before { content: "\f71e"; } +.bi-clipboard-heart::before { content: "\f71f"; } +.bi-clipboard-minus-fill::before { content: "\f720"; } +.bi-clipboard-plus-fill::before { content: "\f721"; } +.bi-clipboard-pulse::before { content: "\f722"; } +.bi-clipboard-x-fill::before { content: "\f723"; } +.bi-clipboard2-check-fill::before { content: "\f724"; } +.bi-clipboard2-check::before { content: "\f725"; } +.bi-clipboard2-data-fill::before { content: "\f726"; } +.bi-clipboard2-data::before { content: "\f727"; } +.bi-clipboard2-fill::before { content: "\f728"; } +.bi-clipboard2-heart-fill::before { content: "\f729"; } +.bi-clipboard2-heart::before { content: "\f72a"; } +.bi-clipboard2-minus-fill::before { content: "\f72b"; } +.bi-clipboard2-minus::before { content: "\f72c"; } +.bi-clipboard2-plus-fill::before { content: "\f72d"; } +.bi-clipboard2-plus::before { content: "\f72e"; } +.bi-clipboard2-pulse-fill::before { content: "\f72f"; } +.bi-clipboard2-pulse::before { content: "\f730"; } +.bi-clipboard2-x-fill::before { content: "\f731"; } +.bi-clipboard2-x::before { content: "\f732"; } +.bi-clipboard2::before { content: "\f733"; } +.bi-emoji-kiss-fill::before { content: "\f734"; } +.bi-emoji-kiss::before { content: "\f735"; } +.bi-envelope-heart-fill::before { content: "\f736"; } +.bi-envelope-heart::before { content: "\f737"; } +.bi-envelope-open-heart-fill::before { content: "\f738"; } +.bi-envelope-open-heart::before { content: "\f739"; } +.bi-envelope-paper-fill::before { content: "\f73a"; } +.bi-envelope-paper-heart-fill::before { content: "\f73b"; } +.bi-envelope-paper-heart::before { content: "\f73c"; } +.bi-envelope-paper::before { content: "\f73d"; } +.bi-filetype-aac::before { content: "\f73e"; } +.bi-filetype-ai::before { content: "\f73f"; } +.bi-filetype-bmp::before { content: "\f740"; } +.bi-filetype-cs::before { content: "\f741"; } +.bi-filetype-css::before { content: "\f742"; } +.bi-filetype-csv::before { content: "\f743"; } +.bi-filetype-doc::before { content: "\f744"; } +.bi-filetype-docx::before { content: "\f745"; } +.bi-filetype-exe::before { content: "\f746"; } +.bi-filetype-gif::before { content: "\f747"; } +.bi-filetype-heic::before { content: "\f748"; } +.bi-filetype-html::before { content: "\f749"; } +.bi-filetype-java::before { content: "\f74a"; } +.bi-filetype-jpg::before { content: "\f74b"; } +.bi-filetype-js::before { content: "\f74c"; } +.bi-filetype-jsx::before { content: "\f74d"; } +.bi-filetype-key::before { content: "\f74e"; } +.bi-filetype-m4p::before { content: "\f74f"; } +.bi-filetype-md::before { content: "\f750"; } +.bi-filetype-mdx::before { content: "\f751"; } +.bi-filetype-mov::before { content: "\f752"; } +.bi-filetype-mp3::before { content: "\f753"; } +.bi-filetype-mp4::before { content: "\f754"; } +.bi-filetype-otf::before { content: "\f755"; } +.bi-filetype-pdf::before { content: "\f756"; } +.bi-filetype-php::before { content: "\f757"; } +.bi-filetype-png::before { content: "\f758"; } +.bi-filetype-ppt-1::before { content: "\f759"; } +.bi-filetype-ppt::before { content: "\f75a"; } +.bi-filetype-psd::before { content: "\f75b"; } +.bi-filetype-py::before { content: "\f75c"; } +.bi-filetype-raw::before { content: "\f75d"; } +.bi-filetype-rb::before { content: "\f75e"; } +.bi-filetype-sass::before { content: "\f75f"; } +.bi-filetype-scss::before { content: "\f760"; } +.bi-filetype-sh::before { content: "\f761"; } +.bi-filetype-svg::before { content: "\f762"; } +.bi-filetype-tiff::before { content: "\f763"; } +.bi-filetype-tsx::before { content: "\f764"; } +.bi-filetype-ttf::before { content: "\f765"; } +.bi-filetype-txt::before { content: "\f766"; } +.bi-filetype-wav::before { content: "\f767"; } +.bi-filetype-woff::before { content: "\f768"; } +.bi-filetype-xls-1::before { content: "\f769"; } +.bi-filetype-xls::before { content: "\f76a"; } +.bi-filetype-xml::before { content: "\f76b"; } +.bi-filetype-yml::before { content: "\f76c"; } +.bi-heart-arrow::before { content: "\f76d"; } +.bi-heart-pulse-fill::before { content: "\f76e"; } +.bi-heart-pulse::before { content: "\f76f"; } +.bi-heartbreak-fill::before { content: "\f770"; } +.bi-heartbreak::before { content: "\f771"; } +.bi-hearts::before { content: "\f772"; } +.bi-hospital-fill::before { content: "\f773"; } +.bi-hospital::before { content: "\f774"; } +.bi-house-heart-fill::before { content: "\f775"; } +.bi-house-heart::before { content: "\f776"; } +.bi-incognito::before { content: "\f777"; } +.bi-magnet-fill::before { content: "\f778"; } +.bi-magnet::before { content: "\f779"; } +.bi-person-heart::before { content: "\f77a"; } +.bi-person-hearts::before { content: "\f77b"; } +.bi-phone-flip::before { content: "\f77c"; } +.bi-plugin::before { content: "\f77d"; } +.bi-postage-fill::before { content: "\f77e"; } +.bi-postage-heart-fill::before { content: "\f77f"; } +.bi-postage-heart::before { content: "\f780"; } +.bi-postage::before { content: "\f781"; } +.bi-postcard-fill::before { content: "\f782"; } +.bi-postcard-heart-fill::before { content: "\f783"; } +.bi-postcard-heart::before { content: "\f784"; } +.bi-postcard::before { content: "\f785"; } +.bi-search-heart-fill::before { content: "\f786"; } +.bi-search-heart::before { content: "\f787"; } +.bi-sliders2-vertical::before { content: "\f788"; } +.bi-sliders2::before { content: "\f789"; } +.bi-trash3-fill::before { content: "\f78a"; } +.bi-trash3::before { content: "\f78b"; } +.bi-valentine::before { content: "\f78c"; } +.bi-valentine2::before { content: "\f78d"; } +.bi-wrench-adjustable-circle-fill::before { content: "\f78e"; } +.bi-wrench-adjustable-circle::before { content: "\f78f"; } +.bi-wrench-adjustable::before { content: "\f790"; } +.bi-filetype-json::before { content: "\f791"; } +.bi-filetype-pptx::before { content: "\f792"; } +.bi-filetype-xlsx::before { content: "\f793"; } diff --git a/pkg/api/static/bootstrap-theme-3.4.1.min.css b/pkg/api/static/bootstrap-theme-3.4.1.min.css deleted file mode 100644 index 2a69f48c7f5..00000000000 --- a/pkg/api/static/bootstrap-theme-3.4.1.min.css +++ /dev/null @@ -1,6 +0,0 @@ -/*! - * Bootstrap v3.4.1 (https://getbootstrap.com/) - * Copyright 2011-2019 Twitter, Inc. - * Licensed under MIT (https://github.com/twbs/bootstrap/blob/master/LICENSE) - */.btn-danger,.btn-default,.btn-info,.btn-primary,.btn-success,.btn-warning{text-shadow:0 -1px 0 rgba(0,0,0,.2);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 1px rgba(0,0,0,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 1px rgba(0,0,0,.075)}.btn-danger.active,.btn-danger:active,.btn-default.active,.btn-default:active,.btn-info.active,.btn-info:active,.btn-primary.active,.btn-primary:active,.btn-success.active,.btn-success:active,.btn-warning.active,.btn-warning:active{-webkit-box-shadow:inset 0 3px 5px rgba(0,0,0,.125);box-shadow:inset 0 3px 5px rgba(0,0,0,.125)}.btn-danger.disabled,.btn-danger[disabled],.btn-default.disabled,.btn-default[disabled],.btn-info.disabled,.btn-info[disabled],.btn-primary.disabled,.btn-primary[disabled],.btn-success.disabled,.btn-success[disabled],.btn-warning.disabled,.btn-warning[disabled],fieldset[disabled] .btn-danger,fieldset[disabled] .btn-default,fieldset[disabled] .btn-info,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-success,fieldset[disabled] .btn-warning{-webkit-box-shadow:none;box-shadow:none}.btn-danger .badge,.btn-default .badge,.btn-info .badge,.btn-primary .badge,.btn-success .badge,.btn-warning .badge{text-shadow:none}.btn.active,.btn:active{background-image:none}.btn-default{background-image:-webkit-linear-gradient(top,#fff 0,#e0e0e0 100%);background-image:-o-linear-gradient(top,#fff 0,#e0e0e0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fff),to(#e0e0e0));background-image:linear-gradient(to bottom,#fff 0,#e0e0e0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#ffe0e0e0', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#dbdbdb;text-shadow:0 1px 0 #fff;border-color:#ccc}.btn-default:focus,.btn-default:hover{background-color:#e0e0e0;background-position:0 -15px}.btn-default.active,.btn-default:active{background-color:#e0e0e0;border-color:#dbdbdb}.btn-default.disabled,.btn-default.disabled.active,.btn-default.disabled.focus,.btn-default.disabled:active,.btn-default.disabled:focus,.btn-default.disabled:hover,.btn-default[disabled],.btn-default[disabled].active,.btn-default[disabled].focus,.btn-default[disabled]:active,.btn-default[disabled]:focus,.btn-default[disabled]:hover,fieldset[disabled] .btn-default,fieldset[disabled] .btn-default.active,fieldset[disabled] .btn-default.focus,fieldset[disabled] .btn-default:active,fieldset[disabled] .btn-default:focus,fieldset[disabled] .btn-default:hover{background-color:#e0e0e0;background-image:none}.btn-primary{background-image:-webkit-linear-gradient(top,#337ab7 0,#265a88 100%);background-image:-o-linear-gradient(top,#337ab7 0,#265a88 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#265a88));background-image:linear-gradient(to bottom,#337ab7 0,#265a88 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff265a88', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#245580}.btn-primary:focus,.btn-primary:hover{background-color:#265a88;background-position:0 -15px}.btn-primary.active,.btn-primary:active{background-color:#265a88;border-color:#245580}.btn-primary.disabled,.btn-primary.disabled.active,.btn-primary.disabled.focus,.btn-primary.disabled:active,.btn-primary.disabled:focus,.btn-primary.disabled:hover,.btn-primary[disabled],.btn-primary[disabled].active,.btn-primary[disabled].focus,.btn-primary[disabled]:active,.btn-primary[disabled]:focus,.btn-primary[disabled]:hover,fieldset[disabled] .btn-primary,fieldset[disabled] .btn-primary.active,fieldset[disabled] .btn-primary.focus,fieldset[disabled] .btn-primary:active,fieldset[disabled] .btn-primary:focus,fieldset[disabled] .btn-primary:hover{background-color:#265a88;background-image:none}.btn-success{background-image:-webkit-linear-gradient(top,#5cb85c 0,#419641 100%);background-image:-o-linear-gradient(top,#5cb85c 0,#419641 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5cb85c),to(#419641));background-image:linear-gradient(to bottom,#5cb85c 0,#419641 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff419641', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#3e8f3e}.btn-success:focus,.btn-success:hover{background-color:#419641;background-position:0 -15px}.btn-success.active,.btn-success:active{background-color:#419641;border-color:#3e8f3e}.btn-success.disabled,.btn-success.disabled.active,.btn-success.disabled.focus,.btn-success.disabled:active,.btn-success.disabled:focus,.btn-success.disabled:hover,.btn-success[disabled],.btn-success[disabled].active,.btn-success[disabled].focus,.btn-success[disabled]:active,.btn-success[disabled]:focus,.btn-success[disabled]:hover,fieldset[disabled] .btn-success,fieldset[disabled] .btn-success.active,fieldset[disabled] .btn-success.focus,fieldset[disabled] .btn-success:active,fieldset[disabled] .btn-success:focus,fieldset[disabled] .btn-success:hover{background-color:#419641;background-image:none}.btn-info{background-image:-webkit-linear-gradient(top,#5bc0de 0,#2aabd2 100%);background-image:-o-linear-gradient(top,#5bc0de 0,#2aabd2 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5bc0de),to(#2aabd2));background-image:linear-gradient(to bottom,#5bc0de 0,#2aabd2 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff2aabd2', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#28a4c9}.btn-info:focus,.btn-info:hover{background-color:#2aabd2;background-position:0 -15px}.btn-info.active,.btn-info:active{background-color:#2aabd2;border-color:#28a4c9}.btn-info.disabled,.btn-info.disabled.active,.btn-info.disabled.focus,.btn-info.disabled:active,.btn-info.disabled:focus,.btn-info.disabled:hover,.btn-info[disabled],.btn-info[disabled].active,.btn-info[disabled].focus,.btn-info[disabled]:active,.btn-info[disabled]:focus,.btn-info[disabled]:hover,fieldset[disabled] .btn-info,fieldset[disabled] .btn-info.active,fieldset[disabled] .btn-info.focus,fieldset[disabled] .btn-info:active,fieldset[disabled] .btn-info:focus,fieldset[disabled] .btn-info:hover{background-color:#2aabd2;background-image:none}.btn-warning{background-image:-webkit-linear-gradient(top,#f0ad4e 0,#eb9316 100%);background-image:-o-linear-gradient(top,#f0ad4e 0,#eb9316 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f0ad4e),to(#eb9316));background-image:linear-gradient(to bottom,#f0ad4e 0,#eb9316 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffeb9316', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#e38d13}.btn-warning:focus,.btn-warning:hover{background-color:#eb9316;background-position:0 -15px}.btn-warning.active,.btn-warning:active{background-color:#eb9316;border-color:#e38d13}.btn-warning.disabled,.btn-warning.disabled.active,.btn-warning.disabled.focus,.btn-warning.disabled:active,.btn-warning.disabled:focus,.btn-warning.disabled:hover,.btn-warning[disabled],.btn-warning[disabled].active,.btn-warning[disabled].focus,.btn-warning[disabled]:active,.btn-warning[disabled]:focus,.btn-warning[disabled]:hover,fieldset[disabled] .btn-warning,fieldset[disabled] .btn-warning.active,fieldset[disabled] .btn-warning.focus,fieldset[disabled] .btn-warning:active,fieldset[disabled] .btn-warning:focus,fieldset[disabled] .btn-warning:hover{background-color:#eb9316;background-image:none}.btn-danger{background-image:-webkit-linear-gradient(top,#d9534f 0,#c12e2a 100%);background-image:-o-linear-gradient(top,#d9534f 0,#c12e2a 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9534f),to(#c12e2a));background-image:linear-gradient(to bottom,#d9534f 0,#c12e2a 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc12e2a', GradientType=0);filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);background-repeat:repeat-x;border-color:#b92c28}.btn-danger:focus,.btn-danger:hover{background-color:#c12e2a;background-position:0 -15px}.btn-danger.active,.btn-danger:active{background-color:#c12e2a;border-color:#b92c28}.btn-danger.disabled,.btn-danger.disabled.active,.btn-danger.disabled.focus,.btn-danger.disabled:active,.btn-danger.disabled:focus,.btn-danger.disabled:hover,.btn-danger[disabled],.btn-danger[disabled].active,.btn-danger[disabled].focus,.btn-danger[disabled]:active,.btn-danger[disabled]:focus,.btn-danger[disabled]:hover,fieldset[disabled] .btn-danger,fieldset[disabled] .btn-danger.active,fieldset[disabled] .btn-danger.focus,fieldset[disabled] .btn-danger:active,fieldset[disabled] .btn-danger:focus,fieldset[disabled] .btn-danger:hover{background-color:#c12e2a;background-image:none}.img-thumbnail,.thumbnail{-webkit-box-shadow:0 1px 2px rgba(0,0,0,.075);box-shadow:0 1px 2px rgba(0,0,0,.075)}.dropdown-menu>li>a:focus,.dropdown-menu>li>a:hover{background-image:-webkit-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f5f5f5),to(#e8e8e8));background-image:linear-gradient(to bottom,#f5f5f5 0,#e8e8e8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);background-repeat:repeat-x;background-color:#e8e8e8}.dropdown-menu>.active>a,.dropdown-menu>.active>a:focus,.dropdown-menu>.active>a:hover{background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x;background-color:#2e6da4}.navbar-default{background-image:-webkit-linear-gradient(top,#fff 0,#f8f8f8 100%);background-image:-o-linear-gradient(top,#fff 0,#f8f8f8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fff),to(#f8f8f8));background-image:linear-gradient(to bottom,#fff 0,#f8f8f8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffffffff', endColorstr='#fff8f8f8', GradientType=0);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);border-radius:4px;-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 5px rgba(0,0,0,.075);box-shadow:inset 0 1px 0 rgba(255,255,255,.15),0 1px 5px rgba(0,0,0,.075)}.navbar-default .navbar-nav>.active>a,.navbar-default .navbar-nav>.open>a{background-image:-webkit-linear-gradient(top,#dbdbdb 0,#e2e2e2 100%);background-image:-o-linear-gradient(top,#dbdbdb 0,#e2e2e2 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dbdbdb),to(#e2e2e2));background-image:linear-gradient(to bottom,#dbdbdb 0,#e2e2e2 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdbdbdb', endColorstr='#ffe2e2e2', GradientType=0);background-repeat:repeat-x;-webkit-box-shadow:inset 0 3px 9px rgba(0,0,0,.075);box-shadow:inset 0 3px 9px rgba(0,0,0,.075)}.navbar-brand,.navbar-nav>li>a{text-shadow:0 1px 0 rgba(255,255,255,.25)}.navbar-inverse{background-image:-webkit-linear-gradient(top,#3c3c3c 0,#222 100%);background-image:-o-linear-gradient(top,#3c3c3c 0,#222 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#3c3c3c),to(#222));background-image:linear-gradient(to bottom,#3c3c3c 0,#222 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff3c3c3c', endColorstr='#ff222222', GradientType=0);background-repeat:repeat-x;filter:progid:DXImageTransform.Microsoft.gradient(enabled=false);border-radius:4px}.navbar-inverse .navbar-nav>.active>a,.navbar-inverse .navbar-nav>.open>a{background-image:-webkit-linear-gradient(top,#080808 0,#0f0f0f 100%);background-image:-o-linear-gradient(top,#080808 0,#0f0f0f 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#080808),to(#0f0f0f));background-image:linear-gradient(to bottom,#080808 0,#0f0f0f 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff080808', endColorstr='#ff0f0f0f', GradientType=0);background-repeat:repeat-x;-webkit-box-shadow:inset 0 3px 9px rgba(0,0,0,.25);box-shadow:inset 0 3px 9px rgba(0,0,0,.25)}.navbar-inverse .navbar-brand,.navbar-inverse .navbar-nav>li>a{text-shadow:0 -1px 0 rgba(0,0,0,.25)}.navbar-fixed-bottom,.navbar-fixed-top,.navbar-static-top{border-radius:0}@media (max-width:767px){.navbar .navbar-nav .open .dropdown-menu>.active>a,.navbar .navbar-nav .open .dropdown-menu>.active>a:focus,.navbar .navbar-nav .open .dropdown-menu>.active>a:hover{color:#fff;background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}}.alert{text-shadow:0 1px 0 rgba(255,255,255,.2);-webkit-box-shadow:inset 0 1px 0 rgba(255,255,255,.25),0 1px 2px rgba(0,0,0,.05);box-shadow:inset 0 1px 0 rgba(255,255,255,.25),0 1px 2px rgba(0,0,0,.05)}.alert-success{background-image:-webkit-linear-gradient(top,#dff0d8 0,#c8e5bc 100%);background-image:-o-linear-gradient(top,#dff0d8 0,#c8e5bc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dff0d8),to(#c8e5bc));background-image:linear-gradient(to bottom,#dff0d8 0,#c8e5bc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffc8e5bc', GradientType=0);background-repeat:repeat-x;border-color:#b2dba1}.alert-info{background-image:-webkit-linear-gradient(top,#d9edf7 0,#b9def0 100%);background-image:-o-linear-gradient(top,#d9edf7 0,#b9def0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9edf7),to(#b9def0));background-image:linear-gradient(to bottom,#d9edf7 0,#b9def0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffb9def0', GradientType=0);background-repeat:repeat-x;border-color:#9acfea}.alert-warning{background-image:-webkit-linear-gradient(top,#fcf8e3 0,#f8efc0 100%);background-image:-o-linear-gradient(top,#fcf8e3 0,#f8efc0 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fcf8e3),to(#f8efc0));background-image:linear-gradient(to bottom,#fcf8e3 0,#f8efc0 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fff8efc0', GradientType=0);background-repeat:repeat-x;border-color:#f5e79e}.alert-danger{background-image:-webkit-linear-gradient(top,#f2dede 0,#e7c3c3 100%);background-image:-o-linear-gradient(top,#f2dede 0,#e7c3c3 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f2dede),to(#e7c3c3));background-image:linear-gradient(to bottom,#f2dede 0,#e7c3c3 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffe7c3c3', GradientType=0);background-repeat:repeat-x;border-color:#dca7a7}.progress{background-image:-webkit-linear-gradient(top,#ebebeb 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#ebebeb 0,#f5f5f5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#ebebeb),to(#f5f5f5));background-image:linear-gradient(to bottom,#ebebeb 0,#f5f5f5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffebebeb', endColorstr='#fff5f5f5', GradientType=0);background-repeat:repeat-x}.progress-bar{background-image:-webkit-linear-gradient(top,#337ab7 0,#286090 100%);background-image:-o-linear-gradient(top,#337ab7 0,#286090 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#286090));background-image:linear-gradient(to bottom,#337ab7 0,#286090 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff286090', GradientType=0);background-repeat:repeat-x}.progress-bar-success{background-image:-webkit-linear-gradient(top,#5cb85c 0,#449d44 100%);background-image:-o-linear-gradient(top,#5cb85c 0,#449d44 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5cb85c),to(#449d44));background-image:linear-gradient(to bottom,#5cb85c 0,#449d44 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5cb85c', endColorstr='#ff449d44', GradientType=0);background-repeat:repeat-x}.progress-bar-info{background-image:-webkit-linear-gradient(top,#5bc0de 0,#31b0d5 100%);background-image:-o-linear-gradient(top,#5bc0de 0,#31b0d5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#5bc0de),to(#31b0d5));background-image:linear-gradient(to bottom,#5bc0de 0,#31b0d5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff5bc0de', endColorstr='#ff31b0d5', GradientType=0);background-repeat:repeat-x}.progress-bar-warning{background-image:-webkit-linear-gradient(top,#f0ad4e 0,#ec971f 100%);background-image:-o-linear-gradient(top,#f0ad4e 0,#ec971f 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f0ad4e),to(#ec971f));background-image:linear-gradient(to bottom,#f0ad4e 0,#ec971f 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff0ad4e', endColorstr='#ffec971f', GradientType=0);background-repeat:repeat-x}.progress-bar-danger{background-image:-webkit-linear-gradient(top,#d9534f 0,#c9302c 100%);background-image:-o-linear-gradient(top,#d9534f 0,#c9302c 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9534f),to(#c9302c));background-image:linear-gradient(to bottom,#d9534f 0,#c9302c 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9534f', endColorstr='#ffc9302c', GradientType=0);background-repeat:repeat-x}.progress-bar-striped{background-image:-webkit-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:-o-linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent);background-image:linear-gradient(45deg,rgba(255,255,255,.15) 25%,transparent 25%,transparent 50%,rgba(255,255,255,.15) 50%,rgba(255,255,255,.15) 75%,transparent 75%,transparent)}.list-group{border-radius:4px;-webkit-box-shadow:0 1px 2px rgba(0,0,0,.075);box-shadow:0 1px 2px rgba(0,0,0,.075)}.list-group-item.active,.list-group-item.active:focus,.list-group-item.active:hover{text-shadow:0 -1px 0 #286090;background-image:-webkit-linear-gradient(top,#337ab7 0,#2b669a 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2b669a 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2b669a));background-image:linear-gradient(to bottom,#337ab7 0,#2b669a 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2b669a', GradientType=0);background-repeat:repeat-x;border-color:#2b669a}.list-group-item.active .badge,.list-group-item.active:focus .badge,.list-group-item.active:hover .badge{text-shadow:none}.panel{-webkit-box-shadow:0 1px 2px rgba(0,0,0,.05);box-shadow:0 1px 2px rgba(0,0,0,.05)}.panel-default>.panel-heading{background-image:-webkit-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-o-linear-gradient(top,#f5f5f5 0,#e8e8e8 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f5f5f5),to(#e8e8e8));background-image:linear-gradient(to bottom,#f5f5f5 0,#e8e8e8 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff5f5f5', endColorstr='#ffe8e8e8', GradientType=0);background-repeat:repeat-x}.panel-primary>.panel-heading{background-image:-webkit-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-o-linear-gradient(top,#337ab7 0,#2e6da4 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#337ab7),to(#2e6da4));background-image:linear-gradient(to bottom,#337ab7 0,#2e6da4 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ff337ab7', endColorstr='#ff2e6da4', GradientType=0);background-repeat:repeat-x}.panel-success>.panel-heading{background-image:-webkit-linear-gradient(top,#dff0d8 0,#d0e9c6 100%);background-image:-o-linear-gradient(top,#dff0d8 0,#d0e9c6 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#dff0d8),to(#d0e9c6));background-image:linear-gradient(to bottom,#dff0d8 0,#d0e9c6 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffdff0d8', endColorstr='#ffd0e9c6', GradientType=0);background-repeat:repeat-x}.panel-info>.panel-heading{background-image:-webkit-linear-gradient(top,#d9edf7 0,#c4e3f3 100%);background-image:-o-linear-gradient(top,#d9edf7 0,#c4e3f3 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#d9edf7),to(#c4e3f3));background-image:linear-gradient(to bottom,#d9edf7 0,#c4e3f3 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffd9edf7', endColorstr='#ffc4e3f3', GradientType=0);background-repeat:repeat-x}.panel-warning>.panel-heading{background-image:-webkit-linear-gradient(top,#fcf8e3 0,#faf2cc 100%);background-image:-o-linear-gradient(top,#fcf8e3 0,#faf2cc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#fcf8e3),to(#faf2cc));background-image:linear-gradient(to bottom,#fcf8e3 0,#faf2cc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fffcf8e3', endColorstr='#fffaf2cc', GradientType=0);background-repeat:repeat-x}.panel-danger>.panel-heading{background-image:-webkit-linear-gradient(top,#f2dede 0,#ebcccc 100%);background-image:-o-linear-gradient(top,#f2dede 0,#ebcccc 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#f2dede),to(#ebcccc));background-image:linear-gradient(to bottom,#f2dede 0,#ebcccc 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#fff2dede', endColorstr='#ffebcccc', GradientType=0);background-repeat:repeat-x}.well{background-image:-webkit-linear-gradient(top,#e8e8e8 0,#f5f5f5 100%);background-image:-o-linear-gradient(top,#e8e8e8 0,#f5f5f5 100%);background-image:-webkit-gradient(linear,left top,left bottom,from(#e8e8e8),to(#f5f5f5));background-image:linear-gradient(to bottom,#e8e8e8 0,#f5f5f5 100%);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr='#ffe8e8e8', endColorstr='#fff5f5f5', GradientType=0);background-repeat:repeat-x;border-color:#dcdcdc;-webkit-box-shadow:inset 0 1px 3px rgba(0,0,0,.05),0 1px 0 rgba(255,255,255,.1);box-shadow:inset 0 1px 3px rgba(0,0,0,.05),0 1px 0 rgba(255,255,255,.1)} -/*# sourceMappingURL=bootstrap-theme.min.css.map */ \ No newline at end of file diff --git a/pkg/api/static/fonts/bootstrap-icons.woff b/pkg/api/static/fonts/bootstrap-icons.woff new file mode 100644 index 00000000000..b26ccd1ac9f Binary files /dev/null and b/pkg/api/static/fonts/bootstrap-icons.woff differ diff --git a/pkg/api/static/fonts/bootstrap-icons.woff2 b/pkg/api/static/fonts/bootstrap-icons.woff2 new file mode 100644 index 00000000000..f865a4b260c Binary files /dev/null and b/pkg/api/static/fonts/bootstrap-icons.woff2 differ diff --git a/pkg/api/static/jquery-1.12.4.min.js b/pkg/api/static/jquery-1.12.4.min.js deleted file mode 100644 index e836475870d..00000000000 --- a/pkg/api/static/jquery-1.12.4.min.js +++ /dev/null @@ -1,5 +0,0 @@ -/*! jQuery v1.12.4 | (c) jQuery Foundation | jquery.org/license */ -!function(a,b){"object"==typeof module&&"object"==typeof module.exports?module.exports=a.document?b(a,!0):function(a){if(!a.document)throw new Error("jQuery requires a window with a document");return b(a)}:b(a)}("undefined"!=typeof window?window:this,function(a,b){var c=[],d=a.document,e=c.slice,f=c.concat,g=c.push,h=c.indexOf,i={},j=i.toString,k=i.hasOwnProperty,l={},m="1.12.4",n=function(a,b){return new n.fn.init(a,b)},o=/^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g,p=/^-ms-/,q=/-([\da-z])/gi,r=function(a,b){return b.toUpperCase()};n.fn=n.prototype={jquery:m,constructor:n,selector:"",length:0,toArray:function(){return e.call(this)},get:function(a){return null!=a?0>a?this[a+this.length]:this[a]:e.call(this)},pushStack:function(a){var b=n.merge(this.constructor(),a);return b.prevObject=this,b.context=this.context,b},each:function(a){return n.each(this,a)},map:function(a){return this.pushStack(n.map(this,function(b,c){return a.call(b,c,b)}))},slice:function(){return this.pushStack(e.apply(this,arguments))},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},eq:function(a){var b=this.length,c=+a+(0>a?b:0);return this.pushStack(c>=0&&b>c?[this[c]]:[])},end:function(){return this.prevObject||this.constructor()},push:g,sort:c.sort,splice:c.splice},n.extend=n.fn.extend=function(){var a,b,c,d,e,f,g=arguments[0]||{},h=1,i=arguments.length,j=!1;for("boolean"==typeof g&&(j=g,g=arguments[h]||{},h++),"object"==typeof g||n.isFunction(g)||(g={}),h===i&&(g=this,h--);i>h;h++)if(null!=(e=arguments[h]))for(d in e)a=g[d],c=e[d],g!==c&&(j&&c&&(n.isPlainObject(c)||(b=n.isArray(c)))?(b?(b=!1,f=a&&n.isArray(a)?a:[]):f=a&&n.isPlainObject(a)?a:{},g[d]=n.extend(j,f,c)):void 0!==c&&(g[d]=c));return g},n.extend({expando:"jQuery"+(m+Math.random()).replace(/\D/g,""),isReady:!0,error:function(a){throw new Error(a)},noop:function(){},isFunction:function(a){return"function"===n.type(a)},isArray:Array.isArray||function(a){return"array"===n.type(a)},isWindow:function(a){return null!=a&&a==a.window},isNumeric:function(a){var b=a&&a.toString();return!n.isArray(a)&&b-parseFloat(b)+1>=0},isEmptyObject:function(a){var b;for(b in a)return!1;return!0},isPlainObject:function(a){var b;if(!a||"object"!==n.type(a)||a.nodeType||n.isWindow(a))return!1;try{if(a.constructor&&!k.call(a,"constructor")&&!k.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}if(!l.ownFirst)for(b in a)return k.call(a,b);for(b in a);return void 0===b||k.call(a,b)},type:function(a){return null==a?a+"":"object"==typeof a||"function"==typeof a?i[j.call(a)]||"object":typeof a},globalEval:function(b){b&&n.trim(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(p,"ms-").replace(q,r)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toLowerCase()===b.toLowerCase()},each:function(a,b){var c,d=0;if(s(a)){for(c=a.length;c>d;d++)if(b.call(a[d],d,a[d])===!1)break}else for(d in a)if(b.call(a[d],d,a[d])===!1)break;return a},trim:function(a){return null==a?"":(a+"").replace(o,"")},makeArray:function(a,b){var c=b||[];return null!=a&&(s(Object(a))?n.merge(c,"string"==typeof a?[a]:a):g.call(c,a)),c},inArray:function(a,b,c){var d;if(b){if(h)return h.call(b,a,c);for(d=b.length,c=c?0>c?Math.max(0,d+c):c:0;d>c;c++)if(c in b&&b[c]===a)return c}return-1},merge:function(a,b){var c=+b.length,d=0,e=a.length;while(c>d)a[e++]=b[d++];if(c!==c)while(void 0!==b[d])a[e++]=b[d++];return a.length=e,a},grep:function(a,b,c){for(var d,e=[],f=0,g=a.length,h=!c;g>f;f++)d=!b(a[f],f),d!==h&&e.push(a[f]);return e},map:function(a,b,c){var d,e,g=0,h=[];if(s(a))for(d=a.length;d>g;g++)e=b(a[g],g,c),null!=e&&h.push(e);else for(g in a)e=b(a[g],g,c),null!=e&&h.push(e);return f.apply([],h)},guid:1,proxy:function(a,b){var c,d,f;return"string"==typeof b&&(f=a[b],b=a,a=f),n.isFunction(a)?(c=e.call(arguments,2),d=function(){return a.apply(b||this,c.concat(e.call(arguments)))},d.guid=a.guid=a.guid||n.guid++,d):void 0},now:function(){return+new Date},support:l}),"function"==typeof Symbol&&(n.fn[Symbol.iterator]=c[Symbol.iterator]),n.each("Boolean Number String Function Array Date RegExp Object Error Symbol".split(" "),function(a,b){i["[object "+b+"]"]=b.toLowerCase()});function s(a){var b=!!a&&"length"in a&&a.length,c=n.type(a);return"function"===c||n.isWindow(a)?!1:"array"===c||0===b||"number"==typeof b&&b>0&&b-1 in a}var t=function(a){var b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u="sizzle"+1*new Date,v=a.document,w=0,x=0,y=ga(),z=ga(),A=ga(),B=function(a,b){return a===b&&(l=!0),0},C=1<<31,D={}.hasOwnProperty,E=[],F=E.pop,G=E.push,H=E.push,I=E.slice,J=function(a,b){for(var c=0,d=a.length;d>c;c++)if(a[c]===b)return c;return-1},K="checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped",L="[\\x20\\t\\r\\n\\f]",M="(?:\\\\.|[\\w-]|[^\\x00-\\xa0])+",N="\\["+L+"*("+M+")(?:"+L+"*([*^$|!~]?=)"+L+"*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|("+M+"))|)"+L+"*\\]",O=":("+M+")(?:\\((('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|((?:\\\\.|[^\\\\()[\\]]|"+N+")*)|.*)\\)|)",P=new RegExp(L+"+","g"),Q=new RegExp("^"+L+"+|((?:^|[^\\\\])(?:\\\\.)*)"+L+"+$","g"),R=new RegExp("^"+L+"*,"+L+"*"),S=new RegExp("^"+L+"*([>+~]|"+L+")"+L+"*"),T=new RegExp("="+L+"*([^\\]'\"]*?)"+L+"*\\]","g"),U=new RegExp(O),V=new RegExp("^"+M+"$"),W={ID:new RegExp("^#("+M+")"),CLASS:new RegExp("^\\.("+M+")"),TAG:new RegExp("^("+M+"|[*])"),ATTR:new RegExp("^"+N),PSEUDO:new RegExp("^"+O),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+L+"*(even|odd|(([+-]|)(\\d*)n|)"+L+"*(?:([+-]|)"+L+"*(\\d+)|))"+L+"*\\)|)","i"),bool:new RegExp("^(?:"+K+")$","i"),needsContext:new RegExp("^"+L+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+L+"*((?:-\\d)?\\d*)"+L+"*\\)|)(?=[^-]|$)","i")},X=/^(?:input|select|textarea|button)$/i,Y=/^h\d$/i,Z=/^[^{]+\{\s*\[native \w/,$=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,_=/[+~]/,aa=/'|\\/g,ba=new RegExp("\\\\([\\da-f]{1,6}"+L+"?|("+L+")|.)","ig"),ca=function(a,b,c){var d="0x"+b-65536;return d!==d||c?b:0>d?String.fromCharCode(d+65536):String.fromCharCode(d>>10|55296,1023&d|56320)},da=function(){m()};try{H.apply(E=I.call(v.childNodes),v.childNodes),E[v.childNodes.length].nodeType}catch(ea){H={apply:E.length?function(a,b){G.apply(a,I.call(b))}:function(a,b){var c=a.length,d=0;while(a[c++]=b[d++]);a.length=c-1}}}function fa(a,b,d,e){var f,h,j,k,l,o,r,s,w=b&&b.ownerDocument,x=b?b.nodeType:9;if(d=d||[],"string"!=typeof a||!a||1!==x&&9!==x&&11!==x)return d;if(!e&&((b?b.ownerDocument||b:v)!==n&&m(b),b=b||n,p)){if(11!==x&&(o=$.exec(a)))if(f=o[1]){if(9===x){if(!(j=b.getElementById(f)))return d;if(j.id===f)return d.push(j),d}else if(w&&(j=w.getElementById(f))&&t(b,j)&&j.id===f)return d.push(j),d}else{if(o[2])return H.apply(d,b.getElementsByTagName(a)),d;if((f=o[3])&&c.getElementsByClassName&&b.getElementsByClassName)return H.apply(d,b.getElementsByClassName(f)),d}if(c.qsa&&!A[a+" "]&&(!q||!q.test(a))){if(1!==x)w=b,s=a;else if("object"!==b.nodeName.toLowerCase()){(k=b.getAttribute("id"))?k=k.replace(aa,"\\$&"):b.setAttribute("id",k=u),r=g(a),h=r.length,l=V.test(k)?"#"+k:"[id='"+k+"']";while(h--)r[h]=l+" "+qa(r[h]);s=r.join(","),w=_.test(a)&&oa(b.parentNode)||b}if(s)try{return H.apply(d,w.querySelectorAll(s)),d}catch(y){}finally{k===u&&b.removeAttribute("id")}}}return i(a.replace(Q,"$1"),b,d,e)}function ga(){var a=[];function b(c,e){return a.push(c+" ")>d.cacheLength&&delete b[a.shift()],b[c+" "]=e}return b}function ha(a){return a[u]=!0,a}function ia(a){var b=n.createElement("div");try{return!!a(b)}catch(c){return!1}finally{b.parentNode&&b.parentNode.removeChild(b),b=null}}function ja(a,b){var c=a.split("|"),e=c.length;while(e--)d.attrHandle[c[e]]=b}function ka(a,b){var c=b&&a,d=c&&1===a.nodeType&&1===b.nodeType&&(~b.sourceIndex||C)-(~a.sourceIndex||C);if(d)return d;if(c)while(c=c.nextSibling)if(c===b)return-1;return a?1:-1}function la(a){return function(b){var c=b.nodeName.toLowerCase();return"input"===c&&b.type===a}}function ma(a){return function(b){var c=b.nodeName.toLowerCase();return("input"===c||"button"===c)&&b.type===a}}function na(a){return ha(function(b){return b=+b,ha(function(c,d){var e,f=a([],c.length,b),g=f.length;while(g--)c[e=f[g]]&&(c[e]=!(d[e]=c[e]))})})}function oa(a){return a&&"undefined"!=typeof a.getElementsByTagName&&a}c=fa.support={},f=fa.isXML=function(a){var b=a&&(a.ownerDocument||a).documentElement;return b?"HTML"!==b.nodeName:!1},m=fa.setDocument=function(a){var b,e,g=a?a.ownerDocument||a:v;return g!==n&&9===g.nodeType&&g.documentElement?(n=g,o=n.documentElement,p=!f(n),(e=n.defaultView)&&e.top!==e&&(e.addEventListener?e.addEventListener("unload",da,!1):e.attachEvent&&e.attachEvent("onunload",da)),c.attributes=ia(function(a){return a.className="i",!a.getAttribute("className")}),c.getElementsByTagName=ia(function(a){return a.appendChild(n.createComment("")),!a.getElementsByTagName("*").length}),c.getElementsByClassName=Z.test(n.getElementsByClassName),c.getById=ia(function(a){return o.appendChild(a).id=u,!n.getElementsByName||!n.getElementsByName(u).length}),c.getById?(d.find.ID=function(a,b){if("undefined"!=typeof b.getElementById&&p){var c=b.getElementById(a);return c?[c]:[]}},d.filter.ID=function(a){var b=a.replace(ba,ca);return function(a){return a.getAttribute("id")===b}}):(delete d.find.ID,d.filter.ID=function(a){var b=a.replace(ba,ca);return function(a){var c="undefined"!=typeof a.getAttributeNode&&a.getAttributeNode("id");return c&&c.value===b}}),d.find.TAG=c.getElementsByTagName?function(a,b){return"undefined"!=typeof b.getElementsByTagName?b.getElementsByTagName(a):c.qsa?b.querySelectorAll(a):void 0}:function(a,b){var c,d=[],e=0,f=b.getElementsByTagName(a);if("*"===a){while(c=f[e++])1===c.nodeType&&d.push(c);return d}return f},d.find.CLASS=c.getElementsByClassName&&function(a,b){return"undefined"!=typeof b.getElementsByClassName&&p?b.getElementsByClassName(a):void 0},r=[],q=[],(c.qsa=Z.test(n.querySelectorAll))&&(ia(function(a){o.appendChild(a).innerHTML="",a.querySelectorAll("[msallowcapture^='']").length&&q.push("[*^$]="+L+"*(?:''|\"\")"),a.querySelectorAll("[selected]").length||q.push("\\["+L+"*(?:value|"+K+")"),a.querySelectorAll("[id~="+u+"-]").length||q.push("~="),a.querySelectorAll(":checked").length||q.push(":checked"),a.querySelectorAll("a#"+u+"+*").length||q.push(".#.+[+~]")}),ia(function(a){var b=n.createElement("input");b.setAttribute("type","hidden"),a.appendChild(b).setAttribute("name","D"),a.querySelectorAll("[name=d]").length&&q.push("name"+L+"*[*^$|!~]?="),a.querySelectorAll(":enabled").length||q.push(":enabled",":disabled"),a.querySelectorAll("*,:x"),q.push(",.*:")})),(c.matchesSelector=Z.test(s=o.matches||o.webkitMatchesSelector||o.mozMatchesSelector||o.oMatchesSelector||o.msMatchesSelector))&&ia(function(a){c.disconnectedMatch=s.call(a,"div"),s.call(a,"[s!='']:x"),r.push("!=",O)}),q=q.length&&new RegExp(q.join("|")),r=r.length&&new RegExp(r.join("|")),b=Z.test(o.compareDocumentPosition),t=b||Z.test(o.contains)?function(a,b){var c=9===a.nodeType?a.documentElement:a,d=b&&b.parentNode;return a===d||!(!d||1!==d.nodeType||!(c.contains?c.contains(d):a.compareDocumentPosition&&16&a.compareDocumentPosition(d)))}:function(a,b){if(b)while(b=b.parentNode)if(b===a)return!0;return!1},B=b?function(a,b){if(a===b)return l=!0,0;var d=!a.compareDocumentPosition-!b.compareDocumentPosition;return d?d:(d=(a.ownerDocument||a)===(b.ownerDocument||b)?a.compareDocumentPosition(b):1,1&d||!c.sortDetached&&b.compareDocumentPosition(a)===d?a===n||a.ownerDocument===v&&t(v,a)?-1:b===n||b.ownerDocument===v&&t(v,b)?1:k?J(k,a)-J(k,b):0:4&d?-1:1)}:function(a,b){if(a===b)return l=!0,0;var c,d=0,e=a.parentNode,f=b.parentNode,g=[a],h=[b];if(!e||!f)return a===n?-1:b===n?1:e?-1:f?1:k?J(k,a)-J(k,b):0;if(e===f)return ka(a,b);c=a;while(c=c.parentNode)g.unshift(c);c=b;while(c=c.parentNode)h.unshift(c);while(g[d]===h[d])d++;return d?ka(g[d],h[d]):g[d]===v?-1:h[d]===v?1:0},n):n},fa.matches=function(a,b){return fa(a,null,null,b)},fa.matchesSelector=function(a,b){if((a.ownerDocument||a)!==n&&m(a),b=b.replace(T,"='$1']"),c.matchesSelector&&p&&!A[b+" "]&&(!r||!r.test(b))&&(!q||!q.test(b)))try{var d=s.call(a,b);if(d||c.disconnectedMatch||a.document&&11!==a.document.nodeType)return d}catch(e){}return fa(b,n,null,[a]).length>0},fa.contains=function(a,b){return(a.ownerDocument||a)!==n&&m(a),t(a,b)},fa.attr=function(a,b){(a.ownerDocument||a)!==n&&m(a);var e=d.attrHandle[b.toLowerCase()],f=e&&D.call(d.attrHandle,b.toLowerCase())?e(a,b,!p):void 0;return void 0!==f?f:c.attributes||!p?a.getAttribute(b):(f=a.getAttributeNode(b))&&f.specified?f.value:null},fa.error=function(a){throw new Error("Syntax error, unrecognized expression: "+a)},fa.uniqueSort=function(a){var b,d=[],e=0,f=0;if(l=!c.detectDuplicates,k=!c.sortStable&&a.slice(0),a.sort(B),l){while(b=a[f++])b===a[f]&&(e=d.push(f));while(e--)a.splice(d[e],1)}return k=null,a},e=fa.getText=function(a){var b,c="",d=0,f=a.nodeType;if(f){if(1===f||9===f||11===f){if("string"==typeof a.textContent)return a.textContent;for(a=a.firstChild;a;a=a.nextSibling)c+=e(a)}else if(3===f||4===f)return a.nodeValue}else while(b=a[d++])c+=e(b);return c},d=fa.selectors={cacheLength:50,createPseudo:ha,match:W,attrHandle:{},find:{},relative:{">":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(a){return a[1]=a[1].replace(ba,ca),a[3]=(a[3]||a[4]||a[5]||"").replace(ba,ca),"~="===a[2]&&(a[3]=" "+a[3]+" "),a.slice(0,4)},CHILD:function(a){return a[1]=a[1].toLowerCase(),"nth"===a[1].slice(0,3)?(a[3]||fa.error(a[0]),a[4]=+(a[4]?a[5]+(a[6]||1):2*("even"===a[3]||"odd"===a[3])),a[5]=+(a[7]+a[8]||"odd"===a[3])):a[3]&&fa.error(a[0]),a},PSEUDO:function(a){var b,c=!a[6]&&a[2];return W.CHILD.test(a[0])?null:(a[3]?a[2]=a[4]||a[5]||"":c&&U.test(c)&&(b=g(c,!0))&&(b=c.indexOf(")",c.length-b)-c.length)&&(a[0]=a[0].slice(0,b),a[2]=c.slice(0,b)),a.slice(0,3))}},filter:{TAG:function(a){var b=a.replace(ba,ca).toLowerCase();return"*"===a?function(){return!0}:function(a){return a.nodeName&&a.nodeName.toLowerCase()===b}},CLASS:function(a){var b=y[a+" "];return b||(b=new RegExp("(^|"+L+")"+a+"("+L+"|$)"))&&y(a,function(a){return b.test("string"==typeof a.className&&a.className||"undefined"!=typeof a.getAttribute&&a.getAttribute("class")||"")})},ATTR:function(a,b,c){return function(d){var e=fa.attr(d,a);return null==e?"!="===b:b?(e+="","="===b?e===c:"!="===b?e!==c:"^="===b?c&&0===e.indexOf(c):"*="===b?c&&e.indexOf(c)>-1:"$="===b?c&&e.slice(-c.length)===c:"~="===b?(" "+e.replace(P," ")+" ").indexOf(c)>-1:"|="===b?e===c||e.slice(0,c.length+1)===c+"-":!1):!0}},CHILD:function(a,b,c,d,e){var f="nth"!==a.slice(0,3),g="last"!==a.slice(-4),h="of-type"===b;return 1===d&&0===e?function(a){return!!a.parentNode}:function(b,c,i){var j,k,l,m,n,o,p=f!==g?"nextSibling":"previousSibling",q=b.parentNode,r=h&&b.nodeName.toLowerCase(),s=!i&&!h,t=!1;if(q){if(f){while(p){m=b;while(m=m[p])if(h?m.nodeName.toLowerCase()===r:1===m.nodeType)return!1;o=p="only"===a&&!o&&"nextSibling"}return!0}if(o=[g?q.firstChild:q.lastChild],g&&s){m=q,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n&&j[2],m=n&&q.childNodes[n];while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if(1===m.nodeType&&++t&&m===b){k[a]=[w,n,t];break}}else if(s&&(m=b,l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),j=k[a]||[],n=j[0]===w&&j[1],t=n),t===!1)while(m=++n&&m&&m[p]||(t=n=0)||o.pop())if((h?m.nodeName.toLowerCase()===r:1===m.nodeType)&&++t&&(s&&(l=m[u]||(m[u]={}),k=l[m.uniqueID]||(l[m.uniqueID]={}),k[a]=[w,t]),m===b))break;return t-=e,t===d||t%d===0&&t/d>=0}}},PSEUDO:function(a,b){var c,e=d.pseudos[a]||d.setFilters[a.toLowerCase()]||fa.error("unsupported pseudo: "+a);return e[u]?e(b):e.length>1?(c=[a,a,"",b],d.setFilters.hasOwnProperty(a.toLowerCase())?ha(function(a,c){var d,f=e(a,b),g=f.length;while(g--)d=J(a,f[g]),a[d]=!(c[d]=f[g])}):function(a){return e(a,0,c)}):e}},pseudos:{not:ha(function(a){var b=[],c=[],d=h(a.replace(Q,"$1"));return d[u]?ha(function(a,b,c,e){var f,g=d(a,null,e,[]),h=a.length;while(h--)(f=g[h])&&(a[h]=!(b[h]=f))}):function(a,e,f){return b[0]=a,d(b,null,f,c),b[0]=null,!c.pop()}}),has:ha(function(a){return function(b){return fa(a,b).length>0}}),contains:ha(function(a){return a=a.replace(ba,ca),function(b){return(b.textContent||b.innerText||e(b)).indexOf(a)>-1}}),lang:ha(function(a){return V.test(a||"")||fa.error("unsupported lang: "+a),a=a.replace(ba,ca).toLowerCase(),function(b){var c;do if(c=p?b.lang:b.getAttribute("xml:lang")||b.getAttribute("lang"))return c=c.toLowerCase(),c===a||0===c.indexOf(a+"-");while((b=b.parentNode)&&1===b.nodeType);return!1}}),target:function(b){var c=a.location&&a.location.hash;return c&&c.slice(1)===b.id},root:function(a){return a===o},focus:function(a){return a===n.activeElement&&(!n.hasFocus||n.hasFocus())&&!!(a.type||a.href||~a.tabIndex)},enabled:function(a){return a.disabled===!1},disabled:function(a){return a.disabled===!0},checked:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&!!a.checked||"option"===b&&!!a.selected},selected:function(a){return a.parentNode&&a.parentNode.selectedIndex,a.selected===!0},empty:function(a){for(a=a.firstChild;a;a=a.nextSibling)if(a.nodeType<6)return!1;return!0},parent:function(a){return!d.pseudos.empty(a)},header:function(a){return Y.test(a.nodeName)},input:function(a){return X.test(a.nodeName)},button:function(a){var b=a.nodeName.toLowerCase();return"input"===b&&"button"===a.type||"button"===b},text:function(a){var b;return"input"===a.nodeName.toLowerCase()&&"text"===a.type&&(null==(b=a.getAttribute("type"))||"text"===b.toLowerCase())},first:na(function(){return[0]}),last:na(function(a,b){return[b-1]}),eq:na(function(a,b,c){return[0>c?c+b:c]}),even:na(function(a,b){for(var c=0;b>c;c+=2)a.push(c);return a}),odd:na(function(a,b){for(var c=1;b>c;c+=2)a.push(c);return a}),lt:na(function(a,b,c){for(var d=0>c?c+b:c;--d>=0;)a.push(d);return a}),gt:na(function(a,b,c){for(var d=0>c?c+b:c;++db;b++)d+=a[b].value;return d}function ra(a,b,c){var d=b.dir,e=c&&"parentNode"===d,f=x++;return b.first?function(b,c,f){while(b=b[d])if(1===b.nodeType||e)return a(b,c,f)}:function(b,c,g){var h,i,j,k=[w,f];if(g){while(b=b[d])if((1===b.nodeType||e)&&a(b,c,g))return!0}else while(b=b[d])if(1===b.nodeType||e){if(j=b[u]||(b[u]={}),i=j[b.uniqueID]||(j[b.uniqueID]={}),(h=i[d])&&h[0]===w&&h[1]===f)return k[2]=h[2];if(i[d]=k,k[2]=a(b,c,g))return!0}}}function sa(a){return a.length>1?function(b,c,d){var e=a.length;while(e--)if(!a[e](b,c,d))return!1;return!0}:a[0]}function ta(a,b,c){for(var d=0,e=b.length;e>d;d++)fa(a,b[d],c);return c}function ua(a,b,c,d,e){for(var f,g=[],h=0,i=a.length,j=null!=b;i>h;h++)(f=a[h])&&(c&&!c(f,d,e)||(g.push(f),j&&b.push(h)));return g}function va(a,b,c,d,e,f){return d&&!d[u]&&(d=va(d)),e&&!e[u]&&(e=va(e,f)),ha(function(f,g,h,i){var j,k,l,m=[],n=[],o=g.length,p=f||ta(b||"*",h.nodeType?[h]:h,[]),q=!a||!f&&b?p:ua(p,m,a,h,i),r=c?e||(f?a:o||d)?[]:g:q;if(c&&c(q,r,h,i),d){j=ua(r,n),d(j,[],h,i),k=j.length;while(k--)(l=j[k])&&(r[n[k]]=!(q[n[k]]=l))}if(f){if(e||a){if(e){j=[],k=r.length;while(k--)(l=r[k])&&j.push(q[k]=l);e(null,r=[],j,i)}k=r.length;while(k--)(l=r[k])&&(j=e?J(f,l):m[k])>-1&&(f[j]=!(g[j]=l))}}else r=ua(r===g?r.splice(o,r.length):r),e?e(null,g,r,i):H.apply(g,r)})}function wa(a){for(var b,c,e,f=a.length,g=d.relative[a[0].type],h=g||d.relative[" "],i=g?1:0,k=ra(function(a){return a===b},h,!0),l=ra(function(a){return J(b,a)>-1},h,!0),m=[function(a,c,d){var e=!g&&(d||c!==j)||((b=c).nodeType?k(a,c,d):l(a,c,d));return b=null,e}];f>i;i++)if(c=d.relative[a[i].type])m=[ra(sa(m),c)];else{if(c=d.filter[a[i].type].apply(null,a[i].matches),c[u]){for(e=++i;f>e;e++)if(d.relative[a[e].type])break;return va(i>1&&sa(m),i>1&&qa(a.slice(0,i-1).concat({value:" "===a[i-2].type?"*":""})).replace(Q,"$1"),c,e>i&&wa(a.slice(i,e)),f>e&&wa(a=a.slice(e)),f>e&&qa(a))}m.push(c)}return sa(m)}function xa(a,b){var c=b.length>0,e=a.length>0,f=function(f,g,h,i,k){var l,o,q,r=0,s="0",t=f&&[],u=[],v=j,x=f||e&&d.find.TAG("*",k),y=w+=null==v?1:Math.random()||.1,z=x.length;for(k&&(j=g===n||g||k);s!==z&&null!=(l=x[s]);s++){if(e&&l){o=0,g||l.ownerDocument===n||(m(l),h=!p);while(q=a[o++])if(q(l,g||n,h)){i.push(l);break}k&&(w=y)}c&&((l=!q&&l)&&r--,f&&t.push(l))}if(r+=s,c&&s!==r){o=0;while(q=b[o++])q(t,u,g,h);if(f){if(r>0)while(s--)t[s]||u[s]||(u[s]=F.call(i));u=ua(u)}H.apply(i,u),k&&!f&&u.length>0&&r+b.length>1&&fa.uniqueSort(i)}return k&&(w=y,j=v),t};return c?ha(f):f}return h=fa.compile=function(a,b){var c,d=[],e=[],f=A[a+" "];if(!f){b||(b=g(a)),c=b.length;while(c--)f=wa(b[c]),f[u]?d.push(f):e.push(f);f=A(a,xa(e,d)),f.selector=a}return f},i=fa.select=function(a,b,e,f){var i,j,k,l,m,n="function"==typeof a&&a,o=!f&&g(a=n.selector||a);if(e=e||[],1===o.length){if(j=o[0]=o[0].slice(0),j.length>2&&"ID"===(k=j[0]).type&&c.getById&&9===b.nodeType&&p&&d.relative[j[1].type]){if(b=(d.find.ID(k.matches[0].replace(ba,ca),b)||[])[0],!b)return e;n&&(b=b.parentNode),a=a.slice(j.shift().value.length)}i=W.needsContext.test(a)?0:j.length;while(i--){if(k=j[i],d.relative[l=k.type])break;if((m=d.find[l])&&(f=m(k.matches[0].replace(ba,ca),_.test(j[0].type)&&oa(b.parentNode)||b))){if(j.splice(i,1),a=f.length&&qa(j),!a)return H.apply(e,f),e;break}}}return(n||h(a,o))(f,b,!p,e,!b||_.test(a)&&oa(b.parentNode)||b),e},c.sortStable=u.split("").sort(B).join("")===u,c.detectDuplicates=!!l,m(),c.sortDetached=ia(function(a){return 1&a.compareDocumentPosition(n.createElement("div"))}),ia(function(a){return a.innerHTML="","#"===a.firstChild.getAttribute("href")})||ja("type|href|height|width",function(a,b,c){return c?void 0:a.getAttribute(b,"type"===b.toLowerCase()?1:2)}),c.attributes&&ia(function(a){return a.innerHTML="",a.firstChild.setAttribute("value",""),""===a.firstChild.getAttribute("value")})||ja("value",function(a,b,c){return c||"input"!==a.nodeName.toLowerCase()?void 0:a.defaultValue}),ia(function(a){return null==a.getAttribute("disabled")})||ja(K,function(a,b,c){var d;return c?void 0:a[b]===!0?b.toLowerCase():(d=a.getAttributeNode(b))&&d.specified?d.value:null}),fa}(a);n.find=t,n.expr=t.selectors,n.expr[":"]=n.expr.pseudos,n.uniqueSort=n.unique=t.uniqueSort,n.text=t.getText,n.isXMLDoc=t.isXML,n.contains=t.contains;var u=function(a,b,c){var d=[],e=void 0!==c;while((a=a[b])&&9!==a.nodeType)if(1===a.nodeType){if(e&&n(a).is(c))break;d.push(a)}return d},v=function(a,b){for(var c=[];a;a=a.nextSibling)1===a.nodeType&&a!==b&&c.push(a);return c},w=n.expr.match.needsContext,x=/^<([\w-]+)\s*\/?>(?:<\/\1>|)$/,y=/^.[^:#\[\.,]*$/;function z(a,b,c){if(n.isFunction(b))return n.grep(a,function(a,d){return!!b.call(a,d,a)!==c});if(b.nodeType)return n.grep(a,function(a){return a===b!==c});if("string"==typeof b){if(y.test(b))return n.filter(b,a,c);b=n.filter(b,a)}return n.grep(a,function(a){return n.inArray(a,b)>-1!==c})}n.filter=function(a,b,c){var d=b[0];return c&&(a=":not("+a+")"),1===b.length&&1===d.nodeType?n.find.matchesSelector(d,a)?[d]:[]:n.find.matches(a,n.grep(b,function(a){return 1===a.nodeType}))},n.fn.extend({find:function(a){var b,c=[],d=this,e=d.length;if("string"!=typeof a)return this.pushStack(n(a).filter(function(){for(b=0;e>b;b++)if(n.contains(d[b],this))return!0}));for(b=0;e>b;b++)n.find(a,d[b],c);return c=this.pushStack(e>1?n.unique(c):c),c.selector=this.selector?this.selector+" "+a:a,c},filter:function(a){return this.pushStack(z(this,a||[],!1))},not:function(a){return this.pushStack(z(this,a||[],!0))},is:function(a){return!!z(this,"string"==typeof a&&w.test(a)?n(a):a||[],!1).length}});var A,B=/^(?:\s*(<[\w\W]+>)[^>]*|#([\w-]*))$/,C=n.fn.init=function(a,b,c){var e,f;if(!a)return this;if(c=c||A,"string"==typeof a){if(e="<"===a.charAt(0)&&">"===a.charAt(a.length-1)&&a.length>=3?[null,a,null]:B.exec(a),!e||!e[1]&&b)return!b||b.jquery?(b||c).find(a):this.constructor(b).find(a);if(e[1]){if(b=b instanceof n?b[0]:b,n.merge(this,n.parseHTML(e[1],b&&b.nodeType?b.ownerDocument||b:d,!0)),x.test(e[1])&&n.isPlainObject(b))for(e in b)n.isFunction(this[e])?this[e](b[e]):this.attr(e,b[e]);return this}if(f=d.getElementById(e[2]),f&&f.parentNode){if(f.id!==e[2])return A.find(a);this.length=1,this[0]=f}return this.context=d,this.selector=a,this}return a.nodeType?(this.context=this[0]=a,this.length=1,this):n.isFunction(a)?"undefined"!=typeof c.ready?c.ready(a):a(n):(void 0!==a.selector&&(this.selector=a.selector,this.context=a.context),n.makeArray(a,this))};C.prototype=n.fn,A=n(d);var D=/^(?:parents|prev(?:Until|All))/,E={children:!0,contents:!0,next:!0,prev:!0};n.fn.extend({has:function(a){var b,c=n(a,this),d=c.length;return this.filter(function(){for(b=0;d>b;b++)if(n.contains(this,c[b]))return!0})},closest:function(a,b){for(var c,d=0,e=this.length,f=[],g=w.test(a)||"string"!=typeof a?n(a,b||this.context):0;e>d;d++)for(c=this[d];c&&c!==b;c=c.parentNode)if(c.nodeType<11&&(g?g.index(c)>-1:1===c.nodeType&&n.find.matchesSelector(c,a))){f.push(c);break}return this.pushStack(f.length>1?n.uniqueSort(f):f)},index:function(a){return a?"string"==typeof a?n.inArray(this[0],n(a)):n.inArray(a.jquery?a[0]:a,this):this[0]&&this[0].parentNode?this.first().prevAll().length:-1},add:function(a,b){return this.pushStack(n.uniqueSort(n.merge(this.get(),n(a,b))))},addBack:function(a){return this.add(null==a?this.prevObject:this.prevObject.filter(a))}});function F(a,b){do a=a[b];while(a&&1!==a.nodeType);return a}n.each({parent:function(a){var b=a.parentNode;return b&&11!==b.nodeType?b:null},parents:function(a){return u(a,"parentNode")},parentsUntil:function(a,b,c){return u(a,"parentNode",c)},next:function(a){return F(a,"nextSibling")},prev:function(a){return F(a,"previousSibling")},nextAll:function(a){return u(a,"nextSibling")},prevAll:function(a){return u(a,"previousSibling")},nextUntil:function(a,b,c){return u(a,"nextSibling",c)},prevUntil:function(a,b,c){return u(a,"previousSibling",c)},siblings:function(a){return v((a.parentNode||{}).firstChild,a)},children:function(a){return v(a.firstChild)},contents:function(a){return n.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:n.merge([],a.childNodes)}},function(a,b){n.fn[a]=function(c,d){var e=n.map(this,b,c);return"Until"!==a.slice(-5)&&(d=c),d&&"string"==typeof d&&(e=n.filter(d,e)),this.length>1&&(E[a]||(e=n.uniqueSort(e)),D.test(a)&&(e=e.reverse())),this.pushStack(e)}});var G=/\S+/g;function H(a){var b={};return n.each(a.match(G)||[],function(a,c){b[c]=!0}),b}n.Callbacks=function(a){a="string"==typeof a?H(a):n.extend({},a);var b,c,d,e,f=[],g=[],h=-1,i=function(){for(e=a.once,d=b=!0;g.length;h=-1){c=g.shift();while(++h-1)f.splice(c,1),h>=c&&h--}),this},has:function(a){return a?n.inArray(a,f)>-1:f.length>0},empty:function(){return f&&(f=[]),this},disable:function(){return e=g=[],f=c="",this},disabled:function(){return!f},lock:function(){return e=!0,c||j.disable(),this},locked:function(){return!!e},fireWith:function(a,c){return e||(c=c||[],c=[a,c.slice?c.slice():c],g.push(c),b||i()),this},fire:function(){return j.fireWith(this,arguments),this},fired:function(){return!!d}};return j},n.extend({Deferred:function(a){var b=[["resolve","done",n.Callbacks("once memory"),"resolved"],["reject","fail",n.Callbacks("once memory"),"rejected"],["notify","progress",n.Callbacks("memory")]],c="pending",d={state:function(){return c},always:function(){return e.done(arguments).fail(arguments),this},then:function(){var a=arguments;return n.Deferred(function(c){n.each(b,function(b,f){var g=n.isFunction(a[b])&&a[b];e[f[1]](function(){var a=g&&g.apply(this,arguments);a&&n.isFunction(a.promise)?a.promise().progress(c.notify).done(c.resolve).fail(c.reject):c[f[0]+"With"](this===d?c.promise():this,g?[a]:arguments)})}),a=null}).promise()},promise:function(a){return null!=a?n.extend(a,d):d}},e={};return d.pipe=d.then,n.each(b,function(a,f){var g=f[2],h=f[3];d[f[1]]=g.add,h&&g.add(function(){c=h},b[1^a][2].disable,b[2][2].lock),e[f[0]]=function(){return e[f[0]+"With"](this===e?d:this,arguments),this},e[f[0]+"With"]=g.fireWith}),d.promise(e),a&&a.call(e,e),e},when:function(a){var b=0,c=e.call(arguments),d=c.length,f=1!==d||a&&n.isFunction(a.promise)?d:0,g=1===f?a:n.Deferred(),h=function(a,b,c){return function(d){b[a]=this,c[a]=arguments.length>1?e.call(arguments):d,c===i?g.notifyWith(b,c):--f||g.resolveWith(b,c)}},i,j,k;if(d>1)for(i=new Array(d),j=new Array(d),k=new Array(d);d>b;b++)c[b]&&n.isFunction(c[b].promise)?c[b].promise().progress(h(b,j,i)).done(h(b,k,c)).fail(g.reject):--f;return f||g.resolveWith(k,c),g.promise()}});var I;n.fn.ready=function(a){return n.ready.promise().done(a),this},n.extend({isReady:!1,readyWait:1,holdReady:function(a){a?n.readyWait++:n.ready(!0)},ready:function(a){(a===!0?--n.readyWait:n.isReady)||(n.isReady=!0,a!==!0&&--n.readyWait>0||(I.resolveWith(d,[n]),n.fn.triggerHandler&&(n(d).triggerHandler("ready"),n(d).off("ready"))))}});function J(){d.addEventListener?(d.removeEventListener("DOMContentLoaded",K),a.removeEventListener("load",K)):(d.detachEvent("onreadystatechange",K),a.detachEvent("onload",K))}function K(){(d.addEventListener||"load"===a.event.type||"complete"===d.readyState)&&(J(),n.ready())}n.ready.promise=function(b){if(!I)if(I=n.Deferred(),"complete"===d.readyState||"loading"!==d.readyState&&!d.documentElement.doScroll)a.setTimeout(n.ready);else if(d.addEventListener)d.addEventListener("DOMContentLoaded",K),a.addEventListener("load",K);else{d.attachEvent("onreadystatechange",K),a.attachEvent("onload",K);var c=!1;try{c=null==a.frameElement&&d.documentElement}catch(e){}c&&c.doScroll&&!function f(){if(!n.isReady){try{c.doScroll("left")}catch(b){return a.setTimeout(f,50)}J(),n.ready()}}()}return I.promise(b)},n.ready.promise();var L;for(L in n(l))break;l.ownFirst="0"===L,l.inlineBlockNeedsLayout=!1,n(function(){var a,b,c,e;c=d.getElementsByTagName("body")[0],c&&c.style&&(b=d.createElement("div"),e=d.createElement("div"),e.style.cssText="position:absolute;border:0;width:0;height:0;top:0;left:-9999px",c.appendChild(e).appendChild(b),"undefined"!=typeof b.style.zoom&&(b.style.cssText="display:inline;margin:0;border:0;padding:1px;width:1px;zoom:1",l.inlineBlockNeedsLayout=a=3===b.offsetWidth,a&&(c.style.zoom=1)),c.removeChild(e))}),function(){var a=d.createElement("div");l.deleteExpando=!0;try{delete a.test}catch(b){l.deleteExpando=!1}a=null}();var M=function(a){var b=n.noData[(a.nodeName+" ").toLowerCase()],c=+a.nodeType||1;return 1!==c&&9!==c?!1:!b||b!==!0&&a.getAttribute("classid")===b},N=/^(?:\{[\w\W]*\}|\[[\w\W]*\])$/,O=/([A-Z])/g;function P(a,b,c){if(void 0===c&&1===a.nodeType){var d="data-"+b.replace(O,"-$1").toLowerCase();if(c=a.getAttribute(d),"string"==typeof c){try{c="true"===c?!0:"false"===c?!1:"null"===c?null:+c+""===c?+c:N.test(c)?n.parseJSON(c):c}catch(e){}n.data(a,b,c)}else c=void 0; -}return c}function Q(a){var b;for(b in a)if(("data"!==b||!n.isEmptyObject(a[b]))&&"toJSON"!==b)return!1;return!0}function R(a,b,d,e){if(M(a)){var f,g,h=n.expando,i=a.nodeType,j=i?n.cache:a,k=i?a[h]:a[h]&&h;if(k&&j[k]&&(e||j[k].data)||void 0!==d||"string"!=typeof b)return k||(k=i?a[h]=c.pop()||n.guid++:h),j[k]||(j[k]=i?{}:{toJSON:n.noop}),"object"!=typeof b&&"function"!=typeof b||(e?j[k]=n.extend(j[k],b):j[k].data=n.extend(j[k].data,b)),g=j[k],e||(g.data||(g.data={}),g=g.data),void 0!==d&&(g[n.camelCase(b)]=d),"string"==typeof b?(f=g[b],null==f&&(f=g[n.camelCase(b)])):f=g,f}}function S(a,b,c){if(M(a)){var d,e,f=a.nodeType,g=f?n.cache:a,h=f?a[n.expando]:n.expando;if(g[h]){if(b&&(d=c?g[h]:g[h].data)){n.isArray(b)?b=b.concat(n.map(b,n.camelCase)):b in d?b=[b]:(b=n.camelCase(b),b=b in d?[b]:b.split(" ")),e=b.length;while(e--)delete d[b[e]];if(c?!Q(d):!n.isEmptyObject(d))return}(c||(delete g[h].data,Q(g[h])))&&(f?n.cleanData([a],!0):l.deleteExpando||g!=g.window?delete g[h]:g[h]=void 0)}}}n.extend({cache:{},noData:{"applet ":!0,"embed ":!0,"object ":"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000"},hasData:function(a){return a=a.nodeType?n.cache[a[n.expando]]:a[n.expando],!!a&&!Q(a)},data:function(a,b,c){return R(a,b,c)},removeData:function(a,b){return S(a,b)},_data:function(a,b,c){return R(a,b,c,!0)},_removeData:function(a,b){return S(a,b,!0)}}),n.fn.extend({data:function(a,b){var c,d,e,f=this[0],g=f&&f.attributes;if(void 0===a){if(this.length&&(e=n.data(f),1===f.nodeType&&!n._data(f,"parsedAttrs"))){c=g.length;while(c--)g[c]&&(d=g[c].name,0===d.indexOf("data-")&&(d=n.camelCase(d.slice(5)),P(f,d,e[d])));n._data(f,"parsedAttrs",!0)}return e}return"object"==typeof a?this.each(function(){n.data(this,a)}):arguments.length>1?this.each(function(){n.data(this,a,b)}):f?P(f,a,n.data(f,a)):void 0},removeData:function(a){return this.each(function(){n.removeData(this,a)})}}),n.extend({queue:function(a,b,c){var d;return a?(b=(b||"fx")+"queue",d=n._data(a,b),c&&(!d||n.isArray(c)?d=n._data(a,b,n.makeArray(c)):d.push(c)),d||[]):void 0},dequeue:function(a,b){b=b||"fx";var c=n.queue(a,b),d=c.length,e=c.shift(),f=n._queueHooks(a,b),g=function(){n.dequeue(a,b)};"inprogress"===e&&(e=c.shift(),d--),e&&("fx"===b&&c.unshift("inprogress"),delete f.stop,e.call(a,g,f)),!d&&f&&f.empty.fire()},_queueHooks:function(a,b){var c=b+"queueHooks";return n._data(a,c)||n._data(a,c,{empty:n.Callbacks("once memory").add(function(){n._removeData(a,b+"queue"),n._removeData(a,c)})})}}),n.fn.extend({queue:function(a,b){var c=2;return"string"!=typeof a&&(b=a,a="fx",c--),arguments.lengthh;h++)b(a[h],c,g?d:d.call(a[h],h,b(a[h],c)));return e?a:j?b.call(a):i?b(a[0],c):f},Z=/^(?:checkbox|radio)$/i,$=/<([\w:-]+)/,_=/^$|\/(?:java|ecma)script/i,aa=/^\s+/,ba="abbr|article|aside|audio|bdi|canvas|data|datalist|details|dialog|figcaption|figure|footer|header|hgroup|main|mark|meter|nav|output|picture|progress|section|summary|template|time|video";function ca(a){var b=ba.split("|"),c=a.createDocumentFragment();if(c.createElement)while(b.length)c.createElement(b.pop());return c}!function(){var a=d.createElement("div"),b=d.createDocumentFragment(),c=d.createElement("input");a.innerHTML="
a",l.leadingWhitespace=3===a.firstChild.nodeType,l.tbody=!a.getElementsByTagName("tbody").length,l.htmlSerialize=!!a.getElementsByTagName("link").length,l.html5Clone="<:nav>"!==d.createElement("nav").cloneNode(!0).outerHTML,c.type="checkbox",c.checked=!0,b.appendChild(c),l.appendChecked=c.checked,a.innerHTML="",l.noCloneChecked=!!a.cloneNode(!0).lastChild.defaultValue,b.appendChild(a),c=d.createElement("input"),c.setAttribute("type","radio"),c.setAttribute("checked","checked"),c.setAttribute("name","t"),a.appendChild(c),l.checkClone=a.cloneNode(!0).cloneNode(!0).lastChild.checked,l.noCloneEvent=!!a.addEventListener,a[n.expando]=1,l.attributes=!a.getAttribute(n.expando)}();var da={option:[1,""],legend:[1,"
","
"],area:[1,"",""],param:[1,"",""],thead:[1,"","
"],tr:[2,"","
"],col:[2,"","
"],td:[3,"","
"],_default:l.htmlSerialize?[0,"",""]:[1,"X
","
"]};da.optgroup=da.option,da.tbody=da.tfoot=da.colgroup=da.caption=da.thead,da.th=da.td;function ea(a,b){var c,d,e=0,f="undefined"!=typeof a.getElementsByTagName?a.getElementsByTagName(b||"*"):"undefined"!=typeof a.querySelectorAll?a.querySelectorAll(b||"*"):void 0;if(!f)for(f=[],c=a.childNodes||a;null!=(d=c[e]);e++)!b||n.nodeName(d,b)?f.push(d):n.merge(f,ea(d,b));return void 0===b||b&&n.nodeName(a,b)?n.merge([a],f):f}function fa(a,b){for(var c,d=0;null!=(c=a[d]);d++)n._data(c,"globalEval",!b||n._data(b[d],"globalEval"))}var ga=/<|&#?\w+;/,ha=/r;r++)if(g=a[r],g||0===g)if("object"===n.type(g))n.merge(q,g.nodeType?[g]:g);else if(ga.test(g)){i=i||p.appendChild(b.createElement("div")),j=($.exec(g)||["",""])[1].toLowerCase(),m=da[j]||da._default,i.innerHTML=m[1]+n.htmlPrefilter(g)+m[2],f=m[0];while(f--)i=i.lastChild;if(!l.leadingWhitespace&&aa.test(g)&&q.push(b.createTextNode(aa.exec(g)[0])),!l.tbody){g="table"!==j||ha.test(g)?""!==m[1]||ha.test(g)?0:i:i.firstChild,f=g&&g.childNodes.length;while(f--)n.nodeName(k=g.childNodes[f],"tbody")&&!k.childNodes.length&&g.removeChild(k)}n.merge(q,i.childNodes),i.textContent="";while(i.firstChild)i.removeChild(i.firstChild);i=p.lastChild}else q.push(b.createTextNode(g));i&&p.removeChild(i),l.appendChecked||n.grep(ea(q,"input"),ia),r=0;while(g=q[r++])if(d&&n.inArray(g,d)>-1)e&&e.push(g);else if(h=n.contains(g.ownerDocument,g),i=ea(p.appendChild(g),"script"),h&&fa(i),c){f=0;while(g=i[f++])_.test(g.type||"")&&c.push(g)}return i=null,p}!function(){var b,c,e=d.createElement("div");for(b in{submit:!0,change:!0,focusin:!0})c="on"+b,(l[b]=c in a)||(e.setAttribute(c,"t"),l[b]=e.attributes[c].expando===!1);e=null}();var ka=/^(?:input|select|textarea)$/i,la=/^key/,ma=/^(?:mouse|pointer|contextmenu|drag|drop)|click/,na=/^(?:focusinfocus|focusoutblur)$/,oa=/^([^.]*)(?:\.(.+)|)/;function pa(){return!0}function qa(){return!1}function ra(){try{return d.activeElement}catch(a){}}function sa(a,b,c,d,e,f){var g,h;if("object"==typeof b){"string"!=typeof c&&(d=d||c,c=void 0);for(h in b)sa(a,h,c,d,b[h],f);return a}if(null==d&&null==e?(e=c,d=c=void 0):null==e&&("string"==typeof c?(e=d,d=void 0):(e=d,d=c,c=void 0)),e===!1)e=qa;else if(!e)return a;return 1===f&&(g=e,e=function(a){return n().off(a),g.apply(this,arguments)},e.guid=g.guid||(g.guid=n.guid++)),a.each(function(){n.event.add(this,b,e,d,c)})}n.event={global:{},add:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n._data(a);if(r){c.handler&&(i=c,c=i.handler,e=i.selector),c.guid||(c.guid=n.guid++),(g=r.events)||(g=r.events={}),(k=r.handle)||(k=r.handle=function(a){return"undefined"==typeof n||a&&n.event.triggered===a.type?void 0:n.event.dispatch.apply(k.elem,arguments)},k.elem=a),b=(b||"").match(G)||[""],h=b.length;while(h--)f=oa.exec(b[h])||[],o=q=f[1],p=(f[2]||"").split(".").sort(),o&&(j=n.event.special[o]||{},o=(e?j.delegateType:j.bindType)||o,j=n.event.special[o]||{},l=n.extend({type:o,origType:q,data:d,handler:c,guid:c.guid,selector:e,needsContext:e&&n.expr.match.needsContext.test(e),namespace:p.join(".")},i),(m=g[o])||(m=g[o]=[],m.delegateCount=0,j.setup&&j.setup.call(a,d,p,k)!==!1||(a.addEventListener?a.addEventListener(o,k,!1):a.attachEvent&&a.attachEvent("on"+o,k))),j.add&&(j.add.call(a,l),l.handler.guid||(l.handler.guid=c.guid)),e?m.splice(m.delegateCount++,0,l):m.push(l),n.event.global[o]=!0);a=null}},remove:function(a,b,c,d,e){var f,g,h,i,j,k,l,m,o,p,q,r=n.hasData(a)&&n._data(a);if(r&&(k=r.events)){b=(b||"").match(G)||[""],j=b.length;while(j--)if(h=oa.exec(b[j])||[],o=q=h[1],p=(h[2]||"").split(".").sort(),o){l=n.event.special[o]||{},o=(d?l.delegateType:l.bindType)||o,m=k[o]||[],h=h[2]&&new RegExp("(^|\\.)"+p.join("\\.(?:.*\\.|)")+"(\\.|$)"),i=f=m.length;while(f--)g=m[f],!e&&q!==g.origType||c&&c.guid!==g.guid||h&&!h.test(g.namespace)||d&&d!==g.selector&&("**"!==d||!g.selector)||(m.splice(f,1),g.selector&&m.delegateCount--,l.remove&&l.remove.call(a,g));i&&!m.length&&(l.teardown&&l.teardown.call(a,p,r.handle)!==!1||n.removeEvent(a,o,r.handle),delete k[o])}else for(o in k)n.event.remove(a,o+b[j],c,d,!0);n.isEmptyObject(k)&&(delete r.handle,n._removeData(a,"events"))}},trigger:function(b,c,e,f){var g,h,i,j,l,m,o,p=[e||d],q=k.call(b,"type")?b.type:b,r=k.call(b,"namespace")?b.namespace.split("."):[];if(i=m=e=e||d,3!==e.nodeType&&8!==e.nodeType&&!na.test(q+n.event.triggered)&&(q.indexOf(".")>-1&&(r=q.split("."),q=r.shift(),r.sort()),h=q.indexOf(":")<0&&"on"+q,b=b[n.expando]?b:new n.Event(q,"object"==typeof b&&b),b.isTrigger=f?2:3,b.namespace=r.join("."),b.rnamespace=b.namespace?new RegExp("(^|\\.)"+r.join("\\.(?:.*\\.|)")+"(\\.|$)"):null,b.result=void 0,b.target||(b.target=e),c=null==c?[b]:n.makeArray(c,[b]),l=n.event.special[q]||{},f||!l.trigger||l.trigger.apply(e,c)!==!1)){if(!f&&!l.noBubble&&!n.isWindow(e)){for(j=l.delegateType||q,na.test(j+q)||(i=i.parentNode);i;i=i.parentNode)p.push(i),m=i;m===(e.ownerDocument||d)&&p.push(m.defaultView||m.parentWindow||a)}o=0;while((i=p[o++])&&!b.isPropagationStopped())b.type=o>1?j:l.bindType||q,g=(n._data(i,"events")||{})[b.type]&&n._data(i,"handle"),g&&g.apply(i,c),g=h&&i[h],g&&g.apply&&M(i)&&(b.result=g.apply(i,c),b.result===!1&&b.preventDefault());if(b.type=q,!f&&!b.isDefaultPrevented()&&(!l._default||l._default.apply(p.pop(),c)===!1)&&M(e)&&h&&e[q]&&!n.isWindow(e)){m=e[h],m&&(e[h]=null),n.event.triggered=q;try{e[q]()}catch(s){}n.event.triggered=void 0,m&&(e[h]=m)}return b.result}},dispatch:function(a){a=n.event.fix(a);var b,c,d,f,g,h=[],i=e.call(arguments),j=(n._data(this,"events")||{})[a.type]||[],k=n.event.special[a.type]||{};if(i[0]=a,a.delegateTarget=this,!k.preDispatch||k.preDispatch.call(this,a)!==!1){h=n.event.handlers.call(this,a,j),b=0;while((f=h[b++])&&!a.isPropagationStopped()){a.currentTarget=f.elem,c=0;while((g=f.handlers[c++])&&!a.isImmediatePropagationStopped())a.rnamespace&&!a.rnamespace.test(g.namespace)||(a.handleObj=g,a.data=g.data,d=((n.event.special[g.origType]||{}).handle||g.handler).apply(f.elem,i),void 0!==d&&(a.result=d)===!1&&(a.preventDefault(),a.stopPropagation()))}return k.postDispatch&&k.postDispatch.call(this,a),a.result}},handlers:function(a,b){var c,d,e,f,g=[],h=b.delegateCount,i=a.target;if(h&&i.nodeType&&("click"!==a.type||isNaN(a.button)||a.button<1))for(;i!=this;i=i.parentNode||this)if(1===i.nodeType&&(i.disabled!==!0||"click"!==a.type)){for(d=[],c=0;h>c;c++)f=b[c],e=f.selector+" ",void 0===d[e]&&(d[e]=f.needsContext?n(e,this).index(i)>-1:n.find(e,this,null,[i]).length),d[e]&&d.push(f);d.length&&g.push({elem:i,handlers:d})}return h]","i"),va=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:-]+)[^>]*)\/>/gi,wa=/\s*$/g,Aa=ca(d),Ba=Aa.appendChild(d.createElement("div"));function Ca(a,b){return n.nodeName(a,"table")&&n.nodeName(11!==b.nodeType?b:b.firstChild,"tr")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function Da(a){return a.type=(null!==n.find.attr(a,"type"))+"/"+a.type,a}function Ea(a){var b=ya.exec(a.type);return b?a.type=b[1]:a.removeAttribute("type"),a}function Fa(a,b){if(1===b.nodeType&&n.hasData(a)){var c,d,e,f=n._data(a),g=n._data(b,f),h=f.events;if(h){delete g.handle,g.events={};for(c in h)for(d=0,e=h[c].length;e>d;d++)n.event.add(b,c,h[c][d])}g.data&&(g.data=n.extend({},g.data))}}function Ga(a,b){var c,d,e;if(1===b.nodeType){if(c=b.nodeName.toLowerCase(),!l.noCloneEvent&&b[n.expando]){e=n._data(b);for(d in e.events)n.removeEvent(b,d,e.handle);b.removeAttribute(n.expando)}"script"===c&&b.text!==a.text?(Da(b).text=a.text,Ea(b)):"object"===c?(b.parentNode&&(b.outerHTML=a.outerHTML),l.html5Clone&&a.innerHTML&&!n.trim(b.innerHTML)&&(b.innerHTML=a.innerHTML)):"input"===c&&Z.test(a.type)?(b.defaultChecked=b.checked=a.checked,b.value!==a.value&&(b.value=a.value)):"option"===c?b.defaultSelected=b.selected=a.defaultSelected:"input"!==c&&"textarea"!==c||(b.defaultValue=a.defaultValue)}}function Ha(a,b,c,d){b=f.apply([],b);var e,g,h,i,j,k,m=0,o=a.length,p=o-1,q=b[0],r=n.isFunction(q);if(r||o>1&&"string"==typeof q&&!l.checkClone&&xa.test(q))return a.each(function(e){var f=a.eq(e);r&&(b[0]=q.call(this,e,f.html())),Ha(f,b,c,d)});if(o&&(k=ja(b,a[0].ownerDocument,!1,a,d),e=k.firstChild,1===k.childNodes.length&&(k=e),e||d)){for(i=n.map(ea(k,"script"),Da),h=i.length;o>m;m++)g=k,m!==p&&(g=n.clone(g,!0,!0),h&&n.merge(i,ea(g,"script"))),c.call(a[m],g,m);if(h)for(j=i[i.length-1].ownerDocument,n.map(i,Ea),m=0;h>m;m++)g=i[m],_.test(g.type||"")&&!n._data(g,"globalEval")&&n.contains(j,g)&&(g.src?n._evalUrl&&n._evalUrl(g.src):n.globalEval((g.text||g.textContent||g.innerHTML||"").replace(za,"")));k=e=null}return a}function Ia(a,b,c){for(var d,e=b?n.filter(b,a):a,f=0;null!=(d=e[f]);f++)c||1!==d.nodeType||n.cleanData(ea(d)),d.parentNode&&(c&&n.contains(d.ownerDocument,d)&&fa(ea(d,"script")),d.parentNode.removeChild(d));return a}n.extend({htmlPrefilter:function(a){return a.replace(va,"<$1>")},clone:function(a,b,c){var d,e,f,g,h,i=n.contains(a.ownerDocument,a);if(l.html5Clone||n.isXMLDoc(a)||!ua.test("<"+a.nodeName+">")?f=a.cloneNode(!0):(Ba.innerHTML=a.outerHTML,Ba.removeChild(f=Ba.firstChild)),!(l.noCloneEvent&&l.noCloneChecked||1!==a.nodeType&&11!==a.nodeType||n.isXMLDoc(a)))for(d=ea(f),h=ea(a),g=0;null!=(e=h[g]);++g)d[g]&&Ga(e,d[g]);if(b)if(c)for(h=h||ea(a),d=d||ea(f),g=0;null!=(e=h[g]);g++)Fa(e,d[g]);else Fa(a,f);return d=ea(f,"script"),d.length>0&&fa(d,!i&&ea(a,"script")),d=h=e=null,f},cleanData:function(a,b){for(var d,e,f,g,h=0,i=n.expando,j=n.cache,k=l.attributes,m=n.event.special;null!=(d=a[h]);h++)if((b||M(d))&&(f=d[i],g=f&&j[f])){if(g.events)for(e in g.events)m[e]?n.event.remove(d,e):n.removeEvent(d,e,g.handle);j[f]&&(delete j[f],k||"undefined"==typeof d.removeAttribute?d[i]=void 0:d.removeAttribute(i),c.push(f))}}}),n.fn.extend({domManip:Ha,detach:function(a){return Ia(this,a,!0)},remove:function(a){return Ia(this,a)},text:function(a){return Y(this,function(a){return void 0===a?n.text(this):this.empty().append((this[0]&&this[0].ownerDocument||d).createTextNode(a))},null,a,arguments.length)},append:function(){return Ha(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ca(this,a);b.appendChild(a)}})},prepend:function(){return Ha(this,arguments,function(a){if(1===this.nodeType||11===this.nodeType||9===this.nodeType){var b=Ca(this,a);b.insertBefore(a,b.firstChild)}})},before:function(){return Ha(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this)})},after:function(){return Ha(this,arguments,function(a){this.parentNode&&this.parentNode.insertBefore(a,this.nextSibling)})},empty:function(){for(var a,b=0;null!=(a=this[b]);b++){1===a.nodeType&&n.cleanData(ea(a,!1));while(a.firstChild)a.removeChild(a.firstChild);a.options&&n.nodeName(a,"select")&&(a.options.length=0)}return this},clone:function(a,b){return a=null==a?!1:a,b=null==b?a:b,this.map(function(){return n.clone(this,a,b)})},html:function(a){return Y(this,function(a){var b=this[0]||{},c=0,d=this.length;if(void 0===a)return 1===b.nodeType?b.innerHTML.replace(ta,""):void 0;if("string"==typeof a&&!wa.test(a)&&(l.htmlSerialize||!ua.test(a))&&(l.leadingWhitespace||!aa.test(a))&&!da[($.exec(a)||["",""])[1].toLowerCase()]){a=n.htmlPrefilter(a);try{for(;d>c;c++)b=this[c]||{},1===b.nodeType&&(n.cleanData(ea(b,!1)),b.innerHTML=a);b=0}catch(e){}}b&&this.empty().append(a)},null,a,arguments.length)},replaceWith:function(){var a=[];return Ha(this,arguments,function(b){var c=this.parentNode;n.inArray(this,a)<0&&(n.cleanData(ea(this)),c&&c.replaceChild(b,this))},a)}}),n.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){n.fn[a]=function(a){for(var c,d=0,e=[],f=n(a),h=f.length-1;h>=d;d++)c=d===h?this:this.clone(!0),n(f[d])[b](c),g.apply(e,c.get());return this.pushStack(e)}});var Ja,Ka={HTML:"block",BODY:"block"};function La(a,b){var c=n(b.createElement(a)).appendTo(b.body),d=n.css(c[0],"display");return c.detach(),d}function Ma(a){var b=d,c=Ka[a];return c||(c=La(a,b),"none"!==c&&c||(Ja=(Ja||n("