Adding OpenStack Swift support to blocks storage

.gitignore

@@ -23,3 +23,7 @@ test-exporter
 
 # This is custom Makefile modification, if it exists.
 Makefile.local
+
+.idea
+compose
+compose-simple

CHANGELOG.md

@@ -52,6 +52,7 @@
 * [FEATURE] Shuffle sharding: added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-tenant` globally, or using per-tenant limit `max_queriers_per_tenant`), each tenants's requests will be handled by different set of queriers. #3113 #3257
 * [FEATURE] Shuffle sharding: added support for shuffle-sharding ingesters on the read path. When ingesters shuffle-sharding is enabled and `-querier.shuffle-sharding-ingesters-lookback-period` is set, queriers will fetch in-memory series from the minimum set of required ingesters, selecting only ingesters which may have received series since 'now - lookback period'. #3252
 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217
+* [FEATURE] Add OpenStack Swift support to blocks storage. #3303
 * [FEATURE] Added support for applying Prometheus relabel configs on series received by the distributor. A `metric_relabel_configs` field has been added to the per-tenant limits configuration. #3329
 * [ENHANCEMENT] Ruler: Introduces two new limits `-ruler.max-rules-per-rule-group` and `-ruler.max-rule-groups-per-tenant` to control the number of rules per rule group and the total number of rule groups for a given user. They are disabled by default. #3366
 * [ENHANCEMENT] Allow to specify multiple comma-separated Cortex services to `-target` CLI option (or its respective YAML config option). For example, `-target=all,compactor` can be used to start Cortex single-binary with compactor as well. #3275

development/tsdb-blocks-storage-swift-single-binary/.data-cortex-1/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

development/tsdb-blocks-storage-swift-single-binary/.data-cortex-2/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

development/tsdb-blocks-storage-swift-single-binary/.dockerignore

@@ -0,0 +1,4 @@
+.data-cortex-1
+.data-cortex-2
+.data-swift
+active-query-tracker

development/tsdb-blocks-storage-swift-single-binary/.gitignore

@@ -0,0 +1,2 @@
+cortex
+.data-swift

development/tsdb-blocks-storage-swift-single-binary/compose-down.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd `dirname $0` && pwd)
+
+docker-compose -f ${SCRIPT_DIR}/docker-compose.yml down

development/tsdb-blocks-storage-swift-single-binary/compose-up.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+SCRIPT_DIR=$(cd `dirname $0` && pwd)
+
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${SCRIPT_DIR}/cortex ${SCRIPT_DIR}/../../cmd/cortex && \
+docker-compose -f ${SCRIPT_DIR}/docker-compose.yml build cortex-1 && \
+docker-compose -f ${SCRIPT_DIR}/docker-compose.yml up $@

development/tsdb-blocks-storage-swift-single-binary/config/cortex.yaml

@@ -0,0 +1,81 @@
+auth_enabled: false
+
+distributor:
+  shard_by_all_labels: true
+  pool:
+    health_check_ingesters: true
+
+ingester_client:
+  grpc_client_config:
+    # Configure the client to allow messages up to 100MB.
+    max_recv_msg_size: 104857600
+    max_send_msg_size: 104857600
+    grpc_compression: gzip
+
+ingester:
+  lifecycler:
+    # We want to start immediately.
+    join_after: 0
+    final_sleep: 0s
+    num_tokens: 512
+
+    ring:
+      kvstore:
+        store: consul
+        consul:
+          host: localhost:8500
+      replication_factor: 1
+
+querier:
+  query_ingesters_within: 3h
+
+store_gateway:
+  sharding_enabled: true
+  sharding_ring:
+    replication_factor: 1
+    kvstore:
+      store: consul
+      consul:
+        host: localhost:8500
+
+blocks_storage:
+  backend: swift
+
+  tsdb:
+    dir: /data/cortex-tsdb-ingester
+    ship_interval: 1m
+    block_ranges_period: [ 2h ]
+    retention_period: 3h
+
+  bucket_store:
+    sync_dir: /data/cortex-tsdb-querier
+
+  swift:
+    auth_url:          http://localhost:5000/v2.0
+    username:          admin
+    password:          s3cr3t
+    project_name:      admin
+    region_name:       RegionOne
+    container_name:    cortex-tsdb
+
+storage:
+  engine: blocks
+
+ruler:
+  enable_api: true
+  enable_sharding: true
+  poll_interval: 2s
+  storage:
+    type: swift
+    swift:
+      auth_url:          http://localhost:5000/v2.0
+      username:          admin
+      password:          s3cr3t
+      project_name:      admin
+      region_name:       RegionOne
+      container_name: cortex-rules
+  ring:
+    kvstore:
+      store: consul
+      consul:
+        host: localhost:8500

development/tsdb-blocks-storage-swift-single-binary/config/grafana-agent.yaml

@@ -0,0 +1,26 @@
+server:
+  log_level: debug
+  http_listen_port: 9091
+
+prometheus:
+  global:
+    scrape_interval: 5s
+    external_labels:
+      scraped_by: grafana-agent
+  configs:
+    - name: local
+      host_filter: false
+      scrape_configs:
+        - job_name: cortex-1
+          static_configs:
+            - targets: ['localhost:8001']
+              labels:
+                container: 'cortex-1'
+        - job_name: cortex-2
+          static_configs:
+            - targets: ['localhost:8002']
+              labels:
+                container: 'cortex-2'
+
+      remote_write:
+        - url: http://localhost:8001/api/prom/push

development/tsdb-blocks-storage-swift-single-binary/config/prometheus.yaml

@@ -0,0 +1,19 @@
+global:
+  scrape_interval: 5s
+  external_labels:
+    scraped_by: prometheus
+
+scrape_configs:
+  - job_name: cortex-1
+    static_configs:
+      - targets: ['localhost:8001']
+        labels:
+          container: 'cortex-1'
+  - job_name: cortex-2
+    static_configs:
+      - targets: ['localhost:8002']
+        labels:
+          container: 'cortex-2'
+
+remote_write:
+  - url: http://localhost:8001/api/prom/push

development/tsdb-blocks-storage-swift-single-binary/dev.dockerfile

@@ -0,0 +1,5 @@
+FROM alpine:3.12
+
+RUN     mkdir /cortex
+WORKDIR /cortex
+ADD     ./cortex ./

development/tsdb-blocks-storage-swift-single-binary/docker-compose.yml

@@ -0,0 +1,100 @@
+version: '3.4'
+services:
+
+  consul:
+    image: consul
+    command: [ "agent", "-dev" ,"-client=0.0.0.0", "-log-level=info" ]
+    ports:
+      - 8500:8500
+
+  swift:
+    image: beaukode/docker-swift-onlyone-authv2-keystone
+    volumes:
+      - .data-swift:/srv:delegated
+    ports:
+      - 5000:5000
+      - 5001:5001
+      - 8080:8080
+      - 35357:35357
+
+  swift-create-container:
+    network_mode: host
+    image: theflockers/swiftclient
+    environment:
+      - OS_USERNAME=admin
+      - OS_PASSWORD=s3cr3t
+      - OS_TENANT_NAME=admin
+      - OS_AUTH_URL=http://localhost:5000/v2.0
+      - OS_REGION_NAME=RegionOne
+    command: ["sh", "-c", "sleep 30 && swift post cortex-rules && \
+                            swift post cortex-tsdb && swift list"]
+
+  prometheus:
+    network_mode: host
+    image: prom/prometheus:v2.16.0
+    command: ["--config.file=/etc/prometheus/prometheus.yaml"]
+    volumes:
+      - ./config:/etc/prometheus
+    ports:
+      - 9090:9090
+
+  # Scrape the metrics also with the Grafana agent (useful to test metadata ingestion
+  # until metadata remote write is not supported by Prometheus).
+  grafana-agent:
+    network_mode: host
+    image: grafana/agent:v0.2.0
+    command: ["-config.file=/etc/agent-config/grafana-agent.yaml", "-prometheus.wal-directory=/tmp"]
+    volumes:
+      - ./config:/etc/agent-config
+    ports:
+      - 9091:9091
+
+  jaeger:
+    image: jaegertracing/all-in-one
+    ports:
+      - 16686:16686
+      - "14268"
+
+  cortex-1:
+    network_mode: host
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: cortex
+    command: ["sh", "-c", "sleep 40 && exec ./cortex -config.file=./config/cortex.yaml -target=all -server.http-listen-port=8001 -server.grpc-listen-port=9001"]
+    depends_on:
+      - consul
+      - swift
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=cortex-1
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8001:8001
+    volumes:
+      - ./config:/cortex/config
+      - .data-cortex-1:/data:delegated
+
+  cortex-2:
+    network_mode: host
+    build:
+      context:    .
+      dockerfile: dev.dockerfile
+    image: cortex
+    command: ["sh", "-c", "sleep 40 && exec ./cortex -config.file=./config/cortex.yaml -target=all -server.http-listen-port=8002 -server.grpc-listen-port=9002"]
+    depends_on:
+      - consul
+      - swift
+    environment:
+      - JAEGER_AGENT_HOST=jaeger
+      - JAEGER_AGENT_PORT=6831
+      - JAEGER_TAGS=app=cortex-2
+      - JAEGER_SAMPLER_TYPE=const
+      - JAEGER_SAMPLER_PARAM=1
+    ports:
+      - 8002:8002
+    volumes:
+      - ./config:/cortex/config
+      - .data-cortex-2:/data:delegated

docs/blocks-storage/querier.md

@@ -190,7 +190,8 @@ The `blocks_storage_config` configures the blocks storage.
 
 ```yaml
 blocks_storage:
-  # Backend storage to use. Supported backends are: s3, gcs, azure, filesystem.
+  # Backend storage to use. Supported backends are: s3, gcs, azure, swift,
+  # filesystem.
   # CLI flag: -blocks-storage.backend
   [backend: <string> | default = "s3"]
 
@@ -266,6 +267,65 @@ blocks_storage:
     # CLI flag: -blocks-storage.azure.max-retries
     [max_retries: <int> | default = 20]
 
+  swift:
+    # OpenStack Swift authentication URL
+    # CLI flag: -blocks-storage.swift.auth-url
+    [auth_url: <string> | default = ""]
+
+    # OpenStack Swift username.
+    # CLI flag: -blocks-storage.swift.username
+    [username: <string> | default = ""]
+
+    # OpenStack Swift user's domain name.
+    # CLI flag: -blocks-storage.swift.user-domain-name
+    [user_domain_name: <string> | default = ""]
+
+    # OpenStack Swift user's domain ID.
+    # CLI flag: -blocks-storage.swift.user-domain-id
+    [user_domain_id: <string> | default = ""]
+
+    # OpenStack Swift user ID.
+    # CLI flag: -blocks-storage.swift.user-id
+    [user_id: <string> | default = ""]
+
+    # OpenStack Swift API key.
+    # CLI flag: -blocks-storage.swift.password
+    [password: <string> | default = ""]
+
+    # OpenStack Swift user's domain ID.
+    # CLI flag: -blocks-storage.swift.domain-id
+    [domain_id: <string> | default = ""]
+
+    # OpenStack Swift user's domain name.
+    # CLI flag: -blocks-storage.swift.domain-name
+    [domain_name: <string> | default = ""]
+
+    # OpenStack Swift project ID (v2,v3 auth only).
+    # CLI flag: -blocks-storage.swift.project-id
+    [project_id: <string> | default = ""]
+
+    # OpenStack Swift project name (v2,v3 auth only).
+    # CLI flag: -blocks-storage.swift.project-name
+    [project_name: <string> | default = ""]
+
+    # ID of the OpenStack Swift project's domain (v3 auth only), only needed if
+    # it differs the from user domain.
+    # CLI flag: -blocks-storage.swift.project-domain-id
+    [project_domain_id: <string> | default = ""]
+
+    # Name of the OpenStack Swift project's domain (v3 auth only), only needed
+    # if it differs from the user domain.
+    # CLI flag: -blocks-storage.swift.project-domain-name
+    [project_domain_name: <string> | default = ""]
+
+    # OpenStack Swift Region to use (v2,v3 auth only).
+    # CLI flag: -blocks-storage.swift.region-name
+    [region_name: <string> | default = ""]
+
+    # Name of the OpenStack Swift container to put chunks in.
+    # CLI flag: -blocks-storage.swift.container-name
+    [container_name: <string> | default = ""]
+
   filesystem:
     # Local filesystem storage directory.
     # CLI flag: -blocks-storage.filesystem.dir