feat: expose S3 HTTP configs for TSDB S3 client

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## master / unreleased
 
+* [CHANGE] Blocks storage: update the default HTTP configuration values for the S3 client to the upstream Thanos default values. #3244
+  - `-blocks-storage.s3.http.idle-conn-timeout` is set 90 seconds.
+  - `-blocks-storage.s3.http.response-header-timeout` is set to 2 minutes.
 * [CHANGE] Improved shuffle sharding support in the write path. This work introduced some config changes: #3090
   * Introduced `-distributor.sharding-strategy` CLI flag (and its respective `sharding_strategy` YAML config option) to explicitly specify which sharding strategy should be used in the write path
   * `-experimental.distributor.user-subring-size` flag renamed to `-distributor.ingestion-tenant-shard-size`
@@ -39,6 +42,10 @@
   - `POST /ingester/push`
 * [FEATURE] Added support for shuffle-sharding queriers in the query-frontend. When configured (`-frontend.max-queriers-per-user` globally, or using per-user limit `max_queriers_per_user`), each user's requests will be handled by different set of queriers. #3113
 * [FEATURE] Query-frontend: added `compression` config to support results cache with compression. #3217
+* [ENHANCEMENT] Expose additional HTTP configs for the S3 backend client. New flag are listed below: #3244
+  - `-blocks-storage.s3.http.idle-conn-timeout`
+  - `-blocks-storage.s3.http.response-header-timeout`
+  - `-blocks-storage.s3.http.insecure-skip-verify`
 * [ENHANCEMENT] Added `cortex_query_frontend_connected_clients` metric to show the number of workers currently connected to the frontend. #3207
 * [ENHANCEMENT] Shuffle sharding: improved shuffle sharding in the write path. Shuffle sharding now should be explicitly enabled via `-distributor.sharding-strategy` CLI flag (or its respective YAML config option) and guarantees stability, consistency, shuffling and balanced zone-awareness properties. #3090 #3214
 * [ENHANCEMENT] Ingester: added new metric `cortex_ingester_active_series` to track active series more accurately. Also added options to control whether active series tracking is enabled (`-ingester.active-series-enabled`, defaults to false), and how often this metric is updated (`-ingester.active-series-update-period`) and max idle time for series to be considered inactive (`-ingester.active-series-idle-timeout`). #3153

docs/blocks-storage/querier.md

@@ -210,6 +210,20 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.insecure
     [insecure: <boolean> | default = false]
 
+    http:
+      # The time an idle connection will remain idle before closing.
+      # CLI flag: -blocks-storage.s3.http.idle-conn-timeout
+      [idle_conn_timeout: <duration> | default = 1m30s]
+
+      # The amount of time the client will wait for a servers response headers.
+      # CLI flag: -blocks-storage.s3.http.response-header-timeout
+      [response_header_timeout: <duration> | default = 2m]
+
+      # If the client connects to S3 via HTTPS and this option is enabled, the
+      # client will accept any certificate and hostname.
+      # CLI flag: -blocks-storage.s3.http.insecure-skip-verify
+      [insecure_skip_verify: <boolean> | default = false]
+
   gcs:
     # GCS bucket name
     # CLI flag: -blocks-storage.gcs.bucket-name

docs/blocks-storage/store-gateway.md

@@ -270,6 +270,20 @@ blocks_storage:
     # CLI flag: -blocks-storage.s3.insecure
     [insecure: <boolean> | default = false]
 
+    http:
+      # The time an idle connection will remain idle before closing.
+      # CLI flag: -blocks-storage.s3.http.idle-conn-timeout
+      [idle_conn_timeout: <duration> | default = 1m30s]
+
+      # The amount of time the client will wait for a servers response headers.
+      # CLI flag: -blocks-storage.s3.http.response-header-timeout
+      [response_header_timeout: <duration> | default = 2m]
+
+      # If the client connects to S3 via HTTPS and this option is enabled, the
+      # client will accept any certificate and hostname.
+      # CLI flag: -blocks-storage.s3.http.insecure-skip-verify
+      [insecure_skip_verify: <boolean> | default = false]
+
   gcs:
     # GCS bucket name
     # CLI flag: -blocks-storage.gcs.bucket-name

docs/configuration/config-file-reference.md

@@ -3154,6 +3154,20 @@ s3:
   # CLI flag: -blocks-storage.s3.insecure
   [insecure: <boolean> | default = false]
 
+  http:
+    # The time an idle connection will remain idle before closing.
+    # CLI flag: -blocks-storage.s3.http.idle-conn-timeout
+    [idle_conn_timeout: <duration> | default = 1m30s]
+
+    # The amount of time the client will wait for a servers response headers.
+    # CLI flag: -blocks-storage.s3.http.response-header-timeout
+    [response_header_timeout: <duration> | default = 2m]
+
+    # If the client connects to S3 via HTTPS and this option is enabled, the
+    # client will accept any certificate and hostname.
+    # CLI flag: -blocks-storage.s3.http.insecure-skip-verify
+    [insecure_skip_verify: <boolean> | default = false]
+
 gcs:
   # GCS bucket name
   # CLI flag: -blocks-storage.gcs.bucket-name

pkg/storage/backend/s3/bucket_client.go

@@ -2,6 +2,7 @@ package s3
 
 import (
 	"github.com/go-kit/kit/log"
+	"github.com/prometheus/common/model"
 	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/objstore/s3"
 )
@@ -23,5 +24,11 @@ func newS3Config(cfg Config) s3.Config {
 		AccessKey: cfg.AccessKeyID,
 		SecretKey: cfg.SecretAccessKey.Value,
 		Insecure:  cfg.Insecure,
+		HTTPConfig: s3.HTTPConfig{
+			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
+			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),
+			InsecureSkipVerify:    cfg.HTTP.InsecureSkipVerify,
+			Transport:             cfg.HTTP.Transport,
+		},
 	}
 }

pkg/storage/backend/s3/config.go

@@ -2,17 +2,38 @@ package s3
 
 import (
 	"flag"
+	"net/http"
+	"time"
 
 	"github.com/cortexproject/cortex/pkg/util/flagext"
 )
 
+// HTTPConfig stores the http.Transport configuration for the s3 minio client.
+type HTTPConfig struct {
+	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout"`
+	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout"`
+	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify"`
+
+	// Allow upstream callers to inject a round tripper
+	Transport http.RoundTripper `yaml:"-"`
+}
+
+// RegisterFlagsWithPrefix registers the flags for TSDB s3 storage with the provided prefix
+func (cfg *HTTPConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.DurationVar(&cfg.IdleConnTimeout, prefix+"s3.http.idle-conn-timeout", 90*time.Second, "The time an idle connection will remain idle before closing.")
+	f.DurationVar(&cfg.ResponseHeaderTimeout, prefix+"s3.http.response-header-timeout", 2*time.Minute, "The amount of time the client will wait for a servers response headers.")
+	f.BoolVar(&cfg.InsecureSkipVerify, prefix+"s3.http.insecure-skip-verify", false, "If the client connects to S3 via HTTPS and this option is enabled, the client will accept any certificate and hostname.")
+}
+
 // Config holds the config options for an S3 backend
 type Config struct {
 	Endpoint        string         `yaml:"endpoint"`
 	BucketName      string         `yaml:"bucket_name"`
 	SecretAccessKey flagext.Secret `yaml:"secret_access_key"`
 	AccessKeyID     string         `yaml:"access_key_id"`
 	Insecure        bool           `yaml:"insecure"`
+
+	HTTP HTTPConfig `yaml:"http"`
 }
 
 // RegisterFlags registers the flags for TSDB s3 storage with the provided prefix
@@ -27,4 +48,5 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.StringVar(&cfg.BucketName, prefix+"s3.bucket-name", "", "S3 bucket name")
 	f.StringVar(&cfg.Endpoint, prefix+"s3.endpoint", "", "The S3 bucket endpoint. It could be an AWS S3 endpoint listed at https://docs.aws.amazon.com/general/latest/gr/s3.html or the address of an S3-compatible service in hostname:port format.")
 	f.BoolVar(&cfg.Insecure, prefix+"s3.insecure", false, "If enabled, use http:// for the S3 endpoint instead of https://. This could be useful in local dev/test environments while using an S3-compatible backend storage, like Minio.")
+	cfg.HTTP.RegisterFlagsWithPrefix(prefix, f)
 }