feat: Add objstore support for Swift using thanos.io/objstore

docs/sources/shared/configuration.md

@@ -5846,6 +5846,10 @@ The `swift_storage_config` block configures the connection to OpenStack Object S
  
 
 ```yaml
+# Set this to true to use the internal OpenStack Swift endpoint URL
+# CLI flag: -<prefix>.swift.internal
+[internal: <boolean> | default = false]
+
 # OpenStack Swift authentication API version. 0 to autodetect.
 # CLI flag: -<prefix>.swift.auth-version
 [auth_version: <int> | default = 0]
@@ -5854,10 +5858,6 @@ The `swift_storage_config` block configures the connection to OpenStack Object S
 # CLI flag: -<prefix>.swift.auth-url
 [auth_url: <string> | default = ""]
 
-# Set this to true to use the internal OpenStack Swift endpoint URL
-# CLI flag: -<prefix>.swift.internal
-[internal: <boolean> | default = false]
-
 # OpenStack Swift username.
 # CLI flag: -<prefix>.swift.username
 [username: <string> | default = ""]
@@ -5925,6 +5925,12 @@ The `swift_storage_config` block configures the connection to OpenStack Object S
 # is received on a request.
 # CLI flag: -<prefix>.swift.request-timeout
 [request_timeout: <duration> | default = 5s]
+
+http_config:
+  # Path to the CA certificates to validate server certificate against. If not
+  # set, the host's root CA certificates are used.
+  # CLI flag: -<prefix>.swift.http.tls-ca-path
+  [tls_ca_path: <string> | default = ""]
 ```
 
 ### table_manager

pkg/loki/config_wrapper_test.go

@@ -553,7 +553,7 @@ memberlist:
 				assert.Equal(t, "example.com", actual.UserDomainName)
 				assert.Equal(t, "1", actual.UserDomainID)
 				assert.Equal(t, "27", actual.UserID)
-				assert.Equal(t, "supersecret", actual.Password)
+				assert.Equal(t, flagext.SecretWithValue("supersecret"), actual.Password)
 				assert.Equal(t, "2", actual.DomainID)
 				assert.Equal(t, "test.com", actual.DomainName)
 				assert.Equal(t, "13", actual.ProjectID)

pkg/storage/bucket/http/config.go

@@ -2,20 +2,38 @@ package http
 
 import (
 	"flag"
+	"net/http"
 	"time"
 )
 
+// NOTE some of the fields are hidden in the documentation due to this struct
+// being by the old Swift storage backend. The hidden fields can be unhidden
+// when we deprecate the old clients.
+
 // Config stores the http.Client configuration for the storage clients.
 type Config struct {
-	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout"`
-	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout"`
-	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify"`
-	TLSHandshakeTimeout   time.Duration `yaml:"tls_handshake_timeout"`
-	ExpectContinueTimeout time.Duration `yaml:"expect_continue_timeout"`
-	MaxIdleConns          int           `yaml:"max_idle_connections"`
-	MaxIdleConnsPerHost   int           `yaml:"max_idle_connections_per_host"`
-	MaxConnsPerHost       int           `yaml:"max_connections_per_host"`
-	CAFile                string        `yaml:"ca_file"`
+	IdleConnTimeout       time.Duration `yaml:"idle_conn_timeout" doc:"hidden"`
+	ResponseHeaderTimeout time.Duration `yaml:"response_header_timeout" doc:"hidden"`
+	InsecureSkipVerify    bool          `yaml:"insecure_skip_verify" doc:"hidden"`
+
+	TLSHandshakeTimeout   time.Duration `yaml:"tls_handshake_timeout" doc:"hidden"`
+	ExpectContinueTimeout time.Duration `yaml:"expect_continue_timeout" doc:"hidden"`
+	MaxIdleConns          int           `yaml:"max_idle_connections" doc:"hidden"`
+	MaxIdleConnsPerHost   int           `yaml:"max_idle_connections_per_host" doc:"hidden"`
+	MaxConnsPerHost       int           `yaml:"max_connections_per_host" doc:"hidden"`
+
+	// Allow upstream callers to inject a round tripper
+	Transport http.RoundTripper `yaml:"-"`
+
+	TLSConfig TLSConfig `yaml:",inline"`
+}
+
+// TLSConfig configures the options for TLS connections.
+type TLSConfig struct {
+	CAPath     string `yaml:"tls_ca_path" category:"advanced"`
+	CertPath   string `yaml:"tls_cert_path" category:"advanced" doc:"hidden"`
+	KeyPath    string `yaml:"tls_key_path" category:"advanced" doc:"hidden"`
+	ServerName string `yaml:"tls_server_name" category:"advanced" doc:"hidden"`
 }
 
 // RegisterFlags registers the flags for the storage HTTP client.
@@ -33,5 +51,13 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxIdleConns, prefix+"max-idle-connections", 100, "Maximum number of idle (keep-alive) connections across all hosts. 0 means no limit.")
 	f.IntVar(&cfg.MaxIdleConnsPerHost, prefix+"max-idle-connections-per-host", 100, "Maximum number of idle (keep-alive) connections to keep per-host. If 0, a built-in default value is used.")
 	f.IntVar(&cfg.MaxConnsPerHost, prefix+"max-connections-per-host", 0, "Maximum number of connections per host. 0 means no limit.")
-	f.StringVar(&cfg.CAFile, prefix+"ca-file", "", "Path to the trusted CA file that signed the SSL certificate of the object storage endpoint.")
+	cfg.TLSConfig.RegisterFlagsWithPrefix(prefix, f)
+}
+
+// RegisterFlagsWithPrefix registers the flags for s3 storage with the provided prefix.
+func (cfg *TLSConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.CAPath, prefix+"http.tls-ca-path", "", "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.")
+	f.StringVar(&cfg.CertPath, prefix+"http.tls-cert-path", "", "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.")
+	f.StringVar(&cfg.KeyPath, prefix+"http.tls-key-path", "", "Path to the key for the client certificate. Also requires the client certificate to be configured.")
+	f.StringVar(&cfg.ServerName, prefix+"http.tls-server-name", "", "Override the expected name on the server certificate.")
 }

pkg/storage/bucket/swift/bucket_client.go

@@ -4,31 +4,52 @@ import (
 	"github.com/go-kit/log"
 	"github.com/prometheus/common/model"
 	"github.com/thanos-io/objstore"
+	"github.com/thanos-io/objstore/exthttp"
 	"github.com/thanos-io/objstore/providers/swift"
 	yaml "gopkg.in/yaml.v2"
 )
 
 // NewBucketClient creates a new Swift bucket client
 func NewBucketClient(cfg Config, _ string, logger log.Logger) (objstore.Bucket, error) {
 	bucketConfig := swift.Config{
-		AuthVersion:       cfg.AuthVersion,
-		AuthUrl:           cfg.AuthURL,
-		Username:          cfg.Username,
-		UserDomainName:    cfg.UserDomainName,
-		UserDomainID:      cfg.UserDomainID,
-		UserId:            cfg.UserID,
-		Password:          cfg.Password,
-		DomainId:          cfg.DomainID,
-		DomainName:        cfg.DomainName,
-		ProjectID:         cfg.ProjectID,
-		ProjectName:       cfg.ProjectName,
-		ProjectDomainID:   cfg.ProjectDomainID,
-		ProjectDomainName: cfg.ProjectDomainName,
-		RegionName:        cfg.RegionName,
-		ContainerName:     cfg.ContainerName,
-		Retries:           cfg.MaxRetries,
-		ConnectTimeout:    model.Duration(cfg.ConnectTimeout),
-		Timeout:           model.Duration(cfg.RequestTimeout),
+		ApplicationCredentialID:     cfg.ApplicationCredentialID,
+		ApplicationCredentialName:   cfg.ApplicationCredentialName,
+		ApplicationCredentialSecret: cfg.ApplicationCredentialSecret.String(),
+		AuthVersion:                 cfg.AuthVersion,
+		AuthUrl:                     cfg.AuthURL,
+		Username:                    cfg.Username,
+		UserDomainName:              cfg.UserDomainName,
+		UserDomainID:                cfg.UserDomainID,
+		UserId:                      cfg.UserID,
+		Password:                    cfg.Password.String(),
+		DomainId:                    cfg.DomainID,
+		DomainName:                  cfg.DomainName,
+		ProjectID:                   cfg.ProjectID,
+		ProjectName:                 cfg.ProjectName,
+		ProjectDomainID:             cfg.ProjectDomainID,
+		ProjectDomainName:           cfg.ProjectDomainName,
+		RegionName:                  cfg.RegionName,
+		ContainerName:               cfg.ContainerName,
+		Retries:                     cfg.MaxRetries,
+		ConnectTimeout:              model.Duration(cfg.ConnectTimeout),
+		Timeout:                     model.Duration(cfg.RequestTimeout),
+		HTTPConfig: exthttp.HTTPConfig{
+			IdleConnTimeout:       model.Duration(cfg.HTTP.IdleConnTimeout),
+			ResponseHeaderTimeout: model.Duration(cfg.HTTP.ResponseHeaderTimeout),
+			InsecureSkipVerify:    cfg.HTTP.InsecureSkipVerify,
+			TLSHandshakeTimeout:   model.Duration(cfg.HTTP.TLSHandshakeTimeout),
+			ExpectContinueTimeout: model.Duration(cfg.HTTP.ExpectContinueTimeout),
+			MaxIdleConns:          cfg.HTTP.MaxIdleConns,
+			MaxIdleConnsPerHost:   cfg.HTTP.MaxIdleConnsPerHost,
+			MaxConnsPerHost:       cfg.HTTP.MaxConnsPerHost,
+			Transport:             cfg.HTTP.Transport,
+			TLSConfig: exthttp.TLSConfig{
+				CAFile:     cfg.HTTP.TLSConfig.CAPath,
+				CertFile:   cfg.HTTP.TLSConfig.CertPath,
+				KeyFile:    cfg.HTTP.TLSConfig.KeyPath,
+				ServerName: cfg.HTTP.TLSConfig.ServerName,
+			},
+		},
 
 		// Hard-coded defaults.
 		ChunkSize:              swift.DefaultConfig.ChunkSize,

pkg/storage/bucket/swift/config.go

@@ -3,29 +3,40 @@ package swift
 import (
 	"flag"
 	"time"
+
+	"github.com/grafana/dskit/flagext"
+
+	"github.com/grafana/loki/v3/pkg/storage/bucket/http"
 )
 
+// NOTE some of the fields are hidden in the documentation due to this struct
+// being by the old Swift storage backend. The hidden fields can be unhidden
+// when we deprecate the old clients.
+
 // Config holds the config options for Swift backend
 type Config struct {
-	AuthVersion       int           `yaml:"auth_version"`
-	AuthURL           string        `yaml:"auth_url"`
-	Internal          bool          `yaml:"internal"`
-	Username          string        `yaml:"username"`
-	UserDomainName    string        `yaml:"user_domain_name"`
-	UserDomainID      string        `yaml:"user_domain_id"`
-	UserID            string        `yaml:"user_id"`
-	Password          string        `yaml:"password"`
-	DomainID          string        `yaml:"domain_id"`
-	DomainName        string        `yaml:"domain_name"`
-	ProjectID         string        `yaml:"project_id"`
-	ProjectName       string        `yaml:"project_name"`
-	ProjectDomainID   string        `yaml:"project_domain_id"`
-	ProjectDomainName string        `yaml:"project_domain_name"`
-	RegionName        string        `yaml:"region_name"`
-	ContainerName     string        `yaml:"container_name"`
-	MaxRetries        int           `yaml:"max_retries"`
-	ConnectTimeout    time.Duration `yaml:"connect_timeout"`
-	RequestTimeout    time.Duration `yaml:"request_timeout"`
+	ApplicationCredentialID     string         `yaml:"application_credential_id" doc:"hidden"`
+	ApplicationCredentialName   string         `yaml:"application_credential_name" doc:"hidden"`
+	ApplicationCredentialSecret flagext.Secret `yaml:"application_credential_secret" doc:"hidden"`
+	AuthVersion                 int            `yaml:"auth_version"`
+	AuthURL                     string         `yaml:"auth_url"`
+	Username                    string         `yaml:"username"`
+	UserDomainName              string         `yaml:"user_domain_name"`
+	UserDomainID                string         `yaml:"user_domain_id"`
+	UserID                      string         `yaml:"user_id"`
+	Password                    flagext.Secret `yaml:"password"`
+	DomainID                    string         `yaml:"domain_id"`
+	DomainName                  string         `yaml:"domain_name"`
+	ProjectID                   string         `yaml:"project_id"`
+	ProjectName                 string         `yaml:"project_name"`
+	ProjectDomainID             string         `yaml:"project_domain_id"`
+	ProjectDomainName           string         `yaml:"project_domain_name"`
+	RegionName                  string         `yaml:"region_name"`
+	ContainerName               string         `yaml:"container_name"`
+	MaxRetries                  int            `yaml:"max_retries" category:"advanced"`
+	ConnectTimeout              time.Duration  `yaml:"connect_timeout" category:"advanced"`
+	RequestTimeout              time.Duration  `yaml:"request_timeout" category:"advanced"`
+	HTTP                        http.Config    `yaml:"http_config"`
 }
 
 // RegisterFlags registers the flags for Swift storage
@@ -35,14 +46,16 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 
 // RegisterFlagsWithPrefix registers the flags for Swift storage with the provided prefix
 func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.StringVar(&cfg.ApplicationCredentialID, prefix+"swift.application-credential-id", "", "OpenStack Swift application credential id")
+	f.StringVar(&cfg.ApplicationCredentialName, prefix+"swift.application-credential-name", "", "OpenStack Swift application credential name")
+	f.Var(&cfg.ApplicationCredentialSecret, prefix+"swift.application-credential-secret", "OpenStack Swift application credential secret")
 	f.IntVar(&cfg.AuthVersion, prefix+"swift.auth-version", 0, "OpenStack Swift authentication API version. 0 to autodetect.")
 	f.StringVar(&cfg.AuthURL, prefix+"swift.auth-url", "", "OpenStack Swift authentication URL")
-	f.BoolVar(&cfg.Internal, prefix+"swift.internal", false, "Set this to true to use the internal OpenStack Swift endpoint URL")
 	f.StringVar(&cfg.Username, prefix+"swift.username", "", "OpenStack Swift username.")
 	f.StringVar(&cfg.UserDomainName, prefix+"swift.user-domain-name", "", "OpenStack Swift user's domain name.")
 	f.StringVar(&cfg.UserDomainID, prefix+"swift.user-domain-id", "", "OpenStack Swift user's domain ID.")
 	f.StringVar(&cfg.UserID, prefix+"swift.user-id", "", "OpenStack Swift user ID.")
-	f.StringVar(&cfg.Password, prefix+"swift.password", "", "OpenStack Swift API key.")
+	f.Var(&cfg.Password, prefix+"swift.password", "OpenStack Swift API key.")
 	f.StringVar(&cfg.DomainID, prefix+"swift.domain-id", "", "OpenStack Swift user's domain ID.")
 	f.StringVar(&cfg.DomainName, prefix+"swift.domain-name", "", "OpenStack Swift user's domain name.")
 	f.StringVar(&cfg.ProjectID, prefix+"swift.project-id", "", "OpenStack Swift project ID (v2,v3 auth only).")
@@ -54,8 +67,5 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
 	f.IntVar(&cfg.MaxRetries, prefix+"swift.max-retries", 3, "Max retries on requests error.")
 	f.DurationVar(&cfg.ConnectTimeout, prefix+"swift.connect-timeout", 10*time.Second, "Time after which a connection attempt is aborted.")
 	f.DurationVar(&cfg.RequestTimeout, prefix+"swift.request-timeout", 5*time.Second, "Time after which an idle request is aborted. The timeout watchdog is reset each time some data is received, so the timeout triggers after X time no data is received on a request.")
-}
-
-func (cfg *Config) Validate() error {
-	return nil
+	cfg.HTTP.RegisterFlagsWithPrefix(prefix+"swift.", f)
 }

pkg/storage/chunk/client/openstack/swift_object_client.go

@@ -3,27 +3,57 @@ package openstack
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"flag"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"time"
 
 	swift "github.com/ncw/swift/v2"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 
+	bucket_http "github.com/grafana/loki/v3/pkg/storage/bucket/http"
 	bucket_swift "github.com/grafana/loki/v3/pkg/storage/bucket/swift"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/client"
 	"github.com/grafana/loki/v3/pkg/storage/chunk/client/hedging"
 	"github.com/grafana/loki/v3/pkg/util/log"
 )
 
-var defaultTransport http.RoundTripper = &http.Transport{
-	Proxy:                 http.ProxyFromEnvironment,
-	MaxIdleConnsPerHost:   200,
-	MaxIdleConns:          200,
-	ExpectContinueTimeout: 5 * time.Second,
+func defaultTransport(config bucket_http.Config) (http.RoundTripper, error) {
+	tlsConfig := &tls.Config{}
+	if len(config.TLSConfig.CAPath) > 0 {
+		caPath := config.TLSConfig.CAPath
+		data, err := os.ReadFile(caPath)
+		if err != nil {
+			return nil, fmt.Errorf("unable to load specified CA cert %s: %s", caPath, err)
+		}
+		caCertPool := x509.NewCertPool()
+		if !caCertPool.AppendCertsFromPEM(data) {
+			return nil, fmt.Errorf("unable to use specified CA cert %s", caPath)
+		}
+		tlsConfig.RootCAs = caCertPool
+	}
+
+	if config.Transport != nil {
+		return config.Transport, nil
+	}
+
+	return &http.Transport{
+		Proxy:                 http.ProxyFromEnvironment,
+		MaxIdleConns:          200,
+		MaxIdleConnsPerHost:   200,
+		ExpectContinueTimeout: 5 * time.Second,
+		// Set this value so that the underlying transport round-tripper
+		// doesn't try to auto decode the body of objects with
+		// content-encoding set to `gzip`.
+		//
+		// Refer: https://golang.org/src/net/http/transport.go?h=roundTrip#L1843.
+		TLSClientConfig: tlsConfig,
+	}, nil
 }
 
 type SwiftObjectClient struct {
@@ -34,6 +64,7 @@ type SwiftObjectClient struct {
 
 // SwiftConfig is config for the Swift Chunk Client.
 type SwiftConfig struct {
+	Internal            bool `yaml:"internal"`
 	bucket_swift.Config `yaml:",inline"`
 }
 
@@ -49,6 +80,7 @@ func (cfg *SwiftConfig) Validate() error {
 
 // RegisterFlagsWithPrefix registers flags with prefix.
 func (cfg *SwiftConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) {
+	f.BoolVar(&cfg.Internal, prefix+"swift.internal", false, "Set this to true to use the internal OpenStack Swift endpoint URL")
 	cfg.Config.RegisterFlagsWithPrefix(prefix, f)
 }
 
@@ -76,12 +108,16 @@ func NewSwiftObjectClient(cfg SwiftConfig, hedgingCfg hedging.Config) (*SwiftObj
 }
 
 func createConnection(cfg SwiftConfig, hedgingCfg hedging.Config, hedging bool) (*swift.Connection, error) {
-	// Create a connection
+	defaultTransport, err := defaultTransport(cfg.Config.HTTP)
+	if err != nil {
+		return nil, err
+	}
+
 	c := &swift.Connection{
 		AuthVersion:    cfg.Config.AuthVersion,
 		AuthUrl:        cfg.Config.AuthURL,
-		Internal:       cfg.Config.Internal,
-		ApiKey:         cfg.Config.Password,
+		Internal:       cfg.Internal,
+		ApiKey:         cfg.Config.Password.String(),
 		UserName:       cfg.Config.Username,
 		UserId:         cfg.Config.UserID,
 		Retries:        cfg.Config.MaxRetries,
@@ -97,12 +133,6 @@ func createConnection(cfg SwiftConfig, hedgingCfg hedging.Config, hedging bool)
 		Transport:      defaultTransport,
 	}
 
-	switch {
-	case cfg.Config.UserDomainName != "":
-		c.Domain = cfg.Config.UserDomainName
-	case cfg.Config.UserDomainID != "":
-		c.DomainId = cfg.Config.UserDomainID
-	}
 	if hedging {
 		var err error
 		c.Transport, err = hedgingCfg.RoundTripperWithRegisterer(c.Transport, prometheus.WrapRegistererWithPrefix("loki_", prometheus.DefaultRegisterer))
@@ -111,7 +141,8 @@ func createConnection(cfg SwiftConfig, hedgingCfg hedging.Config, hedging bool)
 		}
 	}
 
-	err := c.Authenticate(context.TODO())
+	// Create a connection
+	err = c.Authenticate(context.TODO())
 	if err != nil {
 		return nil, err
 	}
@@ -223,4 +254,8 @@ func (s *SwiftObjectClient) IsObjectNotFoundErr(err error) bool {
 }
 
 // TODO(dannyk): implement for client
-func (s *SwiftObjectClient) IsRetryableErr(error) bool { return false }
+func IsRetryableErr(error) bool { return false }
+
+func (s *SwiftObjectClient) IsRetryableErr(err error) bool {
+	return IsRetryableErr(err)
+}