object: support storage class

cmd/config.go

@@ -75,6 +75,10 @@ $ juicefs config redis://localhost --min-client-version 1.0.0 --max-client-versi
 				Name:  "session-token",
 				Usage: "session token for object storage",
 			},
+			&cli.StringFlag{
+				Name:  "storage-class",
+				Usage: "object storage class type",
+			},
 			&cli.BoolFlag{
 				Name:  "encrypt-secret",
 				Usage: "encrypt the secret key if it was previously stored in plain format",
@@ -187,6 +191,12 @@ func config(ctx *cli.Context) error {
 			}
 			format.SessionToken = ctx.String(flag)
 			storage = true
+		case "storage-class": // always update
+			if new := ctx.String(flag); new != format.StorageClass {
+				msg.WriteString(fmt.Sprintf("%10s: %s -> %s\n", flag, format.StorageClass, new))
+				format.StorageClass = new
+				storage = true
+			}
 		case "trash-days":
 			if new := ctx.Int(flag); new != format.TrashDays {
 				if new < 0 {

cmd/flags.go

@@ -84,6 +84,10 @@ func clientFlags() []cli.Flag {
 			Name:  "bucket",
 			Usage: "customized endpoint to access object store",
 		},
+		&cli.StringFlag{
+			Name:  "storage-class",
+			Usage: "object storage class type",
+		},
 		&cli.IntFlag{
 			Name:  "get-timeout",
 			Value: 60,

cmd/format.go

@@ -141,6 +141,10 @@ Details: https://juicefs.com/docs/community/quick_start_guide`,
 				Name:  "session-token",
 				Usage: "session token for object storage",
 			},
+			&cli.StringFlag{
+				Name:  "storage-class",
+				Usage: "object storage class type",
+			},
 			&cli.StringFlag{
 				Name:  "encrypt-rsa-key",
 				Usage: "a path to RSA private key (PEM)",
@@ -219,7 +223,9 @@ func createStorage(format meta.Format) (object.ObjectStorage, error) {
 		return nil, err
 	}
 	blob = object.WithPrefix(blob, format.Name+"/")
-
+	if os, ok := blob.(object.SupportStorageClass); ok {
+		os.SetStorageClass(format.StorageClass)
+	}
 	if format.EncryptKey != "" {
 		passphrase := os.Getenv("JFS_RSA_PASSPHRASE")
 		if passphrase == "" {
@@ -392,6 +398,7 @@ func format(c *cli.Context) error {
 			Name:         name,
 			UUID:         uuid.New().String(),
 			Storage:      c.String("storage"),
+			StorageClass: c.String("storage-class"),
 			Bucket:       c.String("bucket"),
 			AccessKey:    c.String("access-key"),
 			SecretKey:    c.String("secret-key"),

cmd/mount.go

@@ -184,6 +184,9 @@ func updateFormat(c *cli.Context) func(*meta.Format) {
 		if c.IsSet("storage") {
 			format.Storage = c.String("storage")
 		}
+		if c.IsSet("storage-class") {
+			format.StorageClass = c.String("storage-class")
+		}
 	}
 }
 
@@ -434,8 +437,8 @@ func NewReloadableStorage(format *meta.Format, cli meta.Meta, patch func(*meta.F
 			patch(new)
 		}
 		old := &holder.fmt
-		if new.Storage != old.Storage || new.Bucket != old.Bucket || new.AccessKey != old.AccessKey || new.SecretKey != old.SecretKey || new.SessionToken != old.SessionToken {
-			logger.Infof("found new configuration: storage=%s bucket=%s ak=%s", new.Storage, new.Bucket, new.AccessKey)
+		if new.Storage != old.Storage || new.Bucket != old.Bucket || new.AccessKey != old.AccessKey || new.SecretKey != old.SecretKey || new.SessionToken != old.SessionToken || new.StorageClass != old.StorageClass {
+			logger.Infof("found new configuration: storage=%s bucket=%s ak=%s storageClass=%s", new.Storage, new.Bucket, new.AccessKey, new.StorageClass)
 
 			newBlob, err := createStorage(*new)
 			if err != nil {

cmd/sync.go

@@ -73,6 +73,10 @@ $ juicefs sync --include='a1/b1' --exclude='a*' --include='b2' --exclude='b?' s3
 Details: https://juicefs.com/docs/community/administration/sync
 Supported storage systems: https://juicefs.com/docs/community/how_to_setup_object_storage#supported-object-storage`,
 		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:  "storage-class",
+				Usage: "object storage class type",
+			},
 			&cli.StringFlag{
 				Name:    "start",
 				Aliases: []string{"s"},
@@ -349,5 +353,8 @@ func doSync(c *cli.Context) error {
 	if err != nil {
 		return err
 	}
+	if os, ok := dst.(object.SupportStorageClass); ok {
+		os.SetStorageClass(config.StorageClass)
+	}
 	return sync.Sync(src, dst, config)
 }

pkg/meta/config.go

@@ -68,6 +68,7 @@ type Format struct {
 	Name             string
 	UUID             string
 	Storage          string
+	StorageClass     string
 	Bucket           string
 	AccessKey        string `json:",omitempty"`
 	SecretKey        string `json:",omitempty"`

pkg/meta/metadata-sub.sample

@@ -3,6 +3,7 @@
     "Name": "load-dump-test",
     "UUID": "faa27c8f-edab-4791-a4e0-1620b732b343",
     "Storage": "file",
+    "StorageClass": "",
     "Bucket": "/Users/juicefs/.juicefs/local/",
     "SecretKey": "removed",
     "BlockSize": 4096,

pkg/meta/metadata.sample

@@ -3,6 +3,7 @@
     "Name": "load-dump-test",
     "UUID": "faa27c8f-edab-4791-a4e0-1620b732b343",
     "Storage": "file",
+    "StorageClass": "",
     "Bucket": "/Users/juicefs/.juicefs/local/",
     "SecretKey": "removed",
     "BlockSize": 4096,

pkg/object/azure.go

@@ -32,9 +32,10 @@ import (
 
 type wasb struct {
 	DefaultObjectStorage
-	container *azblob.ContainerClient
-	cName     string
-	marker    string
+	container    *azblob.ContainerClient
+	storageClass string
+	cName        string
+	marker       string
 }
 
 func (b *wasb) String() string {
@@ -75,7 +76,11 @@ func (b *wasb) Get(key string, off, limit int64) (io.ReadCloser, error) {
 }
 
 func (b *wasb) Put(key string, data io.Reader) error {
-	_, err := b.container.NewBlockBlobClient(key).UploadStreamToBlockBlob(ctx, data, azblob.UploadStreamToBlockBlobOptions{})
+	options := azblob.UploadStreamToBlockBlobOptions{}
+	if b.storageClass != "" {
+		options.AccessTier = azblob.AccessTier(b.storageClass).ToPtr()
+	}
+	_, err := b.container.NewBlockBlobClient(key).UploadStreamToBlockBlob(ctx, data, options)
 	return err
 }
 
@@ -134,6 +139,10 @@ func (b *wasb) List(prefix, marker, delimiter string, limit int64) ([]Object, er
 	return objs, nil
 }
 
+func (b *wasb) SetStorageClass(sc string) {
+	b.storageClass = sc
+}
+
 func autoWasbEndpoint(containerName, accountName, scheme string, credential *azblob.SharedKeyCredential) (string, error) {
 	baseURLs := []string{"blob.core.windows.net", "blob.core.chinacloudapi.cn"}
 	endpoint := ""

pkg/object/bos.go

@@ -38,8 +38,9 @@ const bosDefaultRegion = "bj"
 
 type bosclient struct {
 	DefaultObjectStorage
-	bucket string
-	c      *bos.Client
+	bucket       string
+	storageClass string
+	c            *bos.Client
 }
 
 func (q *bosclient) String() string {
@@ -61,6 +62,9 @@ func (q *bosclient) Create() error {
 	if err != nil && isExists(err) {
 		err = nil
 	}
+	if q.storageClass != "" && err == nil {
+		err = q.c.PutBucketStorageclass(q.bucket, q.storageClass)
+	}
 	return err
 }
 
@@ -106,7 +110,11 @@ func (q *bosclient) Put(key string, in io.Reader) error {
 	if err != nil {
 		return err
 	}
-	_, err = q.c.BasicPutObject(q.bucket, key, body)
+	args := new(api.PutObjectArgs)
+	if q.storageClass != "" {
+		args.StorageClass = q.storageClass
+	}
+	_, err = q.c.PutObject(q.bucket, key, body, args)
 	return err
 }
 
@@ -149,7 +157,11 @@ func (q *bosclient) List(prefix, marker, delimiter string, limit int64) ([]Objec
 }
 
 func (q *bosclient) CreateMultipartUpload(key string) (*MultipartUpload, error) {
-	r, err := q.c.BasicInitiateMultipartUpload(q.bucket, key)
+	args := new(api.InitiateMultipartUploadArgs)
+	if q.storageClass != "" {
+		args.StorageClass = q.storageClass
+	}
+	r, err := q.c.InitiateMultipartUpload(q.bucket, key, "", args)
 	if err != nil {
 		return nil, err
 	}

pkg/object/cos.go

@@ -39,8 +39,9 @@ import (
 const cosChecksumKey = "x-cos-meta-" + checksumAlgr
 
 type COS struct {
-	c        *cos.Client
-	endpoint string
+	c            *cos.Client
+	endpoint     string
+	storageClass string
 }
 
 func (c *COS) String() string {
@@ -105,14 +106,17 @@ func (c *COS) Get(key string, off, limit int64) (io.ReadCloser, error) {
 }
 
 func (c *COS) Put(key string, in io.Reader) error {
-	var options *cos.ObjectPutOptions
+	var options cos.ObjectPutOptions
 	if ins, ok := in.(io.ReadSeeker); ok {
 		header := http.Header(map[string][]string{
 			cosChecksumKey: {generateChecksum(ins)},
 		})
-		options = &cos.ObjectPutOptions{ObjectPutHeaderOptions: &cos.ObjectPutHeaderOptions{XCosMetaXXX: &header}}
+		options.XCosMetaXXX = &header
 	}
-	_, err := c.c.Object.Put(ctx, key, in, options)
+	if c.storageClass != "" {
+		options.XCosStorageClass = c.storageClass
+	}
+	_, err := c.c.Object.Put(ctx, key, in, &options)
 	return err
 }
 
@@ -174,7 +178,11 @@ func (c *COS) ListAll(prefix, marker string) (<-chan Object, error) {
 }
 
 func (c *COS) CreateMultipartUpload(key string) (*MultipartUpload, error) {
-	resp, _, err := c.c.Object.InitiateMultipartUpload(ctx, key, nil)
+	var options cos.InitiateMultipartUploadOptions
+	if c.storageClass != "" {
+		options.XCosStorageClass = c.storageClass
+	}
+	resp, _, err := c.c.Object.InitiateMultipartUpload(ctx, key, &options)
 	if err != nil {
 		return nil, err
 	}
@@ -228,6 +236,10 @@ func (c *COS) ListUploads(marker string) ([]*PendingPart, string, error) {
 	return parts, result.NextKeyMarker, nil
 }
 
+func (c *COS) SetStorageClass(sc string) {
+	c.storageClass = sc
+}
+
 func autoCOSEndpoint(bucketName, accessKey, secretKey, token string) (string, error) {
 	client := cos.NewClient(nil, &http.Client{
 		Transport: &cos.AuthorizationTransport{
@@ -287,7 +299,7 @@ func newCOS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 		},
 	})
 	client.UserAgent = UserAgent
-	return &COS{client, uri.Host}, nil
+	return &COS{c: client, endpoint: uri.Host}, nil
 }
 
 func init() {

pkg/object/eos.go

@@ -87,7 +87,7 @@ func newEos(endpoint, accessKey, secretKey, token string) (ObjectStorage, error)
 		return nil, fmt.Errorf("aws session: %s", err)
 	}
 	ses.Handlers.Build.PushFront(disableSha256Func)
-	return &eos{s3client{bucket, s3.New(ses), ses}}, nil
+	return &eos{s3client{bucket: bucket, s3: s3.New(ses), ses: ses}}, nil
 }
 
 func init() {

pkg/object/gs.go

@@ -38,10 +38,11 @@ import (
 
 type gs struct {
 	DefaultObjectStorage
-	client    *storage.Client
-	bucket    string
-	region    string
-	pageToken string
+	client       *storage.Client
+	bucket       string
+	region       string
+	pageToken    string
+	storageClass string
 }
 
 func (g *gs) String() string {
@@ -76,8 +77,9 @@ func (g *gs) Create() error {
 	}
 
 	err := g.client.Bucket(g.bucket).Create(ctx, projectID, &storage.BucketAttrs{
-		Name:     g.bucket,
-		Location: g.region,
+		Name:         g.bucket,
+		StorageClass: g.storageClass,
+		Location:     g.region,
 	})
 	if err != nil && strings.Contains(err.Error(), "You already own this bucket") {
 		return nil
@@ -112,6 +114,7 @@ func (g *gs) Get(key string, off, limit int64) (io.ReadCloser, error) {
 
 func (g *gs) Put(key string, data io.Reader) error {
 	writer := g.client.Bucket(g.bucket).Object(key).NewWriter(ctx)
+	writer.StorageClass = g.storageClass
 	_, err := io.Copy(writer, data)
 	if err != nil {
 		return err
@@ -162,6 +165,10 @@ func (g *gs) List(prefix, marker, delimiter string, limit int64) ([]Object, erro
 	return objs, nil
 }
 
+func (g *gs) SetStorageClass(sc string) {
+	g.storageClass = sc
+}
+
 func newGS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {
 	if !strings.Contains(endpoint, "://") {
 		endpoint = fmt.Sprintf("gs://%s", endpoint)