adjust

juicedata · Apr 28, 2023 · 6e0f82f · 6e0f82f
1 parent dd15ef3
commit 6e0f82f
Show file tree

Hide file tree

Showing 17 changed files with 118 additions and 114 deletions.
diff --git a/cmd/config.go b/cmd/config.go
@@ -77,7 +77,7 @@ $ juicefs config redis://localhost --min-client-version 1.0.0 --max-client-versi
 			},
 			&cli.StringFlag{
 				Name:  "storage-class",
-				Usage: "object storage class type",
+				Usage: "the default storage class for data written in future",
 			},
 			&cli.BoolFlag{
 				Name:  "encrypt-secret",

diff --git a/cmd/flags.go b/cmd/flags.go
@@ -86,7 +86,7 @@ func clientFlags() []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:  "storage-class",
-			Usage: "object storage class type",
+			Usage: "the storage class for data written by current client",
 		},
 		&cli.IntFlag{
 			Name:  "get-timeout",

diff --git a/cmd/format.go b/cmd/format.go
@@ -143,7 +143,7 @@ Details: https://juicefs.com/docs/community/quick_start_guide`,
 			},
 			&cli.StringFlag{
 				Name:  "storage-class",
-				Usage: "object storage class type",
+				Usage: "the default storage class",
 			},
 			&cli.StringFlag{
 				Name:  "encrypt-rsa-key",
@@ -223,8 +223,10 @@ func createStorage(format meta.Format) (object.ObjectStorage, error) {
 		return nil, err
 	}
 	blob = object.WithPrefix(blob, format.Name+"/")
-	if os, ok := blob.(object.SupportStorageClass); ok {
-		os.SetStorageClass(format.StorageClass)
+	if format.StorageClass != "" {
+		if os, ok := blob.(object.SupportStorageClass); ok {
+			os.SetStorageClass(format.StorageClass)
+		}
 	}
 	if format.EncryptKey != "" {
 		passphrase := os.Getenv("JFS_RSA_PASSPHRASE")

diff --git a/cmd/sync.go b/cmd/sync.go
@@ -75,7 +75,7 @@ Supported storage systems: https://juicefs.com/docs/community/how_to_setup_objec
 		Flags: []cli.Flag{
 			&cli.StringFlag{
 				Name:  "storage-class",
-				Usage: "object storage class type",
+				Usage: "the storage class for destination",
 			},
 			&cli.StringFlag{
 				Name:    "start",

diff --git a/pkg/meta/config.go b/pkg/meta/config.go
@@ -68,7 +68,7 @@ type Format struct {
 	Name             string
 	UUID             string
 	Storage          string
-	StorageClass     string
+	StorageClass     string `json:",omitempty"`
 	Bucket           string
 	AccessKey        string `json:",omitempty"`
 	SecretKey        string `json:",omitempty"`

diff --git a/pkg/object/azure.go b/pkg/object/azure.go
@@ -32,10 +32,10 @@ import (
 
 type wasb struct {
 	DefaultObjectStorage
-	container    *azblob.ContainerClient
-	storageClass string
-	cName        string
-	marker       string
+	container *azblob.ContainerClient
+	sc        string
+	cName     string
+	marker    string
 }
 
 func (b *wasb) String() string {
@@ -77,8 +77,8 @@ func (b *wasb) Get(key string, off, limit int64) (io.ReadCloser, error) {
 
 func (b *wasb) Put(key string, data io.Reader) error {
 	options := azblob.UploadStreamToBlockBlobOptions{}
-	if b.storageClass != "" {
-		options.AccessTier = azblob.AccessTier(b.storageClass).ToPtr()
+	if b.sc != "" {
+		options.AccessTier = azblob.AccessTier(b.sc).ToPtr()
 	}
 	_, err := b.container.NewBlockBlobClient(key).UploadStreamToBlockBlob(ctx, data, options)
 	return err
@@ -140,7 +140,7 @@ func (b *wasb) List(prefix, marker, delimiter string, limit int64) ([]Object, er
 }
 
 func (b *wasb) SetStorageClass(sc string) {
-	b.storageClass = sc
+	b.sc = sc
 }
 
 func autoWasbEndpoint(containerName, accountName, scheme string, credential *azblob.SharedKeyCredential) (string, error) {

diff --git a/pkg/object/bos.go b/pkg/object/bos.go
@@ -38,9 +38,9 @@ const bosDefaultRegion = "bj"
 
 type bosclient struct {
 	DefaultObjectStorage
-	bucket       string
-	storageClass string
-	c            *bos.Client
+	bucket string
+	sc     string
+	c      *bos.Client
 }
 
 func (q *bosclient) String() string {
@@ -59,12 +59,14 @@ func (q *bosclient) Limits() Limits {
 
 func (q *bosclient) Create() error {
 	_, err := q.c.PutBucket(q.bucket)
+	if err == nil && q.sc != "" {
+		if err := q.c.PutBucketStorageclass(q.bucket, q.sc); err != nil {
+			logger.Warnf("failed to set storage class: %v", err)
+		}
+	}
 	if err != nil && isExists(err) {
 		err = nil
 	}
-	if q.storageClass != "" && err == nil {
-		err = q.c.PutBucketStorageclass(q.bucket, q.storageClass)
-	}
 	return err
 }
 
@@ -111,8 +113,8 @@ func (q *bosclient) Put(key string, in io.Reader) error {
 		return err
 	}
 	args := new(api.PutObjectArgs)
-	if q.storageClass != "" {
-		args.StorageClass = q.storageClass
+	if q.sc != "" {
+		args.StorageClass = q.sc
 	}
 	_, err = q.c.PutObject(q.bucket, key, body, args)
 	return err
@@ -158,8 +160,8 @@ func (q *bosclient) List(prefix, marker, delimiter string, limit int64) ([]Objec
 
 func (q *bosclient) CreateMultipartUpload(key string) (*MultipartUpload, error) {
 	args := new(api.InitiateMultipartUploadArgs)
-	if q.storageClass != "" {
-		args.StorageClass = q.storageClass
+	if q.sc != "" {
+		args.StorageClass = q.sc
 	}
 	r, err := q.c.InitiateMultipartUpload(q.bucket, key, "", args)
 	if err != nil {

diff --git a/pkg/object/cos.go b/pkg/object/cos.go
@@ -39,9 +39,9 @@ import (
 const cosChecksumKey = "x-cos-meta-" + checksumAlgr
 
 type COS struct {
-	c            *cos.Client
-	endpoint     string
-	storageClass string
+	c        *cos.Client
+	endpoint string
+	sc       string
 }
 
 func (c *COS) String() string {
@@ -113,8 +113,8 @@ func (c *COS) Put(key string, in io.Reader) error {
 		})
 		options.XCosMetaXXX = &header
 	}
-	if c.storageClass != "" {
-		options.XCosStorageClass = c.storageClass
+	if c.sc != "" {
+		options.XCosStorageClass = c.sc
 	}
 	_, err := c.c.Object.Put(ctx, key, in, &options)
 	return err
@@ -179,8 +179,8 @@ func (c *COS) ListAll(prefix, marker string) (<-chan Object, error) {
 
 func (c *COS) CreateMultipartUpload(key string) (*MultipartUpload, error) {
 	var options cos.InitiateMultipartUploadOptions
-	if c.storageClass != "" {
-		options.XCosStorageClass = c.storageClass
+	if c.sc != "" {
+		options.XCosStorageClass = c.sc
 	}
 	resp, _, err := c.c.Object.InitiateMultipartUpload(ctx, key, &options)
 	if err != nil {
@@ -237,7 +237,7 @@ func (c *COS) ListUploads(marker string) ([]*PendingPart, string, error) {
 }
 
 func (c *COS) SetStorageClass(sc string) {
-	c.storageClass = sc
+	c.sc = sc
 }
 
 func autoCOSEndpoint(bucketName, accessKey, secretKey, token string) (string, error) {

diff --git a/pkg/object/gs.go b/pkg/object/gs.go
@@ -38,11 +38,11 @@ import (
 
 type gs struct {
 	DefaultObjectStorage
-	client       *storage.Client
-	bucket       string
-	region       string
-	pageToken    string
-	storageClass string
+	client    *storage.Client
+	bucket    string
+	region    string
+	pageToken string
+	sc        string
 }
 
 func (g *gs) String() string {
@@ -78,7 +78,7 @@ func (g *gs) Create() error {
 
 	err := g.client.Bucket(g.bucket).Create(ctx, projectID, &storage.BucketAttrs{
 		Name:         g.bucket,
-		StorageClass: g.storageClass,
+		StorageClass: g.sc,
 		Location:     g.region,
 	})
 	if err != nil && strings.Contains(err.Error(), "You already own this bucket") {
@@ -114,7 +114,7 @@ func (g *gs) Get(key string, off, limit int64) (io.ReadCloser, error) {
 
 func (g *gs) Put(key string, data io.Reader) error {
 	writer := g.client.Bucket(g.bucket).Object(key).NewWriter(ctx)
-	writer.StorageClass = g.storageClass
+	writer.StorageClass = g.sc
 	_, err := io.Copy(writer, data)
 	if err != nil {
 		return err
@@ -166,7 +166,7 @@ func (g *gs) List(prefix, marker, delimiter string, limit int64) ([]Object, erro
 }
 
 func (g *gs) SetStorageClass(sc string) {
-	g.storageClass = sc
+	g.sc = sc
 }
 
 func newGS(endpoint, accessKey, secretKey, token string) (ObjectStorage, error) {

diff --git a/pkg/object/ibmcos.go b/pkg/object/ibmcos.go
@@ -41,9 +41,9 @@ import (
 )
 
 type ibmcos struct {
-	bucket       string
-	s3           *s3.S3
-	storageClass string
+	bucket string
+	s3     *s3.S3
+	sc     string
 }
 
 func (s *ibmcos) String() string {
@@ -53,9 +53,9 @@ func (s *ibmcos) String() string {
 func (s *ibmcos) Create() error {
 	input := &s3.CreateBucketInput{Bucket: &s.bucket}
 	// https://cloud.ibm.com/docs/cloud-object-storage?topic=cloud-object-storage-classes&code=go
-	if s.storageClass != "" {
+	if s.sc != "" {
 		input.CreateBucketConfiguration = &s3.CreateBucketConfiguration{
-			LocationConstraint: &s.storageClass,
+			LocationConstraint: &s.sc,
 		}
 	}
 	_, err := s.s3.CreateBucket(input)
@@ -108,8 +108,8 @@ func (s *ibmcos) Put(key string, in io.Reader) error {
 		Body:        body,
 		ContentType: &mimeType,
 	}
-	if s.storageClass != "" {
-		params.SetStorageClass(s.storageClass)
+	if s.sc != "" {
+		params.SetStorageClass(s.sc)
 	}
 	_, err := s.s3.PutObject(params)
 	return err
@@ -122,8 +122,8 @@ func (s *ibmcos) Copy(dst, src string) error {
 		Key:        &dst,
 		CopySource: &src,
 	}
-	if s.storageClass != "" {
-		params.SetStorageClass(s.storageClass)
+	if s.sc != "" {
+		params.SetStorageClass(s.sc)
 	}
 	_, err := s.s3.CopyObject(params)
 	return err
@@ -205,8 +205,8 @@ func (s *ibmcos) CreateMultipartUpload(key string) (*MultipartUpload, error) {
 		Bucket: &s.bucket,
 		Key:    &key,
 	}
-	if s.storageClass != "" {
-		params.SetStorageClass(s.storageClass)
+	if s.sc != "" {
+		params.SetStorageClass(s.sc)
 	}
 	resp, err := s.s3.CreateMultipartUpload(params)
 	if err != nil {
@@ -283,7 +283,7 @@ func (s *ibmcos) ListUploads(marker string) ([]*PendingPart, string, error) {
 }
 
 func (s *ibmcos) SetStorageClass(sc string) {
-	s.storageClass = sc
+	s.sc = sc
 }
 
 func newIBMCOS(endpoint, apiKey, serviceInstanceID, token string) (ObjectStorage, error) {

diff --git a/pkg/object/jss.go b/pkg/object/jss.go
@@ -45,8 +45,8 @@ func (j *jss) Copy(dst, src string) error {
 		Key:        &dst,
 		CopySource: &src,
 	}
-	if j.storageClass != "" {
-		params.SetStorageClass(j.storageClass)
+	if j.sc != "" {
+		params.SetStorageClass(j.sc)
 	}
 	_, err := j.s3client.s3.CopyObject(params)
 	return err

diff --git a/pkg/object/ks3.go b/pkg/object/ks3.go
@@ -40,9 +40,9 @@ import (
 )
 
 type ks3 struct {
-	bucket       string
-	s3           *s3.S3
-	storageClass string
+	bucket string
+	s3     *s3.S3
+	sc     string
 }
 
 func (s *ks3) String() string {
@@ -51,8 +51,8 @@ func (s *ks3) String() string {
 
 func (s *ks3) Create() error {
 	input := &s3.CreateBucketInput{Bucket: &s.bucket}
-	if s.storageClass != "" {
-		input.BucketType = aws.String(s.storageClass)
+	if s.sc != "" {
+		input.BucketType = aws.String(s.sc)
 	}
 	_, err := s.s3.CreateBucket(input)
 	if err != nil && isExists(err) {
@@ -139,8 +139,8 @@ func (s *ks3) Copy(dst, src string) error {
 		Key:        &dst,
 		CopySource: &src,
 	}
-	if s.storageClass != "" {
-		params.StorageClass = aws.String(s.storageClass)
+	if s.sc != "" {
+		params.StorageClass = aws.String(s.sc)
 	}
 	_, err := s.s3.CopyObject(params)
 	return err
@@ -205,8 +205,8 @@ func (s *ks3) CreateMultipartUpload(key string) (*MultipartUpload, error) {
 		Bucket: &s.bucket,
 		Key:    &key,
 	}
-	if s.storageClass != "" {
-		params.StorageClass = aws.String(s.storageClass)
+	if s.sc != "" {
+		params.StorageClass = aws.String(s.sc)
 	}
 	resp, err := s.s3.CreateMultipartUpload(params)
 	if err != nil {
@@ -294,7 +294,7 @@ func (s *ks3) ListUploads(marker string) ([]*PendingPart, string, error) {
 }
 
 func (s *ks3) SetStorageClass(sc string) {
-	s.storageClass = sc
+	s.sc = sc
 }
 
 var ks3Regions = map[string]string{