Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

object: support storage class #3534

Merged
merged 9 commits into from
May 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions cmd/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ $ juicefs config redis://localhost --min-client-version 1.0.0 --max-client-versi
Name: "session-token",
Usage: "session token for object storage",
},
&cli.StringFlag{
Name: "storage-class",
Usage: "the default storage class for data written in future",
},
&cli.BoolFlag{
Name: "encrypt-secret",
Usage: "encrypt the secret key if it was previously stored in plain format",
Expand Down Expand Up @@ -187,6 +191,12 @@ func config(ctx *cli.Context) error {
}
format.SessionToken = ctx.String(flag)
storage = true
case "storage-class": // always update
if new := ctx.String(flag); new != format.StorageClass {
msg.WriteString(fmt.Sprintf("%10s: %s -> %s\n", flag, format.StorageClass, new))
format.StorageClass = new
storage = true
}
case "trash-days":
if new := ctx.Int(flag); new != format.TrashDays {
if new < 0 {
Expand Down
4 changes: 4 additions & 0 deletions cmd/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ func clientFlags() []cli.Flag {
Name: "bucket",
Usage: "customized endpoint to access object store",
},
&cli.StringFlag{
Name: "storage-class",
Usage: "the storage class for data written by current client",
},
&cli.IntFlag{
Name: "get-timeout",
Value: 60,
Expand Down
11 changes: 10 additions & 1 deletion cmd/format.go
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,10 @@ Details: https://juicefs.com/docs/community/quick_start_guide`,
Name: "session-token",
Usage: "session token for object storage",
},
&cli.StringFlag{
Name: "storage-class",
Usage: "the default storage class",
},
&cli.StringFlag{
Name: "encrypt-rsa-key",
Usage: "a path to RSA private key (PEM)",
Expand Down Expand Up @@ -219,7 +223,11 @@ func createStorage(format meta.Format) (object.ObjectStorage, error) {
return nil, err
}
blob = object.WithPrefix(blob, format.Name+"/")

if format.StorageClass != "" {
if os, ok := blob.(object.SupportStorageClass); ok {
os.SetStorageClass(format.StorageClass)
}
}
if format.EncryptKey != "" {
passphrase := os.Getenv("JFS_RSA_PASSPHRASE")
if passphrase == "" {
Expand Down Expand Up @@ -392,6 +400,7 @@ func format(c *cli.Context) error {
Name: name,
UUID: uuid.New().String(),
Storage: c.String("storage"),
StorageClass: c.String("storage-class"),
Bucket: c.String("bucket"),
AccessKey: c.String("access-key"),
SecretKey: c.String("secret-key"),
Expand Down
7 changes: 5 additions & 2 deletions cmd/mount.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ func updateFormat(c *cli.Context) func(*meta.Format) {
if c.IsSet("storage") {
format.Storage = c.String("storage")
}
if c.IsSet("storage-class") {
format.StorageClass = c.String("storage-class")
}
}
}

Expand Down Expand Up @@ -434,8 +437,8 @@ func NewReloadableStorage(format *meta.Format, cli meta.Meta, patch func(*meta.F
patch(new)
}
old := &holder.fmt
if new.Storage != old.Storage || new.Bucket != old.Bucket || new.AccessKey != old.AccessKey || new.SecretKey != old.SecretKey || new.SessionToken != old.SessionToken {
logger.Infof("found new configuration: storage=%s bucket=%s ak=%s", new.Storage, new.Bucket, new.AccessKey)
if new.Storage != old.Storage || new.Bucket != old.Bucket || new.AccessKey != old.AccessKey || new.SecretKey != old.SecretKey || new.SessionToken != old.SessionToken || new.StorageClass != old.StorageClass {
zhijian-pro marked this conversation as resolved.
Show resolved Hide resolved
logger.Infof("found new configuration: storage=%s bucket=%s ak=%s storageClass=%s", new.Storage, new.Bucket, new.AccessKey, new.StorageClass)

newBlob, err := createStorage(*new)
if err != nil {
Expand Down
13 changes: 7 additions & 6 deletions cmd/object.go
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,13 @@ func (o *jObj) Size() int64 {
}
return o.fi.Size()
}
func (o *jObj) Mtime() time.Time { return o.fi.ModTime() }
func (o *jObj) IsDir() bool { return o.fi.IsDir() }
func (o *jObj) IsSymlink() bool { return o.fi.IsSymlink() }
func (o *jObj) Owner() string { return utils.UserName(o.fi.Uid()) }
func (o *jObj) Group() string { return utils.GroupName(o.fi.Gid()) }
func (o *jObj) Mode() os.FileMode { return o.fi.Mode() }
func (o *jObj) Mtime() time.Time { return o.fi.ModTime() }
func (o *jObj) IsDir() bool { return o.fi.IsDir() }
func (o *jObj) IsSymlink() bool { return o.fi.IsSymlink() }
func (o *jObj) Owner() string { return utils.UserName(o.fi.Uid()) }
func (o *jObj) Group() string { return utils.GroupName(o.fi.Gid()) }
func (o *jObj) Mode() os.FileMode { return o.fi.Mode() }
func (o *jObj) StorageClass() string { return "" }

func (j *juiceFS) Head(key string) (object.Object, error) {
fi, eno := j.jfs.Stat(ctx, j.path(key))
Expand Down
7 changes: 7 additions & 0 deletions cmd/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ $ juicefs sync --include='a1/b1' --exclude='a*' --include='b2' --exclude='b?' s3
Details: https://juicefs.com/docs/community/administration/sync
Supported storage systems: https://juicefs.com/docs/community/how_to_setup_object_storage#supported-object-storage`,
Flags: []cli.Flag{
&cli.StringFlag{
Name: "storage-class",
Usage: "the storage class for destination",
},
&cli.StringFlag{
Name: "start",
Aliases: []string{"s"},
Expand Down Expand Up @@ -349,5 +353,8 @@ func doSync(c *cli.Context) error {
if err != nil {
return err
}
if os, ok := dst.(object.SupportStorageClass); ok {
os.SetStorageClass(config.StorageClass)
}
return sync.Sync(src, dst, config)
}
7 changes: 3 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
cloud.google.com/go/compute v1.5.0
cloud.google.com/go/storage v1.21.0
github.com/Arvintian/scs-go-sdk v1.1.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0
github.com/DataDog/zstd v1.5.0
github.com/IBM/ibm-cos-sdk-go v1.8.0
github.com/agiledragon/gomonkey/v2 v2.6.0
Expand Down Expand Up @@ -80,8 +80,8 @@ require (
cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/iam v0.1.1 // indirect
git.apache.org/thrift.git v0.13.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 // indirect
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
Expand Down Expand Up @@ -206,7 +206,6 @@ require (
github.com/smartystreets/assertions v1.2.0 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stathat/consistent v1.0.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/tidwall/gjson v1.9.3 // indirect
github.com/tidwall/match v1.1.1 // indirect
Expand Down
25 changes: 13 additions & 12 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,15 @@ gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0p
github.com/Arvintian/scs-go-sdk v1.1.0 h1:vqVOfoMD6XSr7eG1a2M9oSiQwhDZYKKdH2rrZRPx6So=
github.com/Arvintian/scs-go-sdk v1.1.0/go.mod h1:DMIkwn27iuTIo9o7INj3L/bcA7bW6QwljWC3ZpxjkXw=
github.com/Azure/azure-sdk-for-go v32.4.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go v32.6.0+incompatible h1:PgaVceWF5idtJajyt1rzq1cep6eRPJ8+8hs4GnNzTo0=
github.com/Azure/azure-sdk-for-go v32.6.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.0 h1:D7l5jspkc4kwBYRWoZE4DQnu6LVpLwDsMZjBKS4wZLQ=
github.com/Azure/azure-sdk-for-go/sdk/azcore v0.23.0/go.mod h1:w5pDIZuawUmY3Bj4tVx3Xb8KS96ToB0j315w9rqpAg0=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.14.0 h1:NVS/4LOQfkBpk+B1VopIzv1ptmYeEskA8w/3K/w7vjo=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2 h1:Px2KVERcYEg2Lv25AqC2hVr0xUWaq94wuEObLIkYzmA=
github.com/Azure/azure-sdk-for-go/sdk/internal v0.9.2/go.mod h1:CdSJQNNzZhCkwDaV27XV1w48ZBPtxe7mlrZAsPNxD5g=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 h1:0nJeKDmB7a1a8RDMjTltahlPsaNlWjq/LpkZleSwINk=
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0/go.mod h1:mbwxKc/fW+IkF0GG591MuXw0KuEQBDkeRoZ9vmVJPxg=
github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.1.0/go.mod h1:AKyIcETwSUFxIcs/Wnq/C+kwCtlEYGUVd7FPNb2slmg=
github.com/Azure/go-autorest/autorest v0.5.0/go.mod h1:9HLKlQjVBH6U3oDfsXOeVc56THsLPw1L03yban4xThw=
Expand All @@ -100,6 +102,7 @@ github.com/Azure/go-autorest/tracing v0.1.0/go.mod h1:ROEEAFwXycQw7Sn3DXNtEedEvd
github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
github.com/AzureAD/microsoft-authentication-library-for-go v0.4.0 h1:WVsrXCnHlDDX8ls+tootqRE87/hL9S/g4ewig9RsD/c=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
Expand Down Expand Up @@ -259,9 +262,8 @@ github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQ
github.com/djherbis/atime v1.0.0 h1:ySLvBAM0EvOGaX7TI4dAM5lWj+RdJUCKtGSEHN8SGBg=
github.com/djherbis/atime v1.0.0/go.mod h1:5W+KBIuTwVGcqjIfaTwt+KSYX1o6uep8dtevevQP/f8=
github.com/dnaeon/go-vcr v0.0.0-20180814043457-aafff18a5cc2/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko=
github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI=
github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/dnsimple/dnsimple-go v0.30.0/go.mod h1:O5TJ0/U6r7AfT8niYNlmohpLbCSG+c71tQlGr9SeGrg=
github.com/dnstap/golang-dnstap v0.0.0-20170829151710-2cf77a2b5e11/go.mod h1:s1PfVYYVmTMgCSPtho4LKBDecEHJWtiVDPNv78Z985U=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
Expand Down Expand Up @@ -645,6 +647,7 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/ks3sdklib/aws-sdk-go v1.2.0 h1:Hhe7Ku2gs/TykWy4hoSVTqlLu2p+AApeatKHysgRgVM=
github.com/ks3sdklib/aws-sdk-go v1.2.0/go.mod h1:DVzr6V4XzDjdy+H+1ptuIDIy1MQgI+28SvUpOkJXJD8=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA=
github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w=
github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY=
Expand Down Expand Up @@ -793,6 +796,7 @@ github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172 h1:FYgKV9znRQmzVrr
github.com/pingcap/kvproto v0.0.0-20221026112947-f8d61344b172/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 h1:URLoJ61DmmY++Sa/yyPEQHG2s/ZBeV1FbIswHEMrdoY=
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
Expand Down Expand Up @@ -920,9 +924,8 @@ github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4Cc
github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
Expand Down Expand Up @@ -1157,8 +1160,6 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
Expand Down
1 change: 1 addition & 0 deletions pkg/meta/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type Format struct {
Name string
UUID string
Storage string
StorageClass string `json:",omitempty"`
Bucket string
AccessKey string `json:",omitempty"`
SecretKey string `json:",omitempty"`
Expand Down
61 changes: 50 additions & 11 deletions pkg/object/azure.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
type wasb struct {
DefaultObjectStorage
container *azblob.ContainerClient
sc string
cName string
marker string
}
Expand All @@ -50,7 +51,11 @@ func (b *wasb) Create() error {
}

func (b *wasb) Head(key string) (Object, error) {
properties, err := b.container.NewBlobClient(key).GetProperties(ctx, &azblob.GetBlobPropertiesOptions{})
client, err := b.container.NewBlobClient(key)
if err != nil {
return nil, err
}
properties, err := client.GetProperties(ctx, &azblob.BlobGetPropertiesOptions{})
if err != nil {
if strings.Contains(err.Error(), string(azblob.StorageErrorCodeBlobNotFound)) {
err = os.ErrNotExist
Expand All @@ -63,30 +68,59 @@ func (b *wasb) Head(key string) (Object, error) {
*properties.ContentLength,
*properties.LastModified,
strings.HasSuffix(key, "/"),
*properties.AccessTier,
}, nil
}

func (b *wasb) Get(key string, off, limit int64) (io.ReadCloser, error) {
download, err := b.container.NewBlockBlobClient(key).Download(ctx, &azblob.DownloadBlobOptions{Offset: &off, Count: &limit})
client, err := b.container.NewBlockBlobClient(key)
if err != nil {
return nil, err
}
return download.BlobDownloadResponse.RawResponse.Body, err
download, err := client.Download(ctx, &azblob.BlobDownloadOptions{Offset: &off, Count: &limit})
if err != nil {
return nil, err
}
return download.RawResponse.Body, err
}

func (b *wasb) Put(key string, data io.Reader) error {
_, err := b.container.NewBlockBlobClient(key).UploadStreamToBlockBlob(ctx, data, azblob.UploadStreamToBlockBlobOptions{})
client, err := b.container.NewBlockBlobClient(key)
if err != nil {
return err
}
options := azblob.UploadStreamOptions{}
if b.sc != "" {
options.AccessTier = azblob.AccessTier(b.sc).ToPtr()
}
_, err = client.UploadStream(ctx, data, options)
return err
}

func (b *wasb) Copy(dst, src string) error {
_, err := b.container.NewBlockBlobClient(dst).CopyFromURL(ctx, b.container.NewBlockBlobClient(src).URL(),
&azblob.CopyBlockBlobFromURLOptions{})
dstCli, err := b.container.NewBlockBlobClient(dst)
if err != nil {
return err
}
srcCli, err := b.container.NewBlockBlobClient(src)
if err != nil {
return err
}
options := &azblob.BlockBlobCopyFromURLOptions{}
if b.sc != "" {
options.Tier = azblob.AccessTier(b.sc).ToPtr()
}
_, err = dstCli.CopyFromURL(ctx, srcCli.URL(),
options)
return err
}

func (b *wasb) Delete(key string) error {
_, err := b.container.NewBlockBlobClient(key).Delete(ctx, &azblob.DeleteBlobOptions{})
client, err := b.container.NewBlockBlobClient(key)
if err != nil {
return err
}
_, err = client.Delete(ctx, &azblob.BlobDeleteOptions{})
if err != nil && strings.Contains(err.Error(), string(azblob.StorageErrorCodeBlobNotFound)) {
err = nil
}
Expand All @@ -107,7 +141,7 @@ func (b *wasb) List(prefix, marker, delimiter string, limit int64) ([]Object, er
}

limit32 := int32(limit)
pager := b.container.ListBlobsFlat(&azblob.ContainerListBlobFlatSegmentOptions{Prefix: &prefix, Marker: &marker, Maxresults: &(limit32)})
pager := b.container.ListBlobsFlat(&azblob.ContainerListBlobsFlatOptions{Prefix: &prefix, Marker: &marker, MaxResults: &(limit32)})
if pager.Err() != nil {
return nil, pager.Err()
}
Expand All @@ -129,11 +163,16 @@ func (b *wasb) List(prefix, marker, delimiter string, limit int64) ([]Object, er
*blob.Properties.ContentLength,
*mtime,
strings.HasSuffix(*blob.Name, "/"),
string(*blob.Properties.AccessTier),
}
}
return objs, nil
}

func (b *wasb) SetStorageClass(sc string) {
b.sc = sc
}

func autoWasbEndpoint(containerName, accountName, scheme string, credential *azblob.SharedKeyCredential) (string, error) {
baseURLs := []string{"blob.core.windows.net", "blob.core.chinacloudapi.cn"}
endpoint := ""
Expand Down Expand Up @@ -172,11 +211,11 @@ func newWasb(endpoint, accountName, accountKey, token string) (ObjectStorage, er
containerName := hostParts[0]
// Connection string support: DefaultEndpointsProtocol=[http|https];AccountName=***;AccountKey=***;EndpointSuffix=[core.windows.net|core.chinacloudapi.cn]
if connString := os.Getenv("AZURE_STORAGE_CONNECTION_STRING"); connString != "" {
var client azblob.ContainerClient
var client *azblob.ContainerClient
if client, err = azblob.NewContainerClientFromConnectionString(connString, containerName, nil); err != nil {
return nil, err
}
return &wasb{container: &client, cName: containerName}, nil
return &wasb{container: client, cName: containerName}, nil
}

credential, err := azblob.NewSharedKeyCredential(accountName, accountKey)
Expand All @@ -198,7 +237,7 @@ func newWasb(endpoint, accountName, accountKey, token string) (ObjectStorage, er
return nil, err
}

return &wasb{container: &client, cName: containerName}, nil
return &wasb{container: client, cName: containerName}, nil
}

func init() {
Expand Down
2 changes: 2 additions & 0 deletions pkg/object/b2.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ func (c *b2client) Head(key string) (Object, error) {
f.ContentLength,
time.Unix(f.UploadTimestamp/1000, 0),
strings.HasSuffix(f.Name, "/"),
"",
}, nil
}

Expand Down Expand Up @@ -142,6 +143,7 @@ func (c *b2client) List(prefix, marker, delimiter string, limit int64) ([]Object
f.ContentLength,
time.Unix(f.UploadTimestamp/1000, 0),
strings.HasSuffix(f.Name, "/"),
"",
}
}
c.nextMarker = resp.NextFileName
Expand Down
Loading