Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: some cleanup work for oss api #746

Merged
merged 9 commits into from
Jul 30, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions cmd/layotto/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -277,12 +277,12 @@ func NewRuntimeGrpcServer(data json.RawMessage, opts ...grpc.ServerOption) (mgrp

// File
runtime.WithFileFactory(
file.NewFileFactory("aliOSS", aliyun_file.NewAliyunFile),
file.NewFileFactory("minioOSS", minio.NewMinioOss),
file.NewFileFactory("awsOSS", aws.NewAwsFile),
file.NewFileFactory("tencentCloudOSS", tencentcloud.NewTencentCloudOSS),
file.NewFileFactory("aliyun.oss", aliyun_file.NewAliyunFile),
file.NewFileFactory("minio", minio.NewMinioOss),
file.NewFileFactory("aws.s3", aws.NewAwsFile),
file.NewFileFactory("tencent.oss", tencentcloud.NewTencentCloudOSS),
file.NewFileFactory("local", local.NewLocalStore),
file.NewFileFactory("qiniuOSS", qiniu.NewQiniuOSS),
file.NewFileFactory("qiniu.oss", qiniu.NewQiniuOSS),
),
runtime.WithOssFactory(
oss.NewFactory("aws.oss", aws_oss.NewAwsOss),
Expand Down
10 changes: 5 additions & 5 deletions cmd/layotto_multiple_api/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -287,12 +287,12 @@ func NewRuntimeGrpcServer(data json.RawMessage, opts ...grpc.ServerOption) (mgrp

// File
runtime.WithFileFactory(
file.NewFileFactory("aliOSS", aliyun_file.NewAliyunFile),
file.NewFileFactory("minioOSS", minio.NewMinioOss),
file.NewFileFactory("awsOSS", aws_file.NewAwsFile),
file.NewFileFactory("tencentCloudOSS", tencentcloud.NewTencentCloudOSS),
file.NewFileFactory("aliyun.oss", aliyun_file.NewAliyunFile),
file.NewFileFactory("minio", minio.NewMinioOss),
file.NewFileFactory("aws.s3", aws_file.NewAwsFile),
file.NewFileFactory("tencent.oss", tencentcloud.NewTencentCloudOSS),
file.NewFileFactory("local", local.NewLocalStore),
file.NewFileFactory("qiniuOSS", qiniu.NewQiniuOSS),
file.NewFileFactory("qiniu.oss", qiniu.NewQiniuOSS),
),

//OSS
Expand Down
10 changes: 5 additions & 5 deletions cmd/layotto_without_xds/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -259,12 +259,12 @@ func NewRuntimeGrpcServer(data json.RawMessage, opts ...grpc.ServerOption) (mgrp

// File
runtime.WithFileFactory(
file.NewFileFactory("aliOSS", aliyun.NewAliyunFile),
file.NewFileFactory("minioOSS", minio.NewMinioOss),
file.NewFileFactory("awsOSS", aws_file.NewAwsFile),
file.NewFileFactory("tencentCloudOSS", tencentcloud.NewTencentCloudOSS),
file.NewFileFactory("aliyun.oss", aliyun.NewAliyunFile),
file.NewFileFactory("minio", minio.NewMinioOss),
file.NewFileFactory("aws.s3", aws_file.NewAwsFile),
file.NewFileFactory("tencent.oss", tencentcloud.NewTencentCloudOSS),
file.NewFileFactory("local", local.NewLocalStore),
file.NewFileFactory("qiniuOSS", qiniu.NewQiniuOSS),
file.NewFileFactory("qiniu.oss", qiniu.NewQiniuOSS),
),

// PubSub
Expand Down
18 changes: 9 additions & 9 deletions components/file/aws/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -100,11 +100,11 @@ func (a *AwsOss) Put(ctx context.Context, st *file.PutFileStu) error {
//var bodySize int64
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return fmt.Errorf("awsoss put file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("aws.s3 put file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return fmt.Errorf("awsoss put file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("aws.s3 put file[%s] fail,err: %s", st.FileName, err.Error())
}
client, err := a.selectClient()
if err != nil {
Expand All @@ -130,11 +130,11 @@ func (a *AwsOss) selectClient() (*s3.Client, error) {
func (a *AwsOss) Get(ctx context.Context, st *file.GetFileStu) (io.ReadCloser, error) {
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return nil, fmt.Errorf("awsoss get file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("aws.s3 get file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return nil, fmt.Errorf("awsoss get file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("aws.s3 get file[%s] fail,err: %s", st.FileName, err.Error())
}
input := &s3.GetObjectInput{
Bucket: &bucket,
Expand Down Expand Up @@ -186,11 +186,11 @@ func (a *AwsOss) List(ctx context.Context, st *file.ListRequest) (*file.ListResp
func (a *AwsOss) Del(ctx context.Context, st *file.DelRequest) error {
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return fmt.Errorf("awsoss put file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("aws.s3 put file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return fmt.Errorf("awsoss put file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("aws.s3 put file[%s] fail,err: %s", st.FileName, err.Error())
}
input := &s3.DeleteObjectInput{
Bucket: &bucket,
Expand All @@ -209,11 +209,11 @@ func (a *AwsOss) Del(ctx context.Context, st *file.DelRequest) error {
func (a *AwsOss) Stat(ctx context.Context, st *file.FileMetaRequest) (*file.FileMetaResp, error) {
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return nil, fmt.Errorf("awsoss stat file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("aws.s3 stat file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return nil, fmt.Errorf("awsoss stat file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("aws.s3 stat file[%s] fail,err: %s", st.FileName, err.Error())
}
input := &s3.HeadObjectInput{
Bucket: &bucket,
Expand All @@ -228,7 +228,7 @@ func (a *AwsOss) Stat(ctx context.Context, st *file.FileMetaRequest) (*file.File
if strings.Contains(err.Error(), "no such key") {
return nil, file.ErrNotExist
}
return nil, fmt.Errorf("awsoss stat file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("aws.s3 stat file[%s] fail,err: %s", st.FileName, err.Error())
}
resp := &file.FileMetaResp{}
resp.Size = out.ContentLength
Expand Down
10 changes: 5 additions & 5 deletions components/file/aws/file_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,11 @@ func TestAwsOss_Put(t *testing.T) {
FileName: "",
}
err = oss.Put(context.Background(), req)
assert.Equal(t, err.Error(), "awsoss put file[] fail,err: invalid fileName format")
assert.Equal(t, err.Error(), "aws.s3 put file[] fail,err: invalid fileName format")

req.FileName = "/a.txt"
err = oss.Put(context.Background(), req)
assert.Equal(t, err.Error(), "awsoss put file[/a.txt] fail,err: invalid fileName format")
assert.Equal(t, err.Error(), "aws.s3 put file[/a.txt] fail,err: invalid fileName format")
}

func TestAwsOss_Get(t *testing.T) {
Expand All @@ -91,17 +91,17 @@ func TestAwsOss_Get(t *testing.T) {
}
err = oss.Put(context.Background(), putReq)

assert.Equal(t, err.Error(), "awsoss put file[/a.txt] fail,err: invalid fileName format")
assert.Equal(t, err.Error(), "aws.s3 put file[/a.txt] fail,err: invalid fileName format")

req := &file.GetFileStu{
FileName: "",
}
_, err = oss.Get(context.Background(), req)
assert.Equal(t, err.Error(), "awsoss get file[] fail,err: invalid fileName format")
assert.Equal(t, err.Error(), "aws.s3 get file[] fail,err: invalid fileName format")

req.FileName = "/a.txt"
_, err = oss.Get(context.Background(), req)
assert.Equal(t, err.Error(), "awsoss get file[/a.txt] fail,err: invalid fileName format")
assert.Equal(t, err.Error(), "aws.s3 get file[/a.txt] fail,err: invalid fileName format")
}

type fun = func() (string, error)
Expand Down
22 changes: 11 additions & 11 deletions components/file/minio/oss.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,11 @@ func (m *MinioOss) Put(ctx context.Context, st *file.PutFileStu) error {
)
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return fmt.Errorf("minioOss put file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("minio put file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return fmt.Errorf("minioOss put file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("minio put file[%s] fail,err: %s", st.FileName, err.Error())
}
core, err := m.selectClient(st.Metadata)
if err != nil {
Expand All @@ -118,11 +118,11 @@ func (m *MinioOss) Put(ctx context.Context, st *file.PutFileStu) error {
func (m *MinioOss) Get(ctx context.Context, st *file.GetFileStu) (io.ReadCloser, error) {
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return nil, fmt.Errorf("minioOss get file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("minio get file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return nil, fmt.Errorf("minioOss get file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("minio get file[%s] fail,err: %s", st.FileName, err.Error())
}
core, err := m.selectClient(st.Metadata)
if err != nil {
Expand All @@ -139,7 +139,7 @@ func (m *MinioOss) List(ctx context.Context, st *file.ListRequest) (*file.ListRe
bucket, err := util.GetBucketName(st.DirectoryName)
marker := ""
if err != nil {
return nil, fmt.Errorf("MinioOss list bucket[%s] fail, err: %s", st.DirectoryName, err.Error())
return nil, fmt.Errorf("minio list bucket[%s] fail, err: %s", st.DirectoryName, err.Error())
}
prefix := util.GetFilePrefixName(st.DirectoryName)

Expand Down Expand Up @@ -168,31 +168,31 @@ func (m *MinioOss) List(ctx context.Context, st *file.ListRequest) (*file.ListRe
func (m *MinioOss) Del(ctx context.Context, st *file.DelRequest) error {
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return fmt.Errorf("minioOss del file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("minio del file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return fmt.Errorf("minioOss del file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("minio del file[%s] fail,err: %s", st.FileName, err.Error())
}
core, err := m.selectClient(st.Metadata)
if err != nil {
return fmt.Errorf("minioOss del file[%s] fail,err: %s", st.FileName, err.Error())
return fmt.Errorf("minio del file[%s] fail,err: %s", st.FileName, err.Error())
}
return core.Client.RemoveObject(ctx, bucket, key, minio.RemoveObjectOptions{})
}

func (m *MinioOss) Stat(ctx context.Context, st *file.FileMetaRequest) (*file.FileMetaResp, error) {
bucket, err := util.GetBucketName(st.FileName)
if err != nil {
return nil, fmt.Errorf("minioOss stat file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("minio stat file[%s] fail,err: %s", st.FileName, err.Error())
}
key, err := util.GetFileName(st.FileName)
if err != nil {
return nil, fmt.Errorf("minioOss stat file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("minio stat file[%s] fail,err: %s", st.FileName, err.Error())
}
core, err := m.selectClient(st.Metadata)
if err != nil {
return nil, fmt.Errorf("minioOss stat file[%s] fail,err: %s", st.FileName, err.Error())
return nil, fmt.Errorf("minio stat file[%s] fail,err: %s", st.FileName, err.Error())
}
info, err := core.Client.StatObject(ctx, bucket, key, minio.GetObjectOptions{})

Expand Down
2 changes: 1 addition & 1 deletion components/file/minio/oss_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ func TestMinioOss_Get(t *testing.T) {
}

_, err = oss.Get(context.TODO(), getReq)
assert.Equal(t, "minioOss get file[file] fail,err: invalid fileName format", err.Error())
assert.Equal(t, "minio get file[file] fail,err: invalid fileName format", err.Error())

// client not exist
getReq.FileName = "bucketName/file"
Expand Down
2 changes: 2 additions & 0 deletions components/oss/oss.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ const (
ServiceName = "oss"
)

// Oss is the interface for ObjectStorageService components.
// For more details about the fields explanation, please refer to the `.proto` file.
type Oss interface {
Init(context.Context, *Config) error
GetObject(context.Context, *GetObjectInput) (*GetObjectOutput, error)
Expand Down
2 changes: 1 addition & 1 deletion configs/config_file.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
},
"file": {
"file_demo": {
"type": "minioOSS",
"type": "minio",
"metadata": [
{
"endpoint": "127.0.0.1:9000",
Expand Down
2 changes: 1 addition & 1 deletion configs/config_file_qiniu_oss.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
},
"file": {
"file_demo": {
"type": "qiniuOSS",
"type": "qiniu.oss",
"metadata": [
{
"endpoint": "<require>",
Expand Down
2 changes: 1 addition & 1 deletion configs/config_file_tencentcloud_oss.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
},
"files": {
"file_demo": {
"type": "tencentCloudOSS",
"type": "tencent.oss",
"metadata": [
{
"endpoint": "<get endpoint from https://console.cloud.tencent.com/cos/bucket configuration management>",
Expand Down
2 changes: 1 addition & 1 deletion docker/layotto-minio/config_minio.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
},
"file": {
"file_demo": {
"type": "minioOSS",
"type": "minio",
"metadata": [
{
"endpoint": "minio:9000",
Expand Down
1 change: 1 addition & 0 deletions docs/_sidebar.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
- API reference
- [spec/proto/runtime/v1/runtime.proto](https://github.com/mosn/layotto/blob/main/docs/en/api_reference/runtime_v1.md)
- [spec/proto/runtime/v1/appcallback.proto](https://github.com/mosn/layotto/blob/main/docs/en/api_reference/appcallback_v1.md)
- [spec/proto/extension/v1/oss.proto](https://github.com/mosn/layotto/blob/main/docs/en/api_reference/oss_v1.md)
- SDK reference
- [java sdk](https://github.com/layotto/java-sdk)
- [.net sdk](https://github.com/layotto/dotnet-sdk)
Expand Down
Loading