From 817d63597ec96cc3994185007efee8b01088aafb Mon Sep 17 00:00:00 2001 From: Noah Hsu Date: Wed, 31 Aug 2022 20:46:19 +0800 Subject: [PATCH] feat: add aliyundrive driver --- CONTRIBUTING.md | 3 + drivers/aliyundrive/driver.go | 292 ++++++++++++++++++++++++++++++++++ drivers/aliyundrive/meta.go | 27 ++++ drivers/aliyundrive/types.go | 54 +++++++ drivers/aliyundrive/util.go | 133 ++++++++++++++++ drivers/all.go | 1 + pkg/cron/cron.go | 35 ++++ pkg/cron/cron_test.go | 15 ++ pkg/utils/hash.go | 19 +++ 9 files changed, 579 insertions(+) create mode 100644 drivers/aliyundrive/driver.go create mode 100644 drivers/aliyundrive/meta.go create mode 100644 drivers/aliyundrive/types.go create mode 100644 drivers/aliyundrive/util.go create mode 100644 pkg/cron/cron.go create mode 100644 pkg/cron/cron_test.go create mode 100644 pkg/utils/hash.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa6f9010e10..bc78475d514 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,9 @@ $ go run main.go $ pnpm dev ``` +## Add a new driver +Copy `drivers/template` folder and rename it, and follow the comments in it. + ## Create a commit Commit messages should be well formatted, and to make that "standardized". diff --git a/drivers/aliyundrive/driver.go b/drivers/aliyundrive/driver.go new file mode 100644 index 00000000000..3fa1ab138e4 --- /dev/null +++ b/drivers/aliyundrive/driver.go @@ -0,0 +1,292 @@ +package local + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "math" + "math/big" + "net/http" + "os" + "time" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/conf" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/cron" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +type AliDrive struct { + model.Storage + Addition + AccessToken string + cron *cron.Cron + DriveId string +} + +func (d *AliDrive) Config() driver.Config { + return config +} + +func (d *AliDrive) GetAddition() driver.Additional { + return d.Addition +} + +func (d *AliDrive) Init(ctx context.Context, storage model.Storage) error { + d.Storage = storage + err := utils.Json.UnmarshalFromString(d.Storage.Addition, &d.Addition) + if err != nil { + return errors.Wrap(err, "error while unmarshal addition") + } + // TODO login / refresh token + //operations.MustSaveDriverStorage(d) + err = d.refreshToken() + if err != nil { + return err + } + // get driver id + res, err, _ := d.request("https://api.aliyundrive.com/v2/user/get", http.MethodPost, nil, nil) + if err != nil { + return err + } + d.DriveId = utils.Json.Get(res, "default_drive_id").ToString() + d.cron = cron.NewCron(time.Hour * 2) + d.cron.Do(func() { + err := d.refreshToken() + if err != nil { + log.Errorf("%+v", err) + } + }) + return err +} + +func (d *AliDrive) Drop(ctx context.Context) error { + if d.cron != nil { + d.cron.Stop() + } + return nil +} + +func (d *AliDrive) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + files, err := d.getFiles(dir.GetID()) + if err != nil { + return nil, err + } + objs := make([]model.Obj, len(files)) + for i := 0; i < len(files); i++ { + objs[i] = fileToObj(files[i]) + } + return objs, nil +} + +//func (d *AliDrive) Get(ctx context.Context, path string) (model.Obj, error) { +// // TODO this is optional +// return nil, errs.NotImplement +//} + +func (d *AliDrive) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + data := base.Json{ + "drive_id": d.DriveId, + "file_id": file.GetID(), + "expire_sec": 14400, + } + res, err, _ := d.request("https://api.aliyundrive.com/v2/file/get_download_url", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, nil) + if err != nil { + return nil, err + } + return &model.Link{ + Header: http.Header{ + "Referer": []string{"https://www.aliyundrive.com/"}, + }, + URL: utils.Json.Get(res, "url").ToString(), + }, nil +} + +func (d *AliDrive) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + _, err, _ := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "check_name_mode": "refuse", + "drive_id": d.DriveId, + "name": dirName, + "parent_file_id": parentDir.GetID(), + "type": "folder", + }) + }, nil) + return err +} + +func (d *AliDrive) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + err := d.batch(srcObj.GetID(), dstDir.GetID(), "/file/move") + return err +} + +func (d *AliDrive) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + _, err, _ := d.request("https://api.aliyundrive.com/v3/file/update", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "check_name_mode": "refuse", + "drive_id": d.DriveId, + "file_id": srcObj.GetID(), + "name": newName, + }) + }, nil) + return err +} + +func (d *AliDrive) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + err := d.batch(srcObj.GetID(), dstDir.GetID(), "/file/copy") + return err +} + +func (d *AliDrive) Remove(ctx context.Context, obj model.Obj) error { + _, err, _ := d.request("https://api.aliyundrive.com/v2/recyclebin/trash", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "drive_id": d.DriveId, + "file_id": obj.GetID(), + }) + }, nil) + return err +} + +func (d *AliDrive) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { + file := model.FileStream{ + Obj: stream, + ReadCloser: stream, + Mimetype: stream.GetMimetype(), + } + const DEFAULT int64 = 10485760 + var count = int(math.Ceil(float64(stream.GetSize()) / float64(DEFAULT))) + + partInfoList := make([]base.Json, 0, count) + for i := 1; i <= count; i++ { + partInfoList = append(partInfoList, base.Json{"part_number": i}) + } + reqBody := base.Json{ + "check_name_mode": "overwrite", + "drive_id": d.DriveId, + "name": file.GetName(), + "parent_file_id": dstDir.GetID(), + "part_info_list": partInfoList, + "size": file.GetSize(), + "type": "file", + } + + if d.RapidUpload { + buf := bytes.NewBuffer(make([]byte, 0, 1024)) + io.CopyN(buf, file, 1024) + reqBody["pre_hash"] = utils.GetSHA1Encode(buf.String()) + // 把头部拼接回去 + file.ReadCloser = struct { + io.Reader + io.Closer + }{ + Reader: io.MultiReader(buf, file), + Closer: file, + } + } else { + reqBody["content_hash_name"] = "none" + reqBody["proof_version"] = "v1" + } + + var resp UploadResp + _, err, e := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + req.SetBody(reqBody) + }, &resp) + + if err != nil && e.Code != "PreHashMatched" { + return err + } + + if d.RapidUpload && e.Code == "PreHashMatched" { + tempFile, err := os.CreateTemp(conf.Conf.TempDir, "file-*") + if err != nil { + return err + } + defer func() { + _ = tempFile.Close() + _ = os.Remove(tempFile.Name()) + }() + delete(reqBody, "pre_hash") + h := sha1.New() + if _, err = io.Copy(io.MultiWriter(tempFile, h), file); err != nil { + return err + } + reqBody["content_hash"] = hex.EncodeToString(h.Sum(nil)) + reqBody["content_hash_name"] = "sha1" + reqBody["proof_version"] = "v1" + + /* + js 隐性转换太坑不知道有没有bug + var n = e.access_token， + r = new BigNumber('0x'.concat(md5(n).slice(0, 16)))， + i = new BigNumber(t.file.size)， + o = i ? r.mod(i) : new gt.BigNumber(0); + (t.file.slice(o.toNumber(), Math.min(o.plus(8).toNumber(), t.file.size))) + */ + buf := make([]byte, 8) + r, _ := new(big.Int).SetString(utils.GetMD5Encode(d.AccessToken)[:16], 16) + i := new(big.Int).SetInt64(file.GetSize()) + o := r.Mod(r, i) + n, _ := io.NewSectionReader(tempFile, o.Int64(), 8).Read(buf[:8]) + reqBody["proof_code"] = base64.StdEncoding.EncodeToString(buf[:n]) + + _, err, e := d.request("https://api.aliyundrive.com/adrive/v2/file/createWithFolders", http.MethodPost, func(req *resty.Request) { + req.SetBody(reqBody) + }, &resp) + if err != nil && e.Code != "PreHashMatched" { + return err + } + if resp.RapidUpload { + return nil + } + // 秒传失败 + if _, err = tempFile.Seek(0, io.SeekStart); err != nil { + return err + } + file.ReadCloser = tempFile + } + + for _, partInfo := range resp.PartInfoList { + req, err := http.NewRequest("PUT", partInfo.UploadUrl, io.LimitReader(file, DEFAULT)) + if err != nil { + return err + } + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + res.Body.Close() + } + var resp2 base.Json + _, err, e = d.request("https://api.aliyundrive.com/v2/file/complete", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "drive_id": d.DriveId, + "file_id": resp.FileId, + "upload_id": resp.UploadId, + }) + }, &resp2) + if err != nil && e.Code != "PreHashMatched" { + return err + } + if resp2["file_id"] == resp.FileId { + return nil + } + return fmt.Errorf("%+v", resp2) +} + +func (d *AliDrive) Other(ctx context.Context, args model.OtherArgs) (interface{}, error) { + return nil, errs.NotSupport +} + +var _ driver.Driver = (*AliDrive)(nil) diff --git a/drivers/aliyundrive/meta.go b/drivers/aliyundrive/meta.go new file mode 100644 index 00000000000..059dd53c905 --- /dev/null +++ b/drivers/aliyundrive/meta.go @@ -0,0 +1,27 @@ +package local + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/operations" +) + +type Addition struct { + driver.RootFolderId + RefreshToken string `json:"refresh_token" required:"true"` + OrderBy string `json:"order_by" type:"select" options:"name,size,updated_at,created_at"` + OrderDirection string `json:"order_direction" type:"select" options:"ASC,DESC"` + RapidUpload bool `json:"rapid_upload"` +} + +var config = driver.Config{ + Name: "aliyundrive", + DefaultRoot: "root", +} + +func New() driver.Driver { + return &AliDrive{} +} + +func init() { + operations.RegisterDriver(config, New) +} diff --git a/drivers/aliyundrive/types.go b/drivers/aliyundrive/types.go new file mode 100644 index 00000000000..6864dceed46 --- /dev/null +++ b/drivers/aliyundrive/types.go @@ -0,0 +1,54 @@ +package local + +import ( + "time" + + "github.com/alist-org/alist/v3/internal/model" +) + +type RespErr struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type Files struct { + Items []File `json:"items"` + NextMarker string `json:"next_marker"` +} + +type File struct { + DriveId string `json:"drive_id"` + CreatedAt *time.Time `json:"created_at"` + FileExtension string `json:"file_extension"` + FileId string `json:"file_id"` + Type string `json:"type"` + Name string `json:"name"` + Category string `json:"category"` + ParentFileId string `json:"parent_file_id"` + UpdatedAt time.Time `json:"updated_at"` + Size int64 `json:"size"` + Thumbnail string `json:"thumbnail"` + Url string `json:"url"` +} + +func fileToObj(f File) model.ObjectThumbnail { + return model.ObjectThumbnail{ + Object: model.Object{ + ID: f.FileId, + Name: f.Name, + Size: f.Size, + Modified: f.UpdatedAt, + IsFolder: f.Type == "folder", + }, + } +} + +type UploadResp struct { + FileId string `json:"file_id"` + UploadId string `json:"upload_id"` + PartInfoList []struct { + UploadUrl string `json:"upload_url"` + } `json:"part_info_list"` + + RapidUpload bool `json:"rapid_upload"` +} diff --git a/drivers/aliyundrive/util.go b/drivers/aliyundrive/util.go new file mode 100644 index 00000000000..113c522e5bd --- /dev/null +++ b/drivers/aliyundrive/util.go @@ -0,0 +1,133 @@ +package local + +import ( + "fmt" + "net/http" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/operations" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "github.com/pkg/errors" +) + +// do others that not defined in Driver interface + +func (d *AliDrive) refreshToken() error { + url := "https://auth.aliyundrive.com/v2/account/token" + var resp base.TokenResp + var e RespErr + _, err := base.RestyClient.R(). + //ForceContentType("application/json"). + SetBody(base.Json{"refresh_token": d.RefreshToken, "grant_type": "refresh_token"}). + SetResult(&resp). + SetError(&e). + Post(url) + if err != nil { + return err + } + if e.Code != "" { + return fmt.Errorf("failed to refresh token: %s", e.Message) + } + d.RefreshToken, d.AccessToken = resp.RefreshToken, resp.AccessToken + operations.MustSaveDriverStorage(d) + return nil +} + +func (d *AliDrive) request(url, method string, callback func(*resty.Request), resp interface{}) ([]byte, error, RespErr) { + req := base.RestyClient.R() + req.SetHeader("Authorization", "Bearer\t"+d.AccessToken) + req.SetHeader("content-type", "application/json") + req.SetHeader("origin", "https://www.aliyundrive.com") + if callback != nil { + callback(req) + } else { + req.SetBody("{}") + } + if resp != nil { + req.SetResult(resp) + } + var e RespErr + req.SetError(&e) + res, err := req.Execute(method, url) + if err != nil { + return nil, errors.WithStack(err), e + } + if e.Code != "" { + if e.Code == "AccessTokenInvalid" { + err = d.refreshToken() + if err != nil { + return nil, err, e + } + return d.request(url, method, callback, resp) + } + return nil, errors.New(e.Message), e + } + return res.Body(), nil, e +} + +func (d *AliDrive) getFiles(fileId string) ([]File, error) { + marker := "first" + res := make([]File, 0) + for marker != "" { + if marker == "first" { + marker = "" + } + var resp Files + data := base.Json{ + "drive_id": d.DriveId, + "fields": "*", + "image_thumbnail_process": "image/resize,w_400/format,jpeg", + "image_url_process": "image/resize,w_1920/format,jpeg", + "limit": 200, + "marker": marker, + "order_by": d.OrderBy, + "order_direction": d.OrderDirection, + "parent_file_id": fileId, + "video_thumbnail_process": "video/snapshot,t_0,f_jpg,ar_auto,w_300", + "url_expire_sec": 14400, + } + _, err, _ := d.request("https://api.aliyundrive.com/v2/file/list", http.MethodPost, func(req *resty.Request) { + req.SetBody(data) + }, &resp) + + if err != nil { + return nil, err + } + marker = resp.NextMarker + res = append(res, resp.Items...) + } + return res, nil +} + +func (d *AliDrive) batch(srcId, dstId string, url string) error { + res, err, _ := d.request("https://api.aliyundrive.com/v3/batch", http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "requests": []base.Json{ + { + "headers": base.Json{ + "Content-Type": "application/json", + }, + "method": "POST", + "id": srcId, + "body": base.Json{ + "drive_id": d.DriveId, + "file_id": srcId, + "to_drive_id": d.DriveId, + "to_parent_file_id": dstId, + }, + "url": url, + }, + }, + "resource": "file", + }) + }, nil) + if err != nil { + return err + } + status := utils.Json.Get(res, "responses", 0, "status").ToInt() + if status < 400 && status >= 100 { + return nil + } + return errors.New(string(res)) +} diff --git a/drivers/all.go b/drivers/all.go index fdbe60b57e4..7fef46ca07c 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -1,6 +1,7 @@ package drivers import ( + _ "github.com/alist-org/alist/v3/drivers/aliyundrive" _ "github.com/alist-org/alist/v3/drivers/local" _ "github.com/alist-org/alist/v3/drivers/onedrive" _ "github.com/alist-org/alist/v3/drivers/pikpak" diff --git a/pkg/cron/cron.go b/pkg/cron/cron.go new file mode 100644 index 00000000000..77d94c74153 --- /dev/null +++ b/pkg/cron/cron.go @@ -0,0 +1,35 @@ +package cron + +import "time" + +type Cron struct { + d time.Duration + ch chan struct{} +} + +func NewCron(d time.Duration) *Cron { + return &Cron{ + d: d, + ch: make(chan struct{}), + } +} + +func (c *Cron) Do(f func()) { + go func() { + ticker := time.NewTicker(c.d) + defer ticker.Stop() + for { + select { + case <-ticker.C: + f() + case <-c.ch: + return + } + } + }() +} + +func (c *Cron) Stop() { + c.ch <- struct{}{} + close(c.ch) +} diff --git a/pkg/cron/cron_test.go b/pkg/cron/cron_test.go new file mode 100644 index 00000000000..6e718762e03 --- /dev/null +++ b/pkg/cron/cron_test.go @@ -0,0 +1,15 @@ +package cron + +import ( + "testing" + "time" +) + +func TestCron(t *testing.T) { + c := NewCron(time.Second) + c.Do(func() { + t.Logf("cron log") + }) + time.Sleep(time.Second * 5) + c.Stop() +} diff --git a/pkg/utils/hash.go b/pkg/utils/hash.go new file mode 100644 index 00000000000..1ebcedf3986 --- /dev/null +++ b/pkg/utils/hash.go @@ -0,0 +1,19 @@ +package utils + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" +) + +func GetSHA1Encode(data string) string { + h := sha1.New() + h.Write([]byte(data)) + return hex.EncodeToString(h.Sum(nil)) +} + +func GetMD5Encode(data string) string { + h := md5.New() + h.Write([]byte(data)) + return hex.EncodeToString(h.Sum(nil)) +}