From 42243b1517d32e4ee4fa7bc6cad69b5e9bb2fdfa Mon Sep 17 00:00:00 2001 From: Jealous Date: Wed, 25 Dec 2024 21:23:58 +0800 Subject: [PATCH] feat(thunder): add offline download tool (#7673) * feat(thunder): add offline download tool * fix(thunder): improve error handling and parse file size in status response --------- Co-authored-by: Andy Hsu --- drivers/thunder/driver.go | 61 +++++++++ drivers/thunder/types.go | 47 +++++++ drivers/thunder/util.go | 1 + internal/offline_download/all.go | 1 + internal/offline_download/thunder/thunder.go | 126 +++++++++++++++++++ internal/offline_download/thunder/util.go | 42 +++++++ internal/offline_download/tool/add.go | 4 + internal/offline_download/tool/download.go | 6 + 8 files changed, 288 insertions(+) create mode 100644 internal/offline_download/thunder/thunder.go create mode 100644 internal/offline_download/thunder/util.go diff --git a/drivers/thunder/driver.go b/drivers/thunder/driver.go index 9ba5dd825f7..8403f2617a6 100644 --- a/drivers/thunder/driver.go +++ b/drivers/thunder/driver.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "strconv" "strings" "github.com/alist-org/alist/v3/drivers/base" @@ -522,3 +523,63 @@ func (xc *XunLeiCommon) IsLogin() bool { _, err := xc.Request(XLUSER_API_URL+"/user/me", http.MethodGet, nil, nil) return err == nil } + +// 离线下载文件 +func (xc *XunLeiCommon) OfflineDownload(ctx context.Context, fileUrl string, parentDir model.Obj, fileName string) (*OfflineTask, error) { + var resp OfflineDownloadResp + _, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) { + r.SetContext(ctx) + r.SetBody(&base.Json{ + "kind": FILE, + "name": fileName, + "parent_id": parentDir.GetID(), + "upload_type": UPLOAD_TYPE_URL, + "url": base.Json{ + "url": fileUrl, + }, + }) + }, &resp) + + if err != nil { + return nil, err + } + + return &resp.Task, err +} + +/* +获取离线下载任务列表 +*/ +func (xc *XunLeiCommon) OfflineList(ctx context.Context, nextPageToken string) ([]OfflineTask, error) { + res := make([]OfflineTask, 0) + + var resp OfflineListResp + _, err := xc.Request(TASK_API_URL, http.MethodGet, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(map[string]string{ + "type": "offline", + "limit": "10000", + "page_token": nextPageToken, + }) + }, &resp) + + if err != nil { + return nil, fmt.Errorf("failed to get offline list: %w", err) + } + res = append(res, resp.Tasks...) + return res, nil +} + +func (xc *XunLeiCommon) DeleteOfflineTasks(ctx context.Context, taskIDs []string, deleteFiles bool) error { + _, err := xc.Request(TASK_API_URL, http.MethodDelete, func(req *resty.Request) { + req.SetContext(ctx). + SetQueryParams(map[string]string{ + "task_ids": strings.Join(taskIDs, ","), + "delete_files": strconv.FormatBool(deleteFiles), + }) + }, nil) + if err != nil { + return fmt.Errorf("failed to delete tasks %v: %w", taskIDs, err) + } + return nil +} diff --git a/drivers/thunder/types.go b/drivers/thunder/types.go index 7c223673448..b7355b2a6fa 100644 --- a/drivers/thunder/types.go +++ b/drivers/thunder/types.go @@ -204,3 +204,50 @@ type UploadTaskResponse struct { File Files `json:"file"` } + +// 添加离线下载响应 +type OfflineDownloadResp struct { + File *string `json:"file"` + Task OfflineTask `json:"task"` + UploadType string `json:"upload_type"` + URL struct { + Kind string `json:"kind"` + } `json:"url"` +} + +// 离线下载列表 +type OfflineListResp struct { + ExpiresIn int64 `json:"expires_in"` + NextPageToken string `json:"next_page_token"` + Tasks []OfflineTask `json:"tasks"` +} + +// offlineTask +type OfflineTask struct { + Callback string `json:"callback"` + CreatedTime string `json:"created_time"` + FileID string `json:"file_id"` + FileName string `json:"file_name"` + FileSize string `json:"file_size"` + IconLink string `json:"icon_link"` + ID string `json:"id"` + Kind string `json:"kind"` + Message string `json:"message"` + Name string `json:"name"` + Params Params `json:"params"` + Phase string `json:"phase"` // PHASE_TYPE_RUNNING, PHASE_TYPE_ERROR, PHASE_TYPE_COMPLETE, PHASE_TYPE_PENDING + Progress int64 `json:"progress"` + Space string `json:"space"` + StatusSize int64 `json:"status_size"` + Statuses []string `json:"statuses"` + ThirdTaskID string `json:"third_task_id"` + Type string `json:"type"` + UpdatedTime string `json:"updated_time"` + UserID string `json:"user_id"` +} + +type Params struct { + FolderType string `json:"folder_type"` + PredictSpeed string `json:"predict_speed"` + PredictType string `json:"predict_type"` +} diff --git a/drivers/thunder/util.go b/drivers/thunder/util.go index 3ec8db58ffe..f509e6b2fbc 100644 --- a/drivers/thunder/util.go +++ b/drivers/thunder/util.go @@ -17,6 +17,7 @@ import ( const ( API_URL = "https://api-pan.xunlei.com/drive/v1" FILE_API_URL = API_URL + "/files" + TASK_API_URL = API_URL + "/tasks" XLUSER_API_URL = "https://xluser-ssl.xunlei.com/v1" ) diff --git a/internal/offline_download/all.go b/internal/offline_download/all.go index 6682155dec8..3d0c7c73a0b 100644 --- a/internal/offline_download/all.go +++ b/internal/offline_download/all.go @@ -6,5 +6,6 @@ import ( _ "github.com/alist-org/alist/v3/internal/offline_download/http" _ "github.com/alist-org/alist/v3/internal/offline_download/pikpak" _ "github.com/alist-org/alist/v3/internal/offline_download/qbit" + _ "github.com/alist-org/alist/v3/internal/offline_download/thunder" _ "github.com/alist-org/alist/v3/internal/offline_download/transmission" ) diff --git a/internal/offline_download/thunder/thunder.go b/internal/offline_download/thunder/thunder.go new file mode 100644 index 00000000000..3ab8b00212b --- /dev/null +++ b/internal/offline_download/thunder/thunder.go @@ -0,0 +1,126 @@ +package thunder + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/offline_download/tool" + "github.com/alist-org/alist/v3/internal/op" +) + +type Thunder struct { + refreshTaskCache bool +} + +func (t *Thunder) Name() string { + return "thunder" +} + +func (t *Thunder) Items() []model.SettingItem { + return nil +} + +func (t *Thunder) Run(task *tool.DownloadTask) error { + return errs.NotSupport +} + +func (t *Thunder) Init() (string, error) { + t.refreshTaskCache = false + return "ok", nil +} + +func (t *Thunder) IsReady() bool { + return true +} + +func (t *Thunder) AddURL(args *tool.AddUrlArgs) (string, error) { + // 添加新任务刷新缓存 + t.refreshTaskCache = true + // args.TempDir 已经被修改为了 DstDirPath + storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir) + if err != nil { + return "", err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return "", fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + + ctx := context.Background() + parentDir, err := op.GetUnwrap(ctx, storage, actualPath) + if err != nil { + return "", err + } + + task, err := thunderDriver.OfflineDownload(ctx, args.Url, parentDir, "") + if err != nil { + return "", fmt.Errorf("failed to add offline download task: %w", err) + } + + return task.ID, nil +} + +func (t *Thunder) Remove(task *tool.DownloadTask) error { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + ctx := context.Background() + err = thunderDriver.DeleteOfflineTasks(ctx, []string{task.GID}, false) + if err != nil { + return err + } + return nil +} + +func (t *Thunder) Status(task *tool.DownloadTask) (*tool.Status, error) { + storage, _, err := op.GetStorageAndActualPath(task.DstDirPath) + if err != nil { + return nil, err + } + thunderDriver, ok := storage.(*thunder.Thunder) + if !ok { + return nil, fmt.Errorf("unsupported storage driver for offline download, only Thunder is supported") + } + tasks, err := t.GetTasks(thunderDriver) + if err != nil { + return nil, err + } + s := &tool.Status{ + Progress: 0, + NewGID: "", + Completed: false, + Status: "the task has been deleted", + Err: nil, + } + for _, t := range tasks { + if t.ID == task.GID { + s.Progress = float64(t.Progress) + s.Status = t.Message + s.Completed = (t.Phase == "PHASE_TYPE_COMPLETE") + s.TotalBytes, err = strconv.ParseInt(t.FileSize, 10, 64) + if err != nil { + s.TotalBytes = 0 + } + if t.Phase == "PHASE_TYPE_ERROR" { + s.Err = errors.New(t.Message) + } + return s, nil + } + } + s.Err = fmt.Errorf("the task has been deleted") + return s, nil +} + +func init() { + tool.Tools.Add(&Thunder{}) +} diff --git a/internal/offline_download/thunder/util.go b/internal/offline_download/thunder/util.go new file mode 100644 index 00000000000..ea400f321d0 --- /dev/null +++ b/internal/offline_download/thunder/util.go @@ -0,0 +1,42 @@ +package thunder + +import ( + "context" + "time" + + "github.com/Xhofe/go-cache" + "github.com/alist-org/alist/v3/drivers/thunder" + "github.com/alist-org/alist/v3/internal/op" + "github.com/alist-org/alist/v3/pkg/singleflight" +) + +var taskCache = cache.NewMemCache(cache.WithShards[[]thunder.OfflineTask](16)) +var taskG singleflight.Group[[]thunder.OfflineTask] + +func (t *Thunder) GetTasks(thunderDriver *thunder.Thunder) ([]thunder.OfflineTask, error) { + key := op.Key(thunderDriver, "/drive/v1/task") + if !t.refreshTaskCache { + if tasks, ok := taskCache.Get(key); ok { + return tasks, nil + } + } + t.refreshTaskCache = false + tasks, err, _ := taskG.Do(key, func() ([]thunder.OfflineTask, error) { + ctx := context.Background() + tasks, err := thunderDriver.OfflineList(ctx, "") + if err != nil { + return nil, err + } + // 添加缓存 10s + if len(tasks) > 0 { + taskCache.Set(key, tasks, cache.WithEx[[]thunder.OfflineTask](time.Second*10)) + } else { + taskCache.Del(key) + } + return tasks, nil + }) + if err != nil { + return nil, err + } + return tasks, nil +} diff --git a/internal/offline_download/tool/add.go b/internal/offline_download/tool/add.go index 42349e2e397..4158051a8f0 100644 --- a/internal/offline_download/tool/add.go +++ b/internal/offline_download/tool/add.go @@ -77,6 +77,10 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro tempDir = args.DstDirPath // 防止将下载好的文件删除 deletePolicy = DeleteNever + case "thunder": + tempDir = args.DstDirPath + // 防止将下载好的文件删除 + deletePolicy = DeleteNever } taskCreator, _ := ctx.Value("user").(*model.User) // taskCreator is nil when convert failed diff --git a/internal/offline_download/tool/download.go b/internal/offline_download/tool/download.go index a0f1a81b3b5..94bf7dbb660 100644 --- a/internal/offline_download/tool/download.go +++ b/internal/offline_download/tool/download.go @@ -83,6 +83,9 @@ outer: if t.tool.Name() == "pikpak" { return nil } + if t.tool.Name() == "thunder" { + return nil + } if t.tool.Name() == "115 Cloud" { // hack for 115 <-time.After(time.Second * 1) @@ -161,6 +164,9 @@ func (t *DownloadTask) Complete() error { if t.tool.Name() == "pikpak" { return nil } + if t.tool.Name() == "thunder" { + return nil + } if t.tool.Name() == "115 Cloud" { return nil }