Add a storage layer for attachments

.drone.yml

@@ -151,6 +151,14 @@ services:
       discovery.type: single-node
     image: elasticsearch:7.5.0
 
+  - name: minio
+    image: minio/minio:RELEASE.2020-05-16T01-33-21Z
+    commands:
+    - minio server /data
+    environment:
+      MINIO_ACCESS_KEY: 123456
+      MINIO_SECRET_KEY: 12345678
+
 steps:
   - name: fetch-tags
     pull: default

cmd/migrate_storage.go

@@ -0,0 +1,147 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+
+	"github.com/urfave/cli"
+)
+
+// CmdMigrateStorage represents the available migrate storage sub-command.
+var CmdMigrateStorage = cli.Command{
+	Name:        "migrate-storage",
+	Usage:       "Migrate the storage",
+	Description: "This is a command for migrating storage.",
+	Action:      runMigrateStorage,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "type, t",
+			Value: "",
+			Usage: "Kinds of files to migrate, currently only 'attachments' is supported",
+		},
+		cli.StringFlag{
+			Name:  "store, s",
+			Value: "local",
+			Usage: "New storage type, local or minio",
+		},
+		cli.StringFlag{
+			Name:  "path, p",
+			Value: "",
+			Usage: "New storage placement if store is local (leave blank for default)",
+		},
+		cli.StringFlag{
+			Name:  "minio-endpoint",
+			Value: "",
+			Usage: "Minio storage endpoint",
+		},
+		cli.StringFlag{
+			Name:  "minio-access-key-id",
+			Value: "",
+			Usage: "Minio storage accessKeyID",
+		},
+		cli.StringFlag{
+			Name:  "minio-secret-access-key",
+			Value: "",
+			Usage: "Minio storage secretAccessKey",
+		},
+		cli.StringFlag{
+			Name:  "minio-bucket",
+			Value: "",
+			Usage: "Minio storage bucket",
+		},
+		cli.StringFlag{
+			Name:  "minio-location",
+			Value: "",
+			Usage: "Minio storage location to create bucket",
+		},
+		cli.StringFlag{
+			Name:  "minio-base-path",
+			Value: "",
+			Usage: "Minio storage basepath on the bucket",
+		},
+		cli.BoolFlag{
+			Name:  "minio-use-ssl",
+			Usage: "Enable SSL for minio",
+		},
+	},
+}
+
+func migrateAttachments(dstStorage storage.ObjectStorage) error {
+	return models.IterateAttachment(func(attach *models.Attachment) error {
+		_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
+		return err
+	})
+}
+
+func runMigrateStorage(ctx *cli.Context) error {
+	if err := initDB(); err != nil {
+		return err
+	}
+
+	log.Trace("AppPath: %s", setting.AppPath)
+	log.Trace("AppWorkPath: %s", setting.AppWorkPath)
+	log.Trace("Custom path: %s", setting.CustomPath)
+	log.Trace("Log path: %s", setting.LogRootPath)
+	setting.InitDBConfig()
+
+	if err := models.NewEngine(context.Background(), migrations.Migrate); err != nil {
+		log.Fatal("Failed to initialize ORM engine: %v", err)
+		return err
+	}
+
+	if err := storage.Init(); err != nil {
+		return err
+	}
+
+	tp := ctx.String("type")
+	switch tp {
+	case "attachments":
+		var dstStorage storage.ObjectStorage
+		var err error
+		switch ctx.String("store") {
+		case "local":
+			p := ctx.String("path")
+			if p == "" {
+				log.Fatal("Path must be given when store is loal")
+				return nil
+			}
+			dstStorage, err = storage.NewLocalStorage(p)
+		case "minio":
+			dstStorage, err = storage.NewMinioStorage(
+				context.Background(),
+				ctx.String("minio-endpoint"),
+				ctx.String("minio-access-key-id"),
+				ctx.String("minio-secret-access-key"),
+				ctx.String("minio-bucket"),
+				ctx.String("minio-location"),
+				ctx.String("minio-base-path"),
+				ctx.Bool("minio-use-ssl"),
+			)
+		default:
+			return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
+		}
+
+		if err != nil {
+			return err
+		}
+		if err := migrateAttachments(dstStorage); err != nil {
+			return err
+		}
+
+		log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
+
+		return nil
+	}
+
+	return nil
+}

custom/conf/app.example.ini

@@ -749,14 +749,35 @@ ENABLE_FEDERATED_AVATAR = false
 [attachment]
 ; Whether attachments are enabled. Defaults to `true`
 ENABLED = true
-; Path for attachments. Defaults to `data/attachments`
-PATH = data/attachments
+
 ; One or more allowed types, e.g. "image/jpeg|image/png". Use "*/*" for all types.
 ALLOWED_TYPES = image/jpeg|image/png|application/zip|application/gzip
 ; Max size of each file. Defaults to 4MB
 MAX_SIZE = 4
 ; Max number of files per upload. Defaults to 5
 MAX_FILES = 5
+; Storage type for attachments, `local` for local disk or `minio` for s3 compatible
+; object storage service, default is `local`.
+STORE_TYPE = local
+; Allows the storage driver to redirect to authenticated URLs to serve files directly
+; Currently, only `minio` is supported.
+SERVE_DIRECT = false
+; Path for attachments. Defaults to `data/attachments` only available when STORE_TYPE is `local`
+PATH = data/attachments
+; Minio endpoint to connect only available when STORE_TYPE is `minio`
+MINIO_ENDPOINT = localhost:9000
+; Minio accessKeyID to connect only available when STORE_TYPE is `minio`
+MINIO_ACCESS_KEY_ID =
+; Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
+MINIO_SECRET_ACCESS_KEY =
+; Minio bucket to store the attachments only available when STORE_TYPE is `minio`
+MINIO_BUCKET = gitea
+; Minio location to create bucket only available when STORE_TYPE is `minio`
+MINIO_LOCATION = us-east-1
+; Minio base path on the bucket only available when STORE_TYPE is `minio`
+MINIO_BASE_PATH = attachments/
+; Minio enabled ssl only available when STORE_TYPE is `minio`
+MINIO_USE_SSL = false
 
 [time]
 ; Specifies the format for fully outputted dates. Defaults to RFC1123

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -470,11 +470,20 @@ set name for unique queues. Individual queues will default to
 ## Attachment (`attachment`)
 
 - `ENABLED`: **true**: Enable this to allow uploading attachments.
-- `PATH`: **data/attachments**: Path to store attachments.
 - `ALLOWED_TYPES`: **see app.example.ini**: Allowed MIME types, e.g. `image/jpeg|image/png`.
    Use `*/*` for all types.
 - `MAX_SIZE`: **4**: Maximum size (MB).
 - `MAX_FILES`: **5**: Maximum number of attachments that can be uploaded at once.
+- `STORE_TYPE`: **local**: Storage type for attachments, `local` for local disk or `minio` for s3 compatible object storage service, default is `local`.
+- `SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
+- `PATH`: **data/attachments**: Path to store attachments only available when STORE_TYPE is `local`
+- `MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when STORE_TYPE is `minio`
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when STORE_TYPE is `minio`
+- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when STORE_TYPE is `minio`
+- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments only available when STORE_TYPE is `minio`
+- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when STORE_TYPE is `minio`
+- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket only available when STORE_TYPE is `minio`
+- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when STORE_TYPE is `minio`
 
 ## Log (`log`)
 

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -180,10 +180,18 @@ menu:
 ## Attachment (`attachment`)
 
 - `ENABLED`: 是否允许用户上传附件。
-- `PATH`: 附件存储路径
 - `ALLOWED_TYPES`: 允许上传的附件类型。比如：`image/jpeg|image/png`，用 `*/*` 表示允许任何类型。
 - `MAX_SIZE`: 附件最大限制，单位 MB，比如： `4`。
 - `MAX_FILES`: 一次最多上传的附件数量，比如： `5`。
+- `STORE_TYPE`: **local**: 附件存储类型，`local` 将存储到本地文件夹， `minio` 将存储到 s3 兼容的对象存储服务中。
+- `PATH`: **data/attachments**: 附件存储路径，仅当 `STORE_TYPE` 为 `local` 时有效。
+- `MINIO_ENDPOINT`: **localhost:9000**: Minio 终端，仅当 `STORE_TYPE` 是 `minio` 时有效。
+- `MINIO_ACCESS_KEY_ID`: Minio accessKeyID ，仅当 `STORE_TYPE` 是 `minio` 时有效。
+- `MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey，仅当 `STORE_TYPE` 是 `minio` 时有效。
+- `MINIO_BUCKET`: **gitea**: Minio bucket to store the attachments，仅当 `STORE_TYPE` 是 `minio` 时有效。
+- `MINIO_LOCATION`: **us-east-1**: Minio location to create bucket，仅当 `STORE_TYPE` 是 `minio` 时有效。
+- `MINIO_BASE_PATH`: **attachments/**: Minio base path on the bucket，仅当 `STORE_TYPE` 是 `minio` 时有效。
+- `MINIO_USE_SSL`: **false**: Minio enabled ssl，仅当 `STORE_TYPE` 是 `minio` 时有效。
 
 关于 `ALLOWED_TYPES`， 在 (*)unix 系统中可以使用`file -I <filename>` 来快速获得对应的 `MIME type`。
 

go.mod

@@ -74,6 +74,7 @@ require (
 	github.com/mgechev/revive v1.0.2
 	github.com/mholt/archiver/v3 v3.3.0
 	github.com/microcosm-cc/bluemonday v1.0.3-0.20191119130333-0a75d7616912
+	github.com/minio/minio-go/v7 v7.0.4
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/msteinert/pam v0.0.0-20151204160544-02ccfbfaf0cc
 	github.com/nfnt/resize v0.0.0-20160724205520-891127d8d1b5
@@ -104,17 +105,17 @@ require (
 	github.com/yuin/goldmark-highlighting v0.0.0-20200307114337-60d527fdb691
 	github.com/yuin/goldmark-meta v0.0.0-20191126180153-f0638e958b60
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de
-	golang.org/x/net v0.0.0-20200625001655-4c5254603344
+	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1
-	golang.org/x/text v0.3.2
+	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
+	golang.org/x/text v0.3.3
 	golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1 // indirect
 	golang.org/x/tools v0.0.0-20200814230902-9882f1d1823d
 	google.golang.org/appengine v1.6.5 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect
 	gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 // indirect
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
-	gopkg.in/ini.v1 v1.52.0
+	gopkg.in/ini.v1 v1.57.0
 	gopkg.in/ldap.v3 v3.0.2
 	gopkg.in/yaml.v2 v2.3.0
 	mvdan.cc/xurls/v2 v2.1.0