feat: migrate an instance's storage between swift and s3

.github/workflows/docker-s3-test.yml

@@ -0,0 +1,61 @@
+name: Build S3-test Docker image
+
+on:
+  push:
+    branches:
+      - feat/s3-vfs-backend
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Image tag to publish under ghcr.io/<repo>"
+        required: false
+        default: "s3-test"
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compute image metadata
+        id: meta
+        run: |
+          repo_lc="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')"
+          tag="${{ github.event.inputs.tag }}"
+          if [ -z "$tag" ]; then
+            tag="s3-test"
+          fi
+          tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')"
+          sha_tag="${tag_lc}-${GITHUB_SHA::7}"
+          echo "image=ghcr.io/${repo_lc}" >> "$GITHUB_OUTPUT"
+          echo "tag=${tag_lc}" >> "$GITHUB_OUTPUT"
+          echo "sha_tag=${sha_tag}" >> "$GITHUB_OUTPUT"
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: scripts/docker/production/Dockerfile
+          push: true
+          tags: |
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }}
+            ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }}
+          build-args: |
+            VERSION_STRING=${{ steps.meta.outputs.tag }}-${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/go-tests.yml

@@ -64,4 +64,4 @@ jobs:
           cache: true
 
       - name: Run tests
-        run: go test -p 1 -timeout 5m ./...
+        run: go test -p 1 -timeout 10m ./...

client/instances.go

@@ -282,6 +282,63 @@ func (ac *AdminClient) ModifyInstance(opts *InstanceOptions) (*Instance, error)
 	return readInstance(res)
 }
 
+// MigrateStorageOptions contains the options for MigrateStorage. It mirrors
+// the fields of storagemigration.Options in the model, without importing
+// that package: the client stays free of any model/... dependency.
+type MigrateStorageOptions struct {
+	To          string
+	DryRun      bool
+	FlagOnly    bool
+	Force       bool
+	PurgeSource bool
+}
+
+// MigrateStorageReport is the report returned by MigrateStorage. It mirrors
+// the fields of storagemigration.Report in the model.
+type MigrateStorageReport struct {
+	Files        int   `json:"Files"`
+	Versions     int   `json:"Versions"`
+	Bytes        int64 `json:"Bytes"`
+	AvatarCopied bool  `json:"AvatarCopied"`
+}
+
+// MigrateStorage moves an instance's object-storage content (files,
+// versions, avatar) from its current backend to the target scheme.
+func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error) {
+	if !validDomain(domain) {
+		return nil, fmt.Errorf("Invalid domain: %s", domain)
+	}
+	q := url.Values{
+		"to": {opts.To},
+	}
+	if opts.DryRun {
+		q.Add("dry_run", "true")
+	}
+	if opts.FlagOnly {
+		q.Add("flag_only", "true")
+	}
+	if opts.Force {
+		q.Add("force", "true")
+	}
+	if opts.PurgeSource {
+		q.Add("purge_source", "true")
+	}
+	res, err := ac.Req(&request.Options{
+		Method:  "POST",
+		Path:    "/instances/" + domain + "/migrate-storage",
+		Queries: q,
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	rep := &MigrateStorageReport{}
+	if err := json.NewDecoder(res.Body).Decode(rep); err != nil {
+		return nil, err
+	}
+	return rep, nil
+}
+
 // DestroyInstance is used to delete an instance and all its data.
 func (ac *AdminClient) DestroyInstance(domain string) error {
 	if !validDomain(domain) {

cmd/instances.go

@@ -45,6 +45,11 @@
 var flagForce bool
 var flagJSON bool
 var flagSwiftLayout int
+var flagMigrateTo string
+var flagMigrateDryRun bool
+var flagMigrateFlagOnly bool
+var flagMigrateForce bool
+var flagMigratePurgeSource bool
 var flagCouchCluster int
 var flagUUID string
 var flagOIDCID string
@@ -248,6 +253,33 @@
 	},
 }
 
+var migrateStorageCmd = &cobra.Command{
+	Use:   "migrate-storage <domain>",
+	Short: "Migrate an instance's file storage to another backend (e.g. s3)",
+	Long: `cozy-stack instances migrate-storage copies an instance's files, file
+versions and avatar to another storage backend and switches the instance to it.
+The source data is kept unless --purge-source is given.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(args) != 1 {
+			return cmd.Usage()
+		}
+		ac := newAdminClient()
+		rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{
+			To:          flagMigrateTo,
+			DryRun:      flagMigrateDryRun,
+			FlagOnly:    flagMigrateFlagOnly,
+			Force:       flagMigrateForce,
+			PurgeSource: flagMigratePurgeSource,
+		})
+		if err != nil {
+			return err
+		}
+		fmt.Printf("migrated: %d files, %d versions, %d bytes, avatar=%v\n",
+			rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied)
+		return nil
+	},
+}
+
 var modifyInstanceCmd = &cobra.Command{
 	Use:   "modify <domain>",
 	Short: "Modify the instance properties",
@@ -1064,6 +1096,7 @@
 	instanceCmdGroup.AddCommand(showInstanceCmd)
 	instanceCmdGroup.AddCommand(showDBPrefixInstanceCmd)
 	instanceCmdGroup.AddCommand(addInstanceCmd)
+	instanceCmdGroup.AddCommand(migrateStorageCmd)
 	instanceCmdGroup.AddCommand(modifyInstanceCmd)
 	instanceCmdGroup.AddCommand(countInstanceCmd)
 	instanceCmdGroup.AddCommand(lsInstanceCmd)
@@ -1101,6 +1134,11 @@
 	addInstanceCmd.Flags().StringVar(&flagPhone, "phone", "", "The phone number of the owner")
 	addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)")
 	addInstanceCmd.Flags().IntVar(&flagSwiftLayout, "swift-layout", -1, "Specify the layout to use for Swift (from 0 for layout V1 to 2 for layout V3, -1 means the default)")
+	migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme")
+	migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching")
+	migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)")
+	migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost")
+	migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch")
 	addInstanceCmd.Flags().IntVar(&flagCouchCluster, "couch-cluster", -1, "Specify the CouchDB cluster where the instance will be created (-1 means the default)")
 	addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS")
 	addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled")

cozy.example.yaml

@@ -67,6 +67,7 @@ fs:
 
   # url: file://localhost/var/lib/cozy
   # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }}
+  # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}&region={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true
 
   # Swift FS can be used with advanced parameters to activate TLS properties.
   # For using swift with https, you must use the "swift+https" scheme.

docs/cli/cozy-stack_instances.md

@@ -50,6 +50,7 @@ cozy-stack instances <command> [flags]
 * [cozy-stack instances fsck](cozy-stack_instances_fsck.md)	 - Check a vfs
 * [cozy-stack instances import](cozy-stack_instances_import.md)	 - Import data from an export link
 * [cozy-stack instances ls](cozy-stack_instances_ls.md)	 - List instances
+* [cozy-stack instances migrate-storage](cozy-stack_instances_migrate-storage.md)	 - Migrate an instance's file storage to another backend (e.g. s3)
 * [cozy-stack instances modify](cozy-stack_instances_modify.md)	 - Modify the instance properties
 * [cozy-stack instances refresh-token-oauth](cozy-stack_instances_refresh-token-oauth.md)	 - Generate a new OAuth refresh token
 * [cozy-stack instances set-disk-quota](cozy-stack_instances_set-disk-quota.md)	 - Change the disk-quota of the instance

docs/cli/cozy-stack_instances_migrate-storage.md

@@ -0,0 +1,39 @@
+## cozy-stack instances migrate-storage
+
+Migrate an instance's file storage to another backend (e.g. s3)
+
+### Synopsis
+
+cozy-stack instances migrate-storage copies an instance's files, file
+versions and avatar to another storage backend and switches the instance to it.
+The source data is kept unless --purge-source is given.
+
+```
+cozy-stack instances migrate-storage <domain> [flags]
+```
+
+### Options
+
+```
+      --dry-run        Report what would be copied without writing or switching
+      --flag-only      Switch the backend pointer without copying (rollback to a retained source)
+      --force          Required with --flag-only; writes since cutover are lost
+  -h, --help           help for migrate-storage
+      --purge-source   Delete source objects after a successful switch
+      --to string      Target storage scheme (default "s3")
+```
+
+### Options inherited from parent commands
+
+```
+      --admin-host string   administration server host (default "localhost")
+      --admin-port int      administration server port (default 6060)
+  -c, --config string       configuration file (default "$HOME/.cozy.yaml")
+      --host string         server host (default "localhost")
+  -p, --port int            server port (default 8080)
+```
+
+### SEE ALSO
+
+* [cozy-stack instances](cozy-stack_instances.md)	 - Manage instances of a stack
+

docs/config.md

@@ -100,6 +100,34 @@ Magick, konnectors and services for example). And they can take several GB for
 the case of importing a Cozy. If needed, it is possible to configure the
 directory where they will be created via the `TMPDIR` environment variable.
 
+## Storage backend migration
+
+The `fs.url` parameter configures the storage backend (`file://`,
+`swift://` or `s3://`) used by all instances, as shown in
+[cozy.example.yaml](https://github.com/cozy/cozy-stack/blob/master/cozy.example.yaml)
+and detailed in the [S3 storage backend](s3.md) documentation.
+
+To move instances to a different backend one at a time, without changing
+the backend used by the rest of the fleet, an optional `fs.migration_target`
+key can be set to a second storage URL. Its connection is initialized
+alongside the default one at startup, so instances can be migrated to it
+while `fs.url` keeps pointing at the previous backend:
+
+```yaml
+fs:
+  url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}
+  migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET&region=rbx&bucket_prefix=cozy&use_ssl=true
+```
+
+As with `fs.url`, S3 credentials are passed as `access_key` and `secret_key`
+query parameters of the URL.
+
+Once `fs.migration_target` is set, the
+[`cozy-stack instances migrate-storage`](cli/cozy-stack_instances_migrate-storage.md)
+command can move individual instances to it. See
+[Migrating an instance from Swift to S3](s3.md#migrating-an-instance-from-swift-to-s3)
+for the full procedure, including rollback.
+
 ## Multiple CouchDB clusters
 
 With a large number of instances, a single CouchDB cluster may not be enough.