diff --git a/.github/workflows/docker-s3-test.yml b/.github/workflows/docker-s3-test.yml new file mode 100644 index 00000000000..2affb3c1594 --- /dev/null +++ b/.github/workflows/docker-s3-test.yml @@ -0,0 +1,61 @@ +name: Build S3-test Docker image + +on: + push: + branches: + - feat/s3-vfs-backend + workflow_dispatch: + inputs: + tag: + description: "Image tag to publish under ghcr.io/" + required: false + default: "s3-test" + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image metadata + id: meta + run: | + repo_lc="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + tag="${{ github.event.inputs.tag }}" + if [ -z "$tag" ]; then + tag="s3-test" + fi + tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" + sha_tag="${tag_lc}-${GITHUB_SHA::7}" + echo "image=ghcr.io/${repo_lc}" >> "$GITHUB_OUTPUT" + echo "tag=${tag_lc}" >> "$GITHUB_OUTPUT" + echo "sha_tag=${sha_tag}" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: scripts/docker/production/Dockerfile + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + build-args: | + VERSION_STRING=${{ steps.meta.outputs.tag }}-${{ github.sha }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 0985e46ddf2..fec5ca6ba5d 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -64,4 +64,4 @@ jobs: cache: true - name: Run tests - run: go test -p 1 -timeout 5m ./... + run: go test -p 1 -timeout 10m ./... diff --git a/client/instances.go b/client/instances.go index 07833c035aa..793479ed6f4 100644 --- a/client/instances.go +++ b/client/instances.go @@ -282,6 +282,63 @@ func (ac *AdminClient) ModifyInstance(opts *InstanceOptions) (*Instance, error) return readInstance(res) } +// MigrateStorageOptions contains the options for MigrateStorage. It mirrors +// the fields of storagemigration.Options in the model, without importing +// that package: the client stays free of any model/... dependency. +type MigrateStorageOptions struct { + To string + DryRun bool + FlagOnly bool + Force bool + PurgeSource bool +} + +// MigrateStorageReport is the report returned by MigrateStorage. It mirrors +// the fields of storagemigration.Report in the model. +type MigrateStorageReport struct { + Files int `json:"Files"` + Versions int `json:"Versions"` + Bytes int64 `json:"Bytes"` + AvatarCopied bool `json:"AvatarCopied"` +} + +// MigrateStorage moves an instance's object-storage content (files, +// versions, avatar) from its current backend to the target scheme. +func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error) { + if !validDomain(domain) { + return nil, fmt.Errorf("Invalid domain: %s", domain) + } + q := url.Values{ + "to": {opts.To}, + } + if opts.DryRun { + q.Add("dry_run", "true") + } + if opts.FlagOnly { + q.Add("flag_only", "true") + } + if opts.Force { + q.Add("force", "true") + } + if opts.PurgeSource { + q.Add("purge_source", "true") + } + res, err := ac.Req(&request.Options{ + Method: "POST", + Path: "/instances/" + domain + "/migrate-storage", + Queries: q, + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + rep := &MigrateStorageReport{} + if err := json.NewDecoder(res.Body).Decode(rep); err != nil { + return nil, err + } + return rep, nil +} + // DestroyInstance is used to delete an instance and all its data. func (ac *AdminClient) DestroyInstance(domain string) error { if !validDomain(domain) { diff --git a/cmd/instances.go b/cmd/instances.go index bf60ab23622..8551929a041 100644 --- a/cmd/instances.go +++ b/cmd/instances.go @@ -45,6 +45,11 @@ var flagPassphrase string var flagForce bool var flagJSON bool var flagSwiftLayout int +var flagMigrateTo string +var flagMigrateDryRun bool +var flagMigrateFlagOnly bool +var flagMigrateForce bool +var flagMigratePurgeSource bool var flagCouchCluster int var flagUUID string var flagOIDCID string @@ -248,6 +253,33 @@ be used as the error message. }, } +var migrateStorageCmd = &cobra.Command{ + Use: "migrate-storage ", + Short: "Migrate an instance's file storage to another backend (e.g. s3)", + Long: `cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Usage() + } + ac := newAdminClient() + rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{ + To: flagMigrateTo, + DryRun: flagMigrateDryRun, + FlagOnly: flagMigrateFlagOnly, + Force: flagMigrateForce, + PurgeSource: flagMigratePurgeSource, + }) + if err != nil { + return err + } + fmt.Printf("migrated: %d files, %d versions, %d bytes, avatar=%v\n", + rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied) + return nil + }, +} + var modifyInstanceCmd = &cobra.Command{ Use: "modify ", Short: "Modify the instance properties", @@ -1064,6 +1096,7 @@ func init() { instanceCmdGroup.AddCommand(showInstanceCmd) instanceCmdGroup.AddCommand(showDBPrefixInstanceCmd) instanceCmdGroup.AddCommand(addInstanceCmd) + instanceCmdGroup.AddCommand(migrateStorageCmd) instanceCmdGroup.AddCommand(modifyInstanceCmd) instanceCmdGroup.AddCommand(countInstanceCmd) instanceCmdGroup.AddCommand(lsInstanceCmd) @@ -1101,6 +1134,11 @@ func init() { addInstanceCmd.Flags().StringVar(&flagPhone, "phone", "", "The phone number of the owner") addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)") addInstanceCmd.Flags().IntVar(&flagSwiftLayout, "swift-layout", -1, "Specify the layout to use for Swift (from 0 for layout V1 to 2 for layout V3, -1 means the default)") + migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme") + migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching") + migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)") + migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost") + migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch") addInstanceCmd.Flags().IntVar(&flagCouchCluster, "couch-cluster", -1, "Specify the CouchDB cluster where the instance will be created (-1 means the default)") addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS") addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled") diff --git a/cozy.example.yaml b/cozy.example.yaml index 1f50fc8c1e6..088ccf30d1e 100644 --- a/cozy.example.yaml +++ b/cozy.example.yaml @@ -67,6 +67,7 @@ fs: # url: file://localhost/var/lib/cozy # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }} + # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}®ion={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true # Swift FS can be used with advanced parameters to activate TLS properties. # For using swift with https, you must use the "swift+https" scheme. diff --git a/docs/cli/cozy-stack_instances.md b/docs/cli/cozy-stack_instances.md index 8a47bfcb0a8..e17f2a7f39d 100644 --- a/docs/cli/cozy-stack_instances.md +++ b/docs/cli/cozy-stack_instances.md @@ -50,6 +50,7 @@ cozy-stack instances [flags] * [cozy-stack instances fsck](cozy-stack_instances_fsck.md) - Check a vfs * [cozy-stack instances import](cozy-stack_instances_import.md) - Import data from an export link * [cozy-stack instances ls](cozy-stack_instances_ls.md) - List instances +* [cozy-stack instances migrate-storage](cozy-stack_instances_migrate-storage.md) - Migrate an instance's file storage to another backend (e.g. s3) * [cozy-stack instances modify](cozy-stack_instances_modify.md) - Modify the instance properties * [cozy-stack instances refresh-token-oauth](cozy-stack_instances_refresh-token-oauth.md) - Generate a new OAuth refresh token * [cozy-stack instances set-disk-quota](cozy-stack_instances_set-disk-quota.md) - Change the disk-quota of the instance diff --git a/docs/cli/cozy-stack_instances_migrate-storage.md b/docs/cli/cozy-stack_instances_migrate-storage.md new file mode 100644 index 00000000000..8bd3876ab81 --- /dev/null +++ b/docs/cli/cozy-stack_instances_migrate-storage.md @@ -0,0 +1,39 @@ +## cozy-stack instances migrate-storage + +Migrate an instance's file storage to another backend (e.g. s3) + +### Synopsis + +cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given. + +``` +cozy-stack instances migrate-storage [flags] +``` + +### Options + +``` + --dry-run Report what would be copied without writing or switching + --flag-only Switch the backend pointer without copying (rollback to a retained source) + --force Required with --flag-only; writes since cutover are lost + -h, --help help for migrate-storage + --purge-source Delete source objects after a successful switch + --to string Target storage scheme (default "s3") +``` + +### Options inherited from parent commands + +``` + --admin-host string administration server host (default "localhost") + --admin-port int administration server port (default 6060) + -c, --config string configuration file (default "$HOME/.cozy.yaml") + --host string server host (default "localhost") + -p, --port int server port (default 8080) +``` + +### SEE ALSO + +* [cozy-stack instances](cozy-stack_instances.md) - Manage instances of a stack + diff --git a/docs/config.md b/docs/config.md index e10d2459803..1bd70e19731 100644 --- a/docs/config.md +++ b/docs/config.md @@ -100,6 +100,34 @@ Magick, konnectors and services for example). And they can take several GB for the case of importing a Cozy. If needed, it is possible to configure the directory where they will be created via the `TMPDIR` environment variable. +## Storage backend migration + +The `fs.url` parameter configures the storage backend (`file://`, +`swift://` or `s3://`) used by all instances, as shown in +[cozy.example.yaml](https://github.com/cozy/cozy-stack/blob/master/cozy.example.yaml) +and detailed in the [S3 storage backend](s3.md) documentation. + +To move instances to a different backend one at a time, without changing +the backend used by the rest of the fleet, an optional `fs.migration_target` +key can be set to a second storage URL. Its connection is initialized +alongside the default one at startup, so instances can be migrated to it +while `fs.url` keeps pointing at the previous backend: + +```yaml +fs: + url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }} + migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +As with `fs.url`, S3 credentials are passed as `access_key` and `secret_key` +query parameters of the URL. + +Once `fs.migration_target` is set, the +[`cozy-stack instances migrate-storage`](cli/cozy-stack_instances_migrate-storage.md) +command can move individual instances to it. See +[Migrating an instance from Swift to S3](s3.md#migrating-an-instance-from-swift-to-s3) +for the full procedure, including rollback. + ## Multiple CouchDB clusters With a large number of instances, a single CouchDB cluster may not be enough. diff --git a/docs/s3.md b/docs/s3.md new file mode 100644 index 00000000000..384c21956a0 --- /dev/null +++ b/docs/s3.md @@ -0,0 +1,339 @@ +[Table of contents](README.md#table-of-contents) + +# S3 Storage Backend + +cozy-stack supports S3-compatible object storage as a file system backend, +alongside the existing local filesystem (afero) and OpenStack Swift backends. +It has been designed to work with any S3-compatible provider (OVH, MinIO, +Scaleway, etc.) and does not depend on the AWS SDK. + +## Configuration + +The S3 backend is configured via the `fs.url` parameter using the `s3://` +scheme. All connection parameters are passed as query parameters: + +```yaml +fs: + url: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +| Parameter | Description | Default | +|-----------------|--------------------------------------|---------| +| `access_key` | S3 access key ID | — | +| `secret_key` | S3 secret access key | — | +| `region` | S3 region | — | +| `bucket_prefix` | Prefix for all bucket names | `cozy` | +| `use_ssl` | Use HTTPS for S3 connections | `true` | + +The host part of the URL is the S3 endpoint (e.g. `s3.rbx.io.cloud.ovh.net` +for OVH, `localhost:9000` for MinIO). + +### Local development with MinIO + +This tutorial explains how to set up a local S3 backend using MinIO for +development and testing. + +**1. Start MinIO with Docker:** + +```bash +docker run -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" +``` + +MinIO is now running: +- S3 API: `http://localhost:9000` +- Web console: `http://localhost:9001` (login: `minioadmin` / `minioadmin`) + +**2. Configure cozy-stack:** + +Buckets are created automatically at startup. No manual bucket creation +is needed. + +Edit your `~/.cozy/cozy.yaml`: + +```yaml +fs: + url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +**3. Build and start:** + +```bash +go build -o ~/go/bin/cozy-stack . +~/go/bin/cozy-stack serve +``` + +You should see in the logs: + +``` +Successfully connected to S3 endpoint localhost:9000 +``` + +**4. (Re)install your apps:** + +When switching from a different storage backend (e.g. `file://`), you need +to reinstall the apps so their assets are stored in S3: + +```bash +cozy-stack apps uninstall drive --domain your.domain.localhost:8080 +cozy-stack apps install drive --domain your.domain.localhost:8080 +cozy-stack apps uninstall home --domain your.domain.localhost:8080 +cozy-stack apps install home --domain your.domain.localhost:8080 +``` + +**5. Verify:** + +Check that objects appear in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-apps-web/ +``` + +Upload a file via the Drive UI or the API: + +```bash +TOKEN=$(cozy-stack instances token-cli your.domain.localhost:8080 io.cozy.files) +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: text/plain" \ + "http://your.domain.localhost:8080/files/io.cozy.files.root-dir?Type=file&Name=test.txt" \ + -d "Hello S3!" +``` + +Verify the file is in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-default/ +``` + +**6. Switching back to local filesystem:** + +Comment out the S3 URL in your config and restart cozy-stack: + +```yaml +fs: + # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +Note: files uploaded to S3 won't be accessible when using the local +filesystem backend, and vice versa. Each backend has its own storage. + +## Migrating an instance from Swift to S3 + +Instances can be moved from Swift to S3 one at a time, without changing the +storage backend for the rest of the fleet. This is useful to validate S3 on +a few instances before committing the whole platform to it. + +### 1. Configure the migration target + +Set `fs.migration_target` to the S3 URL, keeping `fs.url` on `swift://`. +Both connections (Swift and S3) are then initialized at startup: + +```yaml +fs: + url: swift://openstack/?UserName=... + migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +See [`fs.migration_target`](config.md#storage-backend-migration) in the +configuration documentation for details on this key. + +### 2. Run the migration + +```bash +cozy-stack instances migrate-storage --to s3 +``` + +It is recommended to run with `--dry-run` first to see what would be copied +without writing anything or switching the instance. + +The command: + +- blocks the instance (read-only, HTTP traffic only) for the duration of + the copy; +- copies the files, file versions, and the user's avatar to the S3 target + (thumbnails and installed apps are not copied; they regenerate on the + target backend); +- verifies the copied objects against the source; +- flips the instance's storage backend to S3 and unblocks the instance. + +By default the Swift source is kept as-is after a successful migration, so +it stays available for a rollback. + +**Known limitation:** blocking only gates HTTP traffic. Background workers +and triggers can still write to the source during the migration window. +Run migrations during low-activity periods until this is addressed. + +### 3. Roll back if needed + +If something looks wrong shortly after the switch, before any real write +has landed on the S3 target, flip the instance back to the retained Swift +source instantly, without copying anything back: + +```bash +cozy-stack instances migrate-storage --to swift --flag-only --force +``` + +`--force` is required with `--flag-only` because any writes made against S3 +since the cutover are lost. + +If real data now lives on S3 and needs to be preserved, run a full +migration back to Swift instead, which copies the data: + +```bash +cozy-stack instances migrate-storage --to swift +``` + +### 4. Reclaim the source + +Once confident the instance is stable on its new backend, delete the +retained source objects: + +```bash +cozy-stack instances migrate-storage --to s3 --purge-source +``` + +Since the instance already uses `s3`, this runs in purge-only mode: nothing +is copied, verified, or flipped, and the previously retained Swift data is +simply deleted. The instance is not blocked for this step. Running the same +command again is safe and is the way to retry a purge that failed right +after an earlier switch. + +(`--purge-source` can also be supplied on the initial migration if no +rollback window is needed.) + +### 5. Switch the global default + +After the whole fleet has been migrated to S3, change the global `fs.url` +to the S3 URL and remove `fs.migration_target`. From then on, the +per-instance backend flag set by earlier migrations simply matches the +global default. + +## Bucket strategy + +### Design rationale + +Swift uses one container per instance (`cozy-v3-`). This doesn't +scale well for S3 where bucket creation can be limited (AWS limits to 100 +buckets per account by default, OVH to 100 as well). Instead, the S3 backend +uses a **shared bucket per organization** with **key prefixes per instance**. + +### Bucket naming + +Each bucket name is derived from the instance's `OrgID` field: + +``` +- +``` + +- If `OrgID` is empty, `"default"` is used as fallback +- The org ID is sanitized: lowercased, underscores/dots replaced by hyphens, + non-alphanumeric characters stripped, consecutive hyphens collapsed, + truncated to respect the 63-character S3 bucket name limit +- Examples: `cozy-default`, `cozy-acme-corp`, `cozy-org-12345` + +### Dedicated buckets for secondary storage + +In addition to the main VFS bucket, the S3 backend uses dedicated buckets for +other storage needs: + +| Bucket | Content | +|-------------------------------|-----------------------------------------| +| `-` | Main VFS data (files, versions) | +| `-apps-web` | Web application assets (drive, etc.) | +| `-apps-konnectors` | Konnector assets | +| `-assets` | Dynamic assets | +| `-previews` | PDF preview and icon cache | +| `-exports` | Instance export archives | + +Buckets are created automatically on first use. + +## Object key structure + +Within a bucket, each instance's data is isolated by a key prefix derived +from `DBPrefix()` (typically the instance domain or a CouchDB prefix). + +### VFS files + +``` +//// +``` + +The document ID (a 32-character UUID v7 hex string) is split into virtual +subfolders to avoid flat hierarchies: + +``` +cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456 + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^ + first 22 chars 5 ch 5 ch 16-char internalID +``` + +This structure mirrors the Swift V3 layout (`MakeObjectNameV3`). + +### Thumbnails + +``` +/thumbs/- +``` + +Formats: `small`, `medium`, `large`. + +### Avatar + +``` +/avatar +``` + +## Memory consumption + +The S3 backend is designed to have comparable memory usage to Swift: + +| Scenario | Memory per upload | +|-----------------------------|-------------------| +| Known size, file < 5 GiB | ~32 KB (single PUT, stream) | +| Unknown size (rare) | ~5 MiB (multipart, PartSize=5MiB, NumThreads=1) | + +When `ByteSize` is known on the file document (the common case for drive +uploads), the backend passes the exact size to `PutObject`, which uses a +single PUT request that streams directly to S3 with minimal buffering — the +same behavior as Swift's `ObjectCreate`. + +Multipart upload is only used for files with unknown size or exceeding 5 GiB, +with `PartSize=5MiB` and `NumThreads=1` to limit memory. + +## Encryption at rest + +The S3 backend does not implement client-side encryption. Encryption should +be configured at the infrastructure level (S3 bucket default encryption / +SSE-S3), the same approach used for the Swift backend. + +## Differences from Swift + +| Aspect | Swift | S3 | +|--------------------|-------------------------------|----------------------------------------| +| Container/Bucket | One per instance | One per organization (shared) | +| Instance isolation | Container name | Key prefix within bucket | +| Delete instance | Delete entire container | Delete all objects with key prefix | +| File streaming | Native `io.WriteCloser` | `io.Pipe` + `PutObject` goroutine | +| Bulk delete | `BulkDelete` API | `RemoveObjects` channel API | +| Server-side copy | `ObjectCopy` | `CopyObject` (same endpoint only) | + +## Testing + +The VFS integration tests run against all three backends (afero, swift, s3) +using a table-driven approach. The S3 tests use +[testcontainers-go](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/) +with a MinIO container that is started automatically. + +```bash +# Run VFS tests (requires CouchDB + Docker) +COZY_COUCHDB_URL=http://admin:admin@localhost:5984/ \ + go test ./model/vfs/ -run TestVfs -v -count=1 -timeout 300s + +# Run naming unit tests (no external deps) +go test ./model/vfs/vfss3/ -run "TestSanitize|TestBucketName|TestMakeObjectKey|TestMakeDocID" -v +``` diff --git a/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md new file mode 100644 index 00000000000..e8908c92c02 --- /dev/null +++ b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md @@ -0,0 +1,977 @@ +# Per-instance storage backend & Swift→S3 migration — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let an operator migrate a single live instance's file storage from Swift to the S3 backend, one instance at a time, via `cozy-stack instances migrate-storage --to s3`, with rollback. + +**Architecture:** Add a per-instance `FsScheme` field that overrides the global `fs.url` scheme when building the instance's VFS. Run both the Swift and S3 connections during the transition. A server-side migration copies object-storage **content** (files, versions, avatar) source→target **without touching the shared CouchDB index**, inside an HTTP-blocked window, then flips `FsScheme`. Rollback reuses the same engine (`--to swift`) or an instant `--flag-only` flip while the source is retained. + +**Tech Stack:** Go, CouchDB, minio-go/v7 (S3), ncw/swift (Swift), cobra (CLI), echo (admin API), MinIO testcontainer for tests. + +## Global Constraints + +- Backend key layout is identical across Swift-v3 and S3: object name = `docID[:22] + "/" + docID[22:27] + "/" + docID[27:] + "/" + internalID` (fallback `docID + "/" + internalID` when `len(docID)!=32 || len(internalID)!=16`). S3 prefixes it with `keyPrefix = DBPrefix() + "/"`. Use the exported builders `vfsswift.MakeObjectNameV3` and `vfss3.MakeObjectKey` — never re-derive by hand. +- The migration MUST NOT create or modify any CouchDB document (`io.cozy.files`, `io.cozy.files.versions`). Source and target VFS share the same `vfs.NewCouchdbIndexer(i)`. Only object bytes move. +- `FsScheme == ""` means "use global default" and MUST preserve today's behavior for every existing instance (zero data migration of the instance registry). +- No cosign trailers in commits (project convention). +- Only Swift-v3 (`SwiftLayout == 2`) is a supported migration source; reject v1/v2. +- Migrated-but-unverified state is never persisted: `FsScheme` flips only after verification passes. +- **S3 tests require a live MinIO container via Docker.** There is NO per-package `newTestS3VFS` fixture. The real, existing harness is `testutils.StartMinio(t) *MinioFixture` (`tests/testutils/minio_utils.go`) exposing `Client(t) *minio.Client` and `FsURL(bucketPrefix string) *url.URL`. Build an S3 VFS exactly like `makeS3FS` in `model/vfs/vfs_test.go:916`: `mf := testutils.StartMinio(t); config.InitS3Connection(config.Fs{URL: mf.FsURL("test")}); s3fs, _ := vfss3.New(db, index, &diskImpl{}, mutex)`. Any test in this plan that references a `newTestS3VFS`/`newTestAvatarS3`/`minioGetOpts` helper means "build the S3 VFS/avatarer with this StartMinio pattern" — reuse it, do not invent a new container fixture. Tests that must reach unexported `*s3VFS` fields go in `package vfss3` (or an external `vfss3_test` package if importing `tests/testutils` would cycle — the implementer verifies and picks). + +--- + +## File structure + +- `model/instance/instance.go` — add `FsScheme` field + `StorageScheme()` accessor; refactor the triplicated backend `switch` in `MakeVFS`/`AvatarFS`/`ThumbsFS` into one `buildVFS(kind)` helper reading `StorageScheme()`. +- `pkg/config/config/config.go` — parse optional `fs.migration_target` URL into `config.Fs`; add `MigrationTargetURL()` / `HasS3Target()`. +- `model/stack/main.go` — after the default connections, also init the S3 connection from the migration target when the global scheme is not S3. +- `model/vfs/vfs.go` — add `OpenAvatar() (io.ReadCloser, string, error)` to the `Avatarer` interface. +- `model/vfs/vfsswift/avatar_v3.go`, `model/vfs/vfss3/avatar.go`, `model/vfs/vfsafero/avatar.go` — implement `OpenAvatar`. +- `model/vfs/vfss3/impl.go` — add exported index-free `WriteContentAt(docID, internalID string, r io.Reader, size int64) error`. +- `model/instance/storagemigration/migration.go` (new package) — the engine: enumerate, copy, verify, orchestrate (`Migrate`, `Report`). +- `client/instances.go` — `AdminClient.MigrateStorage(opts)`. +- `web/instances/instances.go` — `migrateStorageHandler` + route. +- `cmd/instances.go` — `migrateStorageCmd` + flags. +- Tests colocated: `model/instance/storagemigration/migration_test.go`, plus small unit tests next to touched files. + +--- + +## Task 1: Per-instance `FsScheme` field and effective-scheme accessor + +**Files:** +- Modify: `model/instance/instance.go` (struct ~50-89; `MakeVFS` 250-276; `AvatarFS` 279-304; `ThumbsFS` 306-333) +- Test: `model/instance/instance_storage_scheme_test.go` (create) + +**Interfaces:** +- Produces: `func (i *Instance) StorageScheme() string` — returns `i.FsScheme` if non-empty else `config.FsURL().Scheme`. +- Produces: struct field `FsScheme string json:"fs_scheme,omitempty"`. + +- [ ] **Step 1: Write the failing test** + +Create `model/instance/instance_storage_scheme_test.go`: +```go +package instance + +import ( + "testing" + + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/stretchr/testify/assert" +) + +func TestStorageSchemeFallsBackToGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{} + assert.Equal(t, config.FsURL().Scheme, i.StorageScheme()) +} + +func TestStorageSchemeOverridesGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{FsScheme: config.SchemeS3} + assert.Equal(t, config.SchemeS3, i.StorageScheme()) +} +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/instance/ -run TestStorageScheme -v` +Expected: FAIL — `i.StorageScheme undefined` / `i.FsScheme undefined`. + +- [ ] **Step 3: Add the field** + +In the `Instance` struct (`model/instance/instance.go`), directly under `SwiftLayout int json:"swift_cluster,omitempty"`: +```go + // FsScheme, when non-empty, overrides the global fs.url scheme for this + // instance. Used to migrate a single instance to another storage backend + // (e.g. "s3") without changing the stack-wide default. Empty = global default. + FsScheme string `json:"fs_scheme,omitempty"` +``` + +- [ ] **Step 4: Add the accessor** + +Near `DBPrefix()` (`model/instance/instance.go:183`): +```go +// StorageScheme returns the storage backend scheme effective for this instance: +// the per-instance FsScheme override when set, otherwise the global fs.url scheme. +func (i *Instance) StorageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} +``` + +- [ ] **Step 5: Run test, verify pass** + +Run: `go test ./model/instance/ -run TestStorageScheme -v` +Expected: PASS. + +- [ ] **Step 6: Refactor the triplicated switch to use `StorageScheme()`** + +In `MakeVFS`, `AvatarFS`, and `ThumbsFS`, replace every `fsURL := config.FsURL()` + `switch fsURL.Scheme` with `switch i.StorageScheme()`. Keep `config.FsURL()` only where the afero branch still needs the URL/path (afero uses `fsURL` and `i.DirName()`), fetching it inside that branch: +```go + switch i.StorageScheme() { + case config.SchemeFile, config.SchemeMem: + i.vfs, err = vfsafero.New(i, index, disk, mutex, config.FsURL(), i.DirName()) + case config.SchemeSwift, config.SchemeSwiftSecure: + switch i.SwiftLayout { + case 2: + i.vfs, err = vfsswift.NewV3(i, index, disk, mutex) + default: + err = ErrInvalidSwiftLayout + } + case config.SchemeS3: + i.vfs, err = vfss3.New(i, index, disk, mutex) + default: + err = fmt.Errorf("instance: unknown storage provider %s", i.StorageScheme()) + } +``` +Apply the equivalent change to `AvatarFS` (which builds `vfsafero.NewAvatarFs`/`vfsswift.NewAvatarFsV3`/`vfss3.NewAvatarFs`) and `ThumbsFS`. + +- [ ] **Step 7: Build + full instance package tests** + +Run: `go build ./... && go test ./model/instance/ -run 'TestStorageScheme|TestMakeVFS' -v` +Expected: build OK, tests PASS. + +- [ ] **Step 8: Commit** + +```bash +git add model/instance/instance.go model/instance/instance_storage_scheme_test.go +git commit -m "feat(instance): add per-instance FsScheme override for storage backend" +``` + +--- + +## Task 2: Initialize the S3 connection from a migration target + +**Files:** +- Modify: `pkg/config/config/config.go` (`Fs` struct 225-235; `UseViper` ~830/1156; add accessors near `FsURL` 497) +- Modify: `model/stack/main.go` (83-91) +- Test: `pkg/config/config/s3_target_test.go` (create) + +**Interfaces:** +- Consumes: `config.InitS3Connection(fs Fs) error` (`pkg/config/config/s3.go:23`), which populates the S3 globals when `fs.URL.Scheme == SchemeS3`. +- Produces: `func MigrationTargetURL() *url.URL`; `func HasS3Target() bool`. + +- [ ] **Step 1: Write the failing test** + +Create `pkg/config/config/s3_target_test.go`: +```go +package config + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrationTargetInitsS3WhenGlobalIsSwift(t *testing.T) { + swiftURL, _ := url.Parse("swift://openstack/") + s3URL, _ := url.Parse("s3://key:secret@localhost:9000/?bucket_prefix=cozy&use_ssl=false") + config = &Config{Fs: Fs{URL: swiftURL, MigrationTarget: s3URL}} + + require.True(t, HasS3Target()) + // Init the S3 globals from the target even though the global scheme is swift. + require.NoError(t, InitS3Connection(Fs{URL: MigrationTargetURL()})) + assert.NotNil(t, GetS3Client()) + assert.Equal(t, "cozy", GetS3BucketPrefix()) +} +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./pkg/config/config/ -run TestMigrationTargetInitsS3 -v` +Expected: FAIL — `Fs has no field MigrationTarget` / `HasS3Target undefined` / `MigrationTargetURL undefined`. + +- [ ] **Step 3: Add the config field + accessors** + +In the `Fs` struct (`config.go:225`) add: +```go + // MigrationTarget, when set, is an alternate storage URL (e.g. s3://...) + // whose connection is initialized alongside the default one, so instances + // can be migrated to it while the global scheme stays unchanged. + MigrationTarget *url.URL +``` +Near `FsURL()` (`config.go:497`): +```go +// MigrationTargetURL returns the configured storage migration target URL, or nil. +func MigrationTargetURL() *url.URL { + return config.Fs.MigrationTarget +} + +// HasS3Target reports whether an S3 storage migration target is configured. +func HasS3Target() bool { + u := config.Fs.MigrationTarget + return u != nil && u.Scheme == SchemeS3 +} +``` + +- [ ] **Step 4: Parse `fs.migration_target` in UseViper** + +In `UseViper` where `Fs{...}` is built (`config.go:1156`), parse the optional key and set the field: +```go + var migrationTarget *url.URL + if raw := v.GetString("fs.migration_target"); raw != "" { + migrationTarget, err = url.Parse(raw) + if err != nil { + return err + } + } +``` +and add `MigrationTarget: migrationTarget,` to the `Fs{...}` literal. + +- [ ] **Step 5: Run test, verify pass** + +Run: `go test ./pkg/config/config/ -run TestMigrationTargetInitsS3 -v` +Expected: PASS. + +- [ ] **Step 6: Wire the target init at stack startup** + +In `model/stack/main.go` after the existing `InitDefaultS3Connection()` block (line ~88-91): +```go + // When a storage migration target is configured (e.g. migrating instances + // to S3 while the global default is still Swift), init that connection too. + if config.HasS3Target() { + if err := config.InitS3Connection(config.Fs{URL: config.MigrationTargetURL()}); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 migration target connection: %w", err) + } + } +``` + +- [ ] **Step 7: Build** + +Run: `go build ./...` +Expected: OK. + +- [ ] **Step 8: Commit** + +```bash +git add pkg/config/config/config.go pkg/config/config/s3_target_test.go model/stack/main.go +git commit -m "feat(config): init S3 connection from an optional fs.migration_target" +``` + +--- + +## Task 3: Index-free content writer on the S3 backend + +**Files:** +- Modify: `model/vfs/vfss3/impl.go` (near `ImportFileVersion` 591-626) +- Test: `model/vfs/vfss3/write_content_at_test.go` (create; reuses the package's MinIO testcontainer harness) + +**Interfaces:** +- Produces: `func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error` — puts bytes at `MakeObjectKey(sfs.keyPrefix, docID, internalID)`, creating no CouchDB doc. + +- [ ] **Step 1: Write the failing test** + +Create `model/vfs/vfss3/write_content_at_test.go` (mirror the setup already used in the vfss3 suite to get an `*s3VFS` against MinIO; see the existing test harness in this package for the exact fixture helper name and reuse it): +```go +package vfss3 + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + sfs := newTestS3VFS(t) // existing fixture in this package's tests + s3 := sfs.(*s3VFS) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello s3 migration") + + require.NoError(t, s3.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + objKey := MakeObjectKey(s3.keyPrefix, docID, internalID) + obj, err := s3.client.GetObject(s3.ctx, s3.bucket, objKey, minioGetOpts()) + require.NoError(t, err) + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} +``` +(If the package lacks a `newTestS3VFS`/`minioGetOpts` helper, add minimal ones in the test file based on the container fixture already present in `model/vfs` S3 tests.) + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/vfs/vfss3/ -run TestWriteContentAt -v` +Expected: FAIL — `s3.WriteContentAt undefined`. + +- [ ] **Step 3: Implement** + +Add to `model/vfs/vfss3/impl.go`: +```go +// WriteContentAt streams content into the object backing the (docID, internalID) +// key, creating NO CouchDB document. It is used by storage migration, which +// preserves the shared index and only moves object bytes. size may be -1 when +// unknown (falls back to multipart). +func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, size, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + return err +} +``` + +- [ ] **Step 4: Run test, verify pass** + +Run: `go test ./model/vfs/vfss3/ -run TestWriteContentAt -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/vfs/vfss3/impl.go model/vfs/vfss3/write_content_at_test.go +git commit -m "feat(vfss3): add index-free WriteContentAt for storage migration" +``` + +--- + +## Task 4: `OpenAvatar` on the Avatarer interface + +**Files:** +- Modify: `model/vfs/vfs.go` (`Avatarer` interface 266-273) +- Modify: `model/vfs/vfsswift/avatar_v3.go`, `model/vfs/vfss3/avatar.go`, `model/vfs/vfsafero/avatar.go` +- Test: `model/vfs/vfss3/avatar_test.go` (add a case; or create) + +**Interfaces:** +- Produces: `OpenAvatar() (io.ReadCloser, string, error)` on `vfs.Avatarer` — returns the stored avatar content reader and its content-type; `os.ErrNotExist` when no avatar exists. + +- [ ] **Step 1: Write the failing test (S3 impl)** + +In `model/vfs/vfss3/avatar_test.go`: +```go +func TestOpenAvatarRoundTrip(t *testing.T) { + av := newTestAvatarS3(t) // fixture returning the s3 Avatarer + require.NoError(t, av.CreateAvatar("image/png"). /* write */ , /* ... */) + // (use the package's existing CreateAvatar test flow to store bytes first) + + r, ctype, err := av.OpenAvatar() + require.NoError(t, err) + defer r.Close() + assert.Equal(t, "image/png", ctype) + b, _ := io.ReadAll(r) + assert.NotEmpty(t, b) +} +``` +(Base the write half on the existing `CreateAvatar` test in this package; if none exists, store via `CreateAvatar` returning an `io.WriteCloser` per its current signature.) + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/vfs/vfss3/ -run TestOpenAvatar -v` +Expected: FAIL — `OpenAvatar undefined`. + +- [ ] **Step 3: Extend the interface** + +In `model/vfs/vfs.go` `Avatarer`: +```go + // OpenAvatar returns a reader over the stored avatar content and its + // content-type, or os.ErrNotExist if no avatar is stored. + OpenAvatar() (io.ReadCloser, string, error) +``` + +- [ ] **Step 4: Implement for S3** (`model/vfs/vfss3/avatar.go`) +```go +func (a *avatarS3) OpenAvatar() (io.ReadCloser, string, error) { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + return nil, "", err + } + info, err := obj.Stat() + if err != nil { + if isNoSuchKey(err) { // existing helper used by OpenFile in this pkg + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return obj, info.ContentType, nil +} +``` + +- [ ] **Step 5: Implement for Swift v3** (`model/vfs/vfsswift/avatar_v3.go`) +```go +func (a *avatarV3) OpenAvatar() (io.ReadCloser, string, error) { + f, headers, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) + if err != nil { + if errors.Is(err, swift.ObjectNotFound) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, headers["Content-Type"], nil +} +``` + +- [ ] **Step 6: Implement for afero** (`model/vfs/vfsafero/avatar.go`) +```go +func (a *avatarFS) OpenAvatar() (io.ReadCloser, string, error) { + f, err := a.fs.Open(AvatarFilename) + if err != nil { + if os.IsNotExist(err) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + // content-type is not stored on disk; sniff from the name/content. + return f, "application/octet-stream", nil +} +``` + +- [ ] **Step 7: Run tests + build** + +Run: `go build ./... && go test ./model/vfs/vfss3/ -run TestOpenAvatar -v` +Expected: build OK (all three impls satisfy the interface), test PASS. + +- [ ] **Step 8: Commit** + +```bash +git add model/vfs/vfs.go model/vfs/vfsswift/avatar_v3.go model/vfs/vfss3/avatar.go model/vfs/vfsafero/avatar.go model/vfs/vfss3/avatar_test.go +git commit -m "feat(vfs): add OpenAvatar to the Avatarer interface" +``` + +--- + +## Task 5: Migration engine — enumerate and copy content + +**Files:** +- Create: `model/instance/storagemigration/migration.go` +- Test: `model/instance/storagemigration/migration_test.go` + +**Interfaces:** +- Consumes: `couchdb.ForeachDocs(db prefixer.Prefixer, doctype string, fn func(id string, doc json.RawMessage) error) error`; `vfs.VFS.OpenFile(*vfs.FileDoc)`, `vfs.VFS.OpenFileVersion(*vfs.FileDoc, *vfs.Version)`, `vfs.VFS.FileByID(string)`; `vfsswift.NewV3`, `vfss3.New`; the `contentWriter` interface (Task 3's `WriteContentAt`); `OpenAvatar` (Task 4). +- Produces: `type Report struct { Files, Versions int; Bytes int64; AvatarCopied bool }`; `func CopyContent(inst *instance.Instance, src vfs.VFS, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error)`. + +- [ ] **Step 1: Write the failing test (afero source → S3 target)** + +Create `model/instance/storagemigration/migration_test.go`: +```go +package storagemigration + +import ( + "testing" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyContentMovesFilesVersionsAndAvatar(t *testing.T) { + // Fixture: an instance whose global backend is afero (test default), + // populated with 2 files, 1 extra version, 1 trashed file, and an avatar. + inst, src, dst, srcAv, dstAv := setupMigrationFixture(t) + + rep, err := CopyContent(inst, src, dst, srcAv, dstAv) + require.NoError(t, err) + assert.Equal(t, 3, rep.Files) // 2 live + 1 trashed + assert.Equal(t, 1, rep.Versions) + assert.True(t, rep.AvatarCopied) + + // Every source file's bytes are now readable from the target VFS. + assertAllContentReadableFrom(t, inst, dst) + // The CouchDB index is unchanged (same rev on the files DB). + assertIndexUnchanged(t, inst) +} +``` +(`setupMigrationFixture`, `assertAllContentReadableFrom`, `assertIndexUnchanged` are helpers to write in this test file, building the instance with `lifecycle.Create`, writing files via the source VFS `CreateFile`, and building `dst` via `vfss3.New` against the test MinIO container.) + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/instance/storagemigration/ -run TestCopyContent -v` +Expected: FAIL — package/functions undefined. + +- [ ] **Step 3: Implement the enumerator + copier** + +Create `model/instance/storagemigration/migration.go`: +```go +package storagemigration + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" +) + +// contentWriter is implemented by target VFS backends that can write object +// bytes for a (docID, internalID) key without creating a CouchDB document. +type contentWriter interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error +} + +// Report summarizes a content copy. +type Report struct { + Files int + Versions int + Bytes int64 + AvatarCopied bool +} + +// CopyContent copies all object-storage content (files incl. trashed, versions, +// avatar) for inst from src to dst. It creates/modifies NO CouchDB document. +func CopyContent(inst *instance.Instance, src, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error) { + writer, ok := dst.(contentWriter) + if !ok { + return nil, fmt.Errorf("target backend does not support index-free writes") + } + rep := &Report{} + + // Files (including trashed). + err := couchdb.ForeachDocs(inst, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return err + } + if doc.Type == consts.DirType { + return nil + } + r, err := src.OpenFile(&doc) + if err != nil { + return fmt.Errorf("open source file %s: %w", doc.DocID, err) + } + defer r.Close() + if err := writer.WriteContentAt(doc.DocID, doc.InternalID, r, doc.ByteSize); err != nil { + return fmt.Errorf("write target file %s: %w", doc.DocID, err) + } + rep.Files++ + rep.Bytes += doc.ByteSize + return nil + }) + if err != nil { + return rep, err + } + + // Versions. + err = couchdb.ForeachDocs(inst, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return err + } + fileID, internalID := splitVersionID(ver.DocID) + fileDoc, err := src.FileByID(fileID) + if err != nil { + return fmt.Errorf("file for version %s: %w", ver.DocID, err) + } + r, err := src.OpenFileVersion(fileDoc, &ver) + if err != nil { + return fmt.Errorf("open source version %s: %w", ver.DocID, err) + } + defer r.Close() + if err := writer.WriteContentAt(fileID, internalID, r, ver.ByteSize); err != nil { + return fmt.Errorf("write target version %s: %w", ver.DocID, err) + } + rep.Versions++ + rep.Bytes += ver.ByteSize + return nil + }) + if err != nil { + return rep, err + } + + // Avatar (single optional object). + ar, ctype, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + // no avatar, nothing to do + case err != nil: + return rep, fmt.Errorf("open source avatar: %w", err) + default: + defer ar.Close() + w, err := dstAv.CreateAvatar(ctype) + if err != nil { + return rep, fmt.Errorf("create target avatar: %w", err) + } + if _, err := io.Copy(w, ar); err != nil { + _ = w.Close() + return rep, fmt.Errorf("copy avatar: %w", err) + } + if err := w.Close(); err != nil { + return rep, fmt.Errorf("finalize target avatar: %w", err) + } + rep.AvatarCopied = true + } + + return rep, nil +} + +func splitVersionID(versionDocID string) (fileID, internalID string) { + for i := 0; i < len(versionDocID); i++ { + if versionDocID[i] == '/' { + return versionDocID[:i], versionDocID[i+1:] + } + } + return versionDocID, versionDocID +} +``` +Notes for the implementer: +- Confirm the exact `vfs.FileDoc` field for the doc-type discriminator and the trashed flag; `consts.DirType`/`consts.FileType` are the type values. Directories are skipped; trashed files ARE included (they still carry content). +- Confirm `Avatarer.CreateAvatar` returns an `io.WriteCloser` (as used elsewhere in the code that stores avatars). If its signature differs, adapt the write half accordingly. + +- [ ] **Step 4: Run test, verify pass** + +Run: `go test ./model/instance/storagemigration/ -run TestCopyContent -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/instance/storagemigration/migration.go model/instance/storagemigration/migration_test.go +git commit -m "feat(storagemigration): copy files, versions and avatar between backends" +``` + +--- + +## Task 5b: Index-free content writer on the Swift backend + +Rationale: the rollback design offers a full S3→Swift re-migration (`--to swift`), so Swift must also be a valid copy TARGET. Mirror Task 3 for vfsswift v3. + +**Files:** +- Modify: `model/vfs/vfsswift/impl_v3.go` (near `ImportFileVersion`) +- Modify: `model/instance/storagemigration/migration.go` (package doc comment: Swift↔S3 is now accurate) +- Test: `model/vfs/vfsswift/write_content_at_v3_test.go` (external `package vfsswift_test`, MinIO not needed — Swift uses a swift test server; mirror the existing swift test setup in the package/`model/vfs/vfs_test.go` `makeSwiftFS`) + +**Interfaces:** +- Produces: `func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error` — `ObjectCreate` at `MakeObjectNameV3(docID, internalID)` in the instance container, creating no CouchDB document. + +- [ ] **Step 1: Write the failing test** in `package vfsswift_test`, building the swift VFS the way `makeSwiftFS` (`model/vfs/vfs_test.go:876`) does; write via the interface assertion `sfs.(interface{ WriteContentAt(...) })`; read back the object via the swift connection at `MakeObjectNameV3(docID, internalID)` and assert bytes. Use 32-char docID + 16-char internalID. + +- [ ] **Step 2: Run it, verify RED** — `go test ./model/vfs/vfsswift/ -run TestWriteContentAt -timeout 120s -v` → `WriteContentAt` undefined. + +- [ ] **Step 3: Implement** in `model/vfs/vfsswift/impl_v3.go` (receiver name and container/ctx access mirror `ImportFileVersion`/`CreateFile` in this file): +```go +// WriteContentAt streams content into the object backing the (docID, internalID) +// key in this instance's container, creating NO CouchDB document. Used by +// storage migration, which preserves the shared index and only moves bytes. +func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objName := MakeObjectNameV3(docID, internalID) + f, err := sfs.c.ObjectCreate(sfs.ctx, sfs.container, objName, true, "", "application/octet-stream", nil) + if err != nil { + return err + } + if _, err = io.Copy(f, content); err != nil { + _ = f.Close() + return err + } + return f.Close() +} +``` +(Verify the exact receiver type name — `swiftVFSV3`/`swiftVFS` — and field names `c`/`ctx`/`container` against the file; adapt if they differ. `ObjectCreate` with an empty checksum skips server-side hash verification, matching the streaming path already used in this file.) + +- [ ] **Step 4: Also update** the `storagemigration/migration.go` package doc comment so it no longer implies only-S3-target (Swift↔S3 both now supported as targets). Rebuild. + +- [ ] **Step 5: Run test, verify GREEN**, `go build ./...` clean. + +- [ ] **Step 6: Commit** +```bash +git add model/vfs/vfsswift/impl_v3.go model/vfs/vfsswift/write_content_at_v3_test.go model/instance/storagemigration/migration.go +git commit -m "feat(vfsswift): add index-free WriteContentAt for storage migration" +``` + +--- + +## Task 6: Verification pass + +**Files:** +- Modify: `model/instance/storagemigration/migration.go` +- Test: `model/instance/storagemigration/migration_test.go` (add case) + +**Interfaces:** +- Produces: `func Verify(inst *instance.Instance, dst vfs.VFS, dstAv vfs.Avatarer, expected *Report) error` — re-enumerates and confirms each file/version object is present on the target with a matching byte size; errors listing the first missing/mismatched ID. + +- [ ] **Step 1: Write the failing test** + +Add to `migration_test.go`: +```go +func TestVerifySucceedsAfterCopyAndFailsWhenObjectMissing(t *testing.T) { + inst, src, dst, srcAv, dstAv := setupMigrationFixture(t) + rep, err := CopyContent(inst, src, dst, srcAv, dstAv) + require.NoError(t, err) + require.NoError(t, Verify(inst, dst, dstAv, rep)) + + deleteOneTargetObject(t, inst, dst) // helper: remove a known key + assert.Error(t, Verify(inst, dst, dstAv, rep)) +} +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/instance/storagemigration/ -run TestVerify -v` +Expected: FAIL — `Verify undefined`. + +- [ ] **Step 3: Implement `Verify`** + +Add to `migration.go`. Re-run the same enumeration, but instead of copying, stat the target object via a new `contentStater` capability (add `StatContentAt(docID, internalID string) (int64, error)` to BOTH `vfss3` and `vfsswift` v3, mirroring `WriteContentAt`, returning `os.ErrNotExist` when the object is absent) and compare sizes. Count files+versions and compare totals to `expected`. Return the first discrepancy as an error. (Add the `StatContentAt` methods + a `contentStater` interface here, symmetric to Task 3/5b.) Since either backend can be the target, both must implement it. + +- [ ] **Step 4: Run test, verify pass** + +Run: `go test ./model/instance/storagemigration/ -run TestVerify -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/instance/storagemigration/migration.go model/instance/storagemigration/migration_test.go model/vfs/vfss3/impl.go +git commit -m "feat(storagemigration): verify target objects after copy" +``` + +--- + +## Task 7: Orchestration — `Migrate` with block/flip/dry-run/flag-only/purge + +**Files:** +- Modify: `model/instance/storagemigration/migration.go` +- Test: `model/instance/storagemigration/migration_test.go` (add cases) + +**Interfaces:** +- Consumes: `lifecycle.Block(inst, reason...)`, `lifecycle.Unblock(inst)`; `instance.Update(inst)`; the instance's VFS constructors. +- Produces: +```go +type Options struct { + To string // target scheme, e.g. "s3" (or "swift" for rollback) + DryRun bool + FlagOnly bool // switch pointer to an already-populated backend, no copy + Force bool // required with FlagOnly (data written since cutover is lost) + PurgeSource bool // delete source objects after a successful flip +} +func Migrate(inst *instance.Instance, opts Options) (*Report, error) +``` + +- [ ] **Step 1: Write the failing tests** + +Add cases to `migration_test.go`: +```go +func TestMigrateFlipsSchemeAfterVerify(t *testing.T) { + inst := setupInstanceOnAfero(t) // populated + rep, err := Migrate(inst, Options{To: "s3"}) + require.NoError(t, err) + assert.Equal(t, "s3", inst.FsScheme) + assert.Greater(t, rep.Files, 0) + // Reads now served from S3: + assertReadsServedFrom(t, inst, "s3") +} + +func TestMigrateDryRunDoesNotFlip(t *testing.T) { + inst := setupInstanceOnAfero(t) + _, err := Migrate(inst, Options{To: "s3", DryRun: true}) + require.NoError(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +func TestMigrateFlagOnlyRequiresForce(t *testing.T) { + inst := setupInstanceOnAfero(t) + _, err := Migrate(inst, Options{To: "s3", FlagOnly: true}) + require.Error(t, err) // refuses without --force +} +``` + +- [ ] **Step 2: Run, verify fail** + +Run: `go test ./model/instance/storagemigration/ -run TestMigrate -v` +Expected: FAIL — `Migrate`/`Options` undefined. + +- [ ] **Step 3: Implement `Migrate`** + +Add to `migration.go`. Flow: +1. Guards: `opts.To != inst.StorageScheme()`; if `opts.To == config.SchemeS3` require `config.HasS3Target()` or global S3; Swift source must be `SwiftLayout == 2`. If `opts.FlagOnly && !opts.Force` return an error explaining data-loss risk. +2. Build `src` = VFS for the current scheme, `dst` = VFS for `opts.To`, and the two `Avatarer`s, via the same constructors `MakeVFS`/`AvatarFS` use (`vfsswift.NewV3`, `vfss3.New`, `vfsafero.New`) with a shared `index := vfs.NewCouchdbIndexer(inst)`, `disk := vfs.DiskThresholder(inst)`, `mutex := config.Lock().ReadWrite(inst, "vfs")`. Ensure the target S3 bucket exists via `config.GetS3Client().MakeBucket(...)` guarded by BucketExists (do NOT call `InitFs`, which also touches the index). +3. `FlagOnly`: skip copy; go to step 6 after a presence check (`Verify` with a nil expected, i.e. just confirm target objects exist). +4. `lifecycle.Block(inst, instance.BlockedMoving.Code)`; `defer lifecycle.Unblock(inst)` on all paths. +5. `rep, err := CopyContent(...)`; then `Verify(...)`. On error: return without flipping (instance stays on source), reopened by the deferred Unblock. +6. `DryRun`: return `rep` now without flipping. +7. Flip: `inst.FsScheme = opts.To`; `instance.Update(inst)`. +8. `Unblock` (deferred). +9. `PurgeSource`: after a successful flip, delete source objects (reuse `pkg/s3util.DeletePrefixObjects` for S3 sources, or the Swift container delete for Swift sources). Keep this behind the explicit flag; default off. + +- [ ] **Step 4: Run tests, verify pass** + +Run: `go test ./model/instance/storagemigration/ -run TestMigrate -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/instance/storagemigration/migration.go model/instance/storagemigration/migration_test.go +git commit -m "feat(storagemigration): orchestrate block, copy, verify, flip, rollback" +``` + +--- + +## Task 8: Admin API handler + route + +**Files:** +- Modify: `web/instances/instances.go` (add `migrateStorageHandler`; register in `Routes` ~732) +- Modify: `client/instances.go` (add `MigrateStorage`; extend `InstanceOptions` if needed) +- Test: `web/instances/instances_test.go` (add a case) or an integration test under `tests/` + +**Interfaces:** +- Produces (server): `POST /instances/:domain/migrate-storage` with query params `to`, `dry_run`, `flag_only`, `force`, `purge_source`; returns the JSON `Report`. +- Produces (client): a thin `client.MigrateStorageOptions` struct (mirrors the engine's `Options` fields; keeps the `client` package free of the model import) and `client.MigrateStorageReport`, plus `func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error)`. + +- [ ] **Step 1: Write the failing handler test** + +Add to `web/instances/instances_test.go` a test that creates an instance, POSTs `/instances//migrate-storage?to=s3&dry_run=true`, and asserts 200 + a `Report` body with `files >= 0` and the instance's `fs_scheme` still empty (dry-run). + +- [ ] **Step 2: Run, verify fail** + +Run: `go test ./web/instances/ -run TestMigrateStorage -v` +Expected: FAIL — route/handler missing (404). + +- [ ] **Step 3: Implement the handler + route** + +In `web/instances/instances.go`, mirroring `modifyHandler` (166) and `fsckHandler`: +```go +func migrateStorageHandler(c echo.Context) error { + domain := c.Param("domain") + inst, err := lifecycle.GetInstance(domain) + if err != nil { + return wrapError(err) + } + opts := storagemigration.Options{ + To: c.QueryParam("to"), + DryRun: c.QueryParam("dry_run") == "true", + FlagOnly: c.QueryParam("flag_only") == "true", + Force: c.QueryParam("force") == "true", + PurgeSource: c.QueryParam("purge_source") == "true", + } + rep, err := storagemigration.Migrate(inst, opts) + if err != nil { + return wrapError(err) + } + return c.JSON(http.StatusOK, rep) +} +``` +Register in `Routes`: `router.POST("/:domain/migrate-storage", migrateStorageHandler)`. + +- [ ] **Step 4: Implement the admin client method** + +In `client/instances.go`, mirroring `ModifyInstance` (231): build `url.Values` from the options and `ac.Req(&request.Options{Method: "POST", Path: "/instances/" + domain + "/migrate-storage", Queries: q})`, decode the `Report`. + +- [ ] **Step 5: Run test, verify pass** + +Run: `go test ./web/instances/ -run TestMigrateStorage -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add web/instances/instances.go client/instances.go web/instances/instances_test.go +git commit -m "feat(web/instances): add migrate-storage admin endpoint and client" +``` + +--- + +## Task 9: CLI command + +**Files:** +- Modify: `cmd/instances.go` (add `migrateStorageCmd`, flag vars, register in `init`) +- Test: manual smoke (documented) + reuse existing cmd test harness if present + +**Interfaces:** +- Consumes: `AdminClient.MigrateStorage` (Task 8). + +- [ ] **Step 1: Add flag vars** (near line 47) +```go +var flagMigrateTo string +var flagMigrateDryRun bool +var flagMigrateFlagOnly bool +var flagMigrateForce bool +var flagMigratePurgeSource bool +``` + +- [ ] **Step 2: Define the command** +```go +var migrateStorageCmd = &cobra.Command{ + Use: "migrate-storage ", + Short: "Migrate an instance's file storage to another backend (e.g. s3)", + Long: `cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Usage() + } + ac := newAdminClient() + rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{ + To: flagMigrateTo, + DryRun: flagMigrateDryRun, + FlagOnly: flagMigrateFlagOnly, + Force: flagMigrateForce, + PurgeSource: flagMigratePurgeSource, + }) + if err != nil { + return err + } + fmt.Printf("migrated: %d files, %d versions, %d bytes, avatar=%v\n", + rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied) + return nil + }, +} +``` +(Define a `client.MigrateStorageOptions` mirror struct in `client/instances.go` so the CLI does not import the model package.) + +- [ ] **Step 3: Register + flags** (in `init`) +```go + instanceCmdGroup.AddCommand(migrateStorageCmd) + migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme") + migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching") + migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)") + migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost") + migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch") +``` + +- [ ] **Step 4: Build + smoke** + +Run: `go build ./... && ./cozy-stack instances migrate-storage --help` +Expected: build OK; help lists the flags. + +- [ ] **Step 5: Commit** + +```bash +git add cmd/instances.go client/instances.go +git commit -m "feat(cmd): add instances migrate-storage command" +``` + +--- + +## Task 10: Docs + +**Files:** +- Modify: `docs/config.md` (document `fs.migration_target`) +- Modify: `docs/cli/cozy-stack_instances_migrate-storage.md` (regenerate via the docs generator if the repo autogenerates cobra docs; otherwise add manually) and the S3 doc from the S3 PR (`docs/*s3*`) + +- [ ] **Step 1: Document the config key and the command** + +Add a `fs.migration_target` example to `docs/config.md` and a "Migrating an instance to S3" section describing: configure the target, run `migrate-storage`, verify, `--purge-source` later, and the fleet-wide flip of `fs.url` at the end. + +- [ ] **Step 2: Regenerate CLI docs if applicable** + +Run: `make docs` (or the repo's cobra-doc generator target) and stage the generated file. + +- [ ] **Step 3: Commit** + +```bash +git add docs/ +git commit -m "docs: document fs.migration_target and instances migrate-storage" +``` + +--- + +## Self-review notes (resolved spec open-questions) + +- **Avatar authoritativeness:** confirmed user-uploaded and non-regenerable; a single object at `/avatar` (S3) / `"avatar"` (Swift container). Copied in Task 5 via the new `OpenAvatar` (Task 4). Thumbs are derived → skipped (regenerate on target). +- **Content-copy primitives:** read via exported `OpenFile`/`OpenFileVersion`; write via new index-free `WriteContentAt` (Task 3). Keys via exported `MakeObjectNameV3`/`MakeObjectKey`. Index never rewritten. +- **Read-only mechanism:** `lifecycle.Block`/`Unblock` gate HTTP traffic only (`web/middlewares/instance.go`), not in-process/worker writes. Task 7 blocks with `BlockedMoving` during the window. **Known v1 limitation:** background workers/triggers can still write to the source during the window; run migrations during low activity, or extend later to pause the instance's jobs. Documented as a risk. +- **Config shape:** chose a flat `fs.migration_target` URL (Task 2); the S3 connection is initialized from it at stack startup even while the global scheme stays Swift. +``` diff --git a/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md b/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md new file mode 100644 index 00000000000..7772cdee12c --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md @@ -0,0 +1,253 @@ +# Per-instance storage backend & Swift→S3 migration — Design + +Date: 2026-07-16 +Status: Draft (design approved, spec under review) +Depends on: `feat/s3-vfs-backend` (adds `config.SchemeS3` and the `vfss3` VFS backend) + +## Goal + +Enable migrating a **single live production instance** from the Swift storage +backend to the S3-compatible backend, one instance at a time, without a +big-bang cutover of the whole fleet. The end state is a fleet fully on S3; the +per-instance mechanism is **transitional** and disappears (becomes a no-op) +once the global default is flipped to S3. + +Non-goals (explicitly out of scope): + +- Per-instance S3 **endpoints/credentials**. The target is a single global S3; + we do not build a per-endpoint connection factory. +- Zero-downtime online migration with delta sync. A **read-only maintenance + window** per instance is acceptable for v1. +- Migrating regenerable/ephemeral data (thumbnails, app assets, cache, + exports, archives) — those regenerate or reinstall on the target. + +## Current state (why this is needed) + +Backend selection is **entirely global** today: + +- `Instance.MakeVFS()`, `AvatarFS()`, `ThumbsFS()` + (`model/instance/instance.go:250-333`) switch on + `config.FsURL().Scheme` — the single global `fs.url` scheme. +- There is **no per-instance scheme field**. The only per-instance storage + field is `SwiftLayout` (`swift_cluster`), which selects a Swift *sub-variant*, + not a different backend. +- The S3 connection is a process-wide **singleton** + (`pkg/config/config/s3.go`), keyed by the global `fs.url` query params. + Per-instance isolation on S3 is achieved by bucket-per-org + (`-`) + key-prefix-per-DBPrefix, all against one + global endpoint. +- **No migration tooling between backends exists.** `cozy-stack swift + ls-layouts` only counts; `Fsck` only verifies; `lifecycle` "move" is + cozy-to-cozy server relocation, not backend migration. +- Backend is fixed at instance creation (`lifecycle/create.go:143-153`) and is + **not changeable via patch**. + +## Chosen approach: per-instance flag, in-place, dual-connection stack + +Rejected alternative — *relocate instances to S3-global stacks via the +cozy-to-cozy move machinery*: no core VFS change, but far heavier per instance +(relocation, domains, tokens, routing) for what is only a backend swap. Poor +effort/benefit for a fleet backend migration. + +The stack runs **both** the Swift and the S3 connection during the transition. +Each instance carries a flag selecting its backend; migration copies the +authoritative content to S3 and flips the flag inside a read-only window. + +## Architecture + +### 1. Configuration — dual connection during transition + +`fs.url` stays the **global default** (`swift://…`). Add an optional S3 +migration target so the S3 connection is initialized even while the default is +Swift: + +```yaml +fs: + url: swift://… # global default (unchanged) + migration_target: + url: s3://key:secret@endpoint/?region=…&bucket_prefix=… +``` + +- At startup, if `fs.migration_target` is present, call `InitS3Connection` + (in addition to the default backend init) so both `GetSwiftConnection()` and + `GetS3Client()` singletons are live. +- Add a `config.HasS3Target()` / accessor so the migration command can refuse + to run when no S3 target is configured. +- **End of migration:** operator sets `fs.url` to the `s3://…` URL and removes + `fs.migration_target`. The per-instance flag then equals the global default + and becomes vestigial; migrated instances keep working unchanged. + +The exact config shape (nested `migration_target` vs a flat `fs.s3_url`) is an +implementation detail; the invariant is: **both connections initialized during +the transition period**. + +### 2. Per-instance field + +Add to the `Instance` struct (`model/instance/instance.go`, next to +`SwiftLayout`): + +```go +FsScheme string `json:"fs_scheme,omitempty"` // "" = global default; "s3" = migrated +``` + +- Stored on the `io.cozy.instances` doc (same place as `swift_cluster`). +- Empty for all existing instances → **zero regression**, no data migration of + the instance registry. +- Add an accessor computing the **effective** scheme: + +```go +func (i *Instance) storageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} +``` + +### 3. Backend selection + +`MakeVFS`, `AvatarFS`, `ThumbsFS` (`instance.go:250-333`) read +`i.storageScheme()` instead of `config.FsURL().Scheme`. + +Refactor the currently-triplicated `switch` into a single builder, e.g. +`buildStorage(scheme, kind)` where `kind ∈ {main, avatars, thumbs}`. The +triplication is a real smell in code we are already modifying; consolidating it +keeps the three call sites in lock-step and is the only place the scheme→VFS +mapping lives. + +### 4. Migration engine — object-storage level, index untouched + +The VFS = **CouchDB index/tree** (`io.cozy.files`) + **object storage +content** (bytes keyed by file/version ID). Both backends address content by +ID. Therefore migration **does not copy the index** — only the content bytes, +object by object: + +1. Iterate every content-bearing doc for the instance: + - all `io.cozy.files` **including trashed** files; + - all `io.cozy.files.versions` (old versions live in storage too); + - **uploaded avatars** (authoritative user data — see Open Questions to + confirm the exact storage/doctype). +2. For each: stream content from the **source** storage and `PutObject` into + the **target** (S3) storage under the same logical ID/key. + +This bypasses the high-level `CreateFile` path (which would try to create index +docs) and needs two small primitives per backend: + +- source: "open content by ID" (`vfsswift`/`vfsafero` already expose object + open); +- target: "put content by ID" (`vfss3` already has `PutObject`). + +The engine is **backend-agnostic** (source afero/swift → target s3), which also +makes it testable afero→s3 without a Swift server. + +The migration **does not** copy thumbnails, app/konnector assets, cache, +exports, or archives — these regenerate or reinstall on first access against +the S3 backend after the flip. + +### 5. Migration command + +``` +cozy-stack instances migrate-storage --to s3 [--dry-run] [--purge-source] +``` + +Flow: + +1. Guards: S3 target configured (`config.HasS3Target()`); `--to` differs from + the instance's current effective scheme. +2. **Maintenance ON** (read-only): reject writes for the duration. (Reuse the + existing instance maintenance/blocking mechanism — pin the exact call in the + plan.) +3. Copy content source→S3 (files + versions + trash + avatars). +4. **Verify**: re-walk; every object present in the target with matching size + (and/or MD5); reconcile counts against the index. +5. On success only: set `FsScheme = "s3"`, persist the instance doc. +6. **Maintenance OFF**. + +The flag flips **only after full verification**, so there is never a live, +half-migrated instance. + +`--dry-run`: walk + report counts/sizes without writing or flipping. +`--purge-source`: NOT the default — see Rollback. + +### 6. Rollback command + +Because the source backend is **retained by default**, rollback reuses the same +generic `--to ` engine — there is no separate code path — with two +modes: + +1. **Instant flip-back** — + `cozy-stack instances migrate-storage --to swift --flag-only`: + switches `fs_scheme` back **without copying**, pointing the instance at the + still-present Swift snapshot. Runs in a read-only window and verifies the + source objects are present before flipping. + - Refuses if the source was already purged (`--purge-source` has run). + - Any writes made on S3 **since the cutover are lost** (the Swift snapshot is + stale), so it warns and requires `--force`. Intended for immediate + post-cutover recovery, before real traffic writes to S3. + +2. **Safe re-migration** — + `cozy-stack instances migrate-storage --to swift`: + the same engine in reverse (read-only window, copy S3→Swift, verify, flip). + **No data loss**; use this once real writes have landed on S3. + +`--flag-only` is a general primitive ("switch the pointer to a backend that is +already populated"); it is only meaningful for rollback since the forward +migration must copy first. + +### 7. Safety & idempotence + +- Source data is **kept** by default. `--purge-source` is a **separate, + deferred** step run only after a confidence period. +- Read-only window ⇒ the copied snapshot is consistent (no concurrent writes). +- Copy is **idempotent**: re-running overwrites target objects; a failed run + leaves the flag unchanged (instance still on the source), reopens the + instance, and reports. Partial target objects are overwritten on retry or + removed by `--purge-source` on the target if aborted. + +### 8. Error handling + +- Any failure in copy or verify ⇒ do **not** flip `FsScheme`; reopen the + instance on its original backend; surface the error with the failing + file/version ID. +- Guard against running when the S3 target is unconfigured, or when `--to` + equals the current scheme (no-op). + +### 9. Testing + +- Reuse the MinIO testcontainer already in the VFS suite; use the afero + backend as the migration **source** (no Swift server needed). +- Cases: + - populate an instance with files, multiple versions, trashed files, and an + uploaded avatar; run `migrate-storage --to s3`; assert every content object + exists in S3 with matching size, the CouchDB index is **unchanged**, and + reads are served from S3 after the flip; + - `--dry-run` writes nothing and flips nothing; + - idempotence: running twice yields the same result; + - rollback: `FsScheme` cleared → reads served from source again (while source + retained); + - failure injection mid-copy leaves the instance on the source backend. + +## Open questions to resolve during implementation + +1. **Avatar authoritativeness & storage.** Confirm whether an uploaded avatar + is stored only in `AvatarFS` (authoritative, must be copied) vs derivable. + The design assumes it is authoritative and copies it; verify the exact + doctype/key so the engine enumerates it correctly. +2. **Exact content-copy primitives.** Confirm the object-open (source) and + object-put (target) signatures actually exposed by `vfsswift`, `vfsafero`, + and `vfss3`, and whether a small shared interface is warranted vs + backend-specific helpers. +3. **Maintenance mechanism.** Pin the exact API used to put the instance + read-only for the window (instance blocking/maintenance) and confirm it + rejects writes at the VFS layer, not just the UI. +4. **Config shape.** Decide nested `fs.migration_target.url` vs a flat + `fs.s3_url`, and how `InitS3Connection` is invoked when the global scheme is + still Swift. + +## Rollout sequence (operational) + +1. Deploy stack with `fs.migration_target` configured (both connections live). +2. Migrate instances one at a time with `migrate-storage`, verifying each. +3. After a confidence period, `--purge-source` per migrated instance. +4. Once the whole fleet is on S3, flip `fs.url` to the S3 URL, drop + `fs.migration_target`; `fs_scheme` becomes a no-op that can later be removed. diff --git a/go.mod b/go.mod index bb89af8150d..a0dfd7e5bca 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 github.com/labstack/echo/v4 v4.15.1 github.com/leonelquinteros/gotext v1.7.2 + github.com/minio/minio-go/v7 v7.0.99 github.com/mitchellh/mapstructure v1.5.0 github.com/mssola/user_agent v0.6.0 github.com/ncw/swift/v2 v2.0.3 @@ -103,6 +104,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -121,13 +123,17 @@ require ( github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonas-p/go-shp v0.1.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect @@ -142,6 +148,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -149,6 +156,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -161,6 +169,7 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -185,6 +194,7 @@ require ( go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index c2ebd8c105e..647c2aaad24 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -243,8 +245,13 @@ github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5 github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 h1:abLciEiilfMf19Q1TFWDrp9j5z5one60dnnpvc6eabg= github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666/go.mod h1:xqGOmDZzLOG7+q/CgsbXv10g4tgPsbjhmAxyaTJMvis= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -272,6 +279,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -326,6 +339,8 @@ github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -355,6 +370,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -412,6 +429,8 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -478,6 +497,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/model/app/apps.go b/model/app/apps.go index 34c099a869e..6e4ecb4c74c 100644 --- a/model/app/apps.go +++ b/model/app/apps.go @@ -156,6 +156,10 @@ func Copier(appsType consts.AppType, inst *instance.Instance) appfs.Copier { return appfs.NewAferoCopier(baseFS) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftCopier(config.GetSwiftConnection(), appsType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), appsType) + return appfs.NewS3Copier(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -175,6 +179,10 @@ func AppsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.WebappType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.WebappType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -194,6 +202,10 @@ func KonnectorsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.KonnectorType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.KonnectorType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/instance/instance.go b/model/instance/instance.go index 5a6d4cf6826..761d2806ecd 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -16,6 +16,7 @@ import ( "github.com/cozy/cozy-stack/model/permission" "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" build "github.com/cozy/cozy-stack/pkg/config" "github.com/cozy/cozy-stack/pkg/config/config" @@ -87,6 +88,11 @@ type Instance struct { // See model/vfs/vfsswift for more details. SwiftLayout int `json:"swift_cluster,omitempty"` + // FsScheme, when non-empty, overrides the global fs.url scheme for this + // instance. Used to migrate a single instance to another storage backend + // (e.g. "s3") without changing the stack-wide default. Empty = global default. + FsScheme string `json:"fs_scheme,omitempty"` + CouchCluster int `json:"couch_cluster,omitempty"` // PassphraseHash is a hash of a hash of the user's passphrase: the @@ -186,6 +192,15 @@ func (i *Instance) DBPrefix() string { return i.Domain } +// StorageScheme returns the storage backend scheme effective for this instance: +// the per-instance FsScheme override when set, otherwise the global fs.url scheme. +func (i *Instance) StorageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} + // DomainName returns the main domain name of the instance. func (i *Instance) DomainName() string { return i.Domain @@ -196,6 +211,11 @@ func (i *Instance) GetContextName() string { return i.ContextName } +// GetOrgID returns the organization ID of the instance. +func (i *Instance) GetOrgID() string { + return i.OrgID +} + // SessionSecret returns the session secret. func (i *Instance) SessionSecret() []byte { // The prefix is here to invalidate all the sessions that were created on @@ -246,14 +266,13 @@ func (i *Instance) MakeVFS() error { if i.vfs != nil { return nil } - fsURL := config.FsURL() mutex := config.Lock().ReadWrite(i, "vfs") index := vfs.NewCouchdbIndexer(i) disk := vfs.DiskThresholder(i) var err error - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile, config.SchemeMem: - i.vfs, err = vfsafero.New(i, index, disk, mutex, fsURL, i.DirName()) + i.vfs, err = vfsafero.New(i, index, disk, mutex, config.FsURL(), i.DirName()) case config.SchemeSwift, config.SchemeSwiftSecure: switch i.SwiftLayout { case 2: @@ -261,17 +280,19 @@ func (i *Instance) MakeVFS() error { default: err = ErrInvalidSwiftLayout } + case config.SchemeS3: + i.vfs, err = vfss3.New(i, index, disk, mutex) default: - err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme) + err = fmt.Errorf("instance: unknown storage provider %s", i.StorageScheme()) } return err } // AvatarFS returns the hidden filesystem for storing the avatar. func (i *Instance) AvatarFS() vfs.Avatarer { - fsURL := config.FsURL() - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile: + fsURL := config.FsURL() baseFS := afero.NewBasePathFs(afero.NewOsFs(), path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName)) return vfsafero.NewAvatarFs(baseFS) @@ -285,17 +306,22 @@ func (i *Instance) AvatarFS() vfs.Avatarer { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewAvatarFs(client, bucket, keyPrefix) default: - panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) + panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme())) } } // ThumbsFS returns the hidden filesystem for storing the thumbnails of the // photos/image func (i *Instance) ThumbsFS() vfs.Thumbser { - fsURL := config.FsURL() - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile: + fsURL := config.FsURL() baseFS := afero.NewBasePathFs(afero.NewOsFs(), path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName)) return vfsafero.NewThumbsFs(baseFS) @@ -309,8 +335,13 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewThumbsFs(client, bucket, keyPrefix) default: - panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) + panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme())) } } diff --git a/model/instance/instance_storage_scheme_test.go b/model/instance/instance_storage_scheme_test.go new file mode 100644 index 00000000000..87df01242d2 --- /dev/null +++ b/model/instance/instance_storage_scheme_test.go @@ -0,0 +1,20 @@ +package instance + +import ( + "testing" + + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/stretchr/testify/assert" +) + +func TestStorageSchemeFallsBackToGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{} + assert.Equal(t, config.FsURL().Scheme, i.StorageScheme()) +} + +func TestStorageSchemeOverridesGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{FsScheme: config.SchemeS3} + assert.Equal(t, config.SchemeS3, i.StorageScheme()) +} diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go new file mode 100644 index 00000000000..e6eef52e0a7 --- /dev/null +++ b/model/instance/storagemigration/migration.go @@ -0,0 +1,590 @@ +// Package storagemigration implements the engine that copies an instance's +// object-storage content (files, versions, avatar) from one VFS backend to +// another, without touching the shared CouchDB index. It is used to move an +// instance's files between Swift and S3 — either direction, S3 to Swift as +// well as Swift to S3 — while all other instances sharing the same CouchDB +// cluster keep working against the same io.cozy.files / +// io.cozy.files.versions documents. +package storagemigration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/instance/lifecycle" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/prefixer" + "github.com/cozy/cozy-stack/pkg/s3util" +) + +// contentWriter is implemented by target VFS backends that can write object +// bytes for a (docID, internalID) key without creating a CouchDB document. +type contentWriter interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error +} + +// contentStater is implemented by target VFS backends that can report the +// byte size of the object backing a (docID, internalID) key without +// touching CouchDB. It returns os.ErrNotExist when the object is absent. +type contentStater interface { + StatContentAt(docID, internalID string) (int64, error) +} + +// Report summarizes a content copy performed by CopyContent. +type Report struct { + Files int + Versions int + Bytes int64 + AvatarCopied bool +} + +// CopyContent copies all object-storage content (files including trashed +// ones, file versions, and the avatar) from src to dst. db is only used to +// enumerate the CouchDB documents (io.cozy.files and io.cozy.files.versions) +// that describe what content exists; CopyContent creates or modifies NO +// CouchDB document — it only reads from CouchDB and writes object bytes. +func CopyContent(db prefixer.Prefixer, src, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error) { + writer, ok := dst.(contentWriter) + if !ok { + return nil, fmt.Errorf("storagemigration: target backend does not support index-free writes") + } + + rep := &Report{} + + if err := copyFiles(db, src, writer, rep); err != nil { + return rep, err + } + if err := copyVersions(db, src, writer, rep); err != nil { + return rep, err + } + if err := copyAvatar(srcAv, dstAv, rep); err != nil { + return rep, err + } + + return rep, nil +} + +// Verify re-enumerates the same content that CopyContent copies (files, +// versions, avatar) and confirms each object exists on dst with a byte size +// matching the source CouchDB document, without creating or modifying any +// CouchDB document. It compares the counted totals against expected (the +// Report returned by CopyContent) and returns the FIRST discrepancy found as +// a descriptive error. +func Verify(db prefixer.Prefixer, dst vfs.VFS, dstAv vfs.Avatarer, expected *Report) error { + stater, ok := dst.(contentStater) + if !ok { + return fmt.Errorf("storagemigration: target backend does not support index-free stats") + } + + got := &Report{} + + if err := verifyFiles(db, stater, got); err != nil { + return err + } + if err := verifyVersions(db, stater, got); err != nil { + return err + } + + if got.Files != expected.Files { + return fmt.Errorf("storagemigration: verify: expected %d files, found %d on target", expected.Files, got.Files) + } + if got.Versions != expected.Versions { + return fmt.Errorf("storagemigration: verify: expected %d versions, found %d on target", expected.Versions, got.Versions) + } + + if expected.AvatarCopied { + ar, _, err := dstAv.OpenAvatar() + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: avatar missing on target") + } + if err != nil { + return fmt.Errorf("storagemigration: verify: open target avatar: %w", err) + } + _ = ar.Close() + } + + return nil +} + +// sourceReport enumerates the instance's CouchDB documents (io.cozy.files +// and io.cozy.files.versions) and checks srcAv for an avatar, to compute the +// Report a FlagOnly flip expects the already-populated target to satisfy. It +// does not read or write any object-storage content itself: it describes +// what the source SHOULD have on the target, for Verify to confirm. +func sourceReport(db prefixer.Prefixer, srcAv vfs.Avatarer) (*Report, error) { + rep := &Report{} + + err := couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + rep.Files++ + return nil + }) + if err != nil { + return nil, err + } + + err = couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, _ json.RawMessage) error { + rep.Versions++ + return nil + }) + if err != nil { + return nil, err + } + + ar, _, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + // No avatar on the source: rep.AvatarCopied stays false. + case err != nil: + return nil, fmt.Errorf("storagemigration: open source avatar: %w", err) + default: + _ = ar.Close() + rep.AvatarCopied = true + } + + return rep, nil +} + +// verifyFiles re-enumerates every io.cozy.files document and confirms the +// target object for each non-directory file exists with a matching size. +func verifyFiles(db prefixer.Prefixer, stater contentStater, got *Report) error { + return couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + + size, err := stater.StatContentAt(doc.DocID, doc.InternalID) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: file %s missing on target", doc.DocID) + } + if err != nil { + return fmt.Errorf("storagemigration: verify: stat target file %s: %w", doc.DocID, err) + } + if size != doc.ByteSize { + return fmt.Errorf("storagemigration: verify: file %s size mismatch: expected %d, got %d", doc.DocID, doc.ByteSize, size) + } + + got.Files++ + return nil + }) +} + +// verifyVersions re-enumerates every io.cozy.files.versions document and +// confirms the target object for each version exists with a matching size. +func verifyVersions(db prefixer.Prefixer, stater contentStater, got *Report) error { + return couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return fmt.Errorf("storagemigration: decode version doc: %w", err) + } + + fileID, internalID := splitVersionID(ver.DocID) + + size, err := stater.StatContentAt(fileID, internalID) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: version %s missing on target", ver.DocID) + } + if err != nil { + return fmt.Errorf("storagemigration: verify: stat target version %s: %w", ver.DocID, err) + } + if size != ver.ByteSize { + return fmt.Errorf("storagemigration: verify: version %s size mismatch: expected %d, got %d", ver.DocID, ver.ByteSize, size) + } + + got.Versions++ + return nil + }) +} + +// copyFiles enumerates every io.cozy.files document (ForeachDocs is +// unfiltered, so trashed files are naturally included) and copies the +// content of each file (skipping directories) from src to dst. +func copyFiles(db prefixer.Prefixer, src vfs.VFS, writer contentWriter, rep *Report) error { + return couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + + r, err := src.OpenFile(&doc) + if err != nil { + return fmt.Errorf("storagemigration: open source file %s: %w", doc.DocID, err) + } + defer r.Close() + + if err := writer.WriteContentAt(doc.DocID, doc.InternalID, r, doc.ByteSize); err != nil { + return fmt.Errorf("storagemigration: write target file %s: %w", doc.DocID, err) + } + + rep.Files++ + rep.Bytes += doc.ByteSize + return nil + }) +} + +// copyVersions enumerates every io.cozy.files.versions document and copies +// the content of each version from src to dst. +func copyVersions(db prefixer.Prefixer, src vfs.VFS, writer contentWriter, rep *Report) error { + return couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return fmt.Errorf("storagemigration: decode version doc: %w", err) + } + + fileID, internalID := splitVersionID(ver.DocID) + + fileDoc, err := src.FileByID(fileID) + if err != nil { + return fmt.Errorf("storagemigration: file for version %s: %w", ver.DocID, err) + } + + r, err := src.OpenFileVersion(fileDoc, &ver) + if err != nil { + return fmt.Errorf("storagemigration: open source version %s: %w", ver.DocID, err) + } + defer r.Close() + + if err := writer.WriteContentAt(fileID, internalID, r, ver.ByteSize); err != nil { + return fmt.Errorf("storagemigration: write target version %s: %w", ver.DocID, err) + } + + rep.Versions++ + rep.Bytes += ver.ByteSize + return nil + }) +} + +// copyAvatar copies the instance's avatar, if any, from srcAv to dstAv, +// setting rep.AvatarCopied on success. The absence of an avatar +// (os.ErrNotExist) is not an error. +func copyAvatar(srcAv, dstAv vfs.Avatarer, rep *Report) error { + ar, ctype, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + return nil + case err != nil: + return fmt.Errorf("storagemigration: open source avatar: %w", err) + } + defer ar.Close() + + w, err := dstAv.CreateAvatar(ctype) + if err != nil { + return fmt.Errorf("storagemigration: create target avatar: %w", err) + } + + if _, err := io.Copy(w, ar); err != nil { + _ = w.Close() + return fmt.Errorf("storagemigration: copy avatar: %w", err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("storagemigration: finalize target avatar: %w", err) + } + + rep.AvatarCopied = true + return nil +} + +// splitVersionID splits a io.cozy.files.versions document id +// ("/") into its fileID and internalID parts. +func splitVersionID(versionDocID string) (fileID, internalID string) { + if i := strings.IndexByte(versionDocID, '/'); i >= 0 { + return versionDocID[:i], versionDocID[i+1:] + } + return versionDocID, "" +} + +// Options configures a call to Migrate. +type Options struct { + // To is the target storage scheme: config.SchemeS3 or config.SchemeSwift. + To string + // DryRun copies and verifies the content on the target backend but does + // not flip the instance's FsScheme: the instance keeps serving reads and + // writes from its current (source) backend. Combined with FlagOnly, it + // still verifies the already-populated target but likewise never flips. + DryRun bool + // FlagOnly switches the instance's FsScheme pointer to an already + // populated target backend without copying anything. It is intended for + // rollback (switching back to a backend that a previous migration left + // populated) and requires Force, since any write performed against the + // source since that previous cutover is lost. + FlagOnly bool + // Force is required together with FlagOnly, acknowledging the data-loss + // risk described above. + Force bool + // PurgeSource deletes the source backend's objects after a successful + // flip. It is a best-effort cleanup performed once the instance is + // already fully served from the target: a failure here does not revert + // the flip. + // + // If To already equals the instance's current storage scheme, + // PurgeSource switches Migrate into purge-only mode: nothing is copied, + // verified, or flipped, and only the OTHER backend's leftover data for + // this instance is deleted. This is what makes a deferred reclaim + // (running --purge-source well after the flip, or retrying an inline + // purge that failed) possible. + PurgeSource bool +} + +// containerNamer is implemented by VFS backends (vfsswift's V3 layout) that +// expose the underlying object-storage container name(s) they use, so +// storagemigration can ensure the target container exists without +// hand-rolling the naming scheme itself. +type containerNamer interface { + ContainerNames() map[string]string +} + +// Migrate moves an instance's object-storage content (files, versions, +// avatar) from its current backend to opts.To, verifies the copy, and only +// then flips the instance's FsScheme to the target. The instance is blocked +// (instance.BlockedMoving) for the duration of the copy/verify and unblocked +// on every return path. +// +// FsScheme is updated ONLY after Verify succeeds (or, for FlagOnly, after the +// target backend has been validated); a DryRun or a failed Verify always +// leaves FsScheme unchanged. +// +// If opts.To already equals the instance's current storage scheme AND +// opts.PurgeSource is set, Migrate runs in purge-only mode instead: see +// purgeOnly. Without PurgeSource, opts.To == the current scheme is still an +// error. +func Migrate(inst *instance.Instance, opts Options) (*Report, error) { + switch opts.To { + case config.SchemeS3, config.SchemeSwift: + default: + return nil, fmt.Errorf("storagemigration: unsupported target scheme %q", opts.To) + } + + srcScheme := inst.StorageScheme() + if opts.To == srcScheme { + if opts.PurgeSource { + // Purge-only mode: the instance is already on opts.To (either + // because a previous migration flipped it, or the caller is + // retrying an inline purge that failed after that flip). There + // is nothing to copy, verify, or flip: just reclaim the OTHER + // backend's leftover data for this instance. + return purgeOnly(inst, opts.To) + } + return nil, fmt.Errorf("storagemigration: instance %s already uses %q as its storage scheme", inst.DomainName(), opts.To) + } + + if opts.To == config.SchemeS3 && !config.HasS3Client() { + return nil, errors.New("storagemigration: cannot migrate to s3: no S3 client is configured") + } + if opts.To == config.SchemeSwift && !config.HasSwiftConnection() { + return nil, errors.New("storagemigration: cannot migrate to swift: no swift connection is configured") + } + + if srcScheme == config.SchemeSwift || srcScheme == config.SchemeSwiftSecure { + if inst.SwiftLayout != 2 { + return nil, fmt.Errorf("storagemigration: source swift layout %d is not supported, only layout 2 (v3) can be migrated", inst.SwiftLayout) + } + } + + if opts.FlagOnly && !opts.Force { + return nil, errors.New("storagemigration: flag-only migration requires Force: any write performed against the source since the previous cutover would be lost") + } + + // Build the SOURCE from the instance's current backend before touching + // anything (the instance already knows how to build it for its current + // scheme). + src := inst.VFS() + srcAv := inst.AvatarFS() + + dst, dstAv, err := buildTarget(inst, opts.To) + if err != nil { + return nil, err + } + + if err := lifecycle.Block(inst, instance.BlockedMoving.Code); err != nil { + return nil, fmt.Errorf("storagemigration: block instance: %w", err) + } + defer func() { + _ = lifecycle.Unblock(inst) + }() + + if opts.FlagOnly { + // FlagOnly does not copy anything, but it must not flip onto a + // target that does not already hold the source's content: an + // unpopulated (or merely-existing, e.g. freshly EnsureBucket'd) + // target would otherwise silently strand the instance on zero + // files. Compute the expected counts from the source and verify + // the target really has them before flipping. + expected, err := sourceReport(inst, srcAv) + if err != nil { + return nil, err + } + if err := Verify(inst, dst, dstAv, expected); err != nil { + return expected, err + } + if opts.DryRun { + return expected, nil + } + return flip(inst, opts, srcScheme, expected) + } + + rep, err := CopyContent(inst, src, dst, srcAv, dstAv) + if err != nil { + return rep, err + } + if err := Verify(inst, dst, dstAv, rep); err != nil { + return rep, err + } + + if opts.DryRun { + return rep, nil + } + + return flip(inst, opts, srcScheme, rep) +} + +// buildTarget constructs the VFS + Avatarer pair for the target scheme, +// ensuring the underlying bucket/container exists, without touching the +// CouchDB index (no InitFs: the index is shared with the source and must not +// be reinitialized). +func buildTarget(inst *instance.Instance, to string) (vfs.VFS, vfs.Avatarer, error) { + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migration-target") + + switch to { + case config.SchemeS3: + dst, err := vfss3.New(inst, index, disk, mutex) + if err != nil { + return nil, nil, fmt.Errorf("storagemigration: build s3 target: %w", err) + } + bucket := vfss3.BucketName(inst.GetOrgID(), config.GetS3BucketPrefix()) + if err := s3util.EnsureBucket(context.Background(), config.GetS3Client(), bucket, config.GetS3Region()); err != nil { + return nil, nil, fmt.Errorf("storagemigration: ensure target bucket: %w", err) + } + dstAv := vfss3.NewAvatarFs(config.GetS3Client(), bucket, inst.DBPrefix()+"/") + return dst, dstAv, nil + + case config.SchemeSwift: + dst, err := vfsswift.NewV3(inst, index, disk, mutex) + if err != nil { + return nil, nil, fmt.Errorf("storagemigration: build swift target: %w", err) + } + if cn, ok := dst.(containerNamer); ok { + if container, ok := cn.ContainerNames()["container"]; ok && container != "" { + if err := config.GetSwiftConnection().ContainerCreate(context.Background(), container, nil); err != nil { + return nil, nil, fmt.Errorf("storagemigration: ensure target container: %w", err) + } + } + } + dstAv := vfsswift.NewAvatarFsV3(config.GetSwiftConnection(), inst) + return dst, dstAv, nil + + default: + return nil, nil, fmt.Errorf("storagemigration: unsupported target scheme %q", to) + } +} + +// flip persists the FsScheme change to the target scheme and, if requested, +// purges the source backend's objects on a best-effort basis. It never +// reverts the flip: once the instance points at the target, the target is +// the source of truth for the instance's content. +func flip(inst *instance.Instance, opts Options, srcScheme string, rep *Report) (*Report, error) { + inst.FsScheme = opts.To + if err := instance.Update(inst); err != nil { + return rep, fmt.Errorf("storagemigration: persist storage scheme flip: %w", err) + } + + if opts.PurgeSource { + if err := purgeSource(inst, srcScheme); err != nil { + return rep, fmt.Errorf("storagemigration: purge source after flip: %w", err) + } + } + + return rep, nil +} + +// purgeSource best-effort deletes the source backend's objects after a +// successful flip. The instance already fully serves reads/writes from the +// target at this point, so a purge failure is reported but never reverts the +// flip. +func purgeSource(inst *instance.Instance, srcScheme string) error { + switch srcScheme { + case config.SchemeS3: + bucket := vfss3.BucketName(inst.GetOrgID(), config.GetS3BucketPrefix()) + prefix := inst.DBPrefix() + "/" + return s3util.DeletePrefixObjects(context.Background(), config.GetS3Client(), bucket, prefix) + case config.SchemeSwift, config.SchemeSwiftSecure: + // The v3 swift layout uses a single, per-instance container (see + // vfsswift.NewV3), so purging the source is just deleting that + // container: build the same source VFS instance destroy/reset use + // (see lifecycle.destroy/reset calling inst.VFS().Delete()) and + // reuse its Delete(), which marks the container to-be-deleted and + // removes all its objects before removing the container itself. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migration-purge-source") + src, err := vfsswift.NewV3(inst, index, disk, mutex) + if err != nil { + return fmt.Errorf("storagemigration: build swift source for purge: %w", err) + } + return src.Delete() + default: + return fmt.Errorf("storagemigration: purging source scheme %q is not implemented", srcScheme) + } +} + +// purgeOnly implements Migrate's purge-only mode: opts.To already equals the +// instance's current storage scheme, so there is nothing to copy, verify, or +// flip. It only deletes the OTHER backend's (still-retained) leftover data +// for this instance, which is what makes the documented deferred reclaim +// step (running --purge-source well after the flip) work, and also gives a +// retry path when an inline purge failed after a previous flip. The active +// backend (to) is never touched and the instance is not blocked, since reads +// and writes against it are unaffected. +func purgeOnly(inst *instance.Instance, to string) (*Report, error) { + var otherScheme string + switch to { + case config.SchemeS3: + otherScheme = config.SchemeSwift + case config.SchemeSwift: + otherScheme = config.SchemeS3 + default: + return nil, fmt.Errorf("storagemigration: unsupported target scheme %q", to) + } + + switch otherScheme { + case config.SchemeS3: + if !config.HasS3Client() { + return nil, errors.New("storagemigration: cannot purge s3: no S3 client is configured") + } + case config.SchemeSwift: + if !config.HasSwiftConnection() { + return nil, errors.New("storagemigration: cannot purge swift: no swift connection is configured") + } + } + + if err := purgeSource(inst, otherScheme); err != nil { + return nil, fmt.Errorf("storagemigration: purge-only: %w", err) + } + + return &Report{}, nil +} diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go new file mode 100644 index 00000000000..d50c5542413 --- /dev/null +++ b/model/instance/storagemigration/migration_test.go @@ -0,0 +1,595 @@ +package storagemigration_test + +import ( + "bytes" + "context" + "crypto/md5" + "errors" + "io" + "net/url" + "testing" + "time" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/instance/storagemigration" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + swiftv2 "github.com/ncw/swift/v2" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// migrationPrefixer is a minimal vfs.Prefixer (+ GetOrgID, required by +// vfss3.New's bucket-name derivation) implementation local to this external +// test package, mirroring model/vfs/vfs_test.go's contexter fixture. +type migrationPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *migrationPrefixer) DBCluster() int { return p.cluster } +func (p *migrationPrefixer) DomainName() string { return p.domain } +func (p *migrationPrefixer) DBPrefix() string { return p.prefix } +func (p *migrationPrefixer) GetContextName() string { return p.context } +func (p *migrationPrefixer) GetOrgID() string { return "migrationtestorg" } + +// migrationDisk is a minimal vfs.DiskThresholder (unlimited quota). +type migrationDisk struct{} + +func (migrationDisk) DiskQuota() int64 { return 0 } + +// migrationFixture bundles the source (afero) and target (s3) VFS + Avatarer +// pairs, plus the shared prefixer/db handle used to run CopyContent and to +// inspect CouchDB directly for the index-unchanged assertion. +type migrationFixture struct { + db *migrationPrefixer + src vfs.VFS + dst vfs.VFS + + srcAv vfs.Avatarer + dstAv vfs.Avatarer + + // minioClient and bucket give tests raw access to the S3 target, e.g. to + // delete an object behind the target VFS's back for negative-path checks. + minioClient *minio.Client + bucket string +} + +func setupMigrationFixture(t *testing.T) *migrationFixture { + t.Helper() + + config.UseTestFile(t) + + db := &migrationPrefixer{ + cluster: 0, + domain: "io.cozy.storagemigration.test", + prefix: "io.cozy.storagemigration.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + require.NoError(t, couchdb.ResetDB(db, consts.FilesVersions)) + t.Cleanup(func() { + _ = couchdb.DeleteDB(db, consts.Files) + _ = couchdb.DeleteDB(db, consts.FilesVersions) + }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + + // Source: afero-backed VFS on a temp dir. + tempdir := t.TempDir() + aferoMutex := config.Lock().ReadWrite(db, "storagemigration-test-afero") + aferoURL := &url.URL{Scheme: "file", Host: "localhost", Path: tempdir} + src, err := vfsafero.New(db, index, &migrationDisk{}, aferoMutex, aferoURL, "io.cozy.vfs.test") + require.NoError(t, err) + require.NoError(t, src.InitFs()) + + baseFS := afero.NewBasePathFs(afero.NewOsFs(), tempdir) + srcAv := vfsafero.NewAvatarFs(baseFS) + + // Target: S3-backed VFS against a MinIO test container. + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + s3Mutex := config.Lock().ReadWrite(db, "storagemigration-test-s3") + dst, err := vfss3.New(db, index, &migrationDisk{}, s3Mutex) + require.NoError(t, err) + + bucket := vfss3.BucketName(db.GetOrgID(), config.GetS3BucketPrefix()) + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + keyPrefix := db.DBPrefix() + "/" + dstAv := vfss3.NewAvatarFs(client, bucket, keyPrefix) + + return &migrationFixture{ + db: db, + src: src, + dst: dst, + srcAv: srcAv, + dstAv: dstAv, + minioClient: client, + bucket: bucket, + } +} + +// createSourceFile creates a file of the given name/content on the source +// VFS and returns its FileDoc. +func createSourceFile(t *testing.T, fx *migrationFixture, name string, content []byte) *vfs.FileDoc { + t.Helper() + + doc, err := vfs.NewFileDoc(name, "", int64(len(content)), nil, "text/plain", "text", time.Now(), false, false, false, []string{}) + require.NoError(t, err) + + f, err := fx.src.CreateFile(doc, nil) + require.NoError(t, err) + + _, err = io.Copy(f, bytes.NewReader(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + got, err := fx.src.FileByPath("/" + name) + require.NoError(t, err) + return got +} + +func TestCopyContentMovesFilesVersionsAndAvatar(t *testing.T) { + fx := setupMigrationFixture(t) + + // 2 live files. + file1 := createSourceFile(t, fx, "file1.txt", []byte("hello from file 1")) + file2 := createSourceFile(t, fx, "file2.txt", []byte("hello from file 2, a bit longer")) + + // 1 file that gets trashed (content copy must still include it, since + // ForeachDocs is unfiltered). + file3 := createSourceFile(t, fx, "file3.txt", []byte("this one goes to the trash")) + file3, err := vfs.TrashFile(fx.src, file3) + require.NoError(t, err) + + // 1 extra version on file1. + versionPayload := []byte("an older revision of file 1") + sum := md5.Sum(versionPayload) + internalID := utils.RandomString(16) + version := &vfs.Version{ + DocID: file1.DocID + "/" + internalID, + ByteSize: int64(len(versionPayload)), + MD5Sum: sum[:], + } + version.Rels.File.Data.ID = file1.DocID + require.NoError(t, fx.src.ImportFileVersion(version, io.NopCloser(bytes.NewReader(versionPayload)))) + + // Avatar. + avatarPayload := []byte("fake png bytes for the avatar") + aw, err := fx.srcAv.CreateAvatar("image/png") + require.NoError(t, err) + _, err = aw.Write(avatarPayload) + require.NoError(t, err) + require.NoError(t, aw.Close()) + + // Capture revs before the copy: CopyContent must not touch the index. + revBefore1 := file1.Rev() + revBefore2 := file2.Rev() + revBefore3 := file3.Rev() + + rep, err := storagemigration.CopyContent(fx.db, fx.src, fx.dst, fx.srcAv, fx.dstAv) + require.NoError(t, err) + + assert.Equal(t, 3, rep.Files) // 2 live + 1 trashed + assert.Equal(t, 1, rep.Versions) + assert.True(t, rep.AvatarCopied) + + // Every source file's bytes are now readable from the target VFS. + assertFileContentOn(t, fx.dst, file1, []byte("hello from file 1")) + assertFileContentOn(t, fx.dst, file2, []byte("hello from file 2, a bit longer")) + assertFileContentOn(t, fx.dst, file3, []byte("this one goes to the trash")) + + // The version is readable via the target VFS. + vr, err := fx.dst.OpenFileVersion(file1, version) + require.NoError(t, err) + gotVersion, err := io.ReadAll(vr) + require.NoError(t, err) + require.NoError(t, vr.Close()) + assert.Equal(t, versionPayload, gotVersion) + + // The avatar is readable via the target avatarer. Note: the source + // avatarer is afero-backed, which does not persist a content-type on + // disk and always reports "application/octet-stream" from OpenAvatar + // (see vfsafero's OpenAvatar); CopyContent faithfully forwards whatever + // content-type srcAv.OpenAvatar() reports to dstAv.CreateAvatar(), so + // that is what ends up stored on the target too. + ar, ctype, err := fx.dstAv.OpenAvatar() + require.NoError(t, err) + gotAvatar, err := io.ReadAll(ar) + require.NoError(t, err) + require.NoError(t, ar.Close()) + assert.Equal(t, "application/octet-stream", ctype) + assert.Equal(t, avatarPayload, gotAvatar) + + // The CouchDB index is unchanged: same revs as before the copy. + reread1 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file1.DocID, reread1)) + reread2 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file2.DocID, reread2)) + reread3 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file3.DocID, reread3)) + + assert.Equal(t, revBefore1, reread1.Rev()) + assert.Equal(t, revBefore2, reread2.Rev()) + assert.Equal(t, revBefore3, reread3.Rev()) +} + +func TestVerifySucceedsAfterCopyAndFailsWhenObjectMissing(t *testing.T) { + fx := setupMigrationFixture(t) + + file1 := createSourceFile(t, fx, "file1.txt", []byte("hello from file 1")) + _ = createSourceFile(t, fx, "file2.txt", []byte("hello from file 2, a bit longer")) + + rep, err := storagemigration.CopyContent(fx.db, fx.src, fx.dst, fx.srcAv, fx.dstAv) + require.NoError(t, err) + + require.NoError(t, storagemigration.Verify(fx.db, fx.dst, fx.dstAv, rep)) + + // Remove one known target object directly via the raw MinIO client, then + // confirm Verify now detects the discrepancy. + keyPrefix := fx.db.DBPrefix() + "/" + objKey := vfss3.MakeObjectKey(keyPrefix, file1.DocID, file1.InternalID) + + require.NoError(t, fx.minioClient.RemoveObject(context.Background(), fx.bucket, objKey, minio.RemoveObjectOptions{})) + + assert.Error(t, storagemigration.Verify(fx.db, fx.dst, fx.dstAv, rep)) +} + +func assertFileContentOn(t *testing.T, fs vfs.VFS, doc *vfs.FileDoc, want []byte) { + t.Helper() + r, err := fs.OpenFile(doc) + require.NoError(t, err) + got, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + assert.Equal(t, want, got) +} + +// setupMigrateInstance creates a real instance (via testutils, on the global +// test backend, "mem") and populates it with a couple of files and an +// avatar, then starts a MinIO test server and wires up the global S3 client +// so config.HasS3Client() is true and Migrate can build an S3 target. +func setupMigrateInstance(t *testing.T) *instance.Instance { + t.Helper() + + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + inst := setup.GetTestInstance() + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "migrate-file1.txt", []byte("hello from migrate file 1")) + createInstanceFile(t, inst, "migrate-file2.txt", []byte("hello from migrate file 2, a bit longer")) + + aw, err := inst.AvatarFS().CreateAvatar("image/png") + require.NoError(t, err) + _, err = aw.Write([]byte("fake png bytes for the migrate avatar")) + require.NoError(t, err) + require.NoError(t, aw.Close()) + + return inst +} + +// createInstanceFile creates a file of the given name/content on the +// instance's current VFS. +func createInstanceFile(t *testing.T, inst *instance.Instance, name string, content []byte) *vfs.FileDoc { + t.Helper() + + doc, err := vfs.NewFileDoc(name, "", int64(len(content)), nil, "text/plain", "text", time.Now(), false, false, false, []string{}) + require.NoError(t, err) + + f, err := inst.VFS().CreateFile(doc, nil) + require.NoError(t, err) + + _, err = io.Copy(f, bytes.NewReader(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + got, err := inst.VFS().FileByPath("/" + name) + require.NoError(t, err) + return got +} + +func TestMigrateFlipsSchemeAfterVerify(t *testing.T) { + inst := setupMigrateInstance(t) + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.NotNil(t, rep) + + assert.Equal(t, config.SchemeS3, inst.FsScheme) + assert.Greater(t, rep.Files, 0) + assert.False(t, inst.Blocked, "instance must be unblocked after a successful migration") + + // Reads are now served from S3: build a fresh S3 VFS for the instance + // (mirroring what inst.VFS() would now build) and confirm the migrated + // files are readable from it. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migrate-test-read") + s3fs, err := vfss3.New(inst, index, disk, mutex) + require.NoError(t, err) + + doc1, err := s3fs.FileByPath("/migrate-file1.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc1, []byte("hello from migrate file 1")) + + doc2, err := s3fs.FileByPath("/migrate-file2.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc2, []byte("hello from migrate file 2, a bit longer")) +} + +func TestMigrateDryRunDoesNotFlip(t *testing.T) { + inst := setupMigrateInstance(t) + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, DryRun: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Greater(t, rep.Files, 0) + + assert.Equal(t, "", inst.FsScheme) + assert.False(t, inst.Blocked, "instance must be unblocked after a dry-run migration") +} + +func TestMigrateFlagOnlyRequiresForce(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true}) + require.Error(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +// TestMigrateFlagOnlyFlipsWhenTargetPopulated covers the CRITICAL fix: a +// FlagOnly+Force flip must succeed (and actually flip) once the target +// backend genuinely already holds the source's content. +func TestMigrateFlagOnlyFlipsWhenTargetPopulated(t *testing.T) { + inst := setupMigrateInstance(t) + + // Populate the S3 target for real once, so it already contains the + // instance's full content (2 files + avatar) by the time the flag-only + // flip below relies on it. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Simulate a rollback scenario: the instance is pointed back at its + // (still fully intact, never purged) previous scheme, and we now want + // to flip it back onto the S3 target without recopying anything, since + // that target is already fully populated from the migration above. + inst.FsScheme = "" + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true, Force: true}) + require.NoError(t, err) + require.NotNil(t, rep) + + assert.Equal(t, config.SchemeS3, inst.FsScheme) + assert.Equal(t, 2, rep.Files) + assert.True(t, rep.AvatarCopied) +} + +// TestMigrateFlagOnlyFailsWhenTargetEmpty covers the CRITICAL fix's negative +// path: a FlagOnly+Force flip against a target that only exists (e.g. an +// empty bucket created by buildTarget's EnsureBucket call) but does not +// actually hold the source's content must fail, and must NOT flip +// FsScheme. +func TestMigrateFlagOnlyFailsWhenTargetEmpty(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true, Force: true}) + require.Error(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +// TestMigratePurgeSourceRemovesSourceObjects covers the IMPORTANT fix: a +// swift source must actually be purged (not return a "not implemented" +// error) after a successful flip. It exercises the real swift-source purge +// path end-to-end: an instance is first migrated from mem to a real +// (in-memory swifttest server) swift backend, populating swift for real; +// it is then migrated from swift to S3 with PurgeSource, and the test +// confirms the swift container backing the instance is gone afterward. +func TestMigratePurgeSourceRemovesSourceObjects(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + setup.SetupSwiftTest() + inst := setup.GetTestInstance() + + // GetTestInstance created this instance against the test config's + // default (non-swift) scheme, so it was never assigned a swift layout. + // Migrate requires layout v3 for any swift source (see the SwiftLayout + // guard in Migrate), so set it explicitly here to simulate a real + // swift-scheme instance, as would exist in production. + inst.SwiftLayout = 2 + require.NoError(t, instance.Update(inst)) + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "purge-file1.txt", []byte("hello from purge file 1")) + createInstanceFile(t, inst, "purge-file2.txt", []byte("hello from purge file 2, a bit longer")) + + // Step 1: migrate mem -> swift for real, so the swift container backing + // this instance is genuinely populated. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeSwift}) + require.NoError(t, err) + require.Equal(t, config.SchemeSwift, inst.FsScheme) + + containerName := swiftContainerName(t, inst) + + // Sanity check: the container really exists before the purge. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + require.NoError(t, err, "the swift container must exist after the first migration") + + // Step 2: migrate swift -> S3 with PurgeSource, exercising the swift + // source purge implementation. + _, err = storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, PurgeSource: true}) + require.NoError(t, err) + assert.Equal(t, config.SchemeS3, inst.FsScheme) + + // The swift container must be gone now: purgeSource must have actually + // deleted it, not returned a "not implemented" error after a + // successful (and now unrevertable) flip. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + assert.True(t, errors.Is(err, swiftv2.ContainerNotFound), "expected the swift container to be gone after purge, got err=%v", err) +} + +// swiftContainerName builds the same per-instance swift V3 container that +// buildTarget/purgeSource use, so tests can inspect it directly against the +// swift connection. +func swiftContainerName(t *testing.T, inst *instance.Instance) string { + t.Helper() + + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "storagemigration-test-swift-container-name") + + sfs, err := vfsswift.NewV3(inst, index, disk, mutex) + require.NoError(t, err) + + cn, ok := sfs.(interface{ ContainerNames() map[string]string }) + require.True(t, ok, "vfsswift.NewV3 must expose ContainerNames()") + + return cn.ContainerNames()["container"] +} + +// TestMigratePurgeOnlyReclaimsOtherBackend covers the CRITICAL fix: once an +// instance already sits on its target scheme (a previous migration flipped +// it, and the Swift source was deliberately retained for rollback, as +// docs/s3.md step 4 describes), a later call with PurgeSource and the SAME +// To must not hit the "already uses that scheme" guard. Instead it must run +// in purge-only mode: reclaim the other backend's leftover data without +// copying, verifying, or flipping anything. +func TestMigratePurgeOnlyReclaimsOtherBackend(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + setup.SetupSwiftTest() + inst := setup.GetTestInstance() + + // See TestMigratePurgeSourceRemovesSourceObjects: a swift source requires + // layout v3 to be migrated. + inst.SwiftLayout = 2 + require.NoError(t, instance.Update(inst)) + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "purge-only-file1.txt", []byte("hello from purge-only file 1")) + createInstanceFile(t, inst, "purge-only-file2.txt", []byte("hello from purge-only file 2, a bit longer")) + + // Step 1: migrate mem -> swift for real, so the swift container backing + // this instance is genuinely populated. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeSwift}) + require.NoError(t, err) + require.Equal(t, config.SchemeSwift, inst.FsScheme) + + containerName := swiftContainerName(t, inst) + + // Step 2: migrate swift -> S3 WITHOUT PurgeSource, so the instance ends + // on S3 while the swift source is deliberately retained, exactly as + // docs/s3.md's rollback window describes. + _, err = storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Sanity check: the retained swift container still exists after the + // flip, since PurgeSource was not requested. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + require.NoError(t, err, "the swift container must still exist: PurgeSource was not requested on the flip") + + // Step 3 (the deferred reclaim, run later): call Migrate again with + // To == the instance's CURRENT scheme (s3) and PurgeSource set. This + // must not error out on the "already uses that scheme" guard; it must + // instead purge the other backend (swift) and leave the instance as-is. + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, PurgeSource: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, config.SchemeS3, inst.FsScheme, "purge-only must not change the instance's scheme") + assert.False(t, inst.Blocked, "purge-only must not leave the instance blocked") + + // The swift container must be gone now. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + assert.True(t, errors.Is(err, swiftv2.ContainerNotFound), "expected the swift container to be gone after purge-only, got err=%v", err) + + // The instance's S3 content (the active backend) must be untouched. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migrate-test-purge-only-read") + s3fs, err := vfss3.New(inst, index, disk, mutex) + require.NoError(t, err) + doc1, err := s3fs.FileByPath("/purge-only-file1.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc1, []byte("hello from purge-only file 1")) +} + +// TestMigratePurgeOnlyWithoutPurgeFlagStillErrors covers the guard that must +// still hold for a plain re-run against the current scheme without +// PurgeSource: purge-only mode is only entered when PurgeSource is set. +func TestMigratePurgeOnlyWithoutPurgeFlagStillErrors(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + _, err = storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.Error(t, err) + assert.Equal(t, config.SchemeS3, inst.FsScheme) +} + +// TestMigrateFlagOnlyDryRunDoesNotFlip covers the IMPORTANT fix: combining +// FlagOnly with DryRun must still verify the target, but must NOT flip +// FsScheme, even with Force set. +func TestMigrateFlagOnlyDryRunDoesNotFlip(t *testing.T) { + inst := setupMigrateInstance(t) + + // Populate the S3 target for real once, so it already contains the + // instance's full content by the time the flag-only dry-run below + // relies on it. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Reset the scheme, as a rollback scenario would have it, then attempt a + // flag-only flip back onto S3 as a dry run. + inst.FsScheme = "" + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true, Force: true, DryRun: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, 2, rep.Files) + + assert.Equal(t, "", inst.FsScheme, "a dry-run flag-only migration must not flip FsScheme") + assert.False(t, inst.Blocked, "instance must be unblocked after a dry-run flag-only migration") +} diff --git a/model/move/archiver.go b/model/move/archiver.go index be36ccb9b7e..03bb896ac3a 100644 --- a/model/move/archiver.go +++ b/model/move/archiver.go @@ -14,6 +14,7 @@ import ( "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/crypto" multierror "github.com/hashicorp/go-multierror" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -45,6 +46,8 @@ func SystemArchiver() Archiver { return newAferoArchiver(fs) case config.SchemeSwift, config.SchemeSwiftSecure: return newSwiftArchiver() + case config.SchemeS3: + return newS3Archiver() default: panic(fmt.Errorf("exports: unknown storage provider %s", fsURL.Scheme)) } @@ -149,3 +152,84 @@ func (a *switfArchiver) RemoveArchives(exportDocs []*ExportDoc) error { } return nil } + +func newS3Archiver() Archiver { + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-exports" + return &s3Archiver{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +type s3Archiver struct { + client *minio.Client + bucket string + ctx context.Context +} + +func (a *s3Archiver) ensureBucket() error { + err := a.client.MakeBucket(a.ctx, a.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (a *s3Archiver) OpenArchive(inst *instance.Instance, exportDoc *ExportDoc) (io.ReadCloser, error) { + objectName := exportDoc.Domain + "/" + exportDoc.ID() + obj, err := a.client.GetObject(a.ctx, a.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Verify the object exists by calling Stat; GetObject itself does not + // perform a network request until Read is called, and Stat triggers + // a HEAD that surfaces NoSuchKey early. + if _, err := obj.Stat(); err != nil { + obj.Close() + return nil, err + } + return obj, nil +} + +func (a *s3Archiver) CreateArchive(exportDoc *ExportDoc) (io.WriteCloser, error) { + if err := a.ensureBucket(); err != nil { + return nil, err + } + + objectName := exportDoc.Domain + "/" + exportDoc.ID() + pr, pw := io.Pipe() + + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, objectName, + pr, -1, + minio.PutObjectOptions{ContentType: "application/tar+gzip"}) + // Close the read side so that any error is propagated to the writer. + pr.CloseWithError(err) + }() + + return pw, nil +} + +func (a *s3Archiver) RemoveArchives(exportDocs []*ExportDoc) error { + if len(exportDocs) == 0 { + return nil + } + + objectsCh := make(chan minio.ObjectInfo, len(exportDocs)) + for _, e := range exportDocs { + objectsCh <- minio.ObjectInfo{Key: e.Domain + "/" + e.ID()} + } + close(objectsCh) + + var errm error + for e := range a.client.RemoveObjects(a.ctx, a.bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = multierror.Append(errm, e.Err) + } + return errm +} diff --git a/model/stack/main.go b/model/stack/main.go index 5ddb66f0660..820c36af832 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -85,6 +85,25 @@ security features. Please do not use this binary as your production server. return nil, nil, fmt.Errorf("failed to init the swift connection: %w", err) } + // Init the main global connection to the S3 server + if err := config.InitDefaultS3Connection(); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 connection: %w", err) + } + + // When a storage migration target is configured (e.g. migrating instances + // to S3 while the global default is still Swift), init that connection too. + if config.HasS3Target() { + // If fs.url is already an s3 scheme, initializing the migration + // target here would silently overwrite the global S3 client with the + // migration endpoint: refuse to start instead of risking that. + if config.FsURL().Scheme == config.SchemeS3 { + return nil, nil, fmt.Errorf("fs.migration_target must not be set when fs.url is already an s3 scheme") + } + if err := config.InitS3Connection(config.Fs{URL: config.MigrationTargetURL()}); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 migration target connection: %w", err) + } + } + workersList, err := job.GetWorkersList() if err != nil { return nil, nil, fmt.Errorf("failed to get the workers list: %w", err) diff --git a/model/vfs/vfs.go b/model/vfs/vfs.go index 24f9aaad733..edd85a6e38a 100644 --- a/model/vfs/vfs.go +++ b/model/vfs/vfs.go @@ -270,6 +270,9 @@ type Avatarer interface { // but does if there was a problem deleting it. DeleteAvatar() error ServeAvatarContent(w http.ResponseWriter, req *http.Request) error + // OpenAvatar returns a reader over the stored avatar content and its + // content-type, or os.ErrNotExist if no avatar is stored. + OpenAvatar() (io.ReadCloser, string, error) } // Thumbser defines an interface to define a thumbnail filesystem. diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index 7d20d308d3f..40d9f1e52b0 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -17,6 +17,7 @@ import ( "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/consts" @@ -54,6 +55,7 @@ func TestVfs(t *testing.T) { aferoFS := makeAferoFS(t) swiftFS := makeSwiftFS(t) + s3FS := makeS3FS(t) var tests = []struct { name string @@ -61,6 +63,7 @@ func TestVfs(t *testing.T) { }{ {"afero", aferoFS}, {"swift", swiftFS}, + {"s3", s3FS}, } for _, tt := range tests { @@ -909,3 +912,31 @@ func makeSwiftFS(t *testing.T) vfs.VFS { return swiftFs } + +func makeS3FS(t *testing.T) vfs.VFS { + t.Helper() + + minioFixture := testutils.StartMinio(t) + db := &contexter{0, "io.cozy.vfs.s3.test", "io.cozy.vfs.s3.test", "cozy_beta"} + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{ + URL: minioFixture.FsURL("test"), + })) + + mutex = config.Lock().ReadWrite(db, "vfs-s3-test") + s3Fs, err := vfss3.New(db, index, &diskImpl{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + + require.NoError(t, g.Wait()) + require.NoError(t, s3Fs.InitFs()) + + return s3Fs +} diff --git a/model/vfs/vfsafero/avatar.go b/model/vfs/vfsafero/avatar.go index 86b0d1ab398..44d38587339 100644 --- a/model/vfs/vfsafero/avatar.go +++ b/model/vfs/vfsafero/avatar.go @@ -74,6 +74,20 @@ func (a *avatarFS) AvatarExists() (bool, error) { return infos.Size() > 0, nil } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. The content-type +// is not persisted on disk by this backend, so a generic value is returned. +func (a *avatarFS) OpenAvatar() (io.ReadCloser, string, error) { + f, err := a.fs.Open(AvatarFilename) + if err != nil { + if os.IsNotExist(err) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, "application/octet-stream", nil +} + func (a *avatarFS) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { s, err := a.fs.Stat(AvatarFilename) if err != nil { diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go new file mode 100644 index 00000000000..7ff2feea834 --- /dev/null +++ b/model/vfs/vfss3/avatar.go @@ -0,0 +1,127 @@ +package vfss3 + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/minio/minio-go/v7" +) + +// NewAvatarFs creates a new avatar filesystem backed by S3. +func NewAvatarFs(client *minio.Client, bucket, keyPrefix string) vfs.Avatarer { + return &avatarS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type avatarS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +func (a *avatarS3) avatarKey() string { + return a.keyPrefix + "avatar" +} + +func (a *avatarS3) CreateAvatar(contentType string) (io.WriteCloser, error) { + key := a.avatarKey() + pr, pw := io.Pipe() + + meta := map[string]string{ + "created-at": time.Now().UTC().Format(time.RFC3339), + } + + errCh := make(chan error, 1) + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, key, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &avatarWriter{pw: pw, errCh: errCh}, nil +} + +type avatarWriter struct { + pw *io.PipeWriter + errCh chan error +} + +func (w *avatarWriter) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *avatarWriter) Close() error { + if err := w.pw.Close(); err != nil { + return err + } + return <-w.errCh +} + +func (a *avatarS3) DeleteAvatar() error { + err := a.client.RemoveObject(a.ctx, a.bucket, a.avatarKey(), minio.RemoveObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return err + } + return nil +} + +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. +func (a *avatarS3) OpenAvatar() (io.ReadCloser, string, error) { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + info, err := obj.Stat() + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return obj, info.ContentType, nil +} + +func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + t := time.Time{} + if createdAt, ok := info.UserMetadata["Created-At"]; ok && createdAt != "" { + if createdAtTime, err := time.Parse(time.RFC3339, createdAt); err == nil { + t = createdAtTime + } + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, "avatar", t, obj) + return nil +} diff --git a/model/vfs/vfss3/avatar_open_test.go b/model/vfs/vfss3/avatar_open_test.go new file mode 100644 index 00000000000..2da3c0ebfdd --- /dev/null +++ b/model/vfs/vfss3/avatar_open_test.go @@ -0,0 +1,55 @@ +package vfss3_test + +import ( + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestOpenAvatarRoundTrip verifies that OpenAvatar returns the content and +// content-type previously stored via CreateAvatar, and that it reports +// os.ErrNotExist when no avatar has been stored yet. +func TestOpenAvatarRoundTrip(t *testing.T) { + config.UseTestFile(t) + + mf := testutils.StartMinio(t) + + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + bucket := "io-cozy-vfss3-openavatar-test" + keyPrefix := "io.cozy.vfss3.openavatar.test/" + + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + av := vfss3.NewAvatarFs(client, bucket, keyPrefix) + + // No avatar stored yet: OpenAvatar must report os.ErrNotExist. + _, _, err := av.OpenAvatar() + assert.ErrorIs(t, err, os.ErrNotExist) + + // Store an avatar, then read it back. + w, err := av.CreateAvatar("image/png") + require.NoError(t, err) + payload := []byte("fake png bytes") + _, err = w.Write(payload) + require.NoError(t, err) + require.NoError(t, w.Close()) + + r, ctype, err := av.OpenAvatar() + require.NoError(t, err) + defer r.Close() + assert.Equal(t, "image/png", ctype) + + got, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, payload, got) +} diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go new file mode 100644 index 00000000000..364f6c3bca1 --- /dev/null +++ b/model/vfs/vfss3/fsck.go @@ -0,0 +1,254 @@ +package vfss3 + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "path" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/minio/minio-go/v7" +) + +func (sfs *s3VFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { + if errors.Is(err, vfs.ErrFsckFailFast) { + return nil + } + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + _, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) checkFiles( + entries map[string]*vfs.TreeFile, + accumulate func(log *vfs.FsckLog), + failFast bool, +) error { + versions := make(map[string]*vfs.Version, 1024) + err := couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error { + v := &vfs.Version{} + if erru := json.Unmarshal(data, v); erru != nil { + return erru + } + versions[v.DocID] = v + return nil + }) + if err != nil { + return err + } + + images := make(map[string]struct{}) + err = couchdb.ForeachDocs(sfs, consts.NotesImages, func(_ string, data json.RawMessage) error { + img := make(map[string]interface{}) + if erru := json.Unmarshal(data, &img); erru != nil { + return erru + } + id, _ := img["_id"].(string) + images[id] = struct{}{} + return nil + }) + if err != nil && !couchdb.IsNoDatabaseError(err) { + return err + } + + fileIDs := make(map[string]struct{}, len(entries)) + for _, f := range entries { + fileIDs[f.DocID] = struct{}{} + } + + // List all objects under our key prefix + for obj := range sfs.client.ListObjects(sfs.ctx, sfs.bucket, minio.ListObjectsOptions{ + Prefix: sfs.keyPrefix, + Recursive: true, + }) { + if obj.Err != nil { + return obj.Err + } + + // Strip the key prefix to get the object name + objName := strings.TrimPrefix(obj.Key, sfs.keyPrefix) + + if objName == "avatar" { + continue + } + if strings.HasPrefix(objName, "thumbs/") { + thumbName := strings.TrimPrefix(objName, "thumbs/") + idx := strings.LastIndex(thumbName, "-") + if idx < 0 { + continue + } + thumbName = thumbName[0:idx] // Remove -format suffix + fileID, _ := makeDocID(thumbName) + if _, ok := fileIDs[fileID]; !ok { + if _, ok := images[fileID]; !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.ThumbnailWithNoFile, + IsFile: true, + FileDoc: &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: objName, + }, + }, + }, + }) + if failFast { + return nil + } + } + } + continue + } + + docID, internalID := makeDocID(objName) + if v, ok := versions[docID+"/"+internalID]; ok { + // ETag from S3 may or may not be an MD5 (multipart uploads use composite ETags). + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsVersion: true, + VersionDoc: v, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: v.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: v.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(versions, v.DocID) + continue + } + f, ok := entries[docID+"/"+internalID] + if !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.IndexMissing, + IsFile: true, + FileDoc: objectToFileDoc(obj), + }) + if failFast { + return nil + } + } else { + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsFile: true, + FileDoc: f, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: f.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: f.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(entries, docID+"/"+internalID) + } + } + + // entries should contain only data that does not contain an associated + // object in S3. + for _, f := range entries { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsFile: true, + FileDoc: f, + }) + if failFast { + return nil + } + } + + for _, v := range versions { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsVersion: true, + VersionDoc: v, + }) + if failFast { + return nil + } + } + + return nil +} + +func objectToFileDoc(obj minio.ObjectInfo) *vfs.TreeFile { + md5sum, _ := hex.DecodeString(strings.Trim(obj.ETag, "\"")) + name := "unknown" + mime, class := vfs.ExtractMimeAndClass(obj.ContentType) + // Strip any key prefix — we need to find the object name portion + // which is just the last segments of the key. + objName := obj.Key + if idx := strings.Index(objName, "/"); idx >= 0 { + // The first segment is the key prefix (db prefix); skip it + objName = objName[idx+1:] + } + fileID, internalID := makeDocID(objName) + return &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: name, + DirID: "", + CreatedAt: obj.LastModified, + UpdatedAt: obj.LastModified, + Fullpath: path.Join(vfs.OrphansDirName, name), + }, + ByteSize: obj.Size, + Mime: mime, + Class: class, + MD5Sum: md5sum, + InternalID: internalID, + }, + } +} diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go new file mode 100644 index 00000000000..891c134870d --- /dev/null +++ b/model/vfs/vfss3/impl.go @@ -0,0 +1,1183 @@ +// Package vfss3 is the implementation of the Virtual File System by using +// an S3-compatible object storage. The file contents are saved in S3 buckets, +// and the metadata are indexed in CouchDB. +package vfss3 + +import ( + "bytes" + "context" + "crypto/md5" + + "errors" + "fmt" + "hash" + "io" + "os" + "regexp" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/lock" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/gofrs/uuid/v5" + "github.com/minio/minio-go/v7" +) + +type s3VFS struct { + vfs.Indexer + vfs.DiskThresholder + client *minio.Client + cluster int + domain string + prefix string // DBPrefix — used as key prefix in the bucket + contextName string + ctx context.Context + bucket string + keyPrefix string // prefix + "/" + region string + mu lock.ErrorRWLocker + log *logger.Entry +} + +var bucketNameCleaner = regexp.MustCompile(`[^a-z0-9-]`) + +// sanitizeBucketName produces a valid S3 bucket name component from an arbitrary string. +func sanitizeBucketName(s string) string { + s = strings.ToLower(s) + s = strings.ReplaceAll(s, "_", "-") + s = strings.ReplaceAll(s, ".", "-") + s = bucketNameCleaner.ReplaceAllString(s, "") + // Collapse consecutive hyphens + for strings.Contains(s, "--") { + s = strings.ReplaceAll(s, "--", "-") + } + s = strings.Trim(s, "-") + if len(s) > 37 { + s = s[:37] + } + return s +} + +// BucketName returns the S3 bucket name for a given orgID and bucket prefix. +func BucketName(orgID, bucketPrefix string) string { + if orgID == "" { + orgID = "default" + } + name := bucketPrefix + "-" + sanitizeBucketName(orgID) + if len(name) > 63 { + name = name[:63] + } + name = strings.TrimRight(name, "-") + if len(name) < 3 { + name += strings.Repeat("0", 3-len(name)) + } + return name +} + +// MakeObjectKey builds the S3 object key for a given file. +// It reuses the same virtual subfolder structure as Swift V3. +func MakeObjectKey(keyPrefix, docID, internalID string) string { + return keyPrefix + makeObjectName(docID, internalID) +} + +// makeObjectName builds the object name (without key prefix), identical to +// vfsswift.MakeObjectNameV3. +func makeObjectName(docID, internalID string) string { + if len(docID) != 32 || len(internalID) != 16 { + return docID + "/" + internalID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] + "/" + internalID +} + +func makeDocID(objName string) (string, string) { + if len(objName) != 51 { + parts := strings.SplitN(objName, "/", 2) + if len(parts) < 2 { + return objName, "" + } + return parts[0], parts[1] + } + return objName[:22] + objName[23:28] + objName[29:34], objName[35:] +} + +// NewInternalID returns a random string that can be used as an internal_vfs_id. +func NewInternalID() string { + return utils.RandomString(16) +} + +// New returns a vfs.VFS instance backed by an S3-compatible object store. +func New(db vfs.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.ErrorRWLocker) (vfs.VFS, error) { + client := config.GetS3Client() + bucketPrefix := config.GetS3BucketPrefix() + + orgID := "" + if inst, ok := db.(interface{ GetOrgID() string }); ok { + orgID = inst.GetOrgID() + } + bucket := BucketName(orgID, bucketPrefix) + dbPrefix := db.DBPrefix() + if dbPrefix == "" { + return nil, fmt.Errorf("vfss3: empty DBPrefix") + } + + return &s3VFS{ + Indexer: index, + DiskThresholder: disk, + client: client, + cluster: db.DBCluster(), + domain: db.DomainName(), + prefix: dbPrefix, + contextName: db.GetContextName(), + ctx: context.Background(), + bucket: bucket, + keyPrefix: dbPrefix + "/", + region: config.GetS3Region(), + mu: mu, + log: logger.WithDomain(db.DomainName()).WithNamespace("vfss3"), + }, nil +} + +func (sfs *s3VFS) MaxFileSize() int64 { + return -1 // no per-file limit — S3 multipart handles large files transparently +} + +func (sfs *s3VFS) DBCluster() int { + return sfs.cluster +} + +func (sfs *s3VFS) DBPrefix() string { + return sfs.prefix +} + +func (sfs *s3VFS) DomainName() string { + return sfs.domain +} + +func (sfs *s3VFS) GetContextName() string { + return sfs.contextName +} + +func (sfs *s3VFS) GetIndexer() vfs.Indexer { + return sfs.Indexer +} + +func (sfs *s3VFS) UseSharingIndexer(index vfs.Indexer) vfs.VFS { + return &s3VFS{ + Indexer: index, + DiskThresholder: sfs.DiskThresholder, + client: sfs.client, + cluster: sfs.cluster, + domain: sfs.domain, + prefix: sfs.prefix, + contextName: sfs.contextName, + ctx: context.Background(), + bucket: sfs.bucket, + keyPrefix: sfs.keyPrefix, + region: sfs.region, + mu: sfs.mu, + log: sfs.log, + } +} + +func (sfs *s3VFS) InitFs() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if err := sfs.Indexer.InitIndex(); err != nil { + return err + } + err := sfs.client.MakeBucket(sfs.ctx, sfs.bucket, minio.MakeBucketOptions{ + Region: sfs.region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + sfs.log.Errorf("Could not create bucket %q: %s", sfs.bucket, err.Error()) + return err + } + sfs.log.Infof("Created bucket %q", sfs.bucket) + return nil +} + +func (sfs *s3VFS) Delete() error { + sfs.log.Infof("Deleting all objects with prefix %q in bucket %q", sfs.keyPrefix, sfs.bucket) + return s3util.DeletePrefixObjects(sfs.ctx, sfs.client, sfs.bucket, sfs.keyPrefix) +} + +func (sfs *s3VFS) CreateDir(doc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + exists, err := sfs.Indexer.DirChildExists(doc.DirID, doc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + if doc.ID() == "" { + return sfs.Indexer.CreateDirDoc(doc) + } + return sfs.Indexer.CreateNamedDirDoc(doc) +} + +// putResult is the result sent back from the background PutObject goroutine. +type putResult struct { + info minio.UploadInfo + err error +} + +func (sfs *s3VFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOptions) (vfs.File, error) { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.Unlock() + + newsize, maxsize, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return nil, err + } + // CheckAvailableDiskSpace already returns ErrFileTooBig / ErrMaxFileSize + // when the upload would exceed the per-file or quota budget. The + // streaming-time check below (s3FileCreation.Write) re-checks against + // maxsize once the actual byte count is known. + + if olddoc != nil { + newdoc.SetID(olddoc.ID()) + newdoc.SetRev(olddoc.Rev()) + newdoc.CreatedAt = olddoc.CreatedAt + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return nil, err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + if !vfs.OptionsAllowCreationInTrash(opts) { + return nil, vfs.ErrParentInTrash + } + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return nil, err + } + if exists { + return nil, os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return nil, err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Use a pipe: writes go into pw, the PutObject goroutine reads from pr. + pr, pw := io.Pipe() + + uploadSize := newdoc.ByteSize + if uploadSize < 0 { + uploadSize = -1 + } + + resultCh := make(chan putResult, 1) + go func() { + info, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, pr, uploadSize, minio.PutObjectOptions{ + ContentType: newdoc.Mime, + PartSize: 5 * 1024 * 1024, // 5 MiB + NumThreads: 1, + }) + // Propagate the outcome to the writer side: if PutObject errored before + // draining the pipe, an in-flight Write would otherwise block forever. + _ = pr.CloseWithError(err) + resultCh <- putResult{info: info, err: err} + }() + + extractor := vfs.NewMetaExtractor(newdoc) + + return &s3FileCreation{ + fs: sfs, + pw: pw, + resultCh: resultCh, + newdoc: newdoc, + olddoc: olddoc, + objKey: objKey, + w: 0, + size: newsize, + maxsize: maxsize, + capsize: capsize, + meta: extractor, + md5H: md5.New(), + }, nil +} + +func (sfs *s3VFS) CopyFile(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, olddoc) + if err != nil { + return err + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + newdoc.InternalID = NewInternalID() + + srcKey := MakeObjectKey(sfs.keyPrefix, olddoc.DocID, olddoc.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(newdoc); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +func (sfs *s3VFS) DissociateFile(src, dst *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if src.DirID != dst.DirID || src.DocName != dst.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + dst.DocID = uid.String() + + srcKey := MakeObjectKey(sfs.keyPrefix, src.DocID, src.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, dst.DocID, dst.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(dst); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + return sfs.destroyFileLocked(src) +} + +func (sfs *s3VFS) DissociateDir(src, dst *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if dst.DirID != src.DirID || dst.DocName != src.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if err := sfs.Indexer.CreateDirDoc(dst); err != nil { + return err + } + return sfs.Indexer.DeleteDirDoc(src) +} + +func (sfs *s3VFS) destroyDir(doc *vfs.DirDoc, push func(vfs.TrashJournal) error, onlyContent bool) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + files, destroyed, err := sfs.Indexer.DeleteDirDocAndContent(doc, onlyContent) + if err != nil { + return err + } + if len(files) == 0 { + return nil + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + ids := make([]string, len(files)) + objNames := make([]string, len(files)) + for i, file := range files { + ids[i] = file.DocID + objNames[i] = MakeObjectKey(sfs.keyPrefix, file.DocID, file.InternalID) + } + return push(vfs.TrashJournal{ + FileIDs: ids, + ObjectNames: objNames, + }) +} + +func (sfs *s3VFS) DestroyDirContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, true) +} + +func (sfs *s3VFS) DestroyDirAndContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, false) +} + +func (sfs *s3VFS) DestroyFile(doc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.destroyFileLocked(doc) +} + +func (sfs *s3VFS) destroyFileLocked(doc *vfs.FileDoc) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := []string{ + MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID), + } + if err := sfs.Indexer.DeleteFileDoc(doc); err != nil { + return err + } + destroyed := doc.ByteSize + if versions, errv := vfs.VersionsFor(sfs, doc.DocID); errv == nil { + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID)) + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + sfs.log.Warnf("DestroyFile failed on BatchDeleteVersions: %s", err) + } + } + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("DestroyFile failed on deleteObjects: %s", err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return nil +} + +func (sfs *s3VFS) EnsureErased(journal vfs.TrashJournal) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := journal.ObjectNames + var errm error + var destroyed int64 + var allVersions []*vfs.Version + for _, fileID := range journal.FileIDs { + versions, err := vfs.VersionsFor(sfs, fileID) + if err != nil { + if !couchdb.IsNoDatabaseError(err) { + sfs.log.Warnf("EnsureErased failed on VersionsFor(%s): %s", fileID, err) + errm = errors.Join(errm, err) + } + continue + } + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, fileID, internalID)) + destroyed += v.ByteSize + } + allVersions = append(allVersions, versions...) + } + if err := sfs.Indexer.BatchDeleteVersions(allVersions); err != nil { + sfs.log.Warnf("EnsureErased failed on BatchDeleteVersions: %s", err) + errm = errors.Join(errm, err) + } + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("EnsureErased failed on deleteObjects: %s", err) + errm = errors.Join(errm, err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return errm +} + +func (sfs *s3VFS) OpenFile(doc *vfs.FileDoc) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Stat the object to detect if it exists. + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) OpenFileVersion(doc *vfs.FileDoc, version *vfs.Version) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + internalID := version.DocID + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + diskQuota := sfs.DiskQuota() + if diskQuota > 0 { + diskUsage, err := sfs.DiskUsage() + if err != nil { + return err + } + if diskUsage+version.ByteSize > diskQuota { + return vfs.ErrFileTooBig + } + } + + parts := strings.SplitN(version.DocID, "/", 2) + if len(parts) != 2 { + return vfs.ErrIllegalFilename + } + objKey := MakeObjectKey(sfs.keyPrefix, parts[0], parts[1]) + + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, version.ByteSize, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + if errc := content.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + + return sfs.Indexer.CreateVersion(version) +} + +// WriteContentAt streams content into the object backing the (docID, internalID) +// key, creating NO CouchDB document. It is used by storage migration, which +// preserves the shared index and only moves object bytes. size may be -1 when +// unknown (falls back to multipart). +func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, size, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + return err +} + +// StatContentAt returns the byte size of the object backing the (docID, +// internalID) key, without touching CouchDB. It returns os.ErrNotExist when +// the object is absent. Used by storage migration to verify a copy landed on +// the target before flipping the instance's backend flag. +func (sfs *s3VFS) StatContentAt(docID, internalID string) (int64, error) { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + info, err := sfs.client.StatObject(sfs.ctx, sfs.bucket, objKey, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return 0, os.ErrNotExist + } + return 0, err + } + return info.Size, nil +} + +func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + save := vfs.NewVersion(doc) + if err := sfs.Indexer.CreateVersion(save); err != nil { + return err + } + + newdoc := doc.Clone().(*vfs.FileDoc) + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + newdoc.InternalID = parts[1] + } + vfs.SetMetaFromVersion(newdoc, version) + if err := sfs.Indexer.UpdateFileDoc(doc, newdoc); err != nil { + _ = sfs.Indexer.DeleteVersion(save) + return err + } + + return sfs.Indexer.DeleteVersion(version) +} + +func (sfs *s3VFS) CleanOldVersion(fileID string, v *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.cleanOldVersion(fileID, v) +} + +func (sfs *s3VFS) cleanOldVersion(fileID string, v *vfs.Version) error { + if err := sfs.Indexer.DeleteVersion(v); err != nil { + return err + } + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, fileID, internalID) + return sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) +} + +func (sfs *s3VFS) ClearOldVersions() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + versions, err := sfs.Indexer.AllVersions() + if err != nil { + return err + } + var objNames []string + var destroyed int64 + for _, v := range versions { + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, parts[0], parts[1])) + } + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + return err + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames) +} + +func (sfs *s3VFS) CopyFileFromOtherFS( + newdoc, olddoc *vfs.FileDoc, + srcFS vfs.Fs, + srcDoc *vfs.FileDoc, +) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return err + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + return vfs.ErrParentInTrash + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Try server-side copy if the source is also an s3VFS on the same client. + if srcS3, ok := srcFS.(*s3VFS); ok { + srcKey := MakeObjectKey(srcS3.keyPrefix, srcDoc.DocID, srcDoc.InternalID) + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: srcS3.bucket, Object: srcKey}, + ); err != nil { + return err + } + } else { + // Stream from the source FS. + srcFile, err := srcFS.OpenFile(srcDoc) + if err != nil { + return err + } + _, err = sfs.client.PutObject(sfs.ctx, sfs.bucket, dstKey, srcFile, srcDoc.ByteSize, minio.PutObjectOptions{ + ContentType: srcDoc.Mime, + }) + if errc := srcFile.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + } + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = sfs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else { + err = sfs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(sfs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := sfs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, internalID) + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) + } + for _, old := range toClean { + _ = sfs.cleanOldVersion(newdoc.DocID, old) + } + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +// UpdateFileDoc calls the indexer UpdateFileDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateFileDoc +func (sfs *s3VFS) UpdateFileDoc(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateFileDoc(olddoc, newdoc) +} + +// UpdateDirDoc calls the indexer UpdateDirDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks that we don't move a directory to one of its descendant +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateDirDoc +func (sfs *s3VFS) UpdateDirDoc(olddoc, newdoc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + if strings.HasPrefix(newdoc.Fullpath, olddoc.Fullpath+"/") { + return vfs.ErrForbiddenDocMove + } + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateDirDoc(olddoc, newdoc) +} + +func (sfs *s3VFS) DirByID(fileID string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByID(fileID) +} + +func (sfs *s3VFS) DirByPath(name string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByPath(name) +} + +func (sfs *s3VFS) FileByID(fileID string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByID(fileID) +} + +func (sfs *s3VFS) FileByPath(name string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByPath(name) +} + +func (sfs *s3VFS) FilePath(doc *vfs.FileDoc) (string, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return "", lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FilePath(doc) +} + +func (sfs *s3VFS) DirOrFileByID(fileID string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByID(fileID) +} + +func (sfs *s3VFS) DirOrFileByPath(name string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByPath(name) +} + +// s3FileCreation represents a file open for writing. It is used to create +// a file or to modify the content of a file. +type s3FileCreation struct { + fs *s3VFS + pw *io.PipeWriter + resultCh chan putResult + newdoc *vfs.FileDoc + olddoc *vfs.FileDoc + objKey string + w int64 + size int64 + maxsize int64 + capsize int64 + meta *vfs.MetaExtractor + md5H hash.Hash + err error +} + +func (f *s3FileCreation) Read(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) ReadAt(p []byte, off int64) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Seek(offset int64, whence int) (int64, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Write(p []byte) (int, error) { + if f.err != nil { + return 0, f.err + } + + if f.meta != nil { + if _, err := (*f.meta).Write(p); err != nil && !errors.Is(err, io.ErrClosedPipe) { + (*f.meta).Abort(err) + f.meta = nil + } + } + + // Write to local MD5 hash + _, _ = f.md5H.Write(p) + + n, err := f.pw.Write(p) + if err != nil { + f.err = err + return n, err + } + + f.w += int64(n) + if f.maxsize >= 0 && f.w > f.maxsize { + f.err = vfs.ErrFileTooBig + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + if f.size >= 0 && f.w > f.size { + f.err = vfs.ErrContentLengthMismatch + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + return n, nil +} + +func (f *s3FileCreation) Close() (err error) { + defer func() { + if err != nil { + // Remove the object from S3 if an error occurred + _ = f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, f.objKey, minio.RemoveObjectOptions{}) + // If an error has occurred when creating a new file, we should + // also delete the file from the index. + if f.olddoc == nil { + _ = f.fs.Indexer.DeleteFileDoc(f.newdoc) + } + } + }() + + // Close the pipe writer to signal EOF to PutObject + if err = f.pw.Close(); err != nil { + if f.meta != nil { + (*f.meta).Abort(err) + f.meta = nil + } + if f.err == nil { + f.err = err + } + } + + // Wait for the PutObject goroutine to finish + result := <-f.resultCh + + if result.err != nil { + if f.meta != nil { + (*f.meta).Abort(result.err) + f.meta = nil + } + if f.err == nil { + f.err = result.err + } + } + + newdoc, olddoc, written := f.newdoc, f.olddoc, f.w + + if f.meta != nil { + if errc := (*f.meta).Close(); errc == nil { + vfs.MergeMetadata(newdoc, (*f.meta).Result()) + } + } + + if f.err != nil { + return f.err + } + + // Verify or compute MD5 checksum. + // The local md5H hash is always computed from the same data stream that + // goes to S3 (via the Write method), so it is authoritative. + localMD5 := f.md5H.Sum(nil) + if newdoc.MD5Sum != nil { + // The caller provided an expected hash — verify it matches what was written. + if !bytes.Equal(newdoc.MD5Sum, localMD5) { + return vfs.ErrInvalidHash + } + } else { + newdoc.MD5Sum = localMD5 + } + + if f.size < 0 { + newdoc.ByteSize = written + } + + if newdoc.ByteSize != written { + return vfs.ErrContentLengthMismatch + } + + lockerr := f.fs.mu.Lock() + if lockerr != nil { + return lockerr + } + defer f.fs.mu.Unlock() + + // Check again that a file with the same path does not exist. It can happen + // when the same file is uploaded twice in parallel. + if olddoc == nil { + exists, err := f.fs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + var newpath string + newpath, err = f.fs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + newdoc.Trashed = strings.HasPrefix(newpath, vfs.TrashDirName+"/") + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = f.fs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else if newdoc.ID() == "" { + err = f.fs.Indexer.CreateFileDoc(newdoc) + } else { + err = f.fs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(f.fs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := f.fs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(f.fs.keyPrefix, newdoc.DocID, internalID) + if err := f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, objKey, minio.RemoveObjectOptions{}); err != nil { + f.fs.log.Warnf("Could not delete previous version %q: %s", objKey, err.Error()) + } + } + for _, old := range toClean { + if err := f.fs.cleanOldVersion(newdoc.DocID, old); err != nil { + f.fs.log.Warnf("Could not delete old versions for %s: %s", newdoc.DocID, err.Error()) + } + } + } + + if f.capsize > 0 && f.size >= f.capsize { + vfs.PushDiskQuotaAlert(f.fs, true) + } + + return nil +} + +// s3FileOpen represents a file open for reading. +type s3FileOpen struct { + obj *minio.Object +} + +func (f *s3FileOpen) Read(p []byte) (int, error) { + return f.obj.Read(p) +} + +func (f *s3FileOpen) ReadAt(p []byte, off int64) (int, error) { + return f.obj.ReadAt(p, off) +} + +func (f *s3FileOpen) Seek(offset int64, whence int) (int64, error) { + return f.obj.Seek(offset, whence) +} + +func (f *s3FileOpen) Write(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileOpen) Close() error { + return f.obj.Close() +} + +var ( + _ vfs.VFS = &s3VFS{} + _ vfs.File = &s3FileCreation{} + _ vfs.File = &s3FileOpen{} +) diff --git a/model/vfs/vfss3/naming_test.go b/model/vfs/vfss3/naming_test.go new file mode 100644 index 00000000000..1201dc82988 --- /dev/null +++ b/model/vfs/vfss3/naming_test.go @@ -0,0 +1,71 @@ +package vfss3 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeBucketName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"production", "production"}, + {"my_context", "my-context"}, + {"My.Context", "my-context"}, + {"UPPERCASE", "uppercase"}, + {"with spaces!", "withspaces"}, + {"a--b--c", "a-b-c"}, + {"-leading-trailing-", "leading-trailing"}, + {"very-long-name-that-exceeds-the-maximum-allowed-length", "very-long-name-that-exceeds-the-maxim"}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, sanitizeBucketName(tt.input)) + }) + } +} + +func TestBucketName(t *testing.T) { + tests := []struct { + orgID string + bucketPrefix string + expected string + }{ + {"org-123", "cozy", "cozy-org-123"}, + {"", "cozy", "cozy-default"}, + {"My_Org", "cozy", "cozy-my-org"}, + {"org.example.com", "cozy", "cozy-org-example-com"}, + } + + for _, tt := range tests { + t.Run(tt.orgID, func(t *testing.T) { + assert.Equal(t, tt.expected, BucketName(tt.orgID, tt.bucketPrefix)) + }) + } +} + +func TestMakeObjectKey(t *testing.T) { + // Standard 32-char docID and 16-char internalID + key := MakeObjectKey("alice.example.com/", "abcdefghijklmnopqrstuvwxyz012345", "0123456789abcdef") + assert.Equal(t, "alice.example.com/abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef", key) + + // Non-standard lengths + key = MakeObjectKey("alice.example.com/", "short", "id") + assert.Equal(t, "alice.example.com/short/id", key) +} + +func TestMakeDocID(t *testing.T) { + // Standard 51-char object name + docID, internalID := makeDocID("abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef") + assert.Equal(t, "abcdefghijklmnopqrstuvwxyz012345", docID) + assert.Equal(t, "0123456789abcdef", internalID) + + // Non-standard + docID, internalID = makeDocID("short/id") + assert.Equal(t, "short", docID) + assert.Equal(t, "id", internalID) +} diff --git a/model/vfs/vfss3/thumbs.go b/model/vfs/vfss3/thumbs.go new file mode 100644 index 00000000000..ee9c064ad0d --- /dev/null +++ b/model/vfs/vfss3/thumbs.go @@ -0,0 +1,247 @@ +package vfss3 + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +var unixEpochZero = time.Time{} + +// NewThumbsFs creates a new thumbnail filesystem backed by S3. +func NewThumbsFs(client *minio.Client, bucket, keyPrefix string) vfs.Thumbser { + return &thumbsS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type thumbsS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +type s3Thumb struct { + pw *io.PipeWriter + errCh chan error + client *minio.Client + bucket string + name string + ctx context.Context +} + +func (t *s3Thumb) Write(p []byte) (int, error) { + return t.pw.Write(p) +} + +func (t *s3Thumb) Commit() error { + if err := t.pw.Close(); err != nil { + return err + } + return <-t.errCh +} + +func (t *s3Thumb) Abort() error { + // Close the pipe with an error to cancel the PutObject goroutine. + errc := t.pw.CloseWithError(fmt.Errorf("thumbnail creation aborted")) + // Drain the errCh so the goroutine is not leaked. + <-t.errCh + // Try to remove the possibly partially written object. + errd := t.client.RemoveObject(t.ctx, t.bucket, t.name, minio.RemoveObjectOptions{}) + if errd != nil && minio.ToErrorResponse(errd).Code == "NoSuchKey" { + errd = nil + } + // Write an empty marker object to indicate that the thumbnail generation failed. + _, errp := t.client.PutObject(t.ctx, t.bucket, t.name, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: echo.MIMEOctetStream, + }) + if errc != nil { + return errc + } + if errd != nil { + return errd + } + return errp +} + +func (ts *thumbsS3) createThumbFile(name, contentType string, meta map[string]string) (vfs.ThumbFiler, error) { + pr, pw := io.Pipe() + + errCh := make(chan error, 1) + go func() { + _, err := ts.client.PutObject(ts.ctx, ts.bucket, name, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &s3Thumb{ + pw: pw, + errCh: errCh, + client: ts.client, + bucket: ts.bucket, + name: name, + ctx: ts.ctx, + }, nil +} + +func (ts *thumbsS3) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(img.ID(), format) + meta := map[string]string{ + "file-md5": hex.EncodeToString(img.MD5Sum), + } + return ts.createThumbFile(name, "image/jpeg", meta) +} + +func (ts *thumbsS3) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { + name := ts.makeName(img.ID(), format) + info, err := ts.client.StatObject(ts.ctx, ts.bucket, name, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return false, nil + } + return false, err + } + if md5str, ok := info.UserMetadata["File-Md5"]; ok && md5str != "" { + md5sum, err := hex.DecodeString(md5str) + if err == nil && !bytes.Equal(md5sum, img.MD5Sum) { + return false, nil + } + } + return true, nil +} + +func (ts *thumbsS3) RemoveThumbs(img *vfs.FileDoc, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(img.ID(), format) + } + return s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) +} + +func (ts *thumbsS3) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { + name := ts.makeName(img.ID(), format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + if info.ContentType == echo.MIMEOctetStream { + // We have some old images where the thumbnail has not been correctly + // saved. We should delete the thumbnail to allow another try. + if info.Size > 0 { + _ = ts.RemoveThumbs(img, vfs.ThumbnailFormatNames) + return os.ErrNotExist + } + // Image magick has failed to generate a thumbnail, and retrying would + // be useless. + return os.ErrInvalid + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(id, format) + return ts.createThumbFile(name, mime, nil) +} + +func (ts *thumbsS3) OpenNoteThumb(id, format string) (io.ReadCloser, error) { + name := ts.makeName(id, format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, s3util.WrapNotFound(err) + } + // Stat to verify the object actually exists (GetObject doesn't fail on missing keys). + if _, err := obj.Stat(); err != nil { + obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return obj, nil +} + +func (ts *thumbsS3) RemoveNoteThumb(id string, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(id, format) + } + err := s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) + if err != nil { + logger.WithNamespace("vfss3").Infof("Cannot remove note thumbs: %s", err) + } + return err +} + +func (ts *thumbsS3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { + name := ts.makeName(id, consts.NoteImageThumbFormat) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + + info, err := obj.Stat() + if err != nil { + obj.Close() + // Try the original format as fallback. + name = ts.makeName(id, consts.NoteImageOriginalFormat) + obj, err = ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + info, err = obj.Stat() + if err != nil { + obj.Close() + return s3util.WrapNotFound(err) + } + } + defer obj.Close() + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) makeName(imgID string, format string) string { + return ts.keyPrefix + fmt.Sprintf("thumbs/%s-%s", makeThumbObjectName(imgID), format) +} + +// makeThumbObjectName builds a virtual subfolder structure for thumbnails. +// It splits the 32-char ID into three parts to avoid a flat hierarchy. +// This is the same logic as vfsswift.MakeObjectName (without internalID). +func makeThumbObjectName(docID string) string { + if len(docID) != 32 { + return docID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] +} diff --git a/model/vfs/vfss3/write_content_at_test.go b/model/vfs/vfss3/write_content_at_test.go new file mode 100644 index 00000000000..e513222d0d0 --- /dev/null +++ b/model/vfs/vfss3/write_content_at_test.go @@ -0,0 +1,86 @@ +package vfss3_test + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/require" +) + +// writeContentAtPrefixer is a minimal vfs.Prefixer implementation, local to +// this test, so it can live in an external test package (package +// vfss3_test) without importing anything from the internal vfss3 test +// harness. It also exposes GetOrgID so vfss3.New can build the bucket name. +type writeContentAtPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *writeContentAtPrefixer) DBCluster() int { return p.cluster } +func (p *writeContentAtPrefixer) DomainName() string { return p.domain } +func (p *writeContentAtPrefixer) DBPrefix() string { return p.prefix } +func (p *writeContentAtPrefixer) GetContextName() string { return p.context } +func (p *writeContentAtPrefixer) GetOrgID() string { return "wcatestorg" } + +// writeContentAtDisk is a minimal vfs.DiskThresholder, unused by +// WriteContentAt itself but required by vfss3.New's signature. +type writeContentAtDisk struct{} + +func (writeContentAtDisk) DiskQuota() int64 { return 0 } + +// TestWriteContentAtPutsBytesWithoutIndex verifies that WriteContentAt is a +// pure object-storage primitive: it puts bytes at the object key derived +// from (docID, internalID) and does not touch CouchDB at all (no +// ResetDB/DefineIndexes/InitFs is performed in this test). +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + config.UseTestFile(t) + + mf := testutils.StartMinio(t) + + db := &writeContentAtPrefixer{ + cluster: 0, + domain: "io.cozy.vfss3.writecontentat.test", + prefix: "io.cozy.vfss3.writecontentat.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + mutex := config.Lock().ReadWrite(db, "vfs-s3-writecontentat-test") + sfs, err := vfss3.New(db, index, &writeContentAtDisk{}, mutex) + require.NoError(t, err) + + // WriteContentAt never creates its own bucket (that's InitFs's job, which + // we deliberately skip here since it would also touch CouchDB through + // Indexer.InitIndex). Create the bucket directly against the raw client. + bucket := vfss3.BucketName(db.GetOrgID(), config.GetS3BucketPrefix()) + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + w := sfs.(interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error + }) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello s3 migration") + + require.NoError(t, w.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + objKey := vfss3.MakeObjectKey(db.DBPrefix()+"/", docID, internalID) + obj, err := client.GetObject(context.Background(), bucket, objKey, minio.GetObjectOptions{}) + require.NoError(t, err) + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} diff --git a/model/vfs/vfsswift/avatar_v3.go b/model/vfs/vfsswift/avatar_v3.go index 917c0f0a3a0..b776cd413ca 100644 --- a/model/vfs/vfsswift/avatar_v3.go +++ b/model/vfs/vfsswift/avatar_v3.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "os" "time" "github.com/cozy/cozy-stack/model/vfs" @@ -45,6 +46,19 @@ func (a *avatarV3) DeleteAvatar() error { return err } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. +func (a *avatarV3) OpenAvatar() (io.ReadCloser, string, error) { + f, headers, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) + if err != nil { + if err == swift.ObjectNotFound { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, headers["Content-Type"], nil +} + func (a *avatarV3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { f, o, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) if err != nil { diff --git a/model/vfs/vfsswift/impl_v3.go b/model/vfs/vfsswift/impl_v3.go index e40eab1ef10..e06304c7c2c 100644 --- a/model/vfs/vfsswift/impl_v3.go +++ b/model/vfs/vfsswift/impl_v3.go @@ -579,6 +579,40 @@ func (sfs *swiftVFSV3) ImportFileVersion(version *vfs.Version, content io.ReadCl return sfs.Indexer.CreateVersion(version) } +// WriteContentAt streams content into the object backing the (docID, +// internalID) key in this instance's container, creating NO CouchDB +// document. Used by storage migration, which preserves the shared index and +// only moves bytes. +func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objName := MakeObjectNameV3(docID, internalID) + f, err := sfs.c.ObjectCreate(sfs.ctx, sfs.container, objName, true, "", "application/octet-stream", nil) + if err != nil { + return err + } + if _, err = io.Copy(f, content); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + +// StatContentAt returns the byte size of the object backing the (docID, +// internalID) key in this instance's container, without touching CouchDB. It +// returns os.ErrNotExist when the object is absent. Used by storage +// migration to verify a copy landed on the target before flipping the +// instance's backend flag. +func (sfs *swiftVFSV3) StatContentAt(docID, internalID string) (int64, error) { + objName := MakeObjectNameV3(docID, internalID) + info, _, err := sfs.c.Object(sfs.ctx, sfs.container, objName) + if errors.Is(err, swift.ObjectNotFound) { + return 0, os.ErrNotExist + } + if err != nil { + return 0, err + } + return info.Bytes, nil +} + func (sfs *swiftVFSV3) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr diff --git a/model/vfs/vfsswift/write_content_at_v3_test.go b/model/vfs/vfsswift/write_content_at_v3_test.go new file mode 100644 index 00000000000..42834de0d7a --- /dev/null +++ b/model/vfs/vfsswift/write_content_at_v3_test.go @@ -0,0 +1,107 @@ +package vfsswift_test + +import ( + "bytes" + "context" + "io" + "net/url" + "testing" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/ncw/swift/v2/swifttest" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// writeContentAtPrefixer is a minimal vfs.Prefixer implementation, local to +// this test, so it can live in an external test package (package +// vfsswift_test) without importing anything from the internal vfsswift test +// harness. +type writeContentAtPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *writeContentAtPrefixer) DBCluster() int { return p.cluster } +func (p *writeContentAtPrefixer) DomainName() string { return p.domain } +func (p *writeContentAtPrefixer) DBPrefix() string { return p.prefix } +func (p *writeContentAtPrefixer) GetContextName() string { return p.context } + +// writeContentAtDisk is a minimal vfs.DiskThresholder, unused by +// WriteContentAt itself but required by vfsswift.NewV3's signature. +type writeContentAtDisk struct{} + +func (writeContentAtDisk) DiskQuota() int64 { return 0 } + +// TestWriteContentAtPutsBytesWithoutIndex verifies that WriteContentAt is a +// pure object-storage primitive: it puts bytes at the object key derived +// from (docID, internalID) and does not touch CouchDB (no document is +// created for the write itself). +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + config.UseTestFile(t) + testutils.NeedCouchdb(t) + + db := &writeContentAtPrefixer{ + cluster: 0, + domain: "io.cozy.vfsswift.writecontentat.test", + prefix: "io.cozy.vfsswift.writecontentat.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + swiftSrv, err := swifttest.NewSwiftServer("localhost") + require.NoError(t, err, "failed to create swift server") + t.Cleanup(func() { swiftSrv.Close() }) + + require.NoError(t, config.InitSwiftConnection(config.Fs{ + URL: &url.URL{ + Scheme: "swift", + Host: "localhost", + RawQuery: "UserName=swifttest&Password=swifttest&AuthURL=" + url.QueryEscape(swiftSrv.AuthURL), + }, + })) + + mutex := config.Lock().ReadWrite(db, "vfs-swiftv3-writecontentat-test") + sfs, err := vfsswift.NewV3(db, index, &writeContentAtDisk{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + + require.NoError(t, sfs.InitFs()) + + w := sfs.(interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error + }) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello swift migration") + + require.NoError(t, w.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + cn := sfs.(interface{ ContainerNames() map[string]string }) + container := cn.ContainerNames()["container"] + objName := vfsswift.MakeObjectNameV3(docID, internalID) + + conn := config.GetSwiftConnection() + obj, _, err := conn.ObjectOpen(context.Background(), container, objName, false, nil) + require.NoError(t, err) + defer obj.Close() + + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go new file mode 100644 index 00000000000..33e08f88794 --- /dev/null +++ b/pkg/appfs/s3.go @@ -0,0 +1,333 @@ +package appfs + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strconv" + "strings" + + "github.com/andybalholm/brotli" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/filetype" + "github.com/cozy/cozy-stack/pkg/s3util" + + web_utils "github.com/cozy/cozy-stack/pkg/utils" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +// installedMarkerSuffix is appended to the app's base key to form the +// "installation complete" marker object. The suffix is chosen so the marker +// can't collide with any file under //... — S3 browsers +// would otherwise display the marker as a file sitting next to a folder of +// the same name. +const installedMarkerSuffix = ".cozy-installed" + +// s3Copier implements the Copier interface backed by S3. +type s3Copier struct { + client *minio.Client + bucket string + appObj string + started bool + objectNames []string + ctx context.Context +} + +// NewS3Copier creates a Copier that stores app files in S3. +func NewS3Copier(client *minio.Client, bucket string) Copier { + return &s3Copier{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { + f.appObj = path.Join(slug, version) + if shasum != "" { + f.appObj += "-" + shasum + } + _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, minio.StatObjectOptions{}) + if err == nil { + return true, nil + } + if s3util.IsNotFound(err) { + return false, nil + } + return false, err +} + +func (f *s3Copier) Start(slug, version, shasum string) (bool, error) { + exist, err := f.Exist(slug, version, shasum) + if err != nil || exist { + return exist, err + } + + if err := s3util.EnsureBucket(f.ctx, f.client, f.bucket, ""); err != nil { + return false, err + } + + f.objectNames = []string{} + f.started = true + return false, nil +} + +func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { + if !f.started { + return fmt.Errorf("appfs: copier must call Start() before Copy()") + } + + // Write directly to the final location (appObj/filename). + // Reject path traversal attempts in filenames. + name := stat.Name() + if strings.Contains(name, "..") { + return fmt.Errorf("appfs: invalid filename %q", name) + } + objName := path.Join(f.appObj, name) + + contentType := filetype.ByExtension(path.Ext(stat.Name())) + if contentType == "" { + contentType, src = filetype.FromReader(src) + } + + // Compress with brotli. + var buf bytes.Buffer + bw := brotli.NewWriter(&buf) + if _, err := io.Copy(bw, src); err != nil { + return err + } + if err := bw.Close(); err != nil { + return err + } + + meta := map[string]string{ + "X-Content-Encoding": "br", + "Original-Content-Length": strconv.FormatInt(stat.Size(), 10), + } + + f.objectNames = append(f.objectNames, objName) + _, err := f.client.PutObject(f.ctx, f.bucket, objName, + bytes.NewReader(buf.Bytes()), int64(buf.Len()), + minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + return err +} + +func (f *s3Copier) Abort() error { + return s3util.DeleteObjects(f.ctx, f.client, f.bucket, f.objectNames) +} + +func (f *s3Copier) Commit() (err error) { + // Create the marker object that signals the version is complete. The + // suffix keeps it on a distinct key from the //... files, + // so S3 browsers don't render a folder and a file with the same name. + _, err = f.client.PutObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: "text/plain", + }) + return err +} + +// s3Server implements the FileServer interface backed by S3. +type s3Server struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FileServer creates a FileServer that serves app files from S3. +func NewS3FileServer(client *minio.Client, bucket string) FileServer { + return &s3Server{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (s *s3Server) Open(slug, version, shasum, file string) (io.ReadCloser, error) { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return nil, s3util.WrapNotFound(err) + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, s3util.WrapNotFound(err) + } + contentEncoding := info.UserMetadata["X-Content-Encoding"] + if contentEncoding == "br" { + return newBrotliReadCloser(obj) + } else if contentEncoding == "gzip" { + return newGzipReadCloser(obj) + } + return obj, nil +} + +func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, slug, version, shasum, file string) error { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + if checkETag := req.Header.Get("Cache-Control") == ""; checkETag { + etagVal := info.ETag + if len(etagVal) > 10 { + etagVal = etagVal[:10] + } + etag := fmt.Sprintf(`"%s"`, etagVal) + if web_utils.CheckPreconditions(w, req, etag) { + return nil + } + w.Header().Set("Etag", etag) + } + + // Read the full object to handle brotli decompression. + // Limit to 50 MiB to avoid unbounded memory allocation from corrupted objects. + const maxAppFileSize = 50 << 20 + content, err := io.ReadAll(io.LimitReader(obj, maxAppFileSize)) + if err != nil { + return err + } + + var r io.Reader = bytes.NewReader(content) + contentLength := info.Size + contentType := info.ContentType + + contentEncoding := info.UserMetadata["X-Content-Encoding"] + origContentLength := info.UserMetadata["Original-Content-Length"] + if contentEncoding == "br" { + if acceptBrotliEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "br") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + r = brotli.NewReader(bytes.NewReader(content)) + } + } else if contentEncoding == "gzip" { + if acceptGzipEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "gzip") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + gr, gerr := gzip.NewReader(bytes.NewReader(content)) + if gerr != nil { + return gerr + } + defer gr.Close() + r = gr + } + } + + ext := path.Ext(file) + if contentType == "" { + contentType = mime.TypeByExtension(ext) + } + if contentType == "text/xml" && ext == ".svg" { + contentType = "image/svg+xml" + } + + return serveContent(w, req, contentType, contentLength, r) +} + +func (s *s3Server) ServeCodeTarball(w http.ResponseWriter, req *http.Request, slug, version, shasum string) error { + objName := path.Join(slug, version) + if shasum != "" { + objName += "-" + shasum + } + objName += ".tgz" + + // Try to serve a pre-built tarball first. + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err == nil { + info, serr := obj.Stat() + if serr == nil { + defer obj.Close() + return serveContent(w, req, info.ContentType, info.Size, obj) + } + obj.Close() + } + + buf, err := prepareTarball(s, slug, version, shasum) + if err != nil { + return err + } + content, err := io.ReadAll(buf) + if err != nil { + return err + } + contentType := mime.TypeByExtension(".gz") + + // Store the tarball for future requests. + _, _ = s.client.PutObject(s.ctx, s.bucket, objName, + bytes.NewReader(content), int64(len(content)), + minio.PutObjectOptions{ContentType: contentType}) + + return serveContent(w, req, contentType, int64(len(content)), bytes.NewReader(content)) +} + +func (s *s3Server) makeObjectName(slug, version, shasum, file string) string { + basepath := path.Join(slug, version) + if shasum != "" { + basepath += "-" + shasum + } + // Prevent path traversal + if strings.Contains(file, "..") { + return basepath + "/invalid" + } + return path.Join(basepath, file) +} + +func (s *s3Server) FilesList(slug, version, shasum string) ([]string, error) { + prefix := s.makeObjectName(slug, version, shasum, "") + "/" + var names []string + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + name := strings.TrimPrefix(obj.Key, prefix) + if name != "" { + names = append(names, name) + } + } + return names, nil +} + +// S3AppsBucket returns the S3 bucket name used for storing applications of a +// given type. The bucket is shared across all instances (like Swift containers). +func S3AppsBucket(bucketPrefix string, appsType consts.AppType) string { + switch appsType { + case consts.WebappType: + return bucketPrefix + "-apps-web" + case consts.KonnectorType: + return bucketPrefix + "-apps-konnectors" + } + panic("Unknown AppType") +} + +// prepareTarball is reused from server.go via the FileServer interface (it +// calls Open and FilesList). The function is defined in server.go. +// We reference it here to document that s3Server satisfies prepareTarball's +// requirements. +var _ FileServer = (*s3Server)(nil) diff --git a/pkg/assets/dynamic/fs.go b/pkg/assets/dynamic/fs.go index baf867a7a55..a0764c8a050 100644 --- a/pkg/assets/dynamic/fs.go +++ b/pkg/assets/dynamic/fs.go @@ -53,6 +53,12 @@ func InitDynamicAssetFS(fsURL string) error { return err } + case config.SchemeS3: + assetFS, err = NewS3FS() + if err != nil { + return err + } + default: return fmt.Errorf("Invalid scheme %s for dynamic assets FS", u.Scheme) } diff --git a/pkg/assets/dynamic/impl_s3.go b/pkg/assets/dynamic/impl_s3.go new file mode 100644 index 00000000000..5001c4faa88 --- /dev/null +++ b/pkg/assets/dynamic/impl_s3.go @@ -0,0 +1,126 @@ +package dynamic + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "github.com/cozy/cozy-stack/pkg/assets/model" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/minio/minio-go/v7" +) + +// S3FS is the S3 implementation of [AssetsFS]. +// +// It saves and fetches assets into/from any S3-compatible object store. +type S3FS struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FS instantiates a new S3FS. +func NewS3FS() (*S3FS, error) { + initCacheOnce.Do(func() { + cache = expirable.NewLRU[string, cacheEntry](1024, nil, 1*time.Hour) + }) + + ctx := context.Background() + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-assets" + + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: config.GetS3Region()}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + return nil, fmt.Errorf("Cannot create bucket for dynamic assets: %s", err) + } + } + + return &S3FS{client: client, bucket: bucket, ctx: ctx}, nil +} + +func (s *S3FS) Add(_ string, _ string, asset *model.Asset) error { + objectName := path.Join(asset.Context, asset.Name) + data := asset.GetData() + _, err := s.client.PutObject(s.ctx, s.bucket, objectName, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{}) + return err +} + +func (s *S3FS) Get(ctx string, name string) ([]byte, error) { + objectName := path.Join(ctx, name) + if entry, ok := cache.Get(objectName); ok { + if !entry.found { + return nil, os.ErrNotExist + } + return entry.content, nil + } + + obj, err := s.client.GetObject(s.ctx, s.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + content, err := io.ReadAll(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + cache.Add(objectName, cacheEntry{found: false}) + return nil, os.ErrNotExist + } + return nil, err + } + + cache.Add(objectName, cacheEntry{found: true, content: content}) + return content, nil +} + +func (s *S3FS) Remove(context, name string) error { + objectName := path.Join(context, name) + return s.client.RemoveObject(s.ctx, s.bucket, objectName, minio.RemoveObjectOptions{}) +} + +func (s *S3FS) List() (map[string][]*model.Asset, error) { + objs := map[string][]*model.Asset{} + + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + + splitted := strings.SplitN(obj.Key, "/", 2) + if len(splitted) < 2 { + continue + } + ctx := splitted[0] + assetName := model.NormalizeAssetName(splitted[1]) + + a, err := GetAsset(ctx, assetName) + if err != nil { + return nil, err + } + + objs[ctx] = append(objs[ctx], a) + } + + return objs, nil +} + +func (s *S3FS) CheckStatus(ctx context.Context) (time.Duration, error) { + before := time.Now() + _, err := s.client.ListBuckets(ctx) + if err != nil { + return 0, err + } + return time.Since(before), nil +} diff --git a/pkg/config/config/config.go b/pkg/config/config/config.go index 9be12b28810..924099e8550 100644 --- a/pkg/config/config/config.go +++ b/pkg/config/config/config.go @@ -83,6 +83,8 @@ const ( // SchemeSwiftSecure is the URL scheme used to configure the swift filesystem // in secure mode (HTTPS). SchemeSwiftSecure = "swift+https" + // SchemeS3 is the URL scheme used to configure an S3-compatible filesystem. + SchemeS3 = "s3" ) // defaultAdminSecretFileName is the default name of the file containing the @@ -230,6 +232,10 @@ type Fs struct { AutoCleanTrashedAfter map[string]string Versioning FsVersioning Contexts map[string]interface{} + // MigrationTarget, when set, is an alternate storage URL (e.g. s3://...) + // whose connection is initialized alongside the default one, so instances + // can be migrated to it while the global scheme stays unchanged. + MigrationTarget *url.URL } // FsVersioning contains the configuration for the versioning of files @@ -496,6 +502,17 @@ func FsURL() *url.URL { return config.Fs.URL } +// MigrationTargetURL returns the configured storage migration target URL, or nil. +func MigrationTargetURL() *url.URL { + return config.Fs.MigrationTarget +} + +// HasS3Target reports whether an S3 storage migration target is configured. +func HasS3Target() bool { + u := config.Fs.MigrationTarget + return u != nil && u.Scheme == SchemeS3 +} + // ServerAddr returns the address on which the stack is run func ServerAddr() string { return net.JoinHostPort(config.Host, strconv.Itoa(config.Port)) @@ -847,6 +864,14 @@ func UseViper(v *viper.Viper) error { } } + var migrationTarget *url.URL + if raw := v.GetString("fs.migration_target"); raw != "" { + migrationTarget, err = url.Parse(raw) + if err != nil { + return err + } + } + couch, err := makeCouch(v) if err != nil { return err @@ -1161,7 +1186,8 @@ func UseViper(v *viper.Viper) error { MaxNumberToKeep: v.GetInt("fs.versioning.max_number_of_versions_to_keep"), MinDelayBetweenTwoVersions: v.GetDuration("fs.versioning.min_delay_between_two_versions"), }, - Contexts: v.GetStringMap("fs.contexts"), + Contexts: v.GetStringMap("fs.contexts"), + MigrationTarget: migrationTarget, }, CouchDB: couch, Jobs: jobs, diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go new file mode 100644 index 00000000000..0cf7a1e729a --- /dev/null +++ b/pkg/config/config/s3.go @@ -0,0 +1,114 @@ +package config + +import ( + "context" + "fmt" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var s3Client *minio.Client +var s3BucketPrefix string +var s3Region string + +// InitDefaultS3Connection initializes the default S3 handler. +func InitDefaultS3Connection() error { + return InitS3Connection(config.Fs) +} + +// InitS3Connection initializes the global S3 client connection. This is +// not a thread-safe method. +func InitS3Connection(fs Fs) error { + fsURL := fs.URL + if fsURL.Scheme != SchemeS3 { + return nil + } + + q := fsURL.Query() + endpoint := fsURL.Host + accessKey := q.Get("access_key") + secretKey := q.Get("secret_key") + region := q.Get("region") + useSSL := q.Get("use_ssl") != "false" // default true + + s3BucketPrefix = q.Get("bucket_prefix") + if s3BucketPrefix == "" { + s3BucketPrefix = "cozy" + } + // Sanitize bucket prefix: lowercase, only alphanumeric and hyphens + s3BucketPrefix = strings.ToLower(s3BucketPrefix) + s3BucketPrefix = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + return r + } + return '-' + }, s3BucketPrefix) + s3BucketPrefix = strings.Trim(s3BucketPrefix, "-") + s3Region = region + + var err error + opts := &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: useSSL, + Region: region, + } + if fs.Transport != nil { + opts.Transport = fs.Transport + } + + s3Client, err = minio.New(endpoint, opts) + if err != nil { + return fmt.Errorf("s3: could not create client: %w", err) + } + + // Verify connectivity by listing buckets + if _, err = s3Client.ListBuckets(context.Background()); err != nil { + log.Errorf("Could not connect to S3 endpoint %s: %s", endpoint, err) + return err + } + + log.Infof("Successfully connected to S3 endpoint %s", endpoint) + + // Pre-create the fixed buckets used by secondary storage (apps, assets, + // previews, exports). The per-org VFS bucket is created on instance init. + ctx := context.Background() + for _, suffix := range []string{"-apps-web", "-apps-konnectors", "-assets", "-previews", "-exports"} { + bucket := s3BucketPrefix + suffix + if err := s3Client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + log.Warnf("Could not create bucket %s: %s", bucket, err) + } + } + } + + return nil +} + +// GetS3Client returns the global S3 client. +func GetS3Client() *minio.Client { + if s3Client == nil { + panic("Called GetS3Client() before InitS3Connection()") + } + return s3Client +} + +// HasS3Client reports whether the global S3 client has been initialized, +// without panicking. Callers that need to guard against a missing S3 +// connection (e.g. before attempting a storage migration to S3) should use +// this instead of recovering from GetS3Client's panic. +func HasS3Client() bool { + return s3Client != nil +} + +// GetS3BucketPrefix returns the configured bucket prefix. +func GetS3BucketPrefix() string { + return s3BucketPrefix +} + +// GetS3Region returns the configured S3 region. +func GetS3Region() string { + return s3Region +} diff --git a/pkg/config/config/s3_target_test.go b/pkg/config/config/s3_target_test.go new file mode 100644 index 00000000000..afffacd8b3f --- /dev/null +++ b/pkg/config/config/s3_target_test.go @@ -0,0 +1,42 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrationTargetInitsS3WhenGlobalIsSwift(t *testing.T) { + // A minimal fake S3 endpoint: InitS3Connection only needs a successful + // ListBuckets call (a signed GET on "/") to consider the connection live; + // bucket-creation failures are only logged, so any other response is fine. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/" { + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, ` + + testtest + +`) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + endpoint := srv.Listener.Addr().String() + + swiftURL, _ := url.Parse("swift://openstack/") + s3URL, _ := url.Parse(fmt.Sprintf("s3://%s/?access_key=key&secret_key=secret&bucket_prefix=cozy&use_ssl=false", endpoint)) + config = &Config{Fs: Fs{URL: swiftURL, MigrationTarget: s3URL}} + + require.True(t, HasS3Target()) + // Init the S3 globals from the target even though the global scheme is swift. + require.NoError(t, InitS3Connection(Fs{URL: MigrationTargetURL()})) + assert.NotNil(t, GetS3Client()) + assert.Equal(t, "cozy", GetS3BucketPrefix()) +} diff --git a/pkg/config/config/swift.go b/pkg/config/config/swift.go index 5d7a83a23ca..66c336e2ae8 100644 --- a/pkg/config/config/swift.go +++ b/pkg/config/config/swift.go @@ -102,3 +102,12 @@ func GetSwiftConnection() *swift.Connection { } return swiftConn } + +// HasSwiftConnection reports whether the global swift connection has been +// initialized, without panicking. Callers that need to guard against a +// missing swift connection (e.g. before attempting a storage migration to +// swift) should use this instead of recovering from GetSwiftConnection's +// panic. +func HasSwiftConnection() bool { + return swiftConn != nil +} diff --git a/pkg/previewfs/cache.go b/pkg/previewfs/cache.go index 3d62f95f5e5..04cb1e49a7d 100644 --- a/pkg/previewfs/cache.go +++ b/pkg/previewfs/cache.go @@ -13,6 +13,7 @@ import ( "time" "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -41,6 +42,10 @@ func SystemCache() Cache { conn := config.GetSwiftConnection() ctx := context.Background() return swiftCache{conn, ctx} + case config.SchemeS3: + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-previews" + return newS3Cache(client, bucket) default: panic(fmt.Errorf("previewfs: unknown storage provider %s", fsURL.Scheme)) } @@ -151,6 +156,81 @@ func (s swiftCache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { return err } +type s3Cache struct { + client *minio.Client + bucket string + ctx context.Context +} + +func newS3Cache(client *minio.Client, bucket string) s3Cache { + return s3Cache{client: client, bucket: bucket, ctx: context.Background()} +} + +func (s s3Cache) ensureBucket() error { + err := s.client.MakeBucket(s.ctx, s.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (s s3Cache) getObject(name string) (*bytes.Buffer, error) { + obj, err := s.client.GetObject(s.ctx, s.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return buf, nil +} + +func (s s3Cache) putObject(name string, buffer *bytes.Buffer) error { + data := buffer.Bytes() + _, err := s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "NoSuchBucket" { + if berr := s.ensureBucket(); berr != nil { + return berr + } + _, err = s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + } + } + return err +} + +func (s s3Cache) GetIcon(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(iconFilename(md5sum)) +} + +func (s s3Cache) SetIcon(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(iconFilename(md5sum), buffer) +} + +func (s s3Cache) GetPreview(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(previewFilename(md5sum)) +} + +func (s s3Cache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(previewFilename(md5sum), buffer) +} + func iconFilename(md5sum []byte) string { return "icon-" + hex.EncodeToString(md5sum) + ".jpg" } diff --git a/pkg/s3util/s3util.go b/pkg/s3util/s3util.go new file mode 100644 index 00000000000..592fe7cdaf0 --- /dev/null +++ b/pkg/s3util/s3util.go @@ -0,0 +1,91 @@ +// Package s3util provides shared helpers for interacting with S3-compatible +// object stores via the minio-go client. +package s3util + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/minio/minio-go/v7" +) + +// IsNotFound returns true when the error is an S3 "not found" response +// (NoSuchKey or NoSuchBucket). +func IsNotFound(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchBucket" +} + +// WrapNotFound converts S3 not-found errors to os.ErrNotExist and sanitizes +// other S3 errors to avoid leaking internal bucket/key details. +func WrapNotFound(err error) error { + if IsNotFound(err) { + return os.ErrNotExist + } + code := minio.ToErrorResponse(err).Code + if code != "" { + return fmt.Errorf("s3 storage error: %s", code) + } + return err +} + +// EnsureBucket creates the bucket if it does not already exist. +func EnsureBucket(ctx context.Context, client *minio.Client, bucket, region string) error { + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ + Region: region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +// DeleteObjects deletes a list of named objects from a bucket. +func DeleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { + if len(objNames) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo, len(objNames)) + for _, name := range objNames { + objectsCh <- minio.ObjectInfo{Key: name} + } + close(objectsCh) + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + return errm +} + +// DeletePrefixObjects deletes all objects in a bucket under a given prefix. +func DeletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { + objectsCh := make(chan minio.ObjectInfo) + var listErr error + go func() { + defer close(objectsCh) + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + listErr = obj.Err + return + } + objectsCh <- obj + } + }() + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + if listErr != nil { + return listErr + } + return errm +} diff --git a/pkg/s3util/s3util_test.go b/pkg/s3util/s3util_test.go new file mode 100644 index 00000000000..45789f05dd6 --- /dev/null +++ b/pkg/s3util/s3util_test.go @@ -0,0 +1,120 @@ +package s3util_test + +import ( + "bytes" + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestS3Util(t *testing.T) { + if testing.Short() { + t.Skip("requires minio container: skipped with --short") + } + + fixture := testutils.StartMinio(t) + client := fixture.Client(t) + ctx := context.Background() + bucket := "s3util-test" + + require.NoError(t, client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})) + t.Cleanup(func() { + // Best-effort cleanup: remove all objects then the bucket. + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) { + _ = client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}) + } + _ = client.RemoveBucket(ctx, bucket) + }) + + t.Run("IsNotFound", func(t *testing.T) { + _, err := client.GetObject(ctx, bucket, "does-not-exist", minio.GetObjectOptions{}) + require.NoError(t, err) // GetObject itself doesn't fail; Stat does. + + _, err = client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + assert.True(t, s3util.IsNotFound(err)) + + // A non-not-found error should return false. + assert.False(t, s3util.IsNotFound(io.ErrUnexpectedEOF)) + }) + + t.Run("WrapNotFound", func(t *testing.T) { + _, err := client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + + wrapped := s3util.WrapNotFound(err) + assert.ErrorIs(t, wrapped, os.ErrNotExist) + + // A non-S3 error passes through unchanged. + orig := io.ErrUnexpectedEOF + assert.Equal(t, orig, s3util.WrapNotFound(orig)) + }) + + t.Run("EnsureBucket", func(t *testing.T) { + newBucket := "s3util-ensure-test" + t.Cleanup(func() { _ = client.RemoveBucket(ctx, newBucket) }) + + // First call creates. + err := s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + + // Second call is idempotent. + err = s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + }) + + t.Run("DeleteObjects", func(t *testing.T) { + // Create a few objects. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeleteObjects(ctx, client, bucket, []string{"del-a", "del-b", "del-c"}) + assert.NoError(t, err) + + // Verify they are gone. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + }) + + t.Run("DeleteObjectsEmpty", func(t *testing.T) { + // Should be a no-op, not an error. + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, nil)) + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, []string{})) + }) + + t.Run("DeletePrefixObjects", func(t *testing.T) { + // Create objects under a prefix and one outside. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three", "outside"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeletePrefixObjects(ctx, client, bucket, "pfx/") + assert.NoError(t, err) + + // Prefixed objects should be gone. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + + // Object outside the prefix should still exist. + _, err = client.StatObject(ctx, bucket, "outside", minio.StatObjectOptions{}) + assert.NoError(t, err) + }) +} diff --git a/scripts/docker/production/Dockerfile b/scripts/docker/production/Dockerfile index 98d41bcf726..dc26d9a9cba 100644 --- a/scripts/docker/production/Dockerfile +++ b/scripts/docker/production/Dockerfile @@ -14,7 +14,17 @@ RUN go mod download # Build cozy-stack COPY . . -RUN ./scripts/build.sh release ./cozy-stack +# Build directly instead of going through scripts/build.sh, which runs +# `git describe` / `git rev-parse` and trips on the COPY'd working tree +# inside the container. The version string can be overridden at build +# time via --build-arg VERSION_STRING=... +ARG VERSION_STRING=docker-build +RUN BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + && go build \ + -ldflags "-X github.com/cozy/cozy-stack/pkg/config.Version=${VERSION_STRING} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildTime=${BUILD_TIME} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildMode=production" \ + -o ./cozy-stack # Multi-stage image: the main image diff --git a/tests/testutils/minio_utils.go b/tests/testutils/minio_utils.go new file mode 100644 index 00000000000..8af8bf262ab --- /dev/null +++ b/tests/testutils/minio_utils.go @@ -0,0 +1,98 @@ +package testutils + +import ( + "context" + "fmt" + "net/url" + "testing" + "time" + + "github.com/docker/go-connections/nat" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" + + c "github.com/docker/docker/api/types/container" + tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// MinioFixture holds the state for a running MinIO container. +type MinioFixture struct { + Container tc.Container + Endpoint string // host:port + AccessKey string + SecretKey string + t *testing.T +} + +// StartMinio starts a MinIO container for testing. +func StartMinio(t *testing.T) *MinioFixture { + t.Helper() + + accessKey := "minioadmin" + secretKey := "minioadmin" + hostPort := getFreePort(t) + + req := tc.ContainerRequest{ + Image: "minio/minio:RELEASE.2025-02-28T09-55-16Z", + ExposedPorts: []string{"9000/tcp"}, + Env: map[string]string{ + "MINIO_ROOT_USER": accessKey, + "MINIO_ROOT_PASSWORD": secretKey, + }, + Cmd: []string{"server", "/data"}, + HostConfigModifier: func(hc *c.HostConfig) { + hc.PortBindings = nat.PortMap{ + "9000/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: hostPort}}, + } + }, + WaitingFor: wait.ForHTTP("/minio/health/live"). + WithPort("9000/tcp"). + WithStartupTimeout(60 * time.Second), + } + + container, err := tc.GenericContainer(context.Background(), tc.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err, "failed to start MinIO") + + host, err := container.Host(context.Background()) + require.NoError(t, err) + + endpoint := fmt.Sprintf("%s:%s", host, hostPort) + t.Logf("MinIO endpoint: %s", endpoint) + + t.Cleanup(func() { + _ = container.Terminate(context.Background()) + }) + + return &MinioFixture{ + Container: container, + Endpoint: endpoint, + AccessKey: accessKey, + SecretKey: secretKey, + t: t, + } +} + +// Client returns a minio.Client connected to this fixture. +func (f *MinioFixture) Client(t *testing.T) *minio.Client { + t.Helper() + client, err := minio.New(f.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(f.AccessKey, f.SecretKey, ""), + Secure: false, + }) + require.NoError(t, err) + return client +} + +// FsURL returns a *url.URL suitable for config.InitS3Connection. +func (f *MinioFixture) FsURL(bucketPrefix string) *url.URL { + return &url.URL{ + Scheme: "s3", + Host: f.Endpoint, + RawQuery: fmt.Sprintf("access_key=%s&secret_key=%s&bucket_prefix=%s&use_ssl=false", f.AccessKey, f.SecretKey, bucketPrefix), + } +} diff --git a/web/instances/instances.go b/web/instances/instances.go index 4e716c3aea0..1778ad96a95 100644 --- a/web/instances/instances.go +++ b/web/instances/instances.go @@ -15,6 +15,7 @@ import ( "github.com/cozy/cozy-stack/model/app" "github.com/cozy/cozy-stack/model/instance" "github.com/cozy/cozy-stack/model/instance/lifecycle" + "github.com/cozy/cozy-stack/model/instance/storagemigration" "github.com/cozy/cozy-stack/model/notification" "github.com/cozy/cozy-stack/model/notification/center" "github.com/cozy/cozy-stack/model/oauth" @@ -300,6 +301,26 @@ func deleteHandler(c echo.Context) error { return c.NoContent(http.StatusNoContent) } +func migrateStorageHandler(c echo.Context) error { + domain := c.Param("domain") + inst, err := lifecycle.GetInstance(domain) + if err != nil { + return wrapError(err) + } + opts := storagemigration.Options{ + To: c.QueryParam("to"), + DryRun: c.QueryParam("dry_run") == "true", + FlagOnly: c.QueryParam("flag_only") == "true", + Force: c.QueryParam("force") == "true", + PurgeSource: c.QueryParam("purge_source") == "true", + } + rep, err := storagemigration.Migrate(inst, opts) + if err != nil { + return wrapError(err) + } + return c.JSON(http.StatusOK, rep) +} + func setAuthMode(c echo.Context) error { domain := c.Param("domain") inst, err := lifecycle.GetInstance(domain) @@ -785,6 +806,9 @@ func Routes(router *echo.Group) { router.GET("/contexts/:name", showContext) router.GET("/with-app-version/:slug/:version", appVersion) + // Storage migration + router.POST("/:domain/migrate-storage", migrateStorageHandler) + // Checks router.GET("/:domain/fsck", fsckHandler) router.POST("/:domain/checks/triggers", checkTriggers) diff --git a/web/instances/instances_test.go b/web/instances/instances_test.go index e6bc473bb4c..4b1cb6cd523 100644 --- a/web/instances/instances_test.go +++ b/web/instances/instances_test.go @@ -97,4 +97,27 @@ func TestInstances(t *testing.T) { attrs.HasValue("feature_sets", []string{"71df3022-abd9-11ee-b79b-9cb6d0907fa3", "790789f8-abd9-11ee-ae09-9cb6d0907fa3"}) }) }) + + t.Run("MigrateStorage", func(t *testing.T) { + domain := "migrate-storage.cozy.localhost" + t.Cleanup(func() { _ = lifecycle.Destroy(domain) }) + + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/instances"). + WithQuery("Domain", domain). + WithQuery("Locale", "en"). + WithQuery("SwiftLayout", "-1"). + WithHeader("Authorization", "Bearer "+token). + Expect().Status(201) + + // An unsupported target scheme is rejected by the Migrate guard + // before touching any storage backend, so this exercises the route + // and error propagation without requiring a live S3/Swift target. + e.POST("/instances/"+domain+"/migrate-storage"). + WithQuery("to", "not-a-scheme"). + WithQuery("dry_run", "true"). + WithHeader("Authorization", "Bearer "+token). + Expect().Status(500) + }) } diff --git a/web/settings/capabilities.go b/web/settings/capabilities.go index dcd2104294a..c76fcb1c86f 100644 --- a/web/settings/capabilities.go +++ b/web/settings/capabilities.go @@ -43,6 +43,8 @@ func NewCapabilities(inst *instance.Instance) jsonapi.Object { switch config.FsURL().Scheme { case config.SchemeSwift, config.SchemeSwiftSecure: versioning = inst.SwiftLayout >= 2 + case config.SchemeS3: + versioning = true } flat := config.GetConfig().Subdomains == config.FlatSubdomains diff --git a/web/settings/instance.go b/web/settings/instance.go index fb87bc6f26e..23df1c3aada 100644 --- a/web/settings/instance.go +++ b/web/settings/instance.go @@ -88,7 +88,6 @@ func (h *HTTPHandler) getInstance(c echo.Context) error { } else if url != "" { doc.M["legal_notice_url"] = url } - return jsonapi.Data(c, http.StatusOK, &apiInstance{doc}, nil) }