From 1ea0e09d08c44484cc6516736031e56b1088dd99 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 11:18:42 +0200 Subject: [PATCH 01/23] feat: add S3-compatible storage backend Implement a complete vfs.VFS backend for S3-compatible object storage (OVH, MinIO, Scaleway, ...) using minio-go/v7, mirroring the Swift V3 implementation. - config: add SchemeS3 and an S3 connection singleton, configured via the fs.url query params (access_key, secret_key, region, bucket_prefix, use_ssl) - model/vfs/vfss3: full VFS implementation with a bucket per orgId, a key prefix per instance, streaming CreateFile via io.Pipe, and multipart uploads with bounded memory - wire SchemeS3 into every storage dispatch point: VFS, avatars, thumbnails, apps copier and file servers, archiver, cache, dynamic assets and capabilities - pkg/s3util: shared S3 helpers (EnsureBucket, DeleteObjects, DeletePrefixObjects, IsNotFound, WrapNotFound) - pkg/appfs and pkg/assets/dynamic: S3 app installation and dynamic assets storage - security hardening: path-traversal protection, S3 error sanitization, MD5 integrity verification and bounded reads, plus the code-review fixes - tests: add the S3 backend to the VFS integration suite (MinIO testcontainer) - docs: document the S3 storage backend architecture and configuration --- cozy.example.yaml | 1 + docs/s3.md | 248 +++++++ go.mod | 12 +- go.sum | 25 +- model/app/apps.go | 12 + model/instance/instance.go | 18 + model/move/archiver.go | 84 +++ model/stack/main.go | 5 + model/vfs/vfs_test.go | 31 + model/vfs/vfss3/avatar.go | 106 +++ model/vfs/vfss3/fsck.go | 254 +++++++ model/vfs/vfss3/impl.go | 1154 ++++++++++++++++++++++++++++++++ model/vfs/vfss3/naming_test.go | 71 ++ model/vfs/vfss3/thumbs.go | 247 +++++++ pkg/appfs/s3.go | 333 +++++++++ pkg/assets/dynamic/fs.go | 6 + pkg/assets/dynamic/impl_s3.go | 126 ++++ pkg/config/config/config.go | 2 + pkg/config/config/s3.go | 106 +++ pkg/previewfs/cache.go | 80 +++ pkg/s3util/s3util.go | 91 +++ pkg/s3util/s3util_test.go | 120 ++++ tests/testutils/minio_utils.go | 98 +++ web/settings/capabilities.go | 2 + web/settings/instance.go | 1 - 25 files changed, 3229 insertions(+), 4 deletions(-) create mode 100644 docs/s3.md create mode 100644 model/vfs/vfss3/avatar.go create mode 100644 model/vfs/vfss3/fsck.go create mode 100644 model/vfs/vfss3/impl.go create mode 100644 model/vfs/vfss3/naming_test.go create mode 100644 model/vfs/vfss3/thumbs.go create mode 100644 pkg/appfs/s3.go create mode 100644 pkg/assets/dynamic/impl_s3.go create mode 100644 pkg/config/config/s3.go create mode 100644 pkg/s3util/s3util.go create mode 100644 pkg/s3util/s3util_test.go create mode 100644 tests/testutils/minio_utils.go diff --git a/cozy.example.yaml b/cozy.example.yaml index 1f50fc8c1e6..088ccf30d1e 100644 --- a/cozy.example.yaml +++ b/cozy.example.yaml @@ -67,6 +67,7 @@ fs: # url: file://localhost/var/lib/cozy # url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }}&ProjectName={{ .Env.OS_PROJECT_NAME }}&UserDomainName={{ .Env.OS_USER_DOMAIN_NAME }}&Timeout={{ .Env.GOSWIFT_TIMEOUT }} + # url: s3://{{ .Env.S3_ENDPOINT }}?access_key={{ .Env.S3_ACCESS_KEY }}&secret_key={{ .Env.S3_SECRET_KEY }}®ion={{ .Env.S3_REGION }}&bucket_prefix=cozy&use_ssl=true # Swift FS can be used with advanced parameters to activate TLS properties. # For using swift with https, you must use the "swift+https" scheme. diff --git a/docs/s3.md b/docs/s3.md new file mode 100644 index 00000000000..c757f3b4455 --- /dev/null +++ b/docs/s3.md @@ -0,0 +1,248 @@ +[Table of contents](README.md#table-of-contents) + +# S3 Storage Backend + +cozy-stack supports S3-compatible object storage as a file system backend, +alongside the existing local filesystem (afero) and OpenStack Swift backends. +It has been designed to work with any S3-compatible provider (OVH, MinIO, +Scaleway, etc.) and does not depend on the AWS SDK. + +## Configuration + +The S3 backend is configured via the `fs.url` parameter using the `s3://` +scheme. All connection parameters are passed as query parameters: + +```yaml +fs: + url: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +| Parameter | Description | Default | +|-----------------|--------------------------------------|---------| +| `access_key` | S3 access key ID | — | +| `secret_key` | S3 secret access key | — | +| `region` | S3 region | — | +| `bucket_prefix` | Prefix for all bucket names | `cozy` | +| `use_ssl` | Use HTTPS for S3 connections | `true` | + +The host part of the URL is the S3 endpoint (e.g. `s3.rbx.io.cloud.ovh.net` +for OVH, `localhost:9000` for MinIO). + +### Local development with MinIO + +This tutorial explains how to set up a local S3 backend using MinIO for +development and testing. + +**1. Start MinIO with Docker:** + +```bash +docker run -d --name minio \ + -p 9000:9000 \ + -p 9001:9001 \ + -e MINIO_ROOT_USER=minioadmin \ + -e MINIO_ROOT_PASSWORD=minioadmin \ + minio/minio server /data --console-address ":9001" +``` + +MinIO is now running: +- S3 API: `http://localhost:9000` +- Web console: `http://localhost:9001` (login: `minioadmin` / `minioadmin`) + +**2. Configure cozy-stack:** + +Buckets are created automatically at startup. No manual bucket creation +is needed. + +Edit your `~/.cozy/cozy.yaml`: + +```yaml +fs: + url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +**3. Build and start:** + +```bash +go build -o ~/go/bin/cozy-stack . +~/go/bin/cozy-stack serve +``` + +You should see in the logs: + +``` +Successfully connected to S3 endpoint localhost:9000 +``` + +**4. (Re)install your apps:** + +When switching from a different storage backend (e.g. `file://`), you need +to reinstall the apps so their assets are stored in S3: + +```bash +cozy-stack apps uninstall drive --domain your.domain.localhost:8080 +cozy-stack apps install drive --domain your.domain.localhost:8080 +cozy-stack apps uninstall home --domain your.domain.localhost:8080 +cozy-stack apps install home --domain your.domain.localhost:8080 +``` + +**5. Verify:** + +Check that objects appear in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-apps-web/ +``` + +Upload a file via the Drive UI or the API: + +```bash +TOKEN=$(cozy-stack instances token-cli your.domain.localhost:8080 io.cozy.files) +curl -X POST \ + -H "Authorization: Bearer $TOKEN" \ + -H "Content-Type: text/plain" \ + "http://your.domain.localhost:8080/files/io.cozy.files.root-dir?Type=file&Name=test.txt" \ + -d "Hello S3!" +``` + +Verify the file is in MinIO: + +```bash +docker exec minio mc ls --recursive local/cozy-default/ +``` + +**6. Switching back to local filesystem:** + +Comment out the S3 URL in your config and restart cozy-stack: + +```yaml +fs: + # url: s3://localhost:9000?access_key=minioadmin&secret_key=minioadmin&bucket_prefix=cozy&use_ssl=false +``` + +Note: files uploaded to S3 won't be accessible when using the local +filesystem backend, and vice versa. Each backend has its own storage. + +## Bucket strategy + +### Design rationale + +Swift uses one container per instance (`cozy-v3-`). This doesn't +scale well for S3 where bucket creation can be limited (AWS limits to 100 +buckets per account by default, OVH to 100 as well). Instead, the S3 backend +uses a **shared bucket per organization** with **key prefixes per instance**. + +### Bucket naming + +Each bucket name is derived from the instance's `OrgID` field: + +``` +- +``` + +- If `OrgID` is empty, `"default"` is used as fallback +- The org ID is sanitized: lowercased, underscores/dots replaced by hyphens, + non-alphanumeric characters stripped, consecutive hyphens collapsed, + truncated to respect the 63-character S3 bucket name limit +- Examples: `cozy-default`, `cozy-acme-corp`, `cozy-org-12345` + +### Dedicated buckets for secondary storage + +In addition to the main VFS bucket, the S3 backend uses dedicated buckets for +other storage needs: + +| Bucket | Content | +|-------------------------------|-----------------------------------------| +| `-` | Main VFS data (files, versions) | +| `-apps-web` | Web application assets (drive, etc.) | +| `-apps-konnectors` | Konnector assets | +| `-assets` | Dynamic assets | +| `-previews` | PDF preview and icon cache | +| `-exports` | Instance export archives | + +Buckets are created automatically on first use. + +## Object key structure + +Within a bucket, each instance's data is isolated by a key prefix derived +from `DBPrefix()` (typically the instance domain or a CouchDB prefix). + +### VFS files + +``` +//// +``` + +The document ID (a 32-character UUID v7 hex string) is split into virtual +subfolders to avoid flat hierarchies: + +``` +cozy218def.../019d35b1-9dc3-78ec-994d-f5/44336/7f1b6/e0AbCdEfGh123456 + ^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ ^^^^^ ^^^^^^^^^^^^^^^^ + first 22 chars 5 ch 5 ch 16-char internalID +``` + +This structure mirrors the Swift V3 layout (`MakeObjectNameV3`). + +### Thumbnails + +``` +/thumbs/- +``` + +Formats: `small`, `medium`, `large`. + +### Avatar + +``` +/avatar +``` + +## Memory consumption + +The S3 backend is designed to have comparable memory usage to Swift: + +| Scenario | Memory per upload | +|-----------------------------|-------------------| +| Known size, file < 5 GiB | ~32 KB (single PUT, stream) | +| Unknown size (rare) | ~5 MiB (multipart, PartSize=5MiB, NumThreads=1) | + +When `ByteSize` is known on the file document (the common case for drive +uploads), the backend passes the exact size to `PutObject`, which uses a +single PUT request that streams directly to S3 with minimal buffering — the +same behavior as Swift's `ObjectCreate`. + +Multipart upload is only used for files with unknown size or exceeding 5 GiB, +with `PartSize=5MiB` and `NumThreads=1` to limit memory. + +## Encryption at rest + +The S3 backend does not implement client-side encryption. Encryption should +be configured at the infrastructure level (S3 bucket default encryption / +SSE-S3), the same approach used for the Swift backend. + +## Differences from Swift + +| Aspect | Swift | S3 | +|--------------------|-------------------------------|----------------------------------------| +| Container/Bucket | One per instance | One per organization (shared) | +| Instance isolation | Container name | Key prefix within bucket | +| Delete instance | Delete entire container | Delete all objects with key prefix | +| File streaming | Native `io.WriteCloser` | `io.Pipe` + `PutObject` goroutine | +| Bulk delete | `BulkDelete` API | `RemoveObjects` channel API | +| Server-side copy | `ObjectCopy` | `CopyObject` (same endpoint only) | + +## Testing + +The VFS integration tests run against all three backends (afero, swift, s3) +using a table-driven approach. The S3 tests use +[testcontainers-go](https://testcontainers.com/guides/getting-started-with-testcontainers-for-go/) +with a MinIO container that is started automatically. + +```bash +# Run VFS tests (requires CouchDB + Docker) +COZY_COUCHDB_URL=http://admin:admin@localhost:5984/ \ + go test ./model/vfs/ -run TestVfs -v -count=1 -timeout 300s + +# Run naming unit tests (no external deps) +go test ./model/vfs/vfss3/ -run "TestSanitize|TestBucketName|TestMakeObjectKey|TestMakeDocID" -v +``` diff --git a/go.mod b/go.mod index bb89af8150d..a0dfd7e5bca 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 github.com/labstack/echo/v4 v4.15.1 github.com/leonelquinteros/gotext v1.7.2 + github.com/minio/minio-go/v7 v7.0.99 github.com/mitchellh/mapstructure v1.5.0 github.com/mssola/user_agent v0.6.0 github.com/ncw/swift/v2 v2.0.3 @@ -103,6 +104,7 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -121,13 +123,17 @@ require ( github.com/imkira/go-interpol v1.1.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jonas-p/go-shp v0.1.1 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.2 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect github.com/labstack/gommon v0.4.2 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.1.0 // indirect @@ -142,6 +148,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -149,6 +156,7 @@ require ( github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/rs/xid v1.6.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -161,6 +169,7 @@ require ( github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.6.1 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -185,6 +194,7 @@ require ( go.opentelemetry.io/otel/trace v1.39.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/time v0.14.0 // indirect diff --git a/go.sum b/go.sum index c2ebd8c105e..647c2aaad24 100644 --- a/go.sum +++ b/go.sum @@ -144,6 +144,8 @@ github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyT github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gavv/httpexpect/v2 v2.16.0 h1:Ty2favARiTYTOkCRZGX7ojXXjGyNAIohM1lZ3vqaEwI= github.com/gavv/httpexpect/v2 v2.16.0/go.mod h1:uJLaO+hQ25ukBJtQi750PsztObHybNllN+t+MbbW8PY= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -243,8 +245,13 @@ github.com/jonas-p/go-shp v0.1.1/go.mod h1:MRIhyxDQ6VVp0oYeD7yPGr5RSTNScUFKCDsI5 github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666 h1:abLciEiilfMf19Q1TFWDrp9j5z5one60dnnpvc6eabg= github.com/justincampbell/bigduration v0.0.0-20160531141349-e45bf03c0666/go.mod h1:xqGOmDZzLOG7+q/CgsbXv10g4tgPsbjhmAxyaTJMvis= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= +github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -272,6 +279,12 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE= +github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/mapstructure v0.0.0-20170523030023-d0303fe80992/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -326,6 +339,8 @@ github.com/oschwald/maxminddb-golang v1.13.1/go.mod h1:K4pgV9N/GcK694KSTmVSDTODk github.com/pelletier/go-toml v1.0.1-0.20170904195809-1d6b12b7cb29/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -355,6 +370,8 @@ github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -412,6 +429,8 @@ github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSW github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= +github.com/tinylib/msgp v1.6.1 h1:ESRv8eL3u+DNHUoSAAQRE50Hm162zqAnBoGv9PzScPY= +github.com/tinylib/msgp v1.6.1/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -478,6 +497,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20170512130425-ab89591268e0/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= diff --git a/model/app/apps.go b/model/app/apps.go index 34c099a869e..6e4ecb4c74c 100644 --- a/model/app/apps.go +++ b/model/app/apps.go @@ -156,6 +156,10 @@ func Copier(appsType consts.AppType, inst *instance.Instance) appfs.Copier { return appfs.NewAferoCopier(baseFS) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftCopier(config.GetSwiftConnection(), appsType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), appsType) + return appfs.NewS3Copier(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -175,6 +179,10 @@ func AppsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.WebappType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.WebappType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -194,6 +202,10 @@ func KonnectorsFileServer(i *instance.Instance) appfs.FileServer { return appfs.NewAferoFileServer(baseFS, nil) case config.SchemeSwift, config.SchemeSwiftSecure: return appfs.NewSwiftFileServer(config.GetSwiftConnection(), consts.KonnectorType) + case config.SchemeS3: + client := config.GetS3Client() + bucket := appfs.S3AppsBucket(config.GetS3BucketPrefix(), consts.KonnectorType) + return appfs.NewS3FileServer(client, bucket) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/instance/instance.go b/model/instance/instance.go index 5a6d4cf6826..c3478c66d10 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -16,6 +16,7 @@ import ( "github.com/cozy/cozy-stack/model/permission" "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" build "github.com/cozy/cozy-stack/pkg/config" "github.com/cozy/cozy-stack/pkg/config/config" @@ -196,6 +197,11 @@ func (i *Instance) GetContextName() string { return i.ContextName } +// GetOrgID returns the organization ID of the instance. +func (i *Instance) GetOrgID() string { + return i.OrgID +} + // SessionSecret returns the session secret. func (i *Instance) SessionSecret() []byte { // The prefix is here to invalidate all the sessions that were created on @@ -261,6 +267,8 @@ func (i *Instance) MakeVFS() error { default: err = ErrInvalidSwiftLayout } + case config.SchemeS3: + i.vfs, err = vfss3.New(i, index, disk, mutex) default: err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme) } @@ -285,6 +293,11 @@ func (i *Instance) AvatarFS() vfs.Avatarer { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewAvatarFs(client, bucket, keyPrefix) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } @@ -309,6 +322,11 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { default: panic(ErrInvalidSwiftLayout) } + case config.SchemeS3: + client := config.GetS3Client() + bucket := vfss3.BucketName(i.GetOrgID(), config.GetS3BucketPrefix()) + keyPrefix := i.DBPrefix() + "/" + return vfss3.NewThumbsFs(client, bucket, keyPrefix) default: panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) } diff --git a/model/move/archiver.go b/model/move/archiver.go index be36ccb9b7e..03bb896ac3a 100644 --- a/model/move/archiver.go +++ b/model/move/archiver.go @@ -14,6 +14,7 @@ import ( "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/crypto" multierror "github.com/hashicorp/go-multierror" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -45,6 +46,8 @@ func SystemArchiver() Archiver { return newAferoArchiver(fs) case config.SchemeSwift, config.SchemeSwiftSecure: return newSwiftArchiver() + case config.SchemeS3: + return newS3Archiver() default: panic(fmt.Errorf("exports: unknown storage provider %s", fsURL.Scheme)) } @@ -149,3 +152,84 @@ func (a *switfArchiver) RemoveArchives(exportDocs []*ExportDoc) error { } return nil } + +func newS3Archiver() Archiver { + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-exports" + return &s3Archiver{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +type s3Archiver struct { + client *minio.Client + bucket string + ctx context.Context +} + +func (a *s3Archiver) ensureBucket() error { + err := a.client.MakeBucket(a.ctx, a.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (a *s3Archiver) OpenArchive(inst *instance.Instance, exportDoc *ExportDoc) (io.ReadCloser, error) { + objectName := exportDoc.Domain + "/" + exportDoc.ID() + obj, err := a.client.GetObject(a.ctx, a.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Verify the object exists by calling Stat; GetObject itself does not + // perform a network request until Read is called, and Stat triggers + // a HEAD that surfaces NoSuchKey early. + if _, err := obj.Stat(); err != nil { + obj.Close() + return nil, err + } + return obj, nil +} + +func (a *s3Archiver) CreateArchive(exportDoc *ExportDoc) (io.WriteCloser, error) { + if err := a.ensureBucket(); err != nil { + return nil, err + } + + objectName := exportDoc.Domain + "/" + exportDoc.ID() + pr, pw := io.Pipe() + + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, objectName, + pr, -1, + minio.PutObjectOptions{ContentType: "application/tar+gzip"}) + // Close the read side so that any error is propagated to the writer. + pr.CloseWithError(err) + }() + + return pw, nil +} + +func (a *s3Archiver) RemoveArchives(exportDocs []*ExportDoc) error { + if len(exportDocs) == 0 { + return nil + } + + objectsCh := make(chan minio.ObjectInfo, len(exportDocs)) + for _, e := range exportDocs { + objectsCh <- minio.ObjectInfo{Key: e.Domain + "/" + e.ID()} + } + close(objectsCh) + + var errm error + for e := range a.client.RemoveObjects(a.ctx, a.bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = multierror.Append(errm, e.Err) + } + return errm +} diff --git a/model/stack/main.go b/model/stack/main.go index 5ddb66f0660..f82bdbe87da 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -85,6 +85,11 @@ security features. Please do not use this binary as your production server. return nil, nil, fmt.Errorf("failed to init the swift connection: %w", err) } + // Init the main global connection to the S3 server + if err := config.InitDefaultS3Connection(); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 connection: %w", err) + } + workersList, err := job.GetWorkersList() if err != nil { return nil, nil, fmt.Errorf("failed to get the workers list: %w", err) diff --git a/model/vfs/vfs_test.go b/model/vfs/vfs_test.go index 7d20d308d3f..40d9f1e52b0 100644 --- a/model/vfs/vfs_test.go +++ b/model/vfs/vfs_test.go @@ -17,6 +17,7 @@ import ( "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" "github.com/cozy/cozy-stack/model/vfs/vfsswift" "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/consts" @@ -54,6 +55,7 @@ func TestVfs(t *testing.T) { aferoFS := makeAferoFS(t) swiftFS := makeSwiftFS(t) + s3FS := makeS3FS(t) var tests = []struct { name string @@ -61,6 +63,7 @@ func TestVfs(t *testing.T) { }{ {"afero", aferoFS}, {"swift", swiftFS}, + {"s3", s3FS}, } for _, tt := range tests { @@ -909,3 +912,31 @@ func makeSwiftFS(t *testing.T) vfs.VFS { return swiftFs } + +func makeS3FS(t *testing.T) vfs.VFS { + t.Helper() + + minioFixture := testutils.StartMinio(t) + db := &contexter{0, "io.cozy.vfs.s3.test", "io.cozy.vfs.s3.test", "cozy_beta"} + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{ + URL: minioFixture.FsURL("test"), + })) + + mutex = config.Lock().ReadWrite(db, "vfs-s3-test") + s3Fs, err := vfss3.New(db, index, &diskImpl{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + + require.NoError(t, g.Wait()) + require.NoError(t, s3Fs.InitFs()) + + return s3Fs +} diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go new file mode 100644 index 00000000000..a164ef39c4b --- /dev/null +++ b/model/vfs/vfss3/avatar.go @@ -0,0 +1,106 @@ +package vfss3 + +import ( + "context" + "fmt" + "io" + "net/http" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/minio/minio-go/v7" +) + +// NewAvatarFs creates a new avatar filesystem backed by S3. +func NewAvatarFs(client *minio.Client, bucket, keyPrefix string) vfs.Avatarer { + return &avatarS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type avatarS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +func (a *avatarS3) avatarKey() string { + return a.keyPrefix + "avatar" +} + +func (a *avatarS3) CreateAvatar(contentType string) (io.WriteCloser, error) { + key := a.avatarKey() + pr, pw := io.Pipe() + + meta := map[string]string{ + "created-at": time.Now().UTC().Format(time.RFC3339), + } + + errCh := make(chan error, 1) + go func() { + _, err := a.client.PutObject(a.ctx, a.bucket, key, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &avatarWriter{pw: pw, errCh: errCh}, nil +} + +type avatarWriter struct { + pw *io.PipeWriter + errCh chan error +} + +func (w *avatarWriter) Write(p []byte) (int, error) { + return w.pw.Write(p) +} + +func (w *avatarWriter) Close() error { + if err := w.pw.Close(); err != nil { + return err + } + return <-w.errCh +} + +func (a *avatarS3) DeleteAvatar() error { + err := a.client.RemoveObject(a.ctx, a.bucket, a.avatarKey(), minio.RemoveObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil + } + return err + } + return nil +} + +func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + t := time.Time{} + if createdAt, ok := info.UserMetadata["Created-At"]; ok && createdAt != "" { + if createdAtTime, err := time.Parse(time.RFC3339, createdAt); err == nil { + t = createdAtTime + } + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, "avatar", t, obj) + return nil +} diff --git a/model/vfs/vfss3/fsck.go b/model/vfs/vfss3/fsck.go new file mode 100644 index 00000000000..364f6c3bca1 --- /dev/null +++ b/model/vfs/vfss3/fsck.go @@ -0,0 +1,254 @@ +package vfss3 + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "errors" + "path" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/minio/minio-go/v7" +) + +func (sfs *s3VFS) Fsck(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + tree, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + if err = sfs.CheckTreeIntegrity(tree, accumulate, failFast); err != nil { + if errors.Is(err, vfs.ErrFsckFailFast) { + return nil + } + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) CheckFilesConsistency(accumulate func(log *vfs.FsckLog), failFast bool) error { + entries := make(map[string]*vfs.TreeFile, 1024) + _, err := sfs.BuildTree(func(f *vfs.TreeFile) { + if !f.IsDir { + entries[f.DocID+"/"+f.InternalID] = f + } + }) + if err != nil { + return err + } + return sfs.checkFiles(entries, accumulate, failFast) +} + +func (sfs *s3VFS) checkFiles( + entries map[string]*vfs.TreeFile, + accumulate func(log *vfs.FsckLog), + failFast bool, +) error { + versions := make(map[string]*vfs.Version, 1024) + err := couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error { + v := &vfs.Version{} + if erru := json.Unmarshal(data, v); erru != nil { + return erru + } + versions[v.DocID] = v + return nil + }) + if err != nil { + return err + } + + images := make(map[string]struct{}) + err = couchdb.ForeachDocs(sfs, consts.NotesImages, func(_ string, data json.RawMessage) error { + img := make(map[string]interface{}) + if erru := json.Unmarshal(data, &img); erru != nil { + return erru + } + id, _ := img["_id"].(string) + images[id] = struct{}{} + return nil + }) + if err != nil && !couchdb.IsNoDatabaseError(err) { + return err + } + + fileIDs := make(map[string]struct{}, len(entries)) + for _, f := range entries { + fileIDs[f.DocID] = struct{}{} + } + + // List all objects under our key prefix + for obj := range sfs.client.ListObjects(sfs.ctx, sfs.bucket, minio.ListObjectsOptions{ + Prefix: sfs.keyPrefix, + Recursive: true, + }) { + if obj.Err != nil { + return obj.Err + } + + // Strip the key prefix to get the object name + objName := strings.TrimPrefix(obj.Key, sfs.keyPrefix) + + if objName == "avatar" { + continue + } + if strings.HasPrefix(objName, "thumbs/") { + thumbName := strings.TrimPrefix(objName, "thumbs/") + idx := strings.LastIndex(thumbName, "-") + if idx < 0 { + continue + } + thumbName = thumbName[0:idx] // Remove -format suffix + fileID, _ := makeDocID(thumbName) + if _, ok := fileIDs[fileID]; !ok { + if _, ok := images[fileID]; !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.ThumbnailWithNoFile, + IsFile: true, + FileDoc: &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: objName, + }, + }, + }, + }) + if failFast { + return nil + } + } + } + continue + } + + docID, internalID := makeDocID(objName) + if v, ok := versions[docID+"/"+internalID]; ok { + // ETag from S3 may or may not be an MD5 (multipart uploads use composite ETags). + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsVersion: true, + VersionDoc: v, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: v.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: v.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(versions, v.DocID) + continue + } + f, ok := entries[docID+"/"+internalID] + if !ok { + accumulate(&vfs.FsckLog{ + Type: vfs.IndexMissing, + IsFile: true, + FileDoc: objectToFileDoc(obj), + }) + if failFast { + return nil + } + } else { + etag := strings.Trim(obj.ETag, "\"") + if !strings.Contains(etag, "-") { + md5sum, err := hex.DecodeString(etag) + if err == nil { + if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Size { + accumulate(&vfs.FsckLog{ + Type: vfs.ContentMismatch, + IsFile: true, + FileDoc: f, + ContentMismatch: &vfs.FsckContentMismatch{ + SizeFile: obj.Size, + SizeIndex: f.ByteSize, + MD5SumFile: md5sum, + MD5SumIndex: f.MD5Sum, + }, + }) + if failFast { + return nil + } + } + } + } + delete(entries, docID+"/"+internalID) + } + } + + // entries should contain only data that does not contain an associated + // object in S3. + for _, f := range entries { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsFile: true, + FileDoc: f, + }) + if failFast { + return nil + } + } + + for _, v := range versions { + accumulate(&vfs.FsckLog{ + Type: vfs.FSMissing, + IsVersion: true, + VersionDoc: v, + }) + if failFast { + return nil + } + } + + return nil +} + +func objectToFileDoc(obj minio.ObjectInfo) *vfs.TreeFile { + md5sum, _ := hex.DecodeString(strings.Trim(obj.ETag, "\"")) + name := "unknown" + mime, class := vfs.ExtractMimeAndClass(obj.ContentType) + // Strip any key prefix — we need to find the object name portion + // which is just the last segments of the key. + objName := obj.Key + if idx := strings.Index(objName, "/"); idx >= 0 { + // The first segment is the key prefix (db prefix); skip it + objName = objName[idx+1:] + } + fileID, internalID := makeDocID(objName) + return &vfs.TreeFile{ + DirOrFileDoc: vfs.DirOrFileDoc{ + DirDoc: &vfs.DirDoc{ + Type: consts.FileType, + DocID: fileID, + DocName: name, + DirID: "", + CreatedAt: obj.LastModified, + UpdatedAt: obj.LastModified, + Fullpath: path.Join(vfs.OrphansDirName, name), + }, + ByteSize: obj.Size, + Mime: mime, + Class: class, + MD5Sum: md5sum, + InternalID: internalID, + }, + } +} diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go new file mode 100644 index 00000000000..caa2cbabfa5 --- /dev/null +++ b/model/vfs/vfss3/impl.go @@ -0,0 +1,1154 @@ +// Package vfss3 is the implementation of the Virtual File System by using +// an S3-compatible object storage. The file contents are saved in S3 buckets, +// and the metadata are indexed in CouchDB. +package vfss3 + +import ( + "bytes" + "context" + "crypto/md5" + + "errors" + "fmt" + "hash" + "io" + "os" + "regexp" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/lock" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/gofrs/uuid/v5" + "github.com/minio/minio-go/v7" +) + +type s3VFS struct { + vfs.Indexer + vfs.DiskThresholder + client *minio.Client + cluster int + domain string + prefix string // DBPrefix — used as key prefix in the bucket + contextName string + ctx context.Context + bucket string + keyPrefix string // prefix + "/" + region string + mu lock.ErrorRWLocker + log *logger.Entry +} + +var bucketNameCleaner = regexp.MustCompile(`[^a-z0-9-]`) + +// sanitizeBucketName produces a valid S3 bucket name component from an arbitrary string. +func sanitizeBucketName(s string) string { + s = strings.ToLower(s) + s = strings.ReplaceAll(s, "_", "-") + s = strings.ReplaceAll(s, ".", "-") + s = bucketNameCleaner.ReplaceAllString(s, "") + // Collapse consecutive hyphens + for strings.Contains(s, "--") { + s = strings.ReplaceAll(s, "--", "-") + } + s = strings.Trim(s, "-") + if len(s) > 37 { + s = s[:37] + } + return s +} + +// BucketName returns the S3 bucket name for a given orgID and bucket prefix. +func BucketName(orgID, bucketPrefix string) string { + if orgID == "" { + orgID = "default" + } + name := bucketPrefix + "-" + sanitizeBucketName(orgID) + if len(name) > 63 { + name = name[:63] + } + name = strings.TrimRight(name, "-") + if len(name) < 3 { + name += strings.Repeat("0", 3-len(name)) + } + return name +} + +// MakeObjectKey builds the S3 object key for a given file. +// It reuses the same virtual subfolder structure as Swift V3. +func MakeObjectKey(keyPrefix, docID, internalID string) string { + return keyPrefix + makeObjectName(docID, internalID) +} + +// makeObjectName builds the object name (without key prefix), identical to +// vfsswift.MakeObjectNameV3. +func makeObjectName(docID, internalID string) string { + if len(docID) != 32 || len(internalID) != 16 { + return docID + "/" + internalID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] + "/" + internalID +} + +func makeDocID(objName string) (string, string) { + if len(objName) != 51 { + parts := strings.SplitN(objName, "/", 2) + if len(parts) < 2 { + return objName, "" + } + return parts[0], parts[1] + } + return objName[:22] + objName[23:28] + objName[29:34], objName[35:] +} + +// NewInternalID returns a random string that can be used as an internal_vfs_id. +func NewInternalID() string { + return utils.RandomString(16) +} + +// New returns a vfs.VFS instance backed by an S3-compatible object store. +func New(db vfs.Prefixer, index vfs.Indexer, disk vfs.DiskThresholder, mu lock.ErrorRWLocker) (vfs.VFS, error) { + client := config.GetS3Client() + bucketPrefix := config.GetS3BucketPrefix() + + orgID := "" + if inst, ok := db.(interface{ GetOrgID() string }); ok { + orgID = inst.GetOrgID() + } + bucket := BucketName(orgID, bucketPrefix) + dbPrefix := db.DBPrefix() + if dbPrefix == "" { + return nil, fmt.Errorf("vfss3: empty DBPrefix") + } + + return &s3VFS{ + Indexer: index, + DiskThresholder: disk, + client: client, + cluster: db.DBCluster(), + domain: db.DomainName(), + prefix: dbPrefix, + contextName: db.GetContextName(), + ctx: context.Background(), + bucket: bucket, + keyPrefix: dbPrefix + "/", + region: config.GetS3Region(), + mu: mu, + log: logger.WithDomain(db.DomainName()).WithNamespace("vfss3"), + }, nil +} + +func (sfs *s3VFS) MaxFileSize() int64 { + return -1 // no per-file limit — S3 multipart handles large files transparently +} + +func (sfs *s3VFS) DBCluster() int { + return sfs.cluster +} + +func (sfs *s3VFS) DBPrefix() string { + return sfs.prefix +} + +func (sfs *s3VFS) DomainName() string { + return sfs.domain +} + +func (sfs *s3VFS) GetContextName() string { + return sfs.contextName +} + +func (sfs *s3VFS) GetIndexer() vfs.Indexer { + return sfs.Indexer +} + +func (sfs *s3VFS) UseSharingIndexer(index vfs.Indexer) vfs.VFS { + return &s3VFS{ + Indexer: index, + DiskThresholder: sfs.DiskThresholder, + client: sfs.client, + cluster: sfs.cluster, + domain: sfs.domain, + prefix: sfs.prefix, + contextName: sfs.contextName, + ctx: context.Background(), + bucket: sfs.bucket, + keyPrefix: sfs.keyPrefix, + region: sfs.region, + mu: sfs.mu, + log: sfs.log, + } +} + +func (sfs *s3VFS) InitFs() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if err := sfs.Indexer.InitIndex(); err != nil { + return err + } + err := sfs.client.MakeBucket(sfs.ctx, sfs.bucket, minio.MakeBucketOptions{ + Region: sfs.region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + sfs.log.Errorf("Could not create bucket %q: %s", sfs.bucket, err.Error()) + return err + } + sfs.log.Infof("Created bucket %q", sfs.bucket) + return nil +} + +func (sfs *s3VFS) Delete() error { + sfs.log.Infof("Deleting all objects with prefix %q in bucket %q", sfs.keyPrefix, sfs.bucket) + return s3util.DeletePrefixObjects(sfs.ctx, sfs.client, sfs.bucket, sfs.keyPrefix) +} + +func (sfs *s3VFS) CreateDir(doc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + exists, err := sfs.Indexer.DirChildExists(doc.DirID, doc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + if doc.ID() == "" { + return sfs.Indexer.CreateDirDoc(doc) + } + return sfs.Indexer.CreateNamedDirDoc(doc) +} + +// putResult is the result sent back from the background PutObject goroutine. +type putResult struct { + info minio.UploadInfo + err error +} + +func (sfs *s3VFS) CreateFile(newdoc, olddoc *vfs.FileDoc, opts ...vfs.CreateOptions) (vfs.File, error) { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.Unlock() + + newsize, maxsize, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return nil, err + } + // CheckAvailableDiskSpace already returns ErrFileTooBig / ErrMaxFileSize + // when the upload would exceed the per-file or quota budget. The + // streaming-time check below (s3FileCreation.Write) re-checks against + // maxsize once the actual byte count is known. + + if olddoc != nil { + newdoc.SetID(olddoc.ID()) + newdoc.SetRev(olddoc.Rev()) + newdoc.CreatedAt = olddoc.CreatedAt + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return nil, err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + if !vfs.OptionsAllowCreationInTrash(opts) { + return nil, vfs.ErrParentInTrash + } + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return nil, err + } + if exists { + return nil, os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return nil, err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Use a pipe: writes go into pw, the PutObject goroutine reads from pr. + pr, pw := io.Pipe() + + uploadSize := newdoc.ByteSize + if uploadSize < 0 { + uploadSize = -1 + } + + resultCh := make(chan putResult, 1) + go func() { + info, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, pr, uploadSize, minio.PutObjectOptions{ + ContentType: newdoc.Mime, + PartSize: 5 * 1024 * 1024, // 5 MiB + NumThreads: 1, + }) + // Propagate the outcome to the writer side: if PutObject errored before + // draining the pipe, an in-flight Write would otherwise block forever. + _ = pr.CloseWithError(err) + resultCh <- putResult{info: info, err: err} + }() + + extractor := vfs.NewMetaExtractor(newdoc) + + return &s3FileCreation{ + fs: sfs, + pw: pw, + resultCh: resultCh, + newdoc: newdoc, + olddoc: olddoc, + objKey: objKey, + w: 0, + size: newsize, + maxsize: maxsize, + capsize: capsize, + meta: extractor, + md5H: md5.New(), + }, nil +} + +func (sfs *s3VFS) CopyFile(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, olddoc) + if err != nil { + return err + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + newdoc.InternalID = NewInternalID() + + srcKey := MakeObjectKey(sfs.keyPrefix, olddoc.DocID, olddoc.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(newdoc); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +func (sfs *s3VFS) DissociateFile(src, dst *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if src.DirID != dst.DirID || src.DocName != dst.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + uid, err := uuid.NewV7() + if err != nil { + return err + } + dst.DocID = uid.String() + + srcKey := MakeObjectKey(sfs.keyPrefix, src.DocID, src.InternalID) + dstKey := MakeObjectKey(sfs.keyPrefix, dst.DocID, dst.InternalID) + + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: sfs.bucket, Object: srcKey}, + ); err != nil { + return err + } + if err := sfs.Indexer.CreateNamedFileDoc(dst); err != nil { + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, dstKey, minio.RemoveObjectOptions{}) + return err + } + + return sfs.destroyFileLocked(src) +} + +func (sfs *s3VFS) DissociateDir(src, dst *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + if dst.DirID != src.DirID || dst.DocName != src.DocName { + exists, err := sfs.Indexer.DirChildExists(dst.DirID, dst.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if err := sfs.Indexer.CreateDirDoc(dst); err != nil { + return err + } + return sfs.Indexer.DeleteDirDoc(src) +} + +func (sfs *s3VFS) destroyDir(doc *vfs.DirDoc, push func(vfs.TrashJournal) error, onlyContent bool) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + files, destroyed, err := sfs.Indexer.DeleteDirDocAndContent(doc, onlyContent) + if err != nil { + return err + } + if len(files) == 0 { + return nil + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + ids := make([]string, len(files)) + objNames := make([]string, len(files)) + for i, file := range files { + ids[i] = file.DocID + objNames[i] = MakeObjectKey(sfs.keyPrefix, file.DocID, file.InternalID) + } + return push(vfs.TrashJournal{ + FileIDs: ids, + ObjectNames: objNames, + }) +} + +func (sfs *s3VFS) DestroyDirContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, true) +} + +func (sfs *s3VFS) DestroyDirAndContent(doc *vfs.DirDoc, push func(vfs.TrashJournal) error) error { + return sfs.destroyDir(doc, push, false) +} + +func (sfs *s3VFS) DestroyFile(doc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.destroyFileLocked(doc) +} + +func (sfs *s3VFS) destroyFileLocked(doc *vfs.FileDoc) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := []string{ + MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID), + } + if err := sfs.Indexer.DeleteFileDoc(doc); err != nil { + return err + } + destroyed := doc.ByteSize + if versions, errv := vfs.VersionsFor(sfs, doc.DocID); errv == nil { + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID)) + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + sfs.log.Warnf("DestroyFile failed on BatchDeleteVersions: %s", err) + } + } + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("DestroyFile failed on deleteObjects: %s", err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return nil +} + +func (sfs *s3VFS) EnsureErased(journal vfs.TrashJournal) error { + diskUsage, _ := sfs.Indexer.DiskUsage() + objNames := journal.ObjectNames + var errm error + var destroyed int64 + var allVersions []*vfs.Version + for _, fileID := range journal.FileIDs { + versions, err := vfs.VersionsFor(sfs, fileID) + if err != nil { + if !couchdb.IsNoDatabaseError(err) { + sfs.log.Warnf("EnsureErased failed on VersionsFor(%s): %s", fileID, err) + errm = errors.Join(errm, err) + } + continue + } + for _, v := range versions { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, fileID, internalID)) + destroyed += v.ByteSize + } + allVersions = append(allVersions, versions...) + } + if err := sfs.Indexer.BatchDeleteVersions(allVersions); err != nil { + sfs.log.Warnf("EnsureErased failed on BatchDeleteVersions: %s", err) + errm = errors.Join(errm, err) + } + if err := s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames); err != nil { + sfs.log.Warnf("EnsureErased failed on deleteObjects: %s", err) + errm = errors.Join(errm, err) + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return errm +} + +func (sfs *s3VFS) OpenFile(doc *vfs.FileDoc) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, doc.InternalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // Stat the object to detect if it exists. + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) OpenFileVersion(doc *vfs.FileDoc, version *vfs.Version) (vfs.File, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + internalID := version.DocID + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, doc.DocID, internalID) + obj, err := sfs.client.GetObject(sfs.ctx, sfs.bucket, objKey, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + if _, err := obj.Stat(); err != nil { + _ = obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return &s3FileOpen{obj}, nil +} + +func (sfs *s3VFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + diskQuota := sfs.DiskQuota() + if diskQuota > 0 { + diskUsage, err := sfs.DiskUsage() + if err != nil { + return err + } + if diskUsage+version.ByteSize > diskQuota { + return vfs.ErrFileTooBig + } + } + + parts := strings.SplitN(version.DocID, "/", 2) + if len(parts) != 2 { + return vfs.ErrIllegalFilename + } + objKey := MakeObjectKey(sfs.keyPrefix, parts[0], parts[1]) + + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, version.ByteSize, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + if errc := content.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + + return sfs.Indexer.CreateVersion(version) +} + +func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + save := vfs.NewVersion(doc) + if err := sfs.Indexer.CreateVersion(save); err != nil { + return err + } + + newdoc := doc.Clone().(*vfs.FileDoc) + if parts := strings.SplitN(version.DocID, "/", 2); len(parts) > 1 { + newdoc.InternalID = parts[1] + } + vfs.SetMetaFromVersion(newdoc, version) + if err := sfs.Indexer.UpdateFileDoc(doc, newdoc); err != nil { + _ = sfs.Indexer.DeleteVersion(save) + return err + } + + return sfs.Indexer.DeleteVersion(version) +} + +func (sfs *s3VFS) CleanOldVersion(fileID string, v *vfs.Version) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + return sfs.cleanOldVersion(fileID, v) +} + +func (sfs *s3VFS) cleanOldVersion(fileID string, v *vfs.Version) error { + if err := sfs.Indexer.DeleteVersion(v); err != nil { + return err + } + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, fileID, internalID) + return sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) +} + +func (sfs *s3VFS) ClearOldVersions() error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + diskUsage, _ := sfs.Indexer.DiskUsage() + versions, err := sfs.Indexer.AllVersions() + if err != nil { + return err + } + var objNames []string + var destroyed int64 + for _, v := range versions { + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + objNames = append(objNames, MakeObjectKey(sfs.keyPrefix, parts[0], parts[1])) + } + destroyed += v.ByteSize + } + if err := sfs.Indexer.BatchDeleteVersions(versions); err != nil { + return err + } + vfs.DiskQuotaAfterDestroy(sfs, diskUsage, destroyed) + return s3util.DeleteObjects(sfs.ctx, sfs.client, sfs.bucket, objNames) +} + +func (sfs *s3VFS) CopyFileFromOtherFS( + newdoc, olddoc *vfs.FileDoc, + srcFS vfs.Fs, + srcDoc *vfs.FileDoc, +) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + + newsize, _, capsize, err := vfs.CheckAvailableDiskSpace(sfs, newdoc) + if err != nil { + return err + } + + newpath, err := sfs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + if strings.HasPrefix(newpath, vfs.TrashDirName+"/") { + return vfs.ErrParentInTrash + } + + if olddoc == nil { + var exists bool + exists, err = sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + if newdoc.DocID == "" { + uid, err := uuid.NewV7() + if err != nil { + return err + } + newdoc.DocID = uid.String() + } + + newdoc.InternalID = NewInternalID() + + dstKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, newdoc.InternalID) + + // Try server-side copy if the source is also an s3VFS on the same client. + if srcS3, ok := srcFS.(*s3VFS); ok { + srcKey := MakeObjectKey(srcS3.keyPrefix, srcDoc.DocID, srcDoc.InternalID) + if _, err := sfs.client.CopyObject(sfs.ctx, + minio.CopyDestOptions{Bucket: sfs.bucket, Object: dstKey}, + minio.CopySrcOptions{Bucket: srcS3.bucket, Object: srcKey}, + ); err != nil { + return err + } + } else { + // Stream from the source FS. + srcFile, err := srcFS.OpenFile(srcDoc) + if err != nil { + return err + } + _, err = sfs.client.PutObject(sfs.ctx, sfs.bucket, dstKey, srcFile, srcDoc.ByteSize, minio.PutObjectOptions{ + ContentType: srcDoc.Mime, + }) + if errc := srcFile.Close(); err == nil { + err = errc + } + if err != nil { + return err + } + } + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = sfs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else { + err = sfs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(sfs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := sfs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(sfs.keyPrefix, newdoc.DocID, internalID) + _ = sfs.client.RemoveObject(sfs.ctx, sfs.bucket, objKey, minio.RemoveObjectOptions{}) + } + for _, old := range toClean { + _ = sfs.cleanOldVersion(newdoc.DocID, old) + } + } + + if capsize > 0 && newsize >= capsize { + vfs.PushDiskQuotaAlert(sfs, true) + } + + return nil +} + +// UpdateFileDoc calls the indexer UpdateFileDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateFileDoc +func (sfs *s3VFS) UpdateFileDoc(olddoc, newdoc *vfs.FileDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateFileDoc(olddoc, newdoc) +} + +// UpdateDirDoc calls the indexer UpdateDirDoc function and adds a few checks +// before actually calling this method: +// - locks the filesystem for writing +// - checks that we don't move a directory to one of its descendant +// - checks in case we have a move operation that the new path is available +// +// @override Indexer.UpdateDirDoc +func (sfs *s3VFS) UpdateDirDoc(olddoc, newdoc *vfs.DirDoc) error { + if lockerr := sfs.mu.Lock(); lockerr != nil { + return lockerr + } + defer sfs.mu.Unlock() + if newdoc.DirID != olddoc.DirID || newdoc.DocName != olddoc.DocName { + if strings.HasPrefix(newdoc.Fullpath, olddoc.Fullpath+"/") { + return vfs.ErrForbiddenDocMove + } + exists, err := sfs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + return sfs.Indexer.UpdateDirDoc(olddoc, newdoc) +} + +func (sfs *s3VFS) DirByID(fileID string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByID(fileID) +} + +func (sfs *s3VFS) DirByPath(name string) (*vfs.DirDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirByPath(name) +} + +func (sfs *s3VFS) FileByID(fileID string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByID(fileID) +} + +func (sfs *s3VFS) FileByPath(name string) (*vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FileByPath(name) +} + +func (sfs *s3VFS) FilePath(doc *vfs.FileDoc) (string, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return "", lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.FilePath(doc) +} + +func (sfs *s3VFS) DirOrFileByID(fileID string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByID(fileID) +} + +func (sfs *s3VFS) DirOrFileByPath(name string) (*vfs.DirDoc, *vfs.FileDoc, error) { + if lockerr := sfs.mu.RLock(); lockerr != nil { + return nil, nil, lockerr + } + defer sfs.mu.RUnlock() + return sfs.Indexer.DirOrFileByPath(name) +} + +// s3FileCreation represents a file open for writing. It is used to create +// a file or to modify the content of a file. +type s3FileCreation struct { + fs *s3VFS + pw *io.PipeWriter + resultCh chan putResult + newdoc *vfs.FileDoc + olddoc *vfs.FileDoc + objKey string + w int64 + size int64 + maxsize int64 + capsize int64 + meta *vfs.MetaExtractor + md5H hash.Hash + err error +} + +func (f *s3FileCreation) Read(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) ReadAt(p []byte, off int64) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Seek(offset int64, whence int) (int64, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileCreation) Write(p []byte) (int, error) { + if f.err != nil { + return 0, f.err + } + + if f.meta != nil { + if _, err := (*f.meta).Write(p); err != nil && !errors.Is(err, io.ErrClosedPipe) { + (*f.meta).Abort(err) + f.meta = nil + } + } + + // Write to local MD5 hash + _, _ = f.md5H.Write(p) + + n, err := f.pw.Write(p) + if err != nil { + f.err = err + return n, err + } + + f.w += int64(n) + if f.maxsize >= 0 && f.w > f.maxsize { + f.err = vfs.ErrFileTooBig + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + if f.size >= 0 && f.w > f.size { + f.err = vfs.ErrContentLengthMismatch + _ = f.pw.CloseWithError(f.err) + return n, f.err + } + + return n, nil +} + +func (f *s3FileCreation) Close() (err error) { + defer func() { + if err != nil { + // Remove the object from S3 if an error occurred + _ = f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, f.objKey, minio.RemoveObjectOptions{}) + // If an error has occurred when creating a new file, we should + // also delete the file from the index. + if f.olddoc == nil { + _ = f.fs.Indexer.DeleteFileDoc(f.newdoc) + } + } + }() + + // Close the pipe writer to signal EOF to PutObject + if err = f.pw.Close(); err != nil { + if f.meta != nil { + (*f.meta).Abort(err) + f.meta = nil + } + if f.err == nil { + f.err = err + } + } + + // Wait for the PutObject goroutine to finish + result := <-f.resultCh + + if result.err != nil { + if f.meta != nil { + (*f.meta).Abort(result.err) + f.meta = nil + } + if f.err == nil { + f.err = result.err + } + } + + newdoc, olddoc, written := f.newdoc, f.olddoc, f.w + + if f.meta != nil { + if errc := (*f.meta).Close(); errc == nil { + vfs.MergeMetadata(newdoc, (*f.meta).Result()) + } + } + + if f.err != nil { + return f.err + } + + // Verify or compute MD5 checksum. + // The local md5H hash is always computed from the same data stream that + // goes to S3 (via the Write method), so it is authoritative. + localMD5 := f.md5H.Sum(nil) + if newdoc.MD5Sum != nil { + // The caller provided an expected hash — verify it matches what was written. + if !bytes.Equal(newdoc.MD5Sum, localMD5) { + return vfs.ErrInvalidHash + } + } else { + newdoc.MD5Sum = localMD5 + } + + if f.size < 0 { + newdoc.ByteSize = written + } + + if newdoc.ByteSize != written { + return vfs.ErrContentLengthMismatch + } + + lockerr := f.fs.mu.Lock() + if lockerr != nil { + return lockerr + } + defer f.fs.mu.Unlock() + + // Check again that a file with the same path does not exist. It can happen + // when the same file is uploaded twice in parallel. + if olddoc == nil { + exists, err := f.fs.Indexer.DirChildExists(newdoc.DirID, newdoc.DocName) + if err != nil { + return err + } + if exists { + return os.ErrExist + } + } + + var newpath string + newpath, err = f.fs.Indexer.FilePath(newdoc) + if err != nil { + return err + } + newdoc.Trashed = strings.HasPrefix(newpath, vfs.TrashDirName+"/") + + var v *vfs.Version + if olddoc != nil { + v = vfs.NewVersion(olddoc) + err = f.fs.Indexer.UpdateFileDoc(olddoc, newdoc) + } else if newdoc.ID() == "" { + err = f.fs.Indexer.CreateFileDoc(newdoc) + } else { + err = f.fs.Indexer.CreateNamedFileDoc(newdoc) + } + if err != nil { + return err + } + + if v != nil { + actionV, toClean, _ := vfs.FindVersionsToClean(f.fs, newdoc.DocID, v) + if bytes.Equal(newdoc.MD5Sum, olddoc.MD5Sum) { + actionV = vfs.CleanCandidateVersion + } + if actionV == vfs.KeepCandidateVersion { + if errv := f.fs.Indexer.CreateVersion(v); errv != nil { + actionV = vfs.CleanCandidateVersion + } + } + if actionV == vfs.CleanCandidateVersion { + internalID := v.DocID + if parts := strings.SplitN(v.DocID, "/", 2); len(parts) > 1 { + internalID = parts[1] + } + objKey := MakeObjectKey(f.fs.keyPrefix, newdoc.DocID, internalID) + if err := f.fs.client.RemoveObject(f.fs.ctx, f.fs.bucket, objKey, minio.RemoveObjectOptions{}); err != nil { + f.fs.log.Warnf("Could not delete previous version %q: %s", objKey, err.Error()) + } + } + for _, old := range toClean { + if err := f.fs.cleanOldVersion(newdoc.DocID, old); err != nil { + f.fs.log.Warnf("Could not delete old versions for %s: %s", newdoc.DocID, err.Error()) + } + } + } + + if f.capsize > 0 && f.size >= f.capsize { + vfs.PushDiskQuotaAlert(f.fs, true) + } + + return nil +} + +// s3FileOpen represents a file open for reading. +type s3FileOpen struct { + obj *minio.Object +} + +func (f *s3FileOpen) Read(p []byte) (int, error) { + return f.obj.Read(p) +} + +func (f *s3FileOpen) ReadAt(p []byte, off int64) (int, error) { + return f.obj.ReadAt(p, off) +} + +func (f *s3FileOpen) Seek(offset int64, whence int) (int64, error) { + return f.obj.Seek(offset, whence) +} + +func (f *s3FileOpen) Write(p []byte) (int, error) { + return 0, os.ErrInvalid +} + +func (f *s3FileOpen) Close() error { + return f.obj.Close() +} + +var ( + _ vfs.VFS = &s3VFS{} + _ vfs.File = &s3FileCreation{} + _ vfs.File = &s3FileOpen{} +) diff --git a/model/vfs/vfss3/naming_test.go b/model/vfs/vfss3/naming_test.go new file mode 100644 index 00000000000..1201dc82988 --- /dev/null +++ b/model/vfs/vfss3/naming_test.go @@ -0,0 +1,71 @@ +package vfss3 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeBucketName(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"production", "production"}, + {"my_context", "my-context"}, + {"My.Context", "my-context"}, + {"UPPERCASE", "uppercase"}, + {"with spaces!", "withspaces"}, + {"a--b--c", "a-b-c"}, + {"-leading-trailing-", "leading-trailing"}, + {"very-long-name-that-exceeds-the-maximum-allowed-length", "very-long-name-that-exceeds-the-maxim"}, + {"", ""}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, sanitizeBucketName(tt.input)) + }) + } +} + +func TestBucketName(t *testing.T) { + tests := []struct { + orgID string + bucketPrefix string + expected string + }{ + {"org-123", "cozy", "cozy-org-123"}, + {"", "cozy", "cozy-default"}, + {"My_Org", "cozy", "cozy-my-org"}, + {"org.example.com", "cozy", "cozy-org-example-com"}, + } + + for _, tt := range tests { + t.Run(tt.orgID, func(t *testing.T) { + assert.Equal(t, tt.expected, BucketName(tt.orgID, tt.bucketPrefix)) + }) + } +} + +func TestMakeObjectKey(t *testing.T) { + // Standard 32-char docID and 16-char internalID + key := MakeObjectKey("alice.example.com/", "abcdefghijklmnopqrstuvwxyz012345", "0123456789abcdef") + assert.Equal(t, "alice.example.com/abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef", key) + + // Non-standard lengths + key = MakeObjectKey("alice.example.com/", "short", "id") + assert.Equal(t, "alice.example.com/short/id", key) +} + +func TestMakeDocID(t *testing.T) { + // Standard 51-char object name + docID, internalID := makeDocID("abcdefghijklmnopqrstuv/wxyz0/12345/0123456789abcdef") + assert.Equal(t, "abcdefghijklmnopqrstuvwxyz012345", docID) + assert.Equal(t, "0123456789abcdef", internalID) + + // Non-standard + docID, internalID = makeDocID("short/id") + assert.Equal(t, "short", docID) + assert.Equal(t, "id", internalID) +} diff --git a/model/vfs/vfss3/thumbs.go b/model/vfs/vfss3/thumbs.go new file mode 100644 index 00000000000..ee9c064ad0d --- /dev/null +++ b/model/vfs/vfss3/thumbs.go @@ -0,0 +1,247 @@ +package vfss3 + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "net/http" + "os" + "time" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/logger" + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +var unixEpochZero = time.Time{} + +// NewThumbsFs creates a new thumbnail filesystem backed by S3. +func NewThumbsFs(client *minio.Client, bucket, keyPrefix string) vfs.Thumbser { + return &thumbsS3{ + client: client, + bucket: bucket, + keyPrefix: keyPrefix, + ctx: context.Background(), + } +} + +type thumbsS3 struct { + client *minio.Client + bucket string + keyPrefix string + ctx context.Context +} + +type s3Thumb struct { + pw *io.PipeWriter + errCh chan error + client *minio.Client + bucket string + name string + ctx context.Context +} + +func (t *s3Thumb) Write(p []byte) (int, error) { + return t.pw.Write(p) +} + +func (t *s3Thumb) Commit() error { + if err := t.pw.Close(); err != nil { + return err + } + return <-t.errCh +} + +func (t *s3Thumb) Abort() error { + // Close the pipe with an error to cancel the PutObject goroutine. + errc := t.pw.CloseWithError(fmt.Errorf("thumbnail creation aborted")) + // Drain the errCh so the goroutine is not leaked. + <-t.errCh + // Try to remove the possibly partially written object. + errd := t.client.RemoveObject(t.ctx, t.bucket, t.name, minio.RemoveObjectOptions{}) + if errd != nil && minio.ToErrorResponse(errd).Code == "NoSuchKey" { + errd = nil + } + // Write an empty marker object to indicate that the thumbnail generation failed. + _, errp := t.client.PutObject(t.ctx, t.bucket, t.name, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: echo.MIMEOctetStream, + }) + if errc != nil { + return errc + } + if errd != nil { + return errd + } + return errp +} + +func (ts *thumbsS3) createThumbFile(name, contentType string, meta map[string]string) (vfs.ThumbFiler, error) { + pr, pw := io.Pipe() + + errCh := make(chan error, 1) + go func() { + _, err := ts.client.PutObject(ts.ctx, ts.bucket, name, pr, -1, minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + errCh <- err + }() + + return &s3Thumb{ + pw: pw, + errCh: errCh, + client: ts.client, + bucket: ts.bucket, + name: name, + ctx: ts.ctx, + }, nil +} + +func (ts *thumbsS3) CreateThumb(img *vfs.FileDoc, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(img.ID(), format) + meta := map[string]string{ + "file-md5": hex.EncodeToString(img.MD5Sum), + } + return ts.createThumbFile(name, "image/jpeg", meta) +} + +func (ts *thumbsS3) ThumbExists(img *vfs.FileDoc, format string) (bool, error) { + name := ts.makeName(img.ID(), format) + info, err := ts.client.StatObject(ts.ctx, ts.bucket, name, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return false, nil + } + return false, err + } + if md5str, ok := info.UserMetadata["File-Md5"]; ok && md5str != "" { + md5sum, err := hex.DecodeString(md5str) + if err == nil && !bytes.Equal(md5sum, img.MD5Sum) { + return false, nil + } + } + return true, nil +} + +func (ts *thumbsS3) RemoveThumbs(img *vfs.FileDoc, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(img.ID(), format) + } + return s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) +} + +func (ts *thumbsS3) ServeThumbContent(w http.ResponseWriter, req *http.Request, img *vfs.FileDoc, format string) error { + name := ts.makeName(img.ID(), format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + if info.ContentType == echo.MIMEOctetStream { + // We have some old images where the thumbnail has not been correctly + // saved. We should delete the thumbnail to allow another try. + if info.Size > 0 { + _ = ts.RemoveThumbs(img, vfs.ThumbnailFormatNames) + return os.ErrNotExist + } + // Image magick has failed to generate a thumbnail, and retrying would + // be useless. + return os.ErrInvalid + } + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) CreateNoteThumb(id, mime, format string) (vfs.ThumbFiler, error) { + name := ts.makeName(id, format) + return ts.createThumbFile(name, mime, nil) +} + +func (ts *thumbsS3) OpenNoteThumb(id, format string) (io.ReadCloser, error) { + name := ts.makeName(id, format) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, s3util.WrapNotFound(err) + } + // Stat to verify the object actually exists (GetObject doesn't fail on missing keys). + if _, err := obj.Stat(); err != nil { + obj.Close() + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return obj, nil +} + +func (ts *thumbsS3) RemoveNoteThumb(id string, formats []string) error { + objNames := make([]string, len(formats)) + for i, format := range formats { + objNames[i] = ts.makeName(id, format) + } + err := s3util.DeleteObjects(ts.ctx, ts.client, ts.bucket, objNames) + if err != nil { + logger.WithNamespace("vfss3").Infof("Cannot remove note thumbs: %s", err) + } + return err +} + +func (ts *thumbsS3) ServeNoteThumbContent(w http.ResponseWriter, req *http.Request, id string) error { + name := ts.makeName(id, consts.NoteImageThumbFormat) + obj, err := ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + + info, err := obj.Stat() + if err != nil { + obj.Close() + // Try the original format as fallback. + name = ts.makeName(id, consts.NoteImageOriginalFormat) + obj, err = ts.client.GetObject(ts.ctx, ts.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + info, err = obj.Stat() + if err != nil { + obj.Close() + return s3util.WrapNotFound(err) + } + } + defer obj.Close() + + w.Header().Set("Etag", fmt.Sprintf(`"%s"`, info.ETag)) + w.Header().Set("Content-Type", info.ContentType) + http.ServeContent(w, req, name, unixEpochZero, obj) + return nil +} + +func (ts *thumbsS3) makeName(imgID string, format string) string { + return ts.keyPrefix + fmt.Sprintf("thumbs/%s-%s", makeThumbObjectName(imgID), format) +} + +// makeThumbObjectName builds a virtual subfolder structure for thumbnails. +// It splits the 32-char ID into three parts to avoid a flat hierarchy. +// This is the same logic as vfsswift.MakeObjectName (without internalID). +func makeThumbObjectName(docID string) string { + if len(docID) != 32 { + return docID + } + return docID[:22] + "/" + docID[22:27] + "/" + docID[27:] +} diff --git a/pkg/appfs/s3.go b/pkg/appfs/s3.go new file mode 100644 index 00000000000..33e08f88794 --- /dev/null +++ b/pkg/appfs/s3.go @@ -0,0 +1,333 @@ +package appfs + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "mime" + "net/http" + "os" + "path" + "strconv" + "strings" + + "github.com/andybalholm/brotli" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/filetype" + "github.com/cozy/cozy-stack/pkg/s3util" + + web_utils "github.com/cozy/cozy-stack/pkg/utils" + "github.com/labstack/echo/v4" + "github.com/minio/minio-go/v7" +) + +// installedMarkerSuffix is appended to the app's base key to form the +// "installation complete" marker object. The suffix is chosen so the marker +// can't collide with any file under //... — S3 browsers +// would otherwise display the marker as a file sitting next to a folder of +// the same name. +const installedMarkerSuffix = ".cozy-installed" + +// s3Copier implements the Copier interface backed by S3. +type s3Copier struct { + client *minio.Client + bucket string + appObj string + started bool + objectNames []string + ctx context.Context +} + +// NewS3Copier creates a Copier that stores app files in S3. +func NewS3Copier(client *minio.Client, bucket string) Copier { + return &s3Copier{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (f *s3Copier) Exist(slug, version, shasum string) (bool, error) { + f.appObj = path.Join(slug, version) + if shasum != "" { + f.appObj += "-" + shasum + } + _, err := f.client.StatObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, minio.StatObjectOptions{}) + if err == nil { + return true, nil + } + if s3util.IsNotFound(err) { + return false, nil + } + return false, err +} + +func (f *s3Copier) Start(slug, version, shasum string) (bool, error) { + exist, err := f.Exist(slug, version, shasum) + if err != nil || exist { + return exist, err + } + + if err := s3util.EnsureBucket(f.ctx, f.client, f.bucket, ""); err != nil { + return false, err + } + + f.objectNames = []string{} + f.started = true + return false, nil +} + +func (f *s3Copier) Copy(stat os.FileInfo, src io.Reader) error { + if !f.started { + return fmt.Errorf("appfs: copier must call Start() before Copy()") + } + + // Write directly to the final location (appObj/filename). + // Reject path traversal attempts in filenames. + name := stat.Name() + if strings.Contains(name, "..") { + return fmt.Errorf("appfs: invalid filename %q", name) + } + objName := path.Join(f.appObj, name) + + contentType := filetype.ByExtension(path.Ext(stat.Name())) + if contentType == "" { + contentType, src = filetype.FromReader(src) + } + + // Compress with brotli. + var buf bytes.Buffer + bw := brotli.NewWriter(&buf) + if _, err := io.Copy(bw, src); err != nil { + return err + } + if err := bw.Close(); err != nil { + return err + } + + meta := map[string]string{ + "X-Content-Encoding": "br", + "Original-Content-Length": strconv.FormatInt(stat.Size(), 10), + } + + f.objectNames = append(f.objectNames, objName) + _, err := f.client.PutObject(f.ctx, f.bucket, objName, + bytes.NewReader(buf.Bytes()), int64(buf.Len()), + minio.PutObjectOptions{ + ContentType: contentType, + UserMetadata: meta, + }) + return err +} + +func (f *s3Copier) Abort() error { + return s3util.DeleteObjects(f.ctx, f.client, f.bucket, f.objectNames) +} + +func (f *s3Copier) Commit() (err error) { + // Create the marker object that signals the version is complete. The + // suffix keeps it on a distinct key from the //... files, + // so S3 browsers don't render a folder and a file with the same name. + _, err = f.client.PutObject(f.ctx, f.bucket, f.appObj+installedMarkerSuffix, + bytes.NewReader(nil), 0, minio.PutObjectOptions{ + ContentType: "text/plain", + }) + return err +} + +// s3Server implements the FileServer interface backed by S3. +type s3Server struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FileServer creates a FileServer that serves app files from S3. +func NewS3FileServer(client *minio.Client, bucket string) FileServer { + return &s3Server{ + client: client, + bucket: bucket, + ctx: context.Background(), + } +} + +func (s *s3Server) Open(slug, version, shasum, file string) (io.ReadCloser, error) { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return nil, s3util.WrapNotFound(err) + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, s3util.WrapNotFound(err) + } + contentEncoding := info.UserMetadata["X-Content-Encoding"] + if contentEncoding == "br" { + return newBrotliReadCloser(obj) + } else if contentEncoding == "gzip" { + return newGzipReadCloser(obj) + } + return obj, nil +} + +func (s *s3Server) ServeFileContent(w http.ResponseWriter, req *http.Request, slug, version, shasum, file string) error { + objName := s.makeObjectName(slug, version, shasum, file) + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err != nil { + return s3util.WrapNotFound(err) + } + defer obj.Close() + + info, err := obj.Stat() + if err != nil { + return s3util.WrapNotFound(err) + } + + if checkETag := req.Header.Get("Cache-Control") == ""; checkETag { + etagVal := info.ETag + if len(etagVal) > 10 { + etagVal = etagVal[:10] + } + etag := fmt.Sprintf(`"%s"`, etagVal) + if web_utils.CheckPreconditions(w, req, etag) { + return nil + } + w.Header().Set("Etag", etag) + } + + // Read the full object to handle brotli decompression. + // Limit to 50 MiB to avoid unbounded memory allocation from corrupted objects. + const maxAppFileSize = 50 << 20 + content, err := io.ReadAll(io.LimitReader(obj, maxAppFileSize)) + if err != nil { + return err + } + + var r io.Reader = bytes.NewReader(content) + contentLength := info.Size + contentType := info.ContentType + + contentEncoding := info.UserMetadata["X-Content-Encoding"] + origContentLength := info.UserMetadata["Original-Content-Length"] + if contentEncoding == "br" { + if acceptBrotliEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "br") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + r = brotli.NewReader(bytes.NewReader(content)) + } + } else if contentEncoding == "gzip" { + if acceptGzipEncoding(req) { + w.Header().Set(echo.HeaderContentEncoding, "gzip") + } else { + if origContentLength != "" { + contentLength, _ = strconv.ParseInt(origContentLength, 10, 64) + } + gr, gerr := gzip.NewReader(bytes.NewReader(content)) + if gerr != nil { + return gerr + } + defer gr.Close() + r = gr + } + } + + ext := path.Ext(file) + if contentType == "" { + contentType = mime.TypeByExtension(ext) + } + if contentType == "text/xml" && ext == ".svg" { + contentType = "image/svg+xml" + } + + return serveContent(w, req, contentType, contentLength, r) +} + +func (s *s3Server) ServeCodeTarball(w http.ResponseWriter, req *http.Request, slug, version, shasum string) error { + objName := path.Join(slug, version) + if shasum != "" { + objName += "-" + shasum + } + objName += ".tgz" + + // Try to serve a pre-built tarball first. + obj, err := s.client.GetObject(s.ctx, s.bucket, objName, minio.GetObjectOptions{}) + if err == nil { + info, serr := obj.Stat() + if serr == nil { + defer obj.Close() + return serveContent(w, req, info.ContentType, info.Size, obj) + } + obj.Close() + } + + buf, err := prepareTarball(s, slug, version, shasum) + if err != nil { + return err + } + content, err := io.ReadAll(buf) + if err != nil { + return err + } + contentType := mime.TypeByExtension(".gz") + + // Store the tarball for future requests. + _, _ = s.client.PutObject(s.ctx, s.bucket, objName, + bytes.NewReader(content), int64(len(content)), + minio.PutObjectOptions{ContentType: contentType}) + + return serveContent(w, req, contentType, int64(len(content)), bytes.NewReader(content)) +} + +func (s *s3Server) makeObjectName(slug, version, shasum, file string) string { + basepath := path.Join(slug, version) + if shasum != "" { + basepath += "-" + shasum + } + // Prevent path traversal + if strings.Contains(file, "..") { + return basepath + "/invalid" + } + return path.Join(basepath, file) +} + +func (s *s3Server) FilesList(slug, version, shasum string) ([]string, error) { + prefix := s.makeObjectName(slug, version, shasum, "") + "/" + var names []string + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + name := strings.TrimPrefix(obj.Key, prefix) + if name != "" { + names = append(names, name) + } + } + return names, nil +} + +// S3AppsBucket returns the S3 bucket name used for storing applications of a +// given type. The bucket is shared across all instances (like Swift containers). +func S3AppsBucket(bucketPrefix string, appsType consts.AppType) string { + switch appsType { + case consts.WebappType: + return bucketPrefix + "-apps-web" + case consts.KonnectorType: + return bucketPrefix + "-apps-konnectors" + } + panic("Unknown AppType") +} + +// prepareTarball is reused from server.go via the FileServer interface (it +// calls Open and FilesList). The function is defined in server.go. +// We reference it here to document that s3Server satisfies prepareTarball's +// requirements. +var _ FileServer = (*s3Server)(nil) diff --git a/pkg/assets/dynamic/fs.go b/pkg/assets/dynamic/fs.go index baf867a7a55..a0764c8a050 100644 --- a/pkg/assets/dynamic/fs.go +++ b/pkg/assets/dynamic/fs.go @@ -53,6 +53,12 @@ func InitDynamicAssetFS(fsURL string) error { return err } + case config.SchemeS3: + assetFS, err = NewS3FS() + if err != nil { + return err + } + default: return fmt.Errorf("Invalid scheme %s for dynamic assets FS", u.Scheme) } diff --git a/pkg/assets/dynamic/impl_s3.go b/pkg/assets/dynamic/impl_s3.go new file mode 100644 index 00000000000..5001c4faa88 --- /dev/null +++ b/pkg/assets/dynamic/impl_s3.go @@ -0,0 +1,126 @@ +package dynamic + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "path" + "strings" + "time" + + "github.com/cozy/cozy-stack/pkg/assets/model" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/hashicorp/golang-lru/v2/expirable" + "github.com/minio/minio-go/v7" +) + +// S3FS is the S3 implementation of [AssetsFS]. +// +// It saves and fetches assets into/from any S3-compatible object store. +type S3FS struct { + client *minio.Client + bucket string + ctx context.Context +} + +// NewS3FS instantiates a new S3FS. +func NewS3FS() (*S3FS, error) { + initCacheOnce.Do(func() { + cache = expirable.NewLRU[string, cacheEntry](1024, nil, 1*time.Hour) + }) + + ctx := context.Background() + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-assets" + + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: config.GetS3Region()}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + return nil, fmt.Errorf("Cannot create bucket for dynamic assets: %s", err) + } + } + + return &S3FS{client: client, bucket: bucket, ctx: ctx}, nil +} + +func (s *S3FS) Add(_ string, _ string, asset *model.Asset) error { + objectName := path.Join(asset.Context, asset.Name) + data := asset.GetData() + _, err := s.client.PutObject(s.ctx, s.bucket, objectName, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{}) + return err +} + +func (s *S3FS) Get(ctx string, name string) ([]byte, error) { + objectName := path.Join(ctx, name) + if entry, ok := cache.Get(objectName); ok { + if !entry.found { + return nil, os.ErrNotExist + } + return entry.content, nil + } + + obj, err := s.client.GetObject(s.ctx, s.bucket, objectName, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + content, err := io.ReadAll(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + cache.Add(objectName, cacheEntry{found: false}) + return nil, os.ErrNotExist + } + return nil, err + } + + cache.Add(objectName, cacheEntry{found: true, content: content}) + return content, nil +} + +func (s *S3FS) Remove(context, name string) error { + objectName := path.Join(context, name) + return s.client.RemoveObject(s.ctx, s.bucket, objectName, minio.RemoveObjectOptions{}) +} + +func (s *S3FS) List() (map[string][]*model.Asset, error) { + objs := map[string][]*model.Asset{} + + for obj := range s.client.ListObjects(s.ctx, s.bucket, minio.ListObjectsOptions{ + Recursive: true, + }) { + if obj.Err != nil { + return nil, obj.Err + } + + splitted := strings.SplitN(obj.Key, "/", 2) + if len(splitted) < 2 { + continue + } + ctx := splitted[0] + assetName := model.NormalizeAssetName(splitted[1]) + + a, err := GetAsset(ctx, assetName) + if err != nil { + return nil, err + } + + objs[ctx] = append(objs[ctx], a) + } + + return objs, nil +} + +func (s *S3FS) CheckStatus(ctx context.Context) (time.Duration, error) { + before := time.Now() + _, err := s.client.ListBuckets(ctx) + if err != nil { + return 0, err + } + return time.Since(before), nil +} diff --git a/pkg/config/config/config.go b/pkg/config/config/config.go index 9be12b28810..5c8d687479f 100644 --- a/pkg/config/config/config.go +++ b/pkg/config/config/config.go @@ -83,6 +83,8 @@ const ( // SchemeSwiftSecure is the URL scheme used to configure the swift filesystem // in secure mode (HTTPS). SchemeSwiftSecure = "swift+https" + // SchemeS3 is the URL scheme used to configure an S3-compatible filesystem. + SchemeS3 = "s3" ) // defaultAdminSecretFileName is the default name of the file containing the diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go new file mode 100644 index 00000000000..506fd57afe2 --- /dev/null +++ b/pkg/config/config/s3.go @@ -0,0 +1,106 @@ +package config + +import ( + "context" + "fmt" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var s3Client *minio.Client +var s3BucketPrefix string +var s3Region string + +// InitDefaultS3Connection initializes the default S3 handler. +func InitDefaultS3Connection() error { + return InitS3Connection(config.Fs) +} + +// InitS3Connection initializes the global S3 client connection. This is +// not a thread-safe method. +func InitS3Connection(fs Fs) error { + fsURL := fs.URL + if fsURL.Scheme != SchemeS3 { + return nil + } + + q := fsURL.Query() + endpoint := fsURL.Host + accessKey := q.Get("access_key") + secretKey := q.Get("secret_key") + region := q.Get("region") + useSSL := q.Get("use_ssl") != "false" // default true + + s3BucketPrefix = q.Get("bucket_prefix") + if s3BucketPrefix == "" { + s3BucketPrefix = "cozy" + } + // Sanitize bucket prefix: lowercase, only alphanumeric and hyphens + s3BucketPrefix = strings.ToLower(s3BucketPrefix) + s3BucketPrefix = strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '-' { + return r + } + return '-' + }, s3BucketPrefix) + s3BucketPrefix = strings.Trim(s3BucketPrefix, "-") + s3Region = region + + var err error + opts := &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: useSSL, + Region: region, + } + if fs.Transport != nil { + opts.Transport = fs.Transport + } + + s3Client, err = minio.New(endpoint, opts) + if err != nil { + return fmt.Errorf("s3: could not create client: %w", err) + } + + // Verify connectivity by listing buckets + if _, err = s3Client.ListBuckets(context.Background()); err != nil { + log.Errorf("Could not connect to S3 endpoint %s: %s", endpoint, err) + return err + } + + log.Infof("Successfully connected to S3 endpoint %s", endpoint) + + // Pre-create the fixed buckets used by secondary storage (apps, assets, + // previews, exports). The per-org VFS bucket is created on instance init. + ctx := context.Background() + for _, suffix := range []string{"-apps-web", "-apps-konnectors", "-assets", "-previews", "-exports"} { + bucket := s3BucketPrefix + suffix + if err := s3Client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{Region: region}); err != nil { + code := minio.ToErrorResponse(err).Code + if code != "BucketAlreadyOwnedByYou" && code != "BucketAlreadyExists" { + log.Warnf("Could not create bucket %s: %s", bucket, err) + } + } + } + + return nil +} + +// GetS3Client returns the global S3 client. +func GetS3Client() *minio.Client { + if s3Client == nil { + panic("Called GetS3Client() before InitS3Connection()") + } + return s3Client +} + +// GetS3BucketPrefix returns the configured bucket prefix. +func GetS3BucketPrefix() string { + return s3BucketPrefix +} + +// GetS3Region returns the configured S3 region. +func GetS3Region() string { + return s3Region +} diff --git a/pkg/previewfs/cache.go b/pkg/previewfs/cache.go index 3d62f95f5e5..04cb1e49a7d 100644 --- a/pkg/previewfs/cache.go +++ b/pkg/previewfs/cache.go @@ -13,6 +13,7 @@ import ( "time" "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/minio/minio-go/v7" "github.com/ncw/swift/v2" "github.com/spf13/afero" ) @@ -41,6 +42,10 @@ func SystemCache() Cache { conn := config.GetSwiftConnection() ctx := context.Background() return swiftCache{conn, ctx} + case config.SchemeS3: + client := config.GetS3Client() + bucket := config.GetS3BucketPrefix() + "-previews" + return newS3Cache(client, bucket) default: panic(fmt.Errorf("previewfs: unknown storage provider %s", fsURL.Scheme)) } @@ -151,6 +156,81 @@ func (s swiftCache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { return err } +type s3Cache struct { + client *minio.Client + bucket string + ctx context.Context +} + +func newS3Cache(client *minio.Client, bucket string) s3Cache { + return s3Cache{client: client, bucket: bucket, ctx: context.Background()} +} + +func (s s3Cache) ensureBucket() error { + err := s.client.MakeBucket(s.ctx, s.bucket, minio.MakeBucketOptions{}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +func (s s3Cache) getObject(name string) (*bytes.Buffer, error) { + obj, err := s.client.GetObject(s.ctx, s.bucket, name, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + defer obj.Close() + + buf := &bytes.Buffer{} + _, err = buf.ReadFrom(obj) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, os.ErrNotExist + } + return nil, err + } + return buf, nil +} + +func (s s3Cache) putObject(name string, buffer *bytes.Buffer) error { + data := buffer.Bytes() + _, err := s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "NoSuchBucket" { + if berr := s.ensureBucket(); berr != nil { + return berr + } + _, err = s.client.PutObject(s.ctx, s.bucket, name, + bytes.NewReader(data), int64(len(data)), + minio.PutObjectOptions{ContentType: "image/jpg"}) + } + } + return err +} + +func (s s3Cache) GetIcon(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(iconFilename(md5sum)) +} + +func (s s3Cache) SetIcon(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(iconFilename(md5sum), buffer) +} + +func (s s3Cache) GetPreview(md5sum []byte) (*bytes.Buffer, error) { + return s.getObject(previewFilename(md5sum)) +} + +func (s s3Cache) SetPreview(md5sum []byte, buffer *bytes.Buffer) error { + return s.putObject(previewFilename(md5sum), buffer) +} + func iconFilename(md5sum []byte) string { return "icon-" + hex.EncodeToString(md5sum) + ".jpg" } diff --git a/pkg/s3util/s3util.go b/pkg/s3util/s3util.go new file mode 100644 index 00000000000..592fe7cdaf0 --- /dev/null +++ b/pkg/s3util/s3util.go @@ -0,0 +1,91 @@ +// Package s3util provides shared helpers for interacting with S3-compatible +// object stores via the minio-go client. +package s3util + +import ( + "context" + "errors" + "fmt" + "os" + + "github.com/minio/minio-go/v7" +) + +// IsNotFound returns true when the error is an S3 "not found" response +// (NoSuchKey or NoSuchBucket). +func IsNotFound(err error) bool { + code := minio.ToErrorResponse(err).Code + return code == "NoSuchKey" || code == "NoSuchBucket" +} + +// WrapNotFound converts S3 not-found errors to os.ErrNotExist and sanitizes +// other S3 errors to avoid leaking internal bucket/key details. +func WrapNotFound(err error) error { + if IsNotFound(err) { + return os.ErrNotExist + } + code := minio.ToErrorResponse(err).Code + if code != "" { + return fmt.Errorf("s3 storage error: %s", code) + } + return err +} + +// EnsureBucket creates the bucket if it does not already exist. +func EnsureBucket(ctx context.Context, client *minio.Client, bucket, region string) error { + err := client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ + Region: region, + }) + if err != nil { + code := minio.ToErrorResponse(err).Code + if code == "BucketAlreadyOwnedByYou" || code == "BucketAlreadyExists" { + return nil + } + return err + } + return nil +} + +// DeleteObjects deletes a list of named objects from a bucket. +func DeleteObjects(ctx context.Context, client *minio.Client, bucket string, objNames []string) error { + if len(objNames) == 0 { + return nil + } + objectsCh := make(chan minio.ObjectInfo, len(objNames)) + for _, name := range objNames { + objectsCh <- minio.ObjectInfo{Key: name} + } + close(objectsCh) + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + return errm +} + +// DeletePrefixObjects deletes all objects in a bucket under a given prefix. +func DeletePrefixObjects(ctx context.Context, client *minio.Client, bucket, prefix string) error { + objectsCh := make(chan minio.ObjectInfo) + var listErr error + go func() { + defer close(objectsCh) + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + }) { + if obj.Err != nil { + listErr = obj.Err + return + } + objectsCh <- obj + } + }() + var errm error + for e := range client.RemoveObjects(ctx, bucket, objectsCh, minio.RemoveObjectsOptions{}) { + errm = errors.Join(errm, e.Err) + } + if listErr != nil { + return listErr + } + return errm +} diff --git a/pkg/s3util/s3util_test.go b/pkg/s3util/s3util_test.go new file mode 100644 index 00000000000..45789f05dd6 --- /dev/null +++ b/pkg/s3util/s3util_test.go @@ -0,0 +1,120 @@ +package s3util_test + +import ( + "bytes" + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/pkg/s3util" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestS3Util(t *testing.T) { + if testing.Short() { + t.Skip("requires minio container: skipped with --short") + } + + fixture := testutils.StartMinio(t) + client := fixture.Client(t) + ctx := context.Background() + bucket := "s3util-test" + + require.NoError(t, client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})) + t.Cleanup(func() { + // Best-effort cleanup: remove all objects then the bucket. + for obj := range client.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) { + _ = client.RemoveObject(ctx, bucket, obj.Key, minio.RemoveObjectOptions{}) + } + _ = client.RemoveBucket(ctx, bucket) + }) + + t.Run("IsNotFound", func(t *testing.T) { + _, err := client.GetObject(ctx, bucket, "does-not-exist", minio.GetObjectOptions{}) + require.NoError(t, err) // GetObject itself doesn't fail; Stat does. + + _, err = client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + assert.True(t, s3util.IsNotFound(err)) + + // A non-not-found error should return false. + assert.False(t, s3util.IsNotFound(io.ErrUnexpectedEOF)) + }) + + t.Run("WrapNotFound", func(t *testing.T) { + _, err := client.StatObject(ctx, bucket, "does-not-exist", minio.StatObjectOptions{}) + require.Error(t, err) + + wrapped := s3util.WrapNotFound(err) + assert.ErrorIs(t, wrapped, os.ErrNotExist) + + // A non-S3 error passes through unchanged. + orig := io.ErrUnexpectedEOF + assert.Equal(t, orig, s3util.WrapNotFound(orig)) + }) + + t.Run("EnsureBucket", func(t *testing.T) { + newBucket := "s3util-ensure-test" + t.Cleanup(func() { _ = client.RemoveBucket(ctx, newBucket) }) + + // First call creates. + err := s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + + // Second call is idempotent. + err = s3util.EnsureBucket(ctx, client, newBucket, "") + assert.NoError(t, err) + }) + + t.Run("DeleteObjects", func(t *testing.T) { + // Create a few objects. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeleteObjects(ctx, client, bucket, []string{"del-a", "del-b", "del-c"}) + assert.NoError(t, err) + + // Verify they are gone. + for _, key := range []string{"del-a", "del-b", "del-c"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + }) + + t.Run("DeleteObjectsEmpty", func(t *testing.T) { + // Should be a no-op, not an error. + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, nil)) + assert.NoError(t, s3util.DeleteObjects(ctx, client, bucket, []string{})) + }) + + t.Run("DeletePrefixObjects", func(t *testing.T) { + // Create objects under a prefix and one outside. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three", "outside"} { + _, err := client.PutObject(ctx, bucket, key, + bytes.NewReader([]byte("x")), 1, + minio.PutObjectOptions{}) + require.NoError(t, err) + } + + err := s3util.DeletePrefixObjects(ctx, client, bucket, "pfx/") + assert.NoError(t, err) + + // Prefixed objects should be gone. + for _, key := range []string{"pfx/one", "pfx/two", "pfx/sub/three"} { + _, err := client.StatObject(ctx, bucket, key, minio.StatObjectOptions{}) + assert.True(t, s3util.IsNotFound(err), "object %s should be deleted", key) + } + + // Object outside the prefix should still exist. + _, err = client.StatObject(ctx, bucket, "outside", minio.StatObjectOptions{}) + assert.NoError(t, err) + }) +} diff --git a/tests/testutils/minio_utils.go b/tests/testutils/minio_utils.go new file mode 100644 index 00000000000..8af8bf262ab --- /dev/null +++ b/tests/testutils/minio_utils.go @@ -0,0 +1,98 @@ +package testutils + +import ( + "context" + "fmt" + "net/url" + "testing" + "time" + + "github.com/docker/go-connections/nat" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/require" + + c "github.com/docker/docker/api/types/container" + tc "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +// MinioFixture holds the state for a running MinIO container. +type MinioFixture struct { + Container tc.Container + Endpoint string // host:port + AccessKey string + SecretKey string + t *testing.T +} + +// StartMinio starts a MinIO container for testing. +func StartMinio(t *testing.T) *MinioFixture { + t.Helper() + + accessKey := "minioadmin" + secretKey := "minioadmin" + hostPort := getFreePort(t) + + req := tc.ContainerRequest{ + Image: "minio/minio:RELEASE.2025-02-28T09-55-16Z", + ExposedPorts: []string{"9000/tcp"}, + Env: map[string]string{ + "MINIO_ROOT_USER": accessKey, + "MINIO_ROOT_PASSWORD": secretKey, + }, + Cmd: []string{"server", "/data"}, + HostConfigModifier: func(hc *c.HostConfig) { + hc.PortBindings = nat.PortMap{ + "9000/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: hostPort}}, + } + }, + WaitingFor: wait.ForHTTP("/minio/health/live"). + WithPort("9000/tcp"). + WithStartupTimeout(60 * time.Second), + } + + container, err := tc.GenericContainer(context.Background(), tc.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err, "failed to start MinIO") + + host, err := container.Host(context.Background()) + require.NoError(t, err) + + endpoint := fmt.Sprintf("%s:%s", host, hostPort) + t.Logf("MinIO endpoint: %s", endpoint) + + t.Cleanup(func() { + _ = container.Terminate(context.Background()) + }) + + return &MinioFixture{ + Container: container, + Endpoint: endpoint, + AccessKey: accessKey, + SecretKey: secretKey, + t: t, + } +} + +// Client returns a minio.Client connected to this fixture. +func (f *MinioFixture) Client(t *testing.T) *minio.Client { + t.Helper() + client, err := minio.New(f.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(f.AccessKey, f.SecretKey, ""), + Secure: false, + }) + require.NoError(t, err) + return client +} + +// FsURL returns a *url.URL suitable for config.InitS3Connection. +func (f *MinioFixture) FsURL(bucketPrefix string) *url.URL { + return &url.URL{ + Scheme: "s3", + Host: f.Endpoint, + RawQuery: fmt.Sprintf("access_key=%s&secret_key=%s&bucket_prefix=%s&use_ssl=false", f.AccessKey, f.SecretKey, bucketPrefix), + } +} diff --git a/web/settings/capabilities.go b/web/settings/capabilities.go index dcd2104294a..c76fcb1c86f 100644 --- a/web/settings/capabilities.go +++ b/web/settings/capabilities.go @@ -43,6 +43,8 @@ func NewCapabilities(inst *instance.Instance) jsonapi.Object { switch config.FsURL().Scheme { case config.SchemeSwift, config.SchemeSwiftSecure: versioning = inst.SwiftLayout >= 2 + case config.SchemeS3: + versioning = true } flat := config.GetConfig().Subdomains == config.FlatSubdomains diff --git a/web/settings/instance.go b/web/settings/instance.go index fb87bc6f26e..23df1c3aada 100644 --- a/web/settings/instance.go +++ b/web/settings/instance.go @@ -88,7 +88,6 @@ func (h *HTTPHandler) getInstance(c echo.Context) error { } else if url != "" { doc.M["legal_notice_url"] = url } - return jsonapi.Data(c, http.StatusOK, &apiInstance{doc}, nil) } From 192c5f304524621cd2569577faecb64821508984 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Sun, 10 May 2026 16:22:41 +0200 Subject: [PATCH 02/23] ci: bump go test per-package timeout from 5m to 10m The model/vfs package now exercises three backends (afero, swift, s3) plus a second MinIO container for S3UploadErrorPropagation, pushing the package past the previous 5-minute per-package limit on GH runners. --- .github/workflows/go-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 0985e46ddf2..fec5ca6ba5d 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -64,4 +64,4 @@ jobs: cache: true - name: Run tests - run: go test -p 1 -timeout 5m ./... + run: go test -p 1 -timeout 10m ./... From e2dcb95a051421bb6528dd073ac29da484861c38 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Tue, 12 May 2026 11:05:08 +0200 Subject: [PATCH 03/23] ci: build and push s3-test docker image to GHCR Adds a workflow that builds the production Dockerfile and pushes the resulting image to ghcr.io/:s3-test on every push to the feat/s3-vfs-backend branch, plus a workflow_dispatch trigger that accepts a custom tag. Also publishes a ${tag}- tag for traceability. --- .github/workflows/docker-s3-test.yml | 59 ++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 .github/workflows/docker-s3-test.yml diff --git a/.github/workflows/docker-s3-test.yml b/.github/workflows/docker-s3-test.yml new file mode 100644 index 00000000000..55a1ca8ce76 --- /dev/null +++ b/.github/workflows/docker-s3-test.yml @@ -0,0 +1,59 @@ +name: Build S3-test Docker image + +on: + push: + branches: + - feat/s3-vfs-backend + workflow_dispatch: + inputs: + tag: + description: "Image tag to publish under ghcr.io/" + required: false + default: "s3-test" + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image metadata + id: meta + run: | + repo_lc="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + tag="${{ github.event.inputs.tag }}" + if [ -z "$tag" ]; then + tag="s3-test" + fi + tag_lc="$(echo "$tag" | tr '[:upper:]' '[:lower:]')" + sha_tag="${tag_lc}-${GITHUB_SHA::7}" + echo "image=ghcr.io/${repo_lc}" >> "$GITHUB_OUTPUT" + echo "tag=${tag_lc}" >> "$GITHUB_OUTPUT" + echo "sha_tag=${sha_tag}" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: scripts/docker/production/Dockerfile + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + cache-from: type=gha + cache-to: type=gha,mode=max From bb6303e78dee9308fba91feb553333c5d3b06a67 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Tue, 12 May 2026 11:09:47 +0200 Subject: [PATCH 04/23] fix(docker): mark /app as safe.directory before build.sh scripts/build.sh runs `git describe` / `git rev-parse` to derive the build version string. With recent git versions the COPY'd working tree trips the "detected dubious ownership" safety check inside the container (the host user that owned the source no longer owns the files), causing the build step to exit 128 before producing the binary. Whitelisting /app as safe restores the previous behaviour. --- scripts/docker/production/Dockerfile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/docker/production/Dockerfile b/scripts/docker/production/Dockerfile index 98d41bcf726..8696cacb186 100644 --- a/scripts/docker/production/Dockerfile +++ b/scripts/docker/production/Dockerfile @@ -14,7 +14,13 @@ RUN go mod download # Build cozy-stack COPY . . -RUN ./scripts/build.sh release ./cozy-stack +# scripts/build.sh runs `git describe` / `git rev-parse` to compute the +# version string. When the working tree is COPY-ed into the container its +# ownership no longer matches the host user, so recent git versions reject +# operations on it ("detected dubious ownership"). Mark /app as safe so the +# build step can read git metadata. +RUN git config --global --add safe.directory /app \ + && ./scripts/build.sh release ./cozy-stack # Multi-stage image: the main image From 52655f5a05cb6bd6a123439341bb282b1e63c013 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Tue, 12 May 2026 11:12:40 +0200 Subject: [PATCH 05/23] fix(docker): build cozy-stack directly without scripts/build.sh scripts/build.sh derives the version from git describe / git rev-parse on the COPY'd working tree, which fails inside the buildx container (exit 128 with no captured output). Inline `go build` with a build-arg version string sidesteps the whole bash + git chain. The workflow passes VERSION_STRING=- so the running binary reports a recognizable version. --- .github/workflows/docker-s3-test.yml | 2 ++ scripts/docker/production/Dockerfile | 18 +++++++++++------- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-s3-test.yml b/.github/workflows/docker-s3-test.yml index 55a1ca8ce76..2affb3c1594 100644 --- a/.github/workflows/docker-s3-test.yml +++ b/.github/workflows/docker-s3-test.yml @@ -55,5 +55,7 @@ jobs: tags: | ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.tag }} ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.sha_tag }} + build-args: | + VERSION_STRING=${{ steps.meta.outputs.tag }}-${{ github.sha }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/scripts/docker/production/Dockerfile b/scripts/docker/production/Dockerfile index 8696cacb186..dc26d9a9cba 100644 --- a/scripts/docker/production/Dockerfile +++ b/scripts/docker/production/Dockerfile @@ -14,13 +14,17 @@ RUN go mod download # Build cozy-stack COPY . . -# scripts/build.sh runs `git describe` / `git rev-parse` to compute the -# version string. When the working tree is COPY-ed into the container its -# ownership no longer matches the host user, so recent git versions reject -# operations on it ("detected dubious ownership"). Mark /app as safe so the -# build step can read git metadata. -RUN git config --global --add safe.directory /app \ - && ./scripts/build.sh release ./cozy-stack +# Build directly instead of going through scripts/build.sh, which runs +# `git describe` / `git rev-parse` and trips on the COPY'd working tree +# inside the container. The version string can be overridden at build +# time via --build-arg VERSION_STRING=... +ARG VERSION_STRING=docker-build +RUN BUILD_TIME="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + && go build \ + -ldflags "-X github.com/cozy/cozy-stack/pkg/config.Version=${VERSION_STRING} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildTime=${BUILD_TIME} \ + -X github.com/cozy/cozy-stack/pkg/config.BuildMode=production" \ + -o ./cozy-stack # Multi-stage image: the main image From 8e7809fd1400c9674929eb26e50c4de6b440b433 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 12:21:53 +0200 Subject: [PATCH 06/23] docs: design for per-instance storage backend and swift-to-s3 migration --- ...-07-16-per-instance-s3-migration-design.md | 231 ++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md diff --git a/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md b/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md new file mode 100644 index 00000000000..2c25b56a0e3 --- /dev/null +++ b/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md @@ -0,0 +1,231 @@ +# Per-instance storage backend & Swift→S3 migration — Design + +Date: 2026-07-16 +Status: Draft (design approved, spec under review) +Depends on: `feat/s3-vfs-backend` (adds `config.SchemeS3` and the `vfss3` VFS backend) + +## Goal + +Enable migrating a **single live production instance** from the Swift storage +backend to the S3-compatible backend, one instance at a time, without a +big-bang cutover of the whole fleet. The end state is a fleet fully on S3; the +per-instance mechanism is **transitional** and disappears (becomes a no-op) +once the global default is flipped to S3. + +Non-goals (explicitly out of scope): + +- Per-instance S3 **endpoints/credentials**. The target is a single global S3; + we do not build a per-endpoint connection factory. +- Zero-downtime online migration with delta sync. A **read-only maintenance + window** per instance is acceptable for v1. +- Migrating regenerable/ephemeral data (thumbnails, app assets, cache, + exports, archives) — those regenerate or reinstall on the target. + +## Current state (why this is needed) + +Backend selection is **entirely global** today: + +- `Instance.MakeVFS()`, `AvatarFS()`, `ThumbsFS()` + (`model/instance/instance.go:250-333`) switch on + `config.FsURL().Scheme` — the single global `fs.url` scheme. +- There is **no per-instance scheme field**. The only per-instance storage + field is `SwiftLayout` (`swift_cluster`), which selects a Swift *sub-variant*, + not a different backend. +- The S3 connection is a process-wide **singleton** + (`pkg/config/config/s3.go`), keyed by the global `fs.url` query params. + Per-instance isolation on S3 is achieved by bucket-per-org + (`-`) + key-prefix-per-DBPrefix, all against one + global endpoint. +- **No migration tooling between backends exists.** `cozy-stack swift + ls-layouts` only counts; `Fsck` only verifies; `lifecycle` "move" is + cozy-to-cozy server relocation, not backend migration. +- Backend is fixed at instance creation (`lifecycle/create.go:143-153`) and is + **not changeable via patch**. + +## Chosen approach: per-instance flag, in-place, dual-connection stack + +Rejected alternative — *relocate instances to S3-global stacks via the +cozy-to-cozy move machinery*: no core VFS change, but far heavier per instance +(relocation, domains, tokens, routing) for what is only a backend swap. Poor +effort/benefit for a fleet backend migration. + +The stack runs **both** the Swift and the S3 connection during the transition. +Each instance carries a flag selecting its backend; migration copies the +authoritative content to S3 and flips the flag inside a read-only window. + +## Architecture + +### 1. Configuration — dual connection during transition + +`fs.url` stays the **global default** (`swift://…`). Add an optional S3 +migration target so the S3 connection is initialized even while the default is +Swift: + +```yaml +fs: + url: swift://… # global default (unchanged) + migration_target: + url: s3://key:secret@endpoint/?region=…&bucket_prefix=… +``` + +- At startup, if `fs.migration_target` is present, call `InitS3Connection` + (in addition to the default backend init) so both `GetSwiftConnection()` and + `GetS3Client()` singletons are live. +- Add a `config.HasS3Target()` / accessor so the migration command can refuse + to run when no S3 target is configured. +- **End of migration:** operator sets `fs.url` to the `s3://…` URL and removes + `fs.migration_target`. The per-instance flag then equals the global default + and becomes vestigial; migrated instances keep working unchanged. + +The exact config shape (nested `migration_target` vs a flat `fs.s3_url`) is an +implementation detail; the invariant is: **both connections initialized during +the transition period**. + +### 2. Per-instance field + +Add to the `Instance` struct (`model/instance/instance.go`, next to +`SwiftLayout`): + +```go +FsScheme string `json:"fs_scheme,omitempty"` // "" = global default; "s3" = migrated +``` + +- Stored on the `io.cozy.instances` doc (same place as `swift_cluster`). +- Empty for all existing instances → **zero regression**, no data migration of + the instance registry. +- Add an accessor computing the **effective** scheme: + +```go +func (i *Instance) storageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} +``` + +### 3. Backend selection + +`MakeVFS`, `AvatarFS`, `ThumbsFS` (`instance.go:250-333`) read +`i.storageScheme()` instead of `config.FsURL().Scheme`. + +Refactor the currently-triplicated `switch` into a single builder, e.g. +`buildStorage(scheme, kind)` where `kind ∈ {main, avatars, thumbs}`. The +triplication is a real smell in code we are already modifying; consolidating it +keeps the three call sites in lock-step and is the only place the scheme→VFS +mapping lives. + +### 4. Migration engine — object-storage level, index untouched + +The VFS = **CouchDB index/tree** (`io.cozy.files`) + **object storage +content** (bytes keyed by file/version ID). Both backends address content by +ID. Therefore migration **does not copy the index** — only the content bytes, +object by object: + +1. Iterate every content-bearing doc for the instance: + - all `io.cozy.files` **including trashed** files; + - all `io.cozy.files.versions` (old versions live in storage too); + - **uploaded avatars** (authoritative user data — see Open Questions to + confirm the exact storage/doctype). +2. For each: stream content from the **source** storage and `PutObject` into + the **target** (S3) storage under the same logical ID/key. + +This bypasses the high-level `CreateFile` path (which would try to create index +docs) and needs two small primitives per backend: + +- source: "open content by ID" (`vfsswift`/`vfsafero` already expose object + open); +- target: "put content by ID" (`vfss3` already has `PutObject`). + +The engine is **backend-agnostic** (source afero/swift → target s3), which also +makes it testable afero→s3 without a Swift server. + +The migration **does not** copy thumbnails, app/konnector assets, cache, +exports, or archives — these regenerate or reinstall on first access against +the S3 backend after the flip. + +### 5. Migration command + +``` +cozy-stack instances migrate-storage --to s3 [--dry-run] [--purge-source] +``` + +Flow: + +1. Guards: S3 target configured (`config.HasS3Target()`); `--to` differs from + the instance's current effective scheme. +2. **Maintenance ON** (read-only): reject writes for the duration. (Reuse the + existing instance maintenance/blocking mechanism — pin the exact call in the + plan.) +3. Copy content source→S3 (files + versions + trash + avatars). +4. **Verify**: re-walk; every object present in the target with matching size + (and/or MD5); reconcile counts against the index. +5. On success only: set `FsScheme = "s3"`, persist the instance doc. +6. **Maintenance OFF**. + +The flag flips **only after full verification**, so there is never a live, +half-migrated instance. + +`--dry-run`: walk + report counts/sizes without writing or flipping. +`--purge-source`: NOT the default — see Rollback. + +### 6. Rollback & safety + +- Source Swift data is **kept** by default. `--purge-source` is a **separate, + deferred** step (its own command / explicit flag) run only after a confidence + period. +- Rollback while source is retained: set `FsScheme = ""` (or `swift`) and + reopen. Cheap and instant. +- Read-only window ⇒ the copied snapshot is consistent (no concurrent writes). +- Copy is **idempotent**: re-running overwrites target objects; a failed run + leaves the flag unchanged (instance still on Swift), reopens the instance, + and reports. Partial target objects are overwritten on retry or removed by + `--purge-source` on the target if aborted. + +### 7. Error handling + +- Any failure in copy or verify ⇒ do **not** flip `FsScheme`; reopen the + instance on its original backend; surface the error with the failing + file/version ID. +- Guard against running when the S3 target is unconfigured, or when `--to` + equals the current scheme (no-op). + +### 8. Testing + +- Reuse the MinIO testcontainer already in the VFS suite; use the afero + backend as the migration **source** (no Swift server needed). +- Cases: + - populate an instance with files, multiple versions, trashed files, and an + uploaded avatar; run `migrate-storage --to s3`; assert every content object + exists in S3 with matching size, the CouchDB index is **unchanged**, and + reads are served from S3 after the flip; + - `--dry-run` writes nothing and flips nothing; + - idempotence: running twice yields the same result; + - rollback: `FsScheme` cleared → reads served from source again (while source + retained); + - failure injection mid-copy leaves the instance on the source backend. + +## Open questions to resolve during implementation + +1. **Avatar authoritativeness & storage.** Confirm whether an uploaded avatar + is stored only in `AvatarFS` (authoritative, must be copied) vs derivable. + The design assumes it is authoritative and copies it; verify the exact + doctype/key so the engine enumerates it correctly. +2. **Exact content-copy primitives.** Confirm the object-open (source) and + object-put (target) signatures actually exposed by `vfsswift`, `vfsafero`, + and `vfss3`, and whether a small shared interface is warranted vs + backend-specific helpers. +3. **Maintenance mechanism.** Pin the exact API used to put the instance + read-only for the window (instance blocking/maintenance) and confirm it + rejects writes at the VFS layer, not just the UI. +4. **Config shape.** Decide nested `fs.migration_target.url` vs a flat + `fs.s3_url`, and how `InitS3Connection` is invoked when the global scheme is + still Swift. + +## Rollout sequence (operational) + +1. Deploy stack with `fs.migration_target` configured (both connections live). +2. Migrate instances one at a time with `migrate-storage`, verifying each. +3. After a confidence period, `--purge-source` per migrated instance. +4. Once the whole fleet is on S3, flip `fs.url` to the S3 URL, drop + `fs.migration_target`; `fs_scheme` becomes a no-op that can later be removed. From 4e31dee6729e3e0e97fc8374c852ffe61a2ea8db Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 14:03:47 +0200 Subject: [PATCH 07/23] docs: add rollback command to per-instance s3 migration design --- ...-07-16-per-instance-s3-migration-design.md | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md b/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md index 2c25b56a0e3..7772cdee12c 100644 --- a/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md +++ b/docs/superpowers/specs/2026-07-16-per-instance-s3-migration-design.md @@ -169,20 +169,42 @@ half-migrated instance. `--dry-run`: walk + report counts/sizes without writing or flipping. `--purge-source`: NOT the default — see Rollback. -### 6. Rollback & safety - -- Source Swift data is **kept** by default. `--purge-source` is a **separate, - deferred** step (its own command / explicit flag) run only after a confidence - period. -- Rollback while source is retained: set `FsScheme = ""` (or `swift`) and - reopen. Cheap and instant. +### 6. Rollback command + +Because the source backend is **retained by default**, rollback reuses the same +generic `--to ` engine — there is no separate code path — with two +modes: + +1. **Instant flip-back** — + `cozy-stack instances migrate-storage --to swift --flag-only`: + switches `fs_scheme` back **without copying**, pointing the instance at the + still-present Swift snapshot. Runs in a read-only window and verifies the + source objects are present before flipping. + - Refuses if the source was already purged (`--purge-source` has run). + - Any writes made on S3 **since the cutover are lost** (the Swift snapshot is + stale), so it warns and requires `--force`. Intended for immediate + post-cutover recovery, before real traffic writes to S3. + +2. **Safe re-migration** — + `cozy-stack instances migrate-storage --to swift`: + the same engine in reverse (read-only window, copy S3→Swift, verify, flip). + **No data loss**; use this once real writes have landed on S3. + +`--flag-only` is a general primitive ("switch the pointer to a backend that is +already populated"); it is only meaningful for rollback since the forward +migration must copy first. + +### 7. Safety & idempotence + +- Source data is **kept** by default. `--purge-source` is a **separate, + deferred** step run only after a confidence period. - Read-only window ⇒ the copied snapshot is consistent (no concurrent writes). - Copy is **idempotent**: re-running overwrites target objects; a failed run - leaves the flag unchanged (instance still on Swift), reopens the instance, - and reports. Partial target objects are overwritten on retry or removed by - `--purge-source` on the target if aborted. + leaves the flag unchanged (instance still on the source), reopens the + instance, and reports. Partial target objects are overwritten on retry or + removed by `--purge-source` on the target if aborted. -### 7. Error handling +### 8. Error handling - Any failure in copy or verify ⇒ do **not** flip `FsScheme`; reopen the instance on its original backend; surface the error with the failing @@ -190,7 +212,7 @@ half-migrated instance. - Guard against running when the S3 target is unconfigured, or when `--to` equals the current scheme (no-op). -### 8. Testing +### 9. Testing - Reuse the MinIO testcontainer already in the VFS suite; use the afero backend as the migration **source** (no Swift server needed). From 9bf86b72c5f9a35d8484dbbfcb5eb033aaadeb59 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 14:12:55 +0200 Subject: [PATCH 08/23] docs: implementation plan for per-instance s3 migration --- .../2026-07-16-per-instance-s3-migration.md | 928 ++++++++++++++++++ 1 file changed, 928 insertions(+) create mode 100644 docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md diff --git a/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md new file mode 100644 index 00000000000..eb9782d226a --- /dev/null +++ b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md @@ -0,0 +1,928 @@ +# Per-instance storage backend & Swift→S3 migration — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let an operator migrate a single live instance's file storage from Swift to the S3 backend, one instance at a time, via `cozy-stack instances migrate-storage --to s3`, with rollback. + +**Architecture:** Add a per-instance `FsScheme` field that overrides the global `fs.url` scheme when building the instance's VFS. Run both the Swift and S3 connections during the transition. A server-side migration copies object-storage **content** (files, versions, avatar) source→target **without touching the shared CouchDB index**, inside an HTTP-blocked window, then flips `FsScheme`. Rollback reuses the same engine (`--to swift`) or an instant `--flag-only` flip while the source is retained. + +**Tech Stack:** Go, CouchDB, minio-go/v7 (S3), ncw/swift (Swift), cobra (CLI), echo (admin API), MinIO testcontainer for tests. + +## Global Constraints + +- Backend key layout is identical across Swift-v3 and S3: object name = `docID[:22] + "/" + docID[22:27] + "/" + docID[27:] + "/" + internalID` (fallback `docID + "/" + internalID` when `len(docID)!=32 || len(internalID)!=16`). S3 prefixes it with `keyPrefix = DBPrefix() + "/"`. Use the exported builders `vfsswift.MakeObjectNameV3` and `vfss3.MakeObjectKey` — never re-derive by hand. +- The migration MUST NOT create or modify any CouchDB document (`io.cozy.files`, `io.cozy.files.versions`). Source and target VFS share the same `vfs.NewCouchdbIndexer(i)`. Only object bytes move. +- `FsScheme == ""` means "use global default" and MUST preserve today's behavior for every existing instance (zero data migration of the instance registry). +- No cosign trailers in commits (project convention). +- Only Swift-v3 (`SwiftLayout == 2`) is a supported migration source; reject v1/v2. +- Migrated-but-unverified state is never persisted: `FsScheme` flips only after verification passes. + +--- + +## File structure + +- `model/instance/instance.go` — add `FsScheme` field + `StorageScheme()` accessor; refactor the triplicated backend `switch` in `MakeVFS`/`AvatarFS`/`ThumbsFS` into one `buildVFS(kind)` helper reading `StorageScheme()`. +- `pkg/config/config/config.go` — parse optional `fs.migration_target` URL into `config.Fs`; add `MigrationTargetURL()` / `HasS3Target()`. +- `model/stack/main.go` — after the default connections, also init the S3 connection from the migration target when the global scheme is not S3. +- `model/vfs/vfs.go` — add `OpenAvatar() (io.ReadCloser, string, error)` to the `Avatarer` interface. +- `model/vfs/vfsswift/avatar_v3.go`, `model/vfs/vfss3/avatar.go`, `model/vfs/vfsafero/avatar.go` — implement `OpenAvatar`. +- `model/vfs/vfss3/impl.go` — add exported index-free `WriteContentAt(docID, internalID string, r io.Reader, size int64) error`. +- `model/instance/storagemigration/migration.go` (new package) — the engine: enumerate, copy, verify, orchestrate (`Migrate`, `Report`). +- `client/instances.go` — `AdminClient.MigrateStorage(opts)`. +- `web/instances/instances.go` — `migrateStorageHandler` + route. +- `cmd/instances.go` — `migrateStorageCmd` + flags. +- Tests colocated: `model/instance/storagemigration/migration_test.go`, plus small unit tests next to touched files. + +--- + +## Task 1: Per-instance `FsScheme` field and effective-scheme accessor + +**Files:** +- Modify: `model/instance/instance.go` (struct ~50-89; `MakeVFS` 250-276; `AvatarFS` 279-304; `ThumbsFS` 306-333) +- Test: `model/instance/instance_storage_scheme_test.go` (create) + +**Interfaces:** +- Produces: `func (i *Instance) StorageScheme() string` — returns `i.FsScheme` if non-empty else `config.FsURL().Scheme`. +- Produces: struct field `FsScheme string json:"fs_scheme,omitempty"`. + +- [ ] **Step 1: Write the failing test** + +Create `model/instance/instance_storage_scheme_test.go`: +```go +package instance + +import ( + "testing" + + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/stretchr/testify/assert" +) + +func TestStorageSchemeFallsBackToGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{} + assert.Equal(t, config.FsURL().Scheme, i.StorageScheme()) +} + +func TestStorageSchemeOverridesGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{FsScheme: config.SchemeS3} + assert.Equal(t, config.SchemeS3, i.StorageScheme()) +} +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/instance/ -run TestStorageScheme -v` +Expected: FAIL — `i.StorageScheme undefined` / `i.FsScheme undefined`. + +- [ ] **Step 3: Add the field** + +In the `Instance` struct (`model/instance/instance.go`), directly under `SwiftLayout int json:"swift_cluster,omitempty"`: +```go + // FsScheme, when non-empty, overrides the global fs.url scheme for this + // instance. Used to migrate a single instance to another storage backend + // (e.g. "s3") without changing the stack-wide default. Empty = global default. + FsScheme string `json:"fs_scheme,omitempty"` +``` + +- [ ] **Step 4: Add the accessor** + +Near `DBPrefix()` (`model/instance/instance.go:183`): +```go +// StorageScheme returns the storage backend scheme effective for this instance: +// the per-instance FsScheme override when set, otherwise the global fs.url scheme. +func (i *Instance) StorageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} +``` + +- [ ] **Step 5: Run test, verify pass** + +Run: `go test ./model/instance/ -run TestStorageScheme -v` +Expected: PASS. + +- [ ] **Step 6: Refactor the triplicated switch to use `StorageScheme()`** + +In `MakeVFS`, `AvatarFS`, and `ThumbsFS`, replace every `fsURL := config.FsURL()` + `switch fsURL.Scheme` with `switch i.StorageScheme()`. Keep `config.FsURL()` only where the afero branch still needs the URL/path (afero uses `fsURL` and `i.DirName()`), fetching it inside that branch: +```go + switch i.StorageScheme() { + case config.SchemeFile, config.SchemeMem: + i.vfs, err = vfsafero.New(i, index, disk, mutex, config.FsURL(), i.DirName()) + case config.SchemeSwift, config.SchemeSwiftSecure: + switch i.SwiftLayout { + case 2: + i.vfs, err = vfsswift.NewV3(i, index, disk, mutex) + default: + err = ErrInvalidSwiftLayout + } + case config.SchemeS3: + i.vfs, err = vfss3.New(i, index, disk, mutex) + default: + err = fmt.Errorf("instance: unknown storage provider %s", i.StorageScheme()) + } +``` +Apply the equivalent change to `AvatarFS` (which builds `vfsafero.NewAvatarFs`/`vfsswift.NewAvatarFsV3`/`vfss3.NewAvatarFs`) and `ThumbsFS`. + +- [ ] **Step 7: Build + full instance package tests** + +Run: `go build ./... && go test ./model/instance/ -run 'TestStorageScheme|TestMakeVFS' -v` +Expected: build OK, tests PASS. + +- [ ] **Step 8: Commit** + +```bash +git add model/instance/instance.go model/instance/instance_storage_scheme_test.go +git commit -m "feat(instance): add per-instance FsScheme override for storage backend" +``` + +--- + +## Task 2: Initialize the S3 connection from a migration target + +**Files:** +- Modify: `pkg/config/config/config.go` (`Fs` struct 225-235; `UseViper` ~830/1156; add accessors near `FsURL` 497) +- Modify: `model/stack/main.go` (83-91) +- Test: `pkg/config/config/s3_target_test.go` (create) + +**Interfaces:** +- Consumes: `config.InitS3Connection(fs Fs) error` (`pkg/config/config/s3.go:23`), which populates the S3 globals when `fs.URL.Scheme == SchemeS3`. +- Produces: `func MigrationTargetURL() *url.URL`; `func HasS3Target() bool`. + +- [ ] **Step 1: Write the failing test** + +Create `pkg/config/config/s3_target_test.go`: +```go +package config + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrationTargetInitsS3WhenGlobalIsSwift(t *testing.T) { + swiftURL, _ := url.Parse("swift://openstack/") + s3URL, _ := url.Parse("s3://key:secret@localhost:9000/?bucket_prefix=cozy&use_ssl=false") + config = &Config{Fs: Fs{URL: swiftURL, MigrationTarget: s3URL}} + + require.True(t, HasS3Target()) + // Init the S3 globals from the target even though the global scheme is swift. + require.NoError(t, InitS3Connection(Fs{URL: MigrationTargetURL()})) + assert.NotNil(t, GetS3Client()) + assert.Equal(t, "cozy", GetS3BucketPrefix()) +} +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./pkg/config/config/ -run TestMigrationTargetInitsS3 -v` +Expected: FAIL — `Fs has no field MigrationTarget` / `HasS3Target undefined` / `MigrationTargetURL undefined`. + +- [ ] **Step 3: Add the config field + accessors** + +In the `Fs` struct (`config.go:225`) add: +```go + // MigrationTarget, when set, is an alternate storage URL (e.g. s3://...) + // whose connection is initialized alongside the default one, so instances + // can be migrated to it while the global scheme stays unchanged. + MigrationTarget *url.URL +``` +Near `FsURL()` (`config.go:497`): +```go +// MigrationTargetURL returns the configured storage migration target URL, or nil. +func MigrationTargetURL() *url.URL { + return config.Fs.MigrationTarget +} + +// HasS3Target reports whether an S3 storage migration target is configured. +func HasS3Target() bool { + u := config.Fs.MigrationTarget + return u != nil && u.Scheme == SchemeS3 +} +``` + +- [ ] **Step 4: Parse `fs.migration_target` in UseViper** + +In `UseViper` where `Fs{...}` is built (`config.go:1156`), parse the optional key and set the field: +```go + var migrationTarget *url.URL + if raw := v.GetString("fs.migration_target"); raw != "" { + migrationTarget, err = url.Parse(raw) + if err != nil { + return err + } + } +``` +and add `MigrationTarget: migrationTarget,` to the `Fs{...}` literal. + +- [ ] **Step 5: Run test, verify pass** + +Run: `go test ./pkg/config/config/ -run TestMigrationTargetInitsS3 -v` +Expected: PASS. + +- [ ] **Step 6: Wire the target init at stack startup** + +In `model/stack/main.go` after the existing `InitDefaultS3Connection()` block (line ~88-91): +```go + // When a storage migration target is configured (e.g. migrating instances + // to S3 while the global default is still Swift), init that connection too. + if config.HasS3Target() { + if err := config.InitS3Connection(config.Fs{URL: config.MigrationTargetURL()}); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 migration target connection: %w", err) + } + } +``` + +- [ ] **Step 7: Build** + +Run: `go build ./...` +Expected: OK. + +- [ ] **Step 8: Commit** + +```bash +git add pkg/config/config/config.go pkg/config/config/s3_target_test.go model/stack/main.go +git commit -m "feat(config): init S3 connection from an optional fs.migration_target" +``` + +--- + +## Task 3: Index-free content writer on the S3 backend + +**Files:** +- Modify: `model/vfs/vfss3/impl.go` (near `ImportFileVersion` 591-626) +- Test: `model/vfs/vfss3/write_content_at_test.go` (create; reuses the package's MinIO testcontainer harness) + +**Interfaces:** +- Produces: `func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error` — puts bytes at `MakeObjectKey(sfs.keyPrefix, docID, internalID)`, creating no CouchDB doc. + +- [ ] **Step 1: Write the failing test** + +Create `model/vfs/vfss3/write_content_at_test.go` (mirror the setup already used in the vfss3 suite to get an `*s3VFS` against MinIO; see the existing test harness in this package for the exact fixture helper name and reuse it): +```go +package vfss3 + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + sfs := newTestS3VFS(t) // existing fixture in this package's tests + s3 := sfs.(*s3VFS) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello s3 migration") + + require.NoError(t, s3.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + objKey := MakeObjectKey(s3.keyPrefix, docID, internalID) + obj, err := s3.client.GetObject(s3.ctx, s3.bucket, objKey, minioGetOpts()) + require.NoError(t, err) + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} +``` +(If the package lacks a `newTestS3VFS`/`minioGetOpts` helper, add minimal ones in the test file based on the container fixture already present in `model/vfs` S3 tests.) + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/vfs/vfss3/ -run TestWriteContentAt -v` +Expected: FAIL — `s3.WriteContentAt undefined`. + +- [ ] **Step 3: Implement** + +Add to `model/vfs/vfss3/impl.go`: +```go +// WriteContentAt streams content into the object backing the (docID, internalID) +// key, creating NO CouchDB document. It is used by storage migration, which +// preserves the shared index and only moves object bytes. size may be -1 when +// unknown (falls back to multipart). +func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, size, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + return err +} +``` + +- [ ] **Step 4: Run test, verify pass** + +Run: `go test ./model/vfs/vfss3/ -run TestWriteContentAt -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/vfs/vfss3/impl.go model/vfs/vfss3/write_content_at_test.go +git commit -m "feat(vfss3): add index-free WriteContentAt for storage migration" +``` + +--- + +## Task 4: `OpenAvatar` on the Avatarer interface + +**Files:** +- Modify: `model/vfs/vfs.go` (`Avatarer` interface 266-273) +- Modify: `model/vfs/vfsswift/avatar_v3.go`, `model/vfs/vfss3/avatar.go`, `model/vfs/vfsafero/avatar.go` +- Test: `model/vfs/vfss3/avatar_test.go` (add a case; or create) + +**Interfaces:** +- Produces: `OpenAvatar() (io.ReadCloser, string, error)` on `vfs.Avatarer` — returns the stored avatar content reader and its content-type; `os.ErrNotExist` when no avatar exists. + +- [ ] **Step 1: Write the failing test (S3 impl)** + +In `model/vfs/vfss3/avatar_test.go`: +```go +func TestOpenAvatarRoundTrip(t *testing.T) { + av := newTestAvatarS3(t) // fixture returning the s3 Avatarer + require.NoError(t, av.CreateAvatar("image/png"). /* write */ , /* ... */) + // (use the package's existing CreateAvatar test flow to store bytes first) + + r, ctype, err := av.OpenAvatar() + require.NoError(t, err) + defer r.Close() + assert.Equal(t, "image/png", ctype) + b, _ := io.ReadAll(r) + assert.NotEmpty(t, b) +} +``` +(Base the write half on the existing `CreateAvatar` test in this package; if none exists, store via `CreateAvatar` returning an `io.WriteCloser` per its current signature.) + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/vfs/vfss3/ -run TestOpenAvatar -v` +Expected: FAIL — `OpenAvatar undefined`. + +- [ ] **Step 3: Extend the interface** + +In `model/vfs/vfs.go` `Avatarer`: +```go + // OpenAvatar returns a reader over the stored avatar content and its + // content-type, or os.ErrNotExist if no avatar is stored. + OpenAvatar() (io.ReadCloser, string, error) +``` + +- [ ] **Step 4: Implement for S3** (`model/vfs/vfss3/avatar.go`) +```go +func (a *avatarS3) OpenAvatar() (io.ReadCloser, string, error) { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + return nil, "", err + } + info, err := obj.Stat() + if err != nil { + if isNoSuchKey(err) { // existing helper used by OpenFile in this pkg + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return obj, info.ContentType, nil +} +``` + +- [ ] **Step 5: Implement for Swift v3** (`model/vfs/vfsswift/avatar_v3.go`) +```go +func (a *avatarV3) OpenAvatar() (io.ReadCloser, string, error) { + f, headers, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) + if err != nil { + if errors.Is(err, swift.ObjectNotFound) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, headers["Content-Type"], nil +} +``` + +- [ ] **Step 6: Implement for afero** (`model/vfs/vfsafero/avatar.go`) +```go +func (a *avatarFS) OpenAvatar() (io.ReadCloser, string, error) { + f, err := a.fs.Open(AvatarFilename) + if err != nil { + if os.IsNotExist(err) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + // content-type is not stored on disk; sniff from the name/content. + return f, "application/octet-stream", nil +} +``` + +- [ ] **Step 7: Run tests + build** + +Run: `go build ./... && go test ./model/vfs/vfss3/ -run TestOpenAvatar -v` +Expected: build OK (all three impls satisfy the interface), test PASS. + +- [ ] **Step 8: Commit** + +```bash +git add model/vfs/vfs.go model/vfs/vfsswift/avatar_v3.go model/vfs/vfss3/avatar.go model/vfs/vfsafero/avatar.go model/vfs/vfss3/avatar_test.go +git commit -m "feat(vfs): add OpenAvatar to the Avatarer interface" +``` + +--- + +## Task 5: Migration engine — enumerate and copy content + +**Files:** +- Create: `model/instance/storagemigration/migration.go` +- Test: `model/instance/storagemigration/migration_test.go` + +**Interfaces:** +- Consumes: `couchdb.ForeachDocs(db prefixer.Prefixer, doctype string, fn func(id string, doc json.RawMessage) error) error`; `vfs.VFS.OpenFile(*vfs.FileDoc)`, `vfs.VFS.OpenFileVersion(*vfs.FileDoc, *vfs.Version)`, `vfs.VFS.FileByID(string)`; `vfsswift.NewV3`, `vfss3.New`; the `contentWriter` interface (Task 3's `WriteContentAt`); `OpenAvatar` (Task 4). +- Produces: `type Report struct { Files, Versions int; Bytes int64; AvatarCopied bool }`; `func CopyContent(inst *instance.Instance, src vfs.VFS, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error)`. + +- [ ] **Step 1: Write the failing test (afero source → S3 target)** + +Create `model/instance/storagemigration/migration_test.go`: +```go +package storagemigration + +import ( + "testing" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCopyContentMovesFilesVersionsAndAvatar(t *testing.T) { + // Fixture: an instance whose global backend is afero (test default), + // populated with 2 files, 1 extra version, 1 trashed file, and an avatar. + inst, src, dst, srcAv, dstAv := setupMigrationFixture(t) + + rep, err := CopyContent(inst, src, dst, srcAv, dstAv) + require.NoError(t, err) + assert.Equal(t, 3, rep.Files) // 2 live + 1 trashed + assert.Equal(t, 1, rep.Versions) + assert.True(t, rep.AvatarCopied) + + // Every source file's bytes are now readable from the target VFS. + assertAllContentReadableFrom(t, inst, dst) + // The CouchDB index is unchanged (same rev on the files DB). + assertIndexUnchanged(t, inst) +} +``` +(`setupMigrationFixture`, `assertAllContentReadableFrom`, `assertIndexUnchanged` are helpers to write in this test file, building the instance with `lifecycle.Create`, writing files via the source VFS `CreateFile`, and building `dst` via `vfss3.New` against the test MinIO container.) + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/instance/storagemigration/ -run TestCopyContent -v` +Expected: FAIL — package/functions undefined. + +- [ ] **Step 3: Implement the enumerator + copier** + +Create `model/instance/storagemigration/migration.go`: +```go +package storagemigration + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" +) + +// contentWriter is implemented by target VFS backends that can write object +// bytes for a (docID, internalID) key without creating a CouchDB document. +type contentWriter interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error +} + +// Report summarizes a content copy. +type Report struct { + Files int + Versions int + Bytes int64 + AvatarCopied bool +} + +// CopyContent copies all object-storage content (files incl. trashed, versions, +// avatar) for inst from src to dst. It creates/modifies NO CouchDB document. +func CopyContent(inst *instance.Instance, src, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error) { + writer, ok := dst.(contentWriter) + if !ok { + return nil, fmt.Errorf("target backend does not support index-free writes") + } + rep := &Report{} + + // Files (including trashed). + err := couchdb.ForeachDocs(inst, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return err + } + if doc.Type == consts.DirType { + return nil + } + r, err := src.OpenFile(&doc) + if err != nil { + return fmt.Errorf("open source file %s: %w", doc.DocID, err) + } + defer r.Close() + if err := writer.WriteContentAt(doc.DocID, doc.InternalID, r, doc.ByteSize); err != nil { + return fmt.Errorf("write target file %s: %w", doc.DocID, err) + } + rep.Files++ + rep.Bytes += doc.ByteSize + return nil + }) + if err != nil { + return rep, err + } + + // Versions. + err = couchdb.ForeachDocs(inst, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return err + } + fileID, internalID := splitVersionID(ver.DocID) + fileDoc, err := src.FileByID(fileID) + if err != nil { + return fmt.Errorf("file for version %s: %w", ver.DocID, err) + } + r, err := src.OpenFileVersion(fileDoc, &ver) + if err != nil { + return fmt.Errorf("open source version %s: %w", ver.DocID, err) + } + defer r.Close() + if err := writer.WriteContentAt(fileID, internalID, r, ver.ByteSize); err != nil { + return fmt.Errorf("write target version %s: %w", ver.DocID, err) + } + rep.Versions++ + rep.Bytes += ver.ByteSize + return nil + }) + if err != nil { + return rep, err + } + + // Avatar (single optional object). + ar, ctype, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + // no avatar, nothing to do + case err != nil: + return rep, fmt.Errorf("open source avatar: %w", err) + default: + defer ar.Close() + w, err := dstAv.CreateAvatar(ctype) + if err != nil { + return rep, fmt.Errorf("create target avatar: %w", err) + } + if _, err := io.Copy(w, ar); err != nil { + _ = w.Close() + return rep, fmt.Errorf("copy avatar: %w", err) + } + if err := w.Close(); err != nil { + return rep, fmt.Errorf("finalize target avatar: %w", err) + } + rep.AvatarCopied = true + } + + return rep, nil +} + +func splitVersionID(versionDocID string) (fileID, internalID string) { + for i := 0; i < len(versionDocID); i++ { + if versionDocID[i] == '/' { + return versionDocID[:i], versionDocID[i+1:] + } + } + return versionDocID, versionDocID +} +``` +Notes for the implementer: +- Confirm the exact `vfs.FileDoc` field for the doc-type discriminator and the trashed flag; `consts.DirType`/`consts.FileType` are the type values. Directories are skipped; trashed files ARE included (they still carry content). +- Confirm `Avatarer.CreateAvatar` returns an `io.WriteCloser` (as used elsewhere in the code that stores avatars). If its signature differs, adapt the write half accordingly. + +- [ ] **Step 4: Run test, verify pass** + +Run: `go test ./model/instance/storagemigration/ -run TestCopyContent -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/instance/storagemigration/migration.go model/instance/storagemigration/migration_test.go +git commit -m "feat(storagemigration): copy files, versions and avatar between backends" +``` + +--- + +## Task 6: Verification pass + +**Files:** +- Modify: `model/instance/storagemigration/migration.go` +- Test: `model/instance/storagemigration/migration_test.go` (add case) + +**Interfaces:** +- Produces: `func Verify(inst *instance.Instance, dst vfs.VFS, dstAv vfs.Avatarer, expected *Report) error` — re-enumerates and confirms each file/version object is present on the target with a matching byte size; errors listing the first missing/mismatched ID. + +- [ ] **Step 1: Write the failing test** + +Add to `migration_test.go`: +```go +func TestVerifySucceedsAfterCopyAndFailsWhenObjectMissing(t *testing.T) { + inst, src, dst, srcAv, dstAv := setupMigrationFixture(t) + rep, err := CopyContent(inst, src, dst, srcAv, dstAv) + require.NoError(t, err) + require.NoError(t, Verify(inst, dst, dstAv, rep)) + + deleteOneTargetObject(t, inst, dst) // helper: remove a known key + assert.Error(t, Verify(inst, dst, dstAv, rep)) +} +``` + +- [ ] **Step 2: Run it, verify it fails** + +Run: `go test ./model/instance/storagemigration/ -run TestVerify -v` +Expected: FAIL — `Verify undefined`. + +- [ ] **Step 3: Implement `Verify`** + +Add to `migration.go`. Re-run the same enumeration, but instead of copying, stat the target object via a new `contentStater` capability (add `StatContentAt(docID, internalID string) (int64, error)` to `vfss3` mirroring `WriteContentAt`, returning `os.ErrNotExist` on NoSuchKey) and compare sizes. Count files+versions and compare totals to `expected`. Return the first discrepancy as an error. (Add the `StatContentAt` method + a `contentStater` interface here, symmetric to Task 3.) + +- [ ] **Step 4: Run test, verify pass** + +Run: `go test ./model/instance/storagemigration/ -run TestVerify -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/instance/storagemigration/migration.go model/instance/storagemigration/migration_test.go model/vfs/vfss3/impl.go +git commit -m "feat(storagemigration): verify target objects after copy" +``` + +--- + +## Task 7: Orchestration — `Migrate` with block/flip/dry-run/flag-only/purge + +**Files:** +- Modify: `model/instance/storagemigration/migration.go` +- Test: `model/instance/storagemigration/migration_test.go` (add cases) + +**Interfaces:** +- Consumes: `lifecycle.Block(inst, reason...)`, `lifecycle.Unblock(inst)`; `instance.Update(inst)`; the instance's VFS constructors. +- Produces: +```go +type Options struct { + To string // target scheme, e.g. "s3" (or "swift" for rollback) + DryRun bool + FlagOnly bool // switch pointer to an already-populated backend, no copy + Force bool // required with FlagOnly (data written since cutover is lost) + PurgeSource bool // delete source objects after a successful flip +} +func Migrate(inst *instance.Instance, opts Options) (*Report, error) +``` + +- [ ] **Step 1: Write the failing tests** + +Add cases to `migration_test.go`: +```go +func TestMigrateFlipsSchemeAfterVerify(t *testing.T) { + inst := setupInstanceOnAfero(t) // populated + rep, err := Migrate(inst, Options{To: "s3"}) + require.NoError(t, err) + assert.Equal(t, "s3", inst.FsScheme) + assert.Greater(t, rep.Files, 0) + // Reads now served from S3: + assertReadsServedFrom(t, inst, "s3") +} + +func TestMigrateDryRunDoesNotFlip(t *testing.T) { + inst := setupInstanceOnAfero(t) + _, err := Migrate(inst, Options{To: "s3", DryRun: true}) + require.NoError(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +func TestMigrateFlagOnlyRequiresForce(t *testing.T) { + inst := setupInstanceOnAfero(t) + _, err := Migrate(inst, Options{To: "s3", FlagOnly: true}) + require.Error(t, err) // refuses without --force +} +``` + +- [ ] **Step 2: Run, verify fail** + +Run: `go test ./model/instance/storagemigration/ -run TestMigrate -v` +Expected: FAIL — `Migrate`/`Options` undefined. + +- [ ] **Step 3: Implement `Migrate`** + +Add to `migration.go`. Flow: +1. Guards: `opts.To != inst.StorageScheme()`; if `opts.To == config.SchemeS3` require `config.HasS3Target()` or global S3; Swift source must be `SwiftLayout == 2`. If `opts.FlagOnly && !opts.Force` return an error explaining data-loss risk. +2. Build `src` = VFS for the current scheme, `dst` = VFS for `opts.To`, and the two `Avatarer`s, via the same constructors `MakeVFS`/`AvatarFS` use (`vfsswift.NewV3`, `vfss3.New`, `vfsafero.New`) with a shared `index := vfs.NewCouchdbIndexer(inst)`, `disk := vfs.DiskThresholder(inst)`, `mutex := config.Lock().ReadWrite(inst, "vfs")`. Ensure the target S3 bucket exists via `config.GetS3Client().MakeBucket(...)` guarded by BucketExists (do NOT call `InitFs`, which also touches the index). +3. `FlagOnly`: skip copy; go to step 6 after a presence check (`Verify` with a nil expected, i.e. just confirm target objects exist). +4. `lifecycle.Block(inst, instance.BlockedMoving.Code)`; `defer lifecycle.Unblock(inst)` on all paths. +5. `rep, err := CopyContent(...)`; then `Verify(...)`. On error: return without flipping (instance stays on source), reopened by the deferred Unblock. +6. `DryRun`: return `rep` now without flipping. +7. Flip: `inst.FsScheme = opts.To`; `instance.Update(inst)`. +8. `Unblock` (deferred). +9. `PurgeSource`: after a successful flip, delete source objects (reuse `pkg/s3util.DeletePrefixObjects` for S3 sources, or the Swift container delete for Swift sources). Keep this behind the explicit flag; default off. + +- [ ] **Step 4: Run tests, verify pass** + +Run: `go test ./model/instance/storagemigration/ -run TestMigrate -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add model/instance/storagemigration/migration.go model/instance/storagemigration/migration_test.go +git commit -m "feat(storagemigration): orchestrate block, copy, verify, flip, rollback" +``` + +--- + +## Task 8: Admin API handler + route + +**Files:** +- Modify: `web/instances/instances.go` (add `migrateStorageHandler`; register in `Routes` ~732) +- Modify: `client/instances.go` (add `MigrateStorage`; extend `InstanceOptions` if needed) +- Test: `web/instances/instances_test.go` (add a case) or an integration test under `tests/` + +**Interfaces:** +- Produces (server): `POST /instances/:domain/migrate-storage` with query params `to`, `dry_run`, `flag_only`, `force`, `purge_source`; returns the JSON `Report`. +- Produces (client): a thin `client.MigrateStorageOptions` struct (mirrors the engine's `Options` fields; keeps the `client` package free of the model import) and `client.MigrateStorageReport`, plus `func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error)`. + +- [ ] **Step 1: Write the failing handler test** + +Add to `web/instances/instances_test.go` a test that creates an instance, POSTs `/instances//migrate-storage?to=s3&dry_run=true`, and asserts 200 + a `Report` body with `files >= 0` and the instance's `fs_scheme` still empty (dry-run). + +- [ ] **Step 2: Run, verify fail** + +Run: `go test ./web/instances/ -run TestMigrateStorage -v` +Expected: FAIL — route/handler missing (404). + +- [ ] **Step 3: Implement the handler + route** + +In `web/instances/instances.go`, mirroring `modifyHandler` (166) and `fsckHandler`: +```go +func migrateStorageHandler(c echo.Context) error { + domain := c.Param("domain") + inst, err := lifecycle.GetInstance(domain) + if err != nil { + return wrapError(err) + } + opts := storagemigration.Options{ + To: c.QueryParam("to"), + DryRun: c.QueryParam("dry_run") == "true", + FlagOnly: c.QueryParam("flag_only") == "true", + Force: c.QueryParam("force") == "true", + PurgeSource: c.QueryParam("purge_source") == "true", + } + rep, err := storagemigration.Migrate(inst, opts) + if err != nil { + return wrapError(err) + } + return c.JSON(http.StatusOK, rep) +} +``` +Register in `Routes`: `router.POST("/:domain/migrate-storage", migrateStorageHandler)`. + +- [ ] **Step 4: Implement the admin client method** + +In `client/instances.go`, mirroring `ModifyInstance` (231): build `url.Values` from the options and `ac.Req(&request.Options{Method: "POST", Path: "/instances/" + domain + "/migrate-storage", Queries: q})`, decode the `Report`. + +- [ ] **Step 5: Run test, verify pass** + +Run: `go test ./web/instances/ -run TestMigrateStorage -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add web/instances/instances.go client/instances.go web/instances/instances_test.go +git commit -m "feat(web/instances): add migrate-storage admin endpoint and client" +``` + +--- + +## Task 9: CLI command + +**Files:** +- Modify: `cmd/instances.go` (add `migrateStorageCmd`, flag vars, register in `init`) +- Test: manual smoke (documented) + reuse existing cmd test harness if present + +**Interfaces:** +- Consumes: `AdminClient.MigrateStorage` (Task 8). + +- [ ] **Step 1: Add flag vars** (near line 47) +```go +var flagMigrateTo string +var flagMigrateDryRun bool +var flagMigrateFlagOnly bool +var flagMigrateForce bool +var flagMigratePurgeSource bool +``` + +- [ ] **Step 2: Define the command** +```go +var migrateStorageCmd = &cobra.Command{ + Use: "migrate-storage ", + Short: "Migrate an instance's file storage to another backend (e.g. s3)", + Long: `cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Usage() + } + ac := newAdminClient() + rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{ + To: flagMigrateTo, + DryRun: flagMigrateDryRun, + FlagOnly: flagMigrateFlagOnly, + Force: flagMigrateForce, + PurgeSource: flagMigratePurgeSource, + }) + if err != nil { + return err + } + fmt.Printf("migrated: %d files, %d versions, %d bytes, avatar=%v\n", + rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied) + return nil + }, +} +``` +(Define a `client.MigrateStorageOptions` mirror struct in `client/instances.go` so the CLI does not import the model package.) + +- [ ] **Step 3: Register + flags** (in `init`) +```go + instanceCmdGroup.AddCommand(migrateStorageCmd) + migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme") + migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching") + migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)") + migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost") + migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch") +``` + +- [ ] **Step 4: Build + smoke** + +Run: `go build ./... && ./cozy-stack instances migrate-storage --help` +Expected: build OK; help lists the flags. + +- [ ] **Step 5: Commit** + +```bash +git add cmd/instances.go client/instances.go +git commit -m "feat(cmd): add instances migrate-storage command" +``` + +--- + +## Task 10: Docs + +**Files:** +- Modify: `docs/config.md` (document `fs.migration_target`) +- Modify: `docs/cli/cozy-stack_instances_migrate-storage.md` (regenerate via the docs generator if the repo autogenerates cobra docs; otherwise add manually) and the S3 doc from the S3 PR (`docs/*s3*`) + +- [ ] **Step 1: Document the config key and the command** + +Add a `fs.migration_target` example to `docs/config.md` and a "Migrating an instance to S3" section describing: configure the target, run `migrate-storage`, verify, `--purge-source` later, and the fleet-wide flip of `fs.url` at the end. + +- [ ] **Step 2: Regenerate CLI docs if applicable** + +Run: `make docs` (or the repo's cobra-doc generator target) and stage the generated file. + +- [ ] **Step 3: Commit** + +```bash +git add docs/ +git commit -m "docs: document fs.migration_target and instances migrate-storage" +``` + +--- + +## Self-review notes (resolved spec open-questions) + +- **Avatar authoritativeness:** confirmed user-uploaded and non-regenerable; a single object at `/avatar` (S3) / `"avatar"` (Swift container). Copied in Task 5 via the new `OpenAvatar` (Task 4). Thumbs are derived → skipped (regenerate on target). +- **Content-copy primitives:** read via exported `OpenFile`/`OpenFileVersion`; write via new index-free `WriteContentAt` (Task 3). Keys via exported `MakeObjectNameV3`/`MakeObjectKey`. Index never rewritten. +- **Read-only mechanism:** `lifecycle.Block`/`Unblock` gate HTTP traffic only (`web/middlewares/instance.go`), not in-process/worker writes. Task 7 blocks with `BlockedMoving` during the window. **Known v1 limitation:** background workers/triggers can still write to the source during the window; run migrations during low activity, or extend later to pause the instance's jobs. Documented as a risk. +- **Config shape:** chose a flat `fs.migration_target` URL (Task 2); the S3 connection is initialized from it at stack startup even while the global scheme stays Swift. +``` From 31a576244a564ceee1402a93727cb7c9dd1921d9 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 16:02:27 +0200 Subject: [PATCH 09/23] docs: correct s3 test harness reference in migration plan --- docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md index eb9782d226a..262b7f9d4ff 100644 --- a/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md +++ b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md @@ -16,6 +16,7 @@ - No cosign trailers in commits (project convention). - Only Swift-v3 (`SwiftLayout == 2`) is a supported migration source; reject v1/v2. - Migrated-but-unverified state is never persisted: `FsScheme` flips only after verification passes. +- **S3 tests require a live MinIO container via Docker.** There is NO per-package `newTestS3VFS` fixture. The real, existing harness is `testutils.StartMinio(t) *MinioFixture` (`tests/testutils/minio_utils.go`) exposing `Client(t) *minio.Client` and `FsURL(bucketPrefix string) *url.URL`. Build an S3 VFS exactly like `makeS3FS` in `model/vfs/vfs_test.go:916`: `mf := testutils.StartMinio(t); config.InitS3Connection(config.Fs{URL: mf.FsURL("test")}); s3fs, _ := vfss3.New(db, index, &diskImpl{}, mutex)`. Any test in this plan that references a `newTestS3VFS`/`newTestAvatarS3`/`minioGetOpts` helper means "build the S3 VFS/avatarer with this StartMinio pattern" — reuse it, do not invent a new container fixture. Tests that must reach unexported `*s3VFS` fields go in `package vfss3` (or an external `vfss3_test` package if importing `tests/testutils` would cycle — the implementer verifies and picks). --- From f444898be5ad69e4de4f115f2f685f62b17c8f96 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 20:07:06 +0200 Subject: [PATCH 10/23] feat(instance): add per-instance FsScheme override for storage backend --- model/instance/instance.go | 33 +++++++++++++------ .../instance/instance_storage_scheme_test.go | 20 +++++++++++ 2 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 model/instance/instance_storage_scheme_test.go diff --git a/model/instance/instance.go b/model/instance/instance.go index c3478c66d10..761d2806ecd 100644 --- a/model/instance/instance.go +++ b/model/instance/instance.go @@ -88,6 +88,11 @@ type Instance struct { // See model/vfs/vfsswift for more details. SwiftLayout int `json:"swift_cluster,omitempty"` + // FsScheme, when non-empty, overrides the global fs.url scheme for this + // instance. Used to migrate a single instance to another storage backend + // (e.g. "s3") without changing the stack-wide default. Empty = global default. + FsScheme string `json:"fs_scheme,omitempty"` + CouchCluster int `json:"couch_cluster,omitempty"` // PassphraseHash is a hash of a hash of the user's passphrase: the @@ -187,6 +192,15 @@ func (i *Instance) DBPrefix() string { return i.Domain } +// StorageScheme returns the storage backend scheme effective for this instance: +// the per-instance FsScheme override when set, otherwise the global fs.url scheme. +func (i *Instance) StorageScheme() string { + if i.FsScheme != "" { + return i.FsScheme + } + return config.FsURL().Scheme +} + // DomainName returns the main domain name of the instance. func (i *Instance) DomainName() string { return i.Domain @@ -252,14 +266,13 @@ func (i *Instance) MakeVFS() error { if i.vfs != nil { return nil } - fsURL := config.FsURL() mutex := config.Lock().ReadWrite(i, "vfs") index := vfs.NewCouchdbIndexer(i) disk := vfs.DiskThresholder(i) var err error - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile, config.SchemeMem: - i.vfs, err = vfsafero.New(i, index, disk, mutex, fsURL, i.DirName()) + i.vfs, err = vfsafero.New(i, index, disk, mutex, config.FsURL(), i.DirName()) case config.SchemeSwift, config.SchemeSwiftSecure: switch i.SwiftLayout { case 2: @@ -270,16 +283,16 @@ func (i *Instance) MakeVFS() error { case config.SchemeS3: i.vfs, err = vfss3.New(i, index, disk, mutex) default: - err = fmt.Errorf("instance: unknown storage provider %s", fsURL.Scheme) + err = fmt.Errorf("instance: unknown storage provider %s", i.StorageScheme()) } return err } // AvatarFS returns the hidden filesystem for storing the avatar. func (i *Instance) AvatarFS() vfs.Avatarer { - fsURL := config.FsURL() - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile: + fsURL := config.FsURL() baseFS := afero.NewBasePathFs(afero.NewOsFs(), path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName)) return vfsafero.NewAvatarFs(baseFS) @@ -299,16 +312,16 @@ func (i *Instance) AvatarFS() vfs.Avatarer { keyPrefix := i.DBPrefix() + "/" return vfss3.NewAvatarFs(client, bucket, keyPrefix) default: - panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) + panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme())) } } // ThumbsFS returns the hidden filesystem for storing the thumbnails of the // photos/image func (i *Instance) ThumbsFS() vfs.Thumbser { - fsURL := config.FsURL() - switch fsURL.Scheme { + switch i.StorageScheme() { case config.SchemeFile: + fsURL := config.FsURL() baseFS := afero.NewBasePathFs(afero.NewOsFs(), path.Join(fsURL.Path, i.DirName(), vfs.ThumbsDirName)) return vfsafero.NewThumbsFs(baseFS) @@ -328,7 +341,7 @@ func (i *Instance) ThumbsFS() vfs.Thumbser { keyPrefix := i.DBPrefix() + "/" return vfss3.NewThumbsFs(client, bucket, keyPrefix) default: - panic(fmt.Sprintf("instance: unknown storage provider %s", fsURL.Scheme)) + panic(fmt.Sprintf("instance: unknown storage provider %s", i.StorageScheme())) } } diff --git a/model/instance/instance_storage_scheme_test.go b/model/instance/instance_storage_scheme_test.go new file mode 100644 index 00000000000..87df01242d2 --- /dev/null +++ b/model/instance/instance_storage_scheme_test.go @@ -0,0 +1,20 @@ +package instance + +import ( + "testing" + + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/stretchr/testify/assert" +) + +func TestStorageSchemeFallsBackToGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{} + assert.Equal(t, config.FsURL().Scheme, i.StorageScheme()) +} + +func TestStorageSchemeOverridesGlobal(t *testing.T) { + config.UseTestFile(t) + i := &Instance{FsScheme: config.SchemeS3} + assert.Equal(t, config.SchemeS3, i.StorageScheme()) +} From 101d1aace59659517426869f68669e9e30dae09c Mon Sep 17 00:00:00 2001 From: Crash-- Date: Thu, 16 Jul 2026 20:15:54 +0200 Subject: [PATCH 11/23] feat(config): init S3 connection from an optional fs.migration_target --- model/stack/main.go | 8 ++++++ pkg/config/config/config.go | 26 +++++++++++++++++- pkg/config/config/s3_target_test.go | 42 +++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 pkg/config/config/s3_target_test.go diff --git a/model/stack/main.go b/model/stack/main.go index f82bdbe87da..861b13456ca 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -90,6 +90,14 @@ security features. Please do not use this binary as your production server. return nil, nil, fmt.Errorf("failed to init the S3 connection: %w", err) } + // When a storage migration target is configured (e.g. migrating instances + // to S3 while the global default is still Swift), init that connection too. + if config.HasS3Target() { + if err := config.InitS3Connection(config.Fs{URL: config.MigrationTargetURL()}); err != nil { + return nil, nil, fmt.Errorf("failed to init the S3 migration target connection: %w", err) + } + } + workersList, err := job.GetWorkersList() if err != nil { return nil, nil, fmt.Errorf("failed to get the workers list: %w", err) diff --git a/pkg/config/config/config.go b/pkg/config/config/config.go index 5c8d687479f..924099e8550 100644 --- a/pkg/config/config/config.go +++ b/pkg/config/config/config.go @@ -232,6 +232,10 @@ type Fs struct { AutoCleanTrashedAfter map[string]string Versioning FsVersioning Contexts map[string]interface{} + // MigrationTarget, when set, is an alternate storage URL (e.g. s3://...) + // whose connection is initialized alongside the default one, so instances + // can be migrated to it while the global scheme stays unchanged. + MigrationTarget *url.URL } // FsVersioning contains the configuration for the versioning of files @@ -498,6 +502,17 @@ func FsURL() *url.URL { return config.Fs.URL } +// MigrationTargetURL returns the configured storage migration target URL, or nil. +func MigrationTargetURL() *url.URL { + return config.Fs.MigrationTarget +} + +// HasS3Target reports whether an S3 storage migration target is configured. +func HasS3Target() bool { + u := config.Fs.MigrationTarget + return u != nil && u.Scheme == SchemeS3 +} + // ServerAddr returns the address on which the stack is run func ServerAddr() string { return net.JoinHostPort(config.Host, strconv.Itoa(config.Port)) @@ -849,6 +864,14 @@ func UseViper(v *viper.Viper) error { } } + var migrationTarget *url.URL + if raw := v.GetString("fs.migration_target"); raw != "" { + migrationTarget, err = url.Parse(raw) + if err != nil { + return err + } + } + couch, err := makeCouch(v) if err != nil { return err @@ -1163,7 +1186,8 @@ func UseViper(v *viper.Viper) error { MaxNumberToKeep: v.GetInt("fs.versioning.max_number_of_versions_to_keep"), MinDelayBetweenTwoVersions: v.GetDuration("fs.versioning.min_delay_between_two_versions"), }, - Contexts: v.GetStringMap("fs.contexts"), + Contexts: v.GetStringMap("fs.contexts"), + MigrationTarget: migrationTarget, }, CouchDB: couch, Jobs: jobs, diff --git a/pkg/config/config/s3_target_test.go b/pkg/config/config/s3_target_test.go new file mode 100644 index 00000000000..afffacd8b3f --- /dev/null +++ b/pkg/config/config/s3_target_test.go @@ -0,0 +1,42 @@ +package config + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMigrationTargetInitsS3WhenGlobalIsSwift(t *testing.T) { + // A minimal fake S3 endpoint: InitS3Connection only needs a successful + // ListBuckets call (a signed GET on "/") to consider the connection live; + // bucket-creation failures are only logged, so any other response is fine. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/" { + w.Header().Set("Content-Type", "application/xml") + fmt.Fprint(w, ` + + testtest + +`) + return + } + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + endpoint := srv.Listener.Addr().String() + + swiftURL, _ := url.Parse("swift://openstack/") + s3URL, _ := url.Parse(fmt.Sprintf("s3://%s/?access_key=key&secret_key=secret&bucket_prefix=cozy&use_ssl=false", endpoint)) + config = &Config{Fs: Fs{URL: swiftURL, MigrationTarget: s3URL}} + + require.True(t, HasS3Target()) + // Init the S3 globals from the target even though the global scheme is swift. + require.NoError(t, InitS3Connection(Fs{URL: MigrationTargetURL()})) + assert.NotNil(t, GetS3Client()) + assert.Equal(t, "cozy", GetS3BucketPrefix()) +} From 7f1409e20953721279b11c51a0768bbc48dfd948 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 09:57:45 +0200 Subject: [PATCH 12/23] feat(vfss3): add index-free WriteContentAt for storage migration WriteContentAt puts bytes at the object key for a given (docID, internalID) pair without creating or touching any CouchDB document. It is the S3-side primitive used by Swift-to-S3 storage migration, which moves object bytes while preserving the shared CouchDB index. --- model/vfs/vfss3/impl.go | 13 ++++ model/vfs/vfss3/write_content_at_test.go | 86 ++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 model/vfs/vfss3/write_content_at_test.go diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index caa2cbabfa5..89f5c2b27fa 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -625,6 +625,19 @@ func (sfs *s3VFS) ImportFileVersion(version *vfs.Version, content io.ReadCloser) return sfs.Indexer.CreateVersion(version) } +// WriteContentAt streams content into the object backing the (docID, internalID) +// key, creating NO CouchDB document. It is used by storage migration, which +// preserves the shared index and only moves object bytes. size may be -1 when +// unknown (falls back to multipart). +func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + _, err := sfs.client.PutObject(sfs.ctx, sfs.bucket, objKey, content, size, minio.PutObjectOptions{ + ContentType: "application/octet-stream", + SendContentMd5: true, + }) + return err +} + func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr diff --git a/model/vfs/vfss3/write_content_at_test.go b/model/vfs/vfss3/write_content_at_test.go new file mode 100644 index 00000000000..e513222d0d0 --- /dev/null +++ b/model/vfs/vfss3/write_content_at_test.go @@ -0,0 +1,86 @@ +package vfss3_test + +import ( + "bytes" + "context" + "io" + "testing" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/require" +) + +// writeContentAtPrefixer is a minimal vfs.Prefixer implementation, local to +// this test, so it can live in an external test package (package +// vfss3_test) without importing anything from the internal vfss3 test +// harness. It also exposes GetOrgID so vfss3.New can build the bucket name. +type writeContentAtPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *writeContentAtPrefixer) DBCluster() int { return p.cluster } +func (p *writeContentAtPrefixer) DomainName() string { return p.domain } +func (p *writeContentAtPrefixer) DBPrefix() string { return p.prefix } +func (p *writeContentAtPrefixer) GetContextName() string { return p.context } +func (p *writeContentAtPrefixer) GetOrgID() string { return "wcatestorg" } + +// writeContentAtDisk is a minimal vfs.DiskThresholder, unused by +// WriteContentAt itself but required by vfss3.New's signature. +type writeContentAtDisk struct{} + +func (writeContentAtDisk) DiskQuota() int64 { return 0 } + +// TestWriteContentAtPutsBytesWithoutIndex verifies that WriteContentAt is a +// pure object-storage primitive: it puts bytes at the object key derived +// from (docID, internalID) and does not touch CouchDB at all (no +// ResetDB/DefineIndexes/InitFs is performed in this test). +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + config.UseTestFile(t) + + mf := testutils.StartMinio(t) + + db := &writeContentAtPrefixer{ + cluster: 0, + domain: "io.cozy.vfss3.writecontentat.test", + prefix: "io.cozy.vfss3.writecontentat.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + mutex := config.Lock().ReadWrite(db, "vfs-s3-writecontentat-test") + sfs, err := vfss3.New(db, index, &writeContentAtDisk{}, mutex) + require.NoError(t, err) + + // WriteContentAt never creates its own bucket (that's InitFs's job, which + // we deliberately skip here since it would also touch CouchDB through + // Indexer.InitIndex). Create the bucket directly against the raw client. + bucket := vfss3.BucketName(db.GetOrgID(), config.GetS3BucketPrefix()) + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + w := sfs.(interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error + }) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello s3 migration") + + require.NoError(t, w.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + objKey := vfss3.MakeObjectKey(db.DBPrefix()+"/", docID, internalID) + obj, err := client.GetObject(context.Background(), bucket, objKey, minio.GetObjectOptions{}) + require.NoError(t, err) + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} From b8fcc0a799e7733267d374533c70fa446e7152cb Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 10:06:16 +0200 Subject: [PATCH 13/23] feat(vfs): add OpenAvatar to the Avatarer interface The Swift->S3 migration needs to copy a user-uploaded avatar as-is (it is authoritative, not regenerable). OpenAvatar exposes a generic read-side (content reader + content-type) across all three vfs backends, returning os.ErrNotExist when no avatar is stored so the migration can treat "no avatar" as a skip. --- model/vfs/vfs.go | 3 ++ model/vfs/vfsafero/avatar.go | 14 ++++++++ model/vfs/vfss3/avatar.go | 21 +++++++++++ model/vfs/vfss3/avatar_open_test.go | 55 +++++++++++++++++++++++++++++ model/vfs/vfsswift/avatar_v3.go | 14 ++++++++ 5 files changed, 107 insertions(+) create mode 100644 model/vfs/vfss3/avatar_open_test.go diff --git a/model/vfs/vfs.go b/model/vfs/vfs.go index 24f9aaad733..edd85a6e38a 100644 --- a/model/vfs/vfs.go +++ b/model/vfs/vfs.go @@ -270,6 +270,9 @@ type Avatarer interface { // but does if there was a problem deleting it. DeleteAvatar() error ServeAvatarContent(w http.ResponseWriter, req *http.Request) error + // OpenAvatar returns a reader over the stored avatar content and its + // content-type, or os.ErrNotExist if no avatar is stored. + OpenAvatar() (io.ReadCloser, string, error) } // Thumbser defines an interface to define a thumbnail filesystem. diff --git a/model/vfs/vfsafero/avatar.go b/model/vfs/vfsafero/avatar.go index 86b0d1ab398..44d38587339 100644 --- a/model/vfs/vfsafero/avatar.go +++ b/model/vfs/vfsafero/avatar.go @@ -74,6 +74,20 @@ func (a *avatarFS) AvatarExists() (bool, error) { return infos.Size() > 0, nil } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. The content-type +// is not persisted on disk by this backend, so a generic value is returned. +func (a *avatarFS) OpenAvatar() (io.ReadCloser, string, error) { + f, err := a.fs.Open(AvatarFilename) + if err != nil { + if os.IsNotExist(err) { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, "application/octet-stream", nil +} + func (a *avatarFS) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { s, err := a.fs.Stat(AvatarFilename) if err != nil { diff --git a/model/vfs/vfss3/avatar.go b/model/vfs/vfss3/avatar.go index a164ef39c4b..7ff2feea834 100644 --- a/model/vfs/vfss3/avatar.go +++ b/model/vfs/vfss3/avatar.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "os" "time" "github.com/cozy/cozy-stack/model/vfs" @@ -80,6 +81,26 @@ func (a *avatarS3) DeleteAvatar() error { return nil } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. +func (a *avatarS3) OpenAvatar() (io.ReadCloser, string, error) { + obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + info, err := obj.Stat() + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return obj, info.ContentType, nil +} + func (a *avatarS3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { obj, err := a.client.GetObject(a.ctx, a.bucket, a.avatarKey(), minio.GetObjectOptions{}) if err != nil { diff --git a/model/vfs/vfss3/avatar_open_test.go b/model/vfs/vfss3/avatar_open_test.go new file mode 100644 index 00000000000..2da3c0ebfdd --- /dev/null +++ b/model/vfs/vfss3/avatar_open_test.go @@ -0,0 +1,55 @@ +package vfss3_test + +import ( + "context" + "io" + "os" + "testing" + + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestOpenAvatarRoundTrip verifies that OpenAvatar returns the content and +// content-type previously stored via CreateAvatar, and that it reports +// os.ErrNotExist when no avatar has been stored yet. +func TestOpenAvatarRoundTrip(t *testing.T) { + config.UseTestFile(t) + + mf := testutils.StartMinio(t) + + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + bucket := "io-cozy-vfss3-openavatar-test" + keyPrefix := "io.cozy.vfss3.openavatar.test/" + + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + av := vfss3.NewAvatarFs(client, bucket, keyPrefix) + + // No avatar stored yet: OpenAvatar must report os.ErrNotExist. + _, _, err := av.OpenAvatar() + assert.ErrorIs(t, err, os.ErrNotExist) + + // Store an avatar, then read it back. + w, err := av.CreateAvatar("image/png") + require.NoError(t, err) + payload := []byte("fake png bytes") + _, err = w.Write(payload) + require.NoError(t, err) + require.NoError(t, w.Close()) + + r, ctype, err := av.OpenAvatar() + require.NoError(t, err) + defer r.Close() + assert.Equal(t, "image/png", ctype) + + got, err := io.ReadAll(r) + require.NoError(t, err) + assert.Equal(t, payload, got) +} diff --git a/model/vfs/vfsswift/avatar_v3.go b/model/vfs/vfsswift/avatar_v3.go index 917c0f0a3a0..b776cd413ca 100644 --- a/model/vfs/vfsswift/avatar_v3.go +++ b/model/vfs/vfsswift/avatar_v3.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + "os" "time" "github.com/cozy/cozy-stack/model/vfs" @@ -45,6 +46,19 @@ func (a *avatarV3) DeleteAvatar() error { return err } +// OpenAvatar returns a reader over the stored avatar content and its +// content-type, or os.ErrNotExist if no avatar is stored. +func (a *avatarV3) OpenAvatar() (io.ReadCloser, string, error) { + f, headers, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) + if err != nil { + if err == swift.ObjectNotFound { + return nil, "", os.ErrNotExist + } + return nil, "", err + } + return f, headers["Content-Type"], nil +} + func (a *avatarV3) ServeAvatarContent(w http.ResponseWriter, req *http.Request) error { f, o, err := a.c.ObjectOpen(a.ctx, a.container, "avatar", false, nil) if err != nil { From 2e6f5ebe746dfbc28da3fdcc9cf46b8a2a116f7f Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 10:21:28 +0200 Subject: [PATCH 14/23] feat(storagemigration): copy files, versions and avatar between backends --- model/instance/storagemigration/migration.go | 161 ++++++++++++ .../storagemigration/migration_test.go | 234 ++++++++++++++++++ 2 files changed, 395 insertions(+) create mode 100644 model/instance/storagemigration/migration.go create mode 100644 model/instance/storagemigration/migration_test.go diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go new file mode 100644 index 00000000000..a67c526eecb --- /dev/null +++ b/model/instance/storagemigration/migration.go @@ -0,0 +1,161 @@ +// Package storagemigration implements the engine that copies an instance's +// object-storage content (files, versions, avatar) from one VFS backend to +// another, without touching the shared CouchDB index. It is used to move an +// instance's files from Swift to S3 (or vice versa) while all other +// instances sharing the same CouchDB cluster keep working against the same +// io.cozy.files / io.cozy.files.versions documents. +package storagemigration + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/prefixer" +) + +// contentWriter is implemented by target VFS backends that can write object +// bytes for a (docID, internalID) key without creating a CouchDB document. +type contentWriter interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error +} + +// Report summarizes a content copy performed by CopyContent. +type Report struct { + Files int + Versions int + Bytes int64 + AvatarCopied bool +} + +// CopyContent copies all object-storage content (files including trashed +// ones, file versions, and the avatar) from src to dst. db is only used to +// enumerate the CouchDB documents (io.cozy.files and io.cozy.files.versions) +// that describe what content exists; CopyContent creates or modifies NO +// CouchDB document — it only reads from CouchDB and writes object bytes. +func CopyContent(db prefixer.Prefixer, src, dst vfs.VFS, srcAv, dstAv vfs.Avatarer) (*Report, error) { + writer, ok := dst.(contentWriter) + if !ok { + return nil, fmt.Errorf("storagemigration: target backend does not support index-free writes") + } + + rep := &Report{} + + if err := copyFiles(db, src, writer, rep); err != nil { + return rep, err + } + if err := copyVersions(db, src, writer, rep); err != nil { + return rep, err + } + if err := copyAvatar(srcAv, dstAv, rep); err != nil { + return rep, err + } + + return rep, nil +} + +// copyFiles enumerates every io.cozy.files document (ForeachDocs is +// unfiltered, so trashed files are naturally included) and copies the +// content of each file (skipping directories) from src to dst. +func copyFiles(db prefixer.Prefixer, src vfs.VFS, writer contentWriter, rep *Report) error { + return couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + + r, err := src.OpenFile(&doc) + if err != nil { + return fmt.Errorf("storagemigration: open source file %s: %w", doc.DocID, err) + } + defer r.Close() + + if err := writer.WriteContentAt(doc.DocID, doc.InternalID, r, doc.ByteSize); err != nil { + return fmt.Errorf("storagemigration: write target file %s: %w", doc.DocID, err) + } + + rep.Files++ + rep.Bytes += doc.ByteSize + return nil + }) +} + +// copyVersions enumerates every io.cozy.files.versions document and copies +// the content of each version from src to dst. +func copyVersions(db prefixer.Prefixer, src vfs.VFS, writer contentWriter, rep *Report) error { + return couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return fmt.Errorf("storagemigration: decode version doc: %w", err) + } + + fileID, internalID := splitVersionID(ver.DocID) + + fileDoc, err := src.FileByID(fileID) + if err != nil { + return fmt.Errorf("storagemigration: file for version %s: %w", ver.DocID, err) + } + + r, err := src.OpenFileVersion(fileDoc, &ver) + if err != nil { + return fmt.Errorf("storagemigration: open source version %s: %w", ver.DocID, err) + } + defer r.Close() + + if err := writer.WriteContentAt(fileID, internalID, r, ver.ByteSize); err != nil { + return fmt.Errorf("storagemigration: write target version %s: %w", ver.DocID, err) + } + + rep.Versions++ + rep.Bytes += ver.ByteSize + return nil + }) +} + +// copyAvatar copies the instance's avatar, if any, from srcAv to dstAv, +// setting rep.AvatarCopied on success. The absence of an avatar +// (os.ErrNotExist) is not an error. +func copyAvatar(srcAv, dstAv vfs.Avatarer, rep *Report) error { + ar, ctype, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + return nil + case err != nil: + return fmt.Errorf("storagemigration: open source avatar: %w", err) + } + defer ar.Close() + + w, err := dstAv.CreateAvatar(ctype) + if err != nil { + return fmt.Errorf("storagemigration: create target avatar: %w", err) + } + + if _, err := io.Copy(w, ar); err != nil { + _ = w.Close() + return fmt.Errorf("storagemigration: copy avatar: %w", err) + } + if err := w.Close(); err != nil { + return fmt.Errorf("storagemigration: finalize target avatar: %w", err) + } + + rep.AvatarCopied = true + return nil +} + +// splitVersionID splits a io.cozy.files.versions document id +// ("/") into its fileID and internalID parts. +func splitVersionID(versionDocID string) (fileID, internalID string) { + if i := strings.IndexByte(versionDocID, '/'); i >= 0 { + return versionDocID[:i], versionDocID[i+1:] + } + return versionDocID, "" +} diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go new file mode 100644 index 00000000000..fa07cbf05af --- /dev/null +++ b/model/instance/storagemigration/migration_test.go @@ -0,0 +1,234 @@ +package storagemigration_test + +import ( + "bytes" + "context" + "crypto/md5" + "io" + "net/url" + "testing" + "time" + + "github.com/cozy/cozy-stack/model/instance/storagemigration" + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfsafero" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/pkg/utils" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/minio/minio-go/v7" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// migrationPrefixer is a minimal vfs.Prefixer (+ GetOrgID, required by +// vfss3.New's bucket-name derivation) implementation local to this external +// test package, mirroring model/vfs/vfs_test.go's contexter fixture. +type migrationPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *migrationPrefixer) DBCluster() int { return p.cluster } +func (p *migrationPrefixer) DomainName() string { return p.domain } +func (p *migrationPrefixer) DBPrefix() string { return p.prefix } +func (p *migrationPrefixer) GetContextName() string { return p.context } +func (p *migrationPrefixer) GetOrgID() string { return "migrationtestorg" } + +// migrationDisk is a minimal vfs.DiskThresholder (unlimited quota). +type migrationDisk struct{} + +func (migrationDisk) DiskQuota() int64 { return 0 } + +// migrationFixture bundles the source (afero) and target (s3) VFS + Avatarer +// pairs, plus the shared prefixer/db handle used to run CopyContent and to +// inspect CouchDB directly for the index-unchanged assertion. +type migrationFixture struct { + db *migrationPrefixer + src vfs.VFS + dst vfs.VFS + + srcAv vfs.Avatarer + dstAv vfs.Avatarer +} + +func setupMigrationFixture(t *testing.T) *migrationFixture { + t.Helper() + + config.UseTestFile(t) + + db := &migrationPrefixer{ + cluster: 0, + domain: "io.cozy.storagemigration.test", + prefix: "io.cozy.storagemigration.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + require.NoError(t, couchdb.ResetDB(db, consts.FilesVersions)) + t.Cleanup(func() { + _ = couchdb.DeleteDB(db, consts.Files) + _ = couchdb.DeleteDB(db, consts.FilesVersions) + }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + + // Source: afero-backed VFS on a temp dir. + tempdir := t.TempDir() + aferoMutex := config.Lock().ReadWrite(db, "storagemigration-test-afero") + aferoURL := &url.URL{Scheme: "file", Host: "localhost", Path: tempdir} + src, err := vfsafero.New(db, index, &migrationDisk{}, aferoMutex, aferoURL, "io.cozy.vfs.test") + require.NoError(t, err) + require.NoError(t, src.InitFs()) + + baseFS := afero.NewBasePathFs(afero.NewOsFs(), tempdir) + srcAv := vfsafero.NewAvatarFs(baseFS) + + // Target: S3-backed VFS against a MinIO test container. + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + s3Mutex := config.Lock().ReadWrite(db, "storagemigration-test-s3") + dst, err := vfss3.New(db, index, &migrationDisk{}, s3Mutex) + require.NoError(t, err) + + bucket := vfss3.BucketName(db.GetOrgID(), config.GetS3BucketPrefix()) + client := mf.Client(t) + require.NoError(t, client.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{})) + + keyPrefix := db.DBPrefix() + "/" + dstAv := vfss3.NewAvatarFs(client, bucket, keyPrefix) + + return &migrationFixture{ + db: db, + src: src, + dst: dst, + srcAv: srcAv, + dstAv: dstAv, + } +} + +// createSourceFile creates a file of the given name/content on the source +// VFS and returns its FileDoc. +func createSourceFile(t *testing.T, fx *migrationFixture, name string, content []byte) *vfs.FileDoc { + t.Helper() + + doc, err := vfs.NewFileDoc(name, "", int64(len(content)), nil, "text/plain", "text", time.Now(), false, false, false, []string{}) + require.NoError(t, err) + + f, err := fx.src.CreateFile(doc, nil) + require.NoError(t, err) + + _, err = io.Copy(f, bytes.NewReader(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + got, err := fx.src.FileByPath("/" + name) + require.NoError(t, err) + return got +} + +func TestCopyContentMovesFilesVersionsAndAvatar(t *testing.T) { + fx := setupMigrationFixture(t) + + // 2 live files. + file1 := createSourceFile(t, fx, "file1.txt", []byte("hello from file 1")) + file2 := createSourceFile(t, fx, "file2.txt", []byte("hello from file 2, a bit longer")) + + // 1 file that gets trashed (content copy must still include it, since + // ForeachDocs is unfiltered). + file3 := createSourceFile(t, fx, "file3.txt", []byte("this one goes to the trash")) + file3, err := vfs.TrashFile(fx.src, file3) + require.NoError(t, err) + + // 1 extra version on file1. + versionPayload := []byte("an older revision of file 1") + sum := md5.Sum(versionPayload) + internalID := utils.RandomString(16) + version := &vfs.Version{ + DocID: file1.DocID + "/" + internalID, + ByteSize: int64(len(versionPayload)), + MD5Sum: sum[:], + } + version.Rels.File.Data.ID = file1.DocID + require.NoError(t, fx.src.ImportFileVersion(version, io.NopCloser(bytes.NewReader(versionPayload)))) + + // Avatar. + avatarPayload := []byte("fake png bytes for the avatar") + aw, err := fx.srcAv.CreateAvatar("image/png") + require.NoError(t, err) + _, err = aw.Write(avatarPayload) + require.NoError(t, err) + require.NoError(t, aw.Close()) + + // Capture revs before the copy: CopyContent must not touch the index. + revBefore1 := file1.Rev() + revBefore2 := file2.Rev() + revBefore3 := file3.Rev() + + rep, err := storagemigration.CopyContent(fx.db, fx.src, fx.dst, fx.srcAv, fx.dstAv) + require.NoError(t, err) + + assert.Equal(t, 3, rep.Files) // 2 live + 1 trashed + assert.Equal(t, 1, rep.Versions) + assert.True(t, rep.AvatarCopied) + + // Every source file's bytes are now readable from the target VFS. + assertFileContentOn(t, fx.dst, file1, []byte("hello from file 1")) + assertFileContentOn(t, fx.dst, file2, []byte("hello from file 2, a bit longer")) + assertFileContentOn(t, fx.dst, file3, []byte("this one goes to the trash")) + + // The version is readable via the target VFS. + vr, err := fx.dst.OpenFileVersion(file1, version) + require.NoError(t, err) + gotVersion, err := io.ReadAll(vr) + require.NoError(t, err) + require.NoError(t, vr.Close()) + assert.Equal(t, versionPayload, gotVersion) + + // The avatar is readable via the target avatarer. Note: the source + // avatarer is afero-backed, which does not persist a content-type on + // disk and always reports "application/octet-stream" from OpenAvatar + // (see vfsafero's OpenAvatar); CopyContent faithfully forwards whatever + // content-type srcAv.OpenAvatar() reports to dstAv.CreateAvatar(), so + // that is what ends up stored on the target too. + ar, ctype, err := fx.dstAv.OpenAvatar() + require.NoError(t, err) + gotAvatar, err := io.ReadAll(ar) + require.NoError(t, err) + require.NoError(t, ar.Close()) + assert.Equal(t, "application/octet-stream", ctype) + assert.Equal(t, avatarPayload, gotAvatar) + + // The CouchDB index is unchanged: same revs as before the copy. + reread1 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file1.DocID, reread1)) + reread2 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file2.DocID, reread2)) + reread3 := &vfs.FileDoc{} + require.NoError(t, couchdb.GetDoc(fx.db, consts.Files, file3.DocID, reread3)) + + assert.Equal(t, revBefore1, reread1.Rev()) + assert.Equal(t, revBefore2, reread2.Rev()) + assert.Equal(t, revBefore3, reread3.Rev()) +} + +func assertFileContentOn(t *testing.T, fs vfs.VFS, doc *vfs.FileDoc, want []byte) { + t.Helper() + r, err := fs.OpenFile(doc) + require.NoError(t, err) + got, err := io.ReadAll(r) + require.NoError(t, err) + require.NoError(t, r.Close()) + assert.Equal(t, want, got) +} From b48f7b3f3ba54e3b89b902d8c846877b96f527dd Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 10:36:16 +0200 Subject: [PATCH 15/23] docs: add task 5b (swift write target) and dual-backend verify to plan --- .../2026-07-16-per-instance-s3-migration.md | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md index 262b7f9d4ff..e8908c92c02 100644 --- a/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md +++ b/docs/superpowers/plans/2026-07-16-per-instance-s3-migration.md @@ -633,6 +633,54 @@ git commit -m "feat(storagemigration): copy files, versions and avatar between b --- +## Task 5b: Index-free content writer on the Swift backend + +Rationale: the rollback design offers a full S3→Swift re-migration (`--to swift`), so Swift must also be a valid copy TARGET. Mirror Task 3 for vfsswift v3. + +**Files:** +- Modify: `model/vfs/vfsswift/impl_v3.go` (near `ImportFileVersion`) +- Modify: `model/instance/storagemigration/migration.go` (package doc comment: Swift↔S3 is now accurate) +- Test: `model/vfs/vfsswift/write_content_at_v3_test.go` (external `package vfsswift_test`, MinIO not needed — Swift uses a swift test server; mirror the existing swift test setup in the package/`model/vfs/vfs_test.go` `makeSwiftFS`) + +**Interfaces:** +- Produces: `func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error` — `ObjectCreate` at `MakeObjectNameV3(docID, internalID)` in the instance container, creating no CouchDB document. + +- [ ] **Step 1: Write the failing test** in `package vfsswift_test`, building the swift VFS the way `makeSwiftFS` (`model/vfs/vfs_test.go:876`) does; write via the interface assertion `sfs.(interface{ WriteContentAt(...) })`; read back the object via the swift connection at `MakeObjectNameV3(docID, internalID)` and assert bytes. Use 32-char docID + 16-char internalID. + +- [ ] **Step 2: Run it, verify RED** — `go test ./model/vfs/vfsswift/ -run TestWriteContentAt -timeout 120s -v` → `WriteContentAt` undefined. + +- [ ] **Step 3: Implement** in `model/vfs/vfsswift/impl_v3.go` (receiver name and container/ctx access mirror `ImportFileVersion`/`CreateFile` in this file): +```go +// WriteContentAt streams content into the object backing the (docID, internalID) +// key in this instance's container, creating NO CouchDB document. Used by +// storage migration, which preserves the shared index and only moves bytes. +func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objName := MakeObjectNameV3(docID, internalID) + f, err := sfs.c.ObjectCreate(sfs.ctx, sfs.container, objName, true, "", "application/octet-stream", nil) + if err != nil { + return err + } + if _, err = io.Copy(f, content); err != nil { + _ = f.Close() + return err + } + return f.Close() +} +``` +(Verify the exact receiver type name — `swiftVFSV3`/`swiftVFS` — and field names `c`/`ctx`/`container` against the file; adapt if they differ. `ObjectCreate` with an empty checksum skips server-side hash verification, matching the streaming path already used in this file.) + +- [ ] **Step 4: Also update** the `storagemigration/migration.go` package doc comment so it no longer implies only-S3-target (Swift↔S3 both now supported as targets). Rebuild. + +- [ ] **Step 5: Run test, verify GREEN**, `go build ./...` clean. + +- [ ] **Step 6: Commit** +```bash +git add model/vfs/vfsswift/impl_v3.go model/vfs/vfsswift/write_content_at_v3_test.go model/instance/storagemigration/migration.go +git commit -m "feat(vfsswift): add index-free WriteContentAt for storage migration" +``` + +--- + ## Task 6: Verification pass **Files:** @@ -664,7 +712,7 @@ Expected: FAIL — `Verify undefined`. - [ ] **Step 3: Implement `Verify`** -Add to `migration.go`. Re-run the same enumeration, but instead of copying, stat the target object via a new `contentStater` capability (add `StatContentAt(docID, internalID string) (int64, error)` to `vfss3` mirroring `WriteContentAt`, returning `os.ErrNotExist` on NoSuchKey) and compare sizes. Count files+versions and compare totals to `expected`. Return the first discrepancy as an error. (Add the `StatContentAt` method + a `contentStater` interface here, symmetric to Task 3.) +Add to `migration.go`. Re-run the same enumeration, but instead of copying, stat the target object via a new `contentStater` capability (add `StatContentAt(docID, internalID string) (int64, error)` to BOTH `vfss3` and `vfsswift` v3, mirroring `WriteContentAt`, returning `os.ErrNotExist` when the object is absent) and compare sizes. Count files+versions and compare totals to `expected`. Return the first discrepancy as an error. (Add the `StatContentAt` methods + a `contentStater` interface here, symmetric to Task 3/5b.) Since either backend can be the target, both must implement it. - [ ] **Step 4: Run test, verify pass** From 2c15e4bfaadbabaca293557903f5a93921063071 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 10:42:41 +0200 Subject: [PATCH 16/23] feat(vfsswift): add index-free WriteContentAt for storage migration rollback Mirrors Task 3's vfss3.WriteContentAt so the Swift v3 backend can also act as a copy target for storagemigration.CopyContent, enabling full S3->Swift re-migration (rollback). Also corrects the storagemigration package doc comment, which previously oversold "Swift<->S3" before Swift-as-target was actually implemented. --- model/instance/storagemigration/migration.go | 7 +- model/vfs/vfsswift/impl_v3.go | 17 +++ .../vfs/vfsswift/write_content_at_v3_test.go | 107 ++++++++++++++++++ 3 files changed, 128 insertions(+), 3 deletions(-) create mode 100644 model/vfs/vfsswift/write_content_at_v3_test.go diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go index a67c526eecb..aaf3fd8b5f1 100644 --- a/model/instance/storagemigration/migration.go +++ b/model/instance/storagemigration/migration.go @@ -1,9 +1,10 @@ // Package storagemigration implements the engine that copies an instance's // object-storage content (files, versions, avatar) from one VFS backend to // another, without touching the shared CouchDB index. It is used to move an -// instance's files from Swift to S3 (or vice versa) while all other -// instances sharing the same CouchDB cluster keep working against the same -// io.cozy.files / io.cozy.files.versions documents. +// instance's files between Swift and S3 — either direction, S3 to Swift as +// well as Swift to S3 — while all other instances sharing the same CouchDB +// cluster keep working against the same io.cozy.files / +// io.cozy.files.versions documents. package storagemigration import ( diff --git a/model/vfs/vfsswift/impl_v3.go b/model/vfs/vfsswift/impl_v3.go index e40eab1ef10..cada518fb9a 100644 --- a/model/vfs/vfsswift/impl_v3.go +++ b/model/vfs/vfsswift/impl_v3.go @@ -579,6 +579,23 @@ func (sfs *swiftVFSV3) ImportFileVersion(version *vfs.Version, content io.ReadCl return sfs.Indexer.CreateVersion(version) } +// WriteContentAt streams content into the object backing the (docID, +// internalID) key in this instance's container, creating NO CouchDB +// document. Used by storage migration, which preserves the shared index and +// only moves bytes. +func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reader, size int64) error { + objName := MakeObjectNameV3(docID, internalID) + f, err := sfs.c.ObjectCreate(sfs.ctx, sfs.container, objName, true, "", "application/octet-stream", nil) + if err != nil { + return err + } + if _, err = io.Copy(f, content); err != nil { + _ = f.Close() + return err + } + return f.Close() +} + func (sfs *swiftVFSV3) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr diff --git a/model/vfs/vfsswift/write_content_at_v3_test.go b/model/vfs/vfsswift/write_content_at_v3_test.go new file mode 100644 index 00000000000..42834de0d7a --- /dev/null +++ b/model/vfs/vfsswift/write_content_at_v3_test.go @@ -0,0 +1,107 @@ +package vfsswift_test + +import ( + "bytes" + "context" + "io" + "net/url" + "testing" + + "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" + "github.com/cozy/cozy-stack/pkg/consts" + "github.com/cozy/cozy-stack/pkg/couchdb" + "github.com/cozy/cozy-stack/tests/testutils" + "github.com/ncw/swift/v2/swifttest" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +// writeContentAtPrefixer is a minimal vfs.Prefixer implementation, local to +// this test, so it can live in an external test package (package +// vfsswift_test) without importing anything from the internal vfsswift test +// harness. +type writeContentAtPrefixer struct { + cluster int + domain string + prefix string + context string +} + +func (p *writeContentAtPrefixer) DBCluster() int { return p.cluster } +func (p *writeContentAtPrefixer) DomainName() string { return p.domain } +func (p *writeContentAtPrefixer) DBPrefix() string { return p.prefix } +func (p *writeContentAtPrefixer) GetContextName() string { return p.context } + +// writeContentAtDisk is a minimal vfs.DiskThresholder, unused by +// WriteContentAt itself but required by vfsswift.NewV3's signature. +type writeContentAtDisk struct{} + +func (writeContentAtDisk) DiskQuota() int64 { return 0 } + +// TestWriteContentAtPutsBytesWithoutIndex verifies that WriteContentAt is a +// pure object-storage primitive: it puts bytes at the object key derived +// from (docID, internalID) and does not touch CouchDB (no document is +// created for the write itself). +func TestWriteContentAtPutsBytesWithoutIndex(t *testing.T) { + config.UseTestFile(t) + testutils.NeedCouchdb(t) + + db := &writeContentAtPrefixer{ + cluster: 0, + domain: "io.cozy.vfsswift.writecontentat.test", + prefix: "io.cozy.vfsswift.writecontentat.test", + context: "cozy_beta", + } + index := vfs.NewCouchdbIndexer(db) + + swiftSrv, err := swifttest.NewSwiftServer("localhost") + require.NoError(t, err, "failed to create swift server") + t.Cleanup(func() { swiftSrv.Close() }) + + require.NoError(t, config.InitSwiftConnection(config.Fs{ + URL: &url.URL{ + Scheme: "swift", + Host: "localhost", + RawQuery: "UserName=swifttest&Password=swifttest&AuthURL=" + url.QueryEscape(swiftSrv.AuthURL), + }, + })) + + mutex := config.Lock().ReadWrite(db, "vfs-swiftv3-writecontentat-test") + sfs, err := vfsswift.NewV3(db, index, &writeContentAtDisk{}, mutex) + require.NoError(t, err) + + require.NoError(t, couchdb.ResetDB(db, consts.Files)) + t.Cleanup(func() { _ = couchdb.DeleteDB(db, consts.Files) }) + + g, _ := errgroup.WithContext(context.Background()) + couchdb.DefineIndexes(g, db, couchdb.IndexesByDoctype(consts.Files)) + couchdb.DefineViews(g, db, couchdb.ViewsByDoctype(consts.Files)) + require.NoError(t, g.Wait()) + + require.NoError(t, sfs.InitFs()) + + w := sfs.(interface { + WriteContentAt(docID, internalID string, content io.Reader, size int64) error + }) + + docID := "0123456789012345678901234567890a" // 32 chars + internalID := "abcdef0123456789" // 16 chars + payload := []byte("hello swift migration") + + require.NoError(t, w.WriteContentAt(docID, internalID, bytes.NewReader(payload), int64(len(payload)))) + + cn := sfs.(interface{ ContainerNames() map[string]string }) + container := cn.ContainerNames()["container"] + objName := vfsswift.MakeObjectNameV3(docID, internalID) + + conn := config.GetSwiftConnection() + obj, _, err := conn.ObjectOpen(context.Background(), container, objName, false, nil) + require.NoError(t, err) + defer obj.Close() + + got, err := io.ReadAll(obj) + require.NoError(t, err) + require.Equal(t, payload, got) +} From ee9e7c3548df690c2fe5725cd8bd409005d4709c Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 10:52:49 +0200 Subject: [PATCH 17/23] feat(storagemigration): verify target objects after copy Add Verify, which re-enumerates the same files/versions/avatar that CopyContent copies and confirms each object exists on the target with a matching byte size, without touching CouchDB. Backed by a new StatContentAt primitive on both the S3 and Swift v3 VFS backends, since either can be the copy target. The orchestrator (a later task) will only flip an instance's backend flag once Verify passes. --- model/instance/storagemigration/migration.go | 104 ++++++++++++++++++ .../storagemigration/migration_test.go | 38 ++++++- model/vfs/vfss3/impl.go | 16 +++ model/vfs/vfsswift/impl_v3.go | 17 +++ 4 files changed, 170 insertions(+), 5 deletions(-) diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go index aaf3fd8b5f1..619152aa6e9 100644 --- a/model/instance/storagemigration/migration.go +++ b/model/instance/storagemigration/migration.go @@ -27,6 +27,13 @@ type contentWriter interface { WriteContentAt(docID, internalID string, content io.Reader, size int64) error } +// contentStater is implemented by target VFS backends that can report the +// byte size of the object backing a (docID, internalID) key without +// touching CouchDB. It returns os.ErrNotExist when the object is absent. +type contentStater interface { + StatContentAt(docID, internalID string) (int64, error) +} + // Report summarizes a content copy performed by CopyContent. type Report struct { Files int @@ -61,6 +68,103 @@ func CopyContent(db prefixer.Prefixer, src, dst vfs.VFS, srcAv, dstAv vfs.Avatar return rep, nil } +// Verify re-enumerates the same content that CopyContent copies (files, +// versions, avatar) and confirms each object exists on dst with a byte size +// matching the source CouchDB document, without creating or modifying any +// CouchDB document. It compares the counted totals against expected (the +// Report returned by CopyContent) and returns the FIRST discrepancy found as +// a descriptive error. +func Verify(db prefixer.Prefixer, dst vfs.VFS, dstAv vfs.Avatarer, expected *Report) error { + stater, ok := dst.(contentStater) + if !ok { + return fmt.Errorf("storagemigration: target backend does not support index-free stats") + } + + got := &Report{} + + if err := verifyFiles(db, stater, got); err != nil { + return err + } + if err := verifyVersions(db, stater, got); err != nil { + return err + } + + if got.Files != expected.Files { + return fmt.Errorf("storagemigration: verify: expected %d files, found %d on target", expected.Files, got.Files) + } + if got.Versions != expected.Versions { + return fmt.Errorf("storagemigration: verify: expected %d versions, found %d on target", expected.Versions, got.Versions) + } + + if expected.AvatarCopied { + ar, _, err := dstAv.OpenAvatar() + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: avatar missing on target") + } + if err != nil { + return fmt.Errorf("storagemigration: verify: open target avatar: %w", err) + } + _ = ar.Close() + } + + return nil +} + +// verifyFiles re-enumerates every io.cozy.files document and confirms the +// target object for each non-directory file exists with a matching size. +func verifyFiles(db prefixer.Prefixer, stater contentStater, got *Report) error { + return couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + + size, err := stater.StatContentAt(doc.DocID, doc.InternalID) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: file %s missing on target", doc.DocID) + } + if err != nil { + return fmt.Errorf("storagemigration: verify: stat target file %s: %w", doc.DocID, err) + } + if size != doc.ByteSize { + return fmt.Errorf("storagemigration: verify: file %s size mismatch: expected %d, got %d", doc.DocID, doc.ByteSize, size) + } + + got.Files++ + return nil + }) +} + +// verifyVersions re-enumerates every io.cozy.files.versions document and +// confirms the target object for each version exists with a matching size. +func verifyVersions(db prefixer.Prefixer, stater contentStater, got *Report) error { + return couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, raw json.RawMessage) error { + var ver vfs.Version + if err := json.Unmarshal(raw, &ver); err != nil { + return fmt.Errorf("storagemigration: decode version doc: %w", err) + } + + fileID, internalID := splitVersionID(ver.DocID) + + size, err := stater.StatContentAt(fileID, internalID) + if errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("storagemigration: verify: version %s missing on target", ver.DocID) + } + if err != nil { + return fmt.Errorf("storagemigration: verify: stat target version %s: %w", ver.DocID, err) + } + if size != ver.ByteSize { + return fmt.Errorf("storagemigration: verify: version %s size mismatch: expected %d, got %d", ver.DocID, ver.ByteSize, size) + } + + got.Versions++ + return nil + }) +} + // copyFiles enumerates every io.cozy.files document (ForeachDocs is // unfiltered, so trashed files are naturally included) and copies the // content of each file (skipping directories) from src to dst. diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go index fa07cbf05af..6c7a4e9a9e1 100644 --- a/model/instance/storagemigration/migration_test.go +++ b/model/instance/storagemigration/migration_test.go @@ -56,6 +56,11 @@ type migrationFixture struct { srcAv vfs.Avatarer dstAv vfs.Avatarer + + // minioClient and bucket give tests raw access to the S3 target, e.g. to + // delete an object behind the target VFS's back for negative-path checks. + minioClient *minio.Client + bucket string } func setupMigrationFixture(t *testing.T) *migrationFixture { @@ -110,11 +115,13 @@ func setupMigrationFixture(t *testing.T) *migrationFixture { dstAv := vfss3.NewAvatarFs(client, bucket, keyPrefix) return &migrationFixture{ - db: db, - src: src, - dst: dst, - srcAv: srcAv, - dstAv: dstAv, + db: db, + src: src, + dst: dst, + srcAv: srcAv, + dstAv: dstAv, + minioClient: client, + bucket: bucket, } } @@ -223,6 +230,27 @@ func TestCopyContentMovesFilesVersionsAndAvatar(t *testing.T) { assert.Equal(t, revBefore3, reread3.Rev()) } +func TestVerifySucceedsAfterCopyAndFailsWhenObjectMissing(t *testing.T) { + fx := setupMigrationFixture(t) + + file1 := createSourceFile(t, fx, "file1.txt", []byte("hello from file 1")) + _ = createSourceFile(t, fx, "file2.txt", []byte("hello from file 2, a bit longer")) + + rep, err := storagemigration.CopyContent(fx.db, fx.src, fx.dst, fx.srcAv, fx.dstAv) + require.NoError(t, err) + + require.NoError(t, storagemigration.Verify(fx.db, fx.dst, fx.dstAv, rep)) + + // Remove one known target object directly via the raw MinIO client, then + // confirm Verify now detects the discrepancy. + keyPrefix := fx.db.DBPrefix() + "/" + objKey := vfss3.MakeObjectKey(keyPrefix, file1.DocID, file1.InternalID) + + require.NoError(t, fx.minioClient.RemoveObject(context.Background(), fx.bucket, objKey, minio.RemoveObjectOptions{})) + + assert.Error(t, storagemigration.Verify(fx.db, fx.dst, fx.dstAv, rep)) +} + func assertFileContentOn(t *testing.T, fs vfs.VFS, doc *vfs.FileDoc, want []byte) { t.Helper() r, err := fs.OpenFile(doc) diff --git a/model/vfs/vfss3/impl.go b/model/vfs/vfss3/impl.go index 89f5c2b27fa..891c134870d 100644 --- a/model/vfs/vfss3/impl.go +++ b/model/vfs/vfss3/impl.go @@ -638,6 +638,22 @@ func (sfs *s3VFS) WriteContentAt(docID, internalID string, content io.Reader, si return err } +// StatContentAt returns the byte size of the object backing the (docID, +// internalID) key, without touching CouchDB. It returns os.ErrNotExist when +// the object is absent. Used by storage migration to verify a copy landed on +// the target before flipping the instance's backend flag. +func (sfs *s3VFS) StatContentAt(docID, internalID string) (int64, error) { + objKey := MakeObjectKey(sfs.keyPrefix, docID, internalID) + info, err := sfs.client.StatObject(sfs.ctx, sfs.bucket, objKey, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + return 0, os.ErrNotExist + } + return 0, err + } + return info.Size, nil +} + func (sfs *s3VFS) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr diff --git a/model/vfs/vfsswift/impl_v3.go b/model/vfs/vfsswift/impl_v3.go index cada518fb9a..e06304c7c2c 100644 --- a/model/vfs/vfsswift/impl_v3.go +++ b/model/vfs/vfsswift/impl_v3.go @@ -596,6 +596,23 @@ func (sfs *swiftVFSV3) WriteContentAt(docID, internalID string, content io.Reade return f.Close() } +// StatContentAt returns the byte size of the object backing the (docID, +// internalID) key in this instance's container, without touching CouchDB. It +// returns os.ErrNotExist when the object is absent. Used by storage +// migration to verify a copy landed on the target before flipping the +// instance's backend flag. +func (sfs *swiftVFSV3) StatContentAt(docID, internalID string) (int64, error) { + objName := MakeObjectNameV3(docID, internalID) + info, _, err := sfs.c.Object(sfs.ctx, sfs.container, objName) + if errors.Is(err, swift.ObjectNotFound) { + return 0, os.ErrNotExist + } + if err != nil { + return 0, err + } + return info.Bytes, nil +} + func (sfs *swiftVFSV3) RevertFileVersion(doc *vfs.FileDoc, version *vfs.Version) error { if lockerr := sfs.mu.Lock(); lockerr != nil { return lockerr From 628d83998adb68f16db2211d752ba8b7aaaae9a7 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 11:08:55 +0200 Subject: [PATCH 18/23] feat(storagemigration): orchestrate block, copy, verify, flip, rollback Add Migrate, which ties CopyContent/Verify together into the full migration flow: guard the target scheme and instance state, block the instance, copy and verify the content, then flip FsScheme only once Verify passes (DryRun and a failed Verify always leave FsScheme unchanged). FlagOnly supports rollback onto an already-populated backend and requires Force, since writes since cutover would be lost. PurgeSource best-effort deletes the source objects after a successful flip (S3-source only for now; swift-source purge reports "not implemented" rather than silently no-op'ing). Also add config.HasS3Client/HasSwiftConnection, non-panicking readiness checks needed to guard Migrate against a target backend that was never initialized. --- model/instance/storagemigration/migration.go | 192 ++++++++++++++++++ .../storagemigration/migration_test.go | 100 +++++++++ pkg/config/config/s3.go | 8 + pkg/config/config/swift.go | 9 + 4 files changed, 309 insertions(+) diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go index 619152aa6e9..b96537c1ae4 100644 --- a/model/instance/storagemigration/migration.go +++ b/model/instance/storagemigration/migration.go @@ -8,6 +8,7 @@ package storagemigration import ( + "context" "encoding/json" "errors" "fmt" @@ -15,10 +16,16 @@ import ( "os" "strings" + "github.com/cozy/cozy-stack/model/instance" + "github.com/cozy/cozy-stack/model/instance/lifecycle" "github.com/cozy/cozy-stack/model/vfs" + "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" + "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/consts" "github.com/cozy/cozy-stack/pkg/couchdb" "github.com/cozy/cozy-stack/pkg/prefixer" + "github.com/cozy/cozy-stack/pkg/s3util" ) // contentWriter is implemented by target VFS backends that can write object @@ -264,3 +271,188 @@ func splitVersionID(versionDocID string) (fileID, internalID string) { } return versionDocID, "" } + +// Options configures a call to Migrate. +type Options struct { + // To is the target storage scheme: config.SchemeS3 or config.SchemeSwift. + To string + // DryRun copies and verifies the content on the target backend but does + // not flip the instance's FsScheme: the instance keeps serving reads and + // writes from its current (source) backend. + DryRun bool + // FlagOnly switches the instance's FsScheme pointer to an already + // populated target backend without copying anything. It is intended for + // rollback (switching back to a backend that a previous migration left + // populated) and requires Force, since any write performed against the + // source since that previous cutover is lost. + FlagOnly bool + // Force is required together with FlagOnly, acknowledging the data-loss + // risk described above. + Force bool + // PurgeSource deletes the source backend's objects after a successful + // flip. It is a best-effort cleanup performed once the instance is + // already fully served from the target: a failure here does not revert + // the flip. + PurgeSource bool +} + +// containerNamer is implemented by VFS backends (vfsswift's V3 layout) that +// expose the underlying object-storage container name(s) they use, so +// storagemigration can ensure the target container exists without +// hand-rolling the naming scheme itself. +type containerNamer interface { + ContainerNames() map[string]string +} + +// Migrate moves an instance's object-storage content (files, versions, +// avatar) from its current backend to opts.To, verifies the copy, and only +// then flips the instance's FsScheme to the target. The instance is blocked +// (instance.BlockedMoving) for the duration of the copy/verify and unblocked +// on every return path. +// +// FsScheme is updated ONLY after Verify succeeds (or, for FlagOnly, after the +// target backend has been validated); a DryRun or a failed Verify always +// leaves FsScheme unchanged. +func Migrate(inst *instance.Instance, opts Options) (*Report, error) { + switch opts.To { + case config.SchemeS3, config.SchemeSwift: + default: + return nil, fmt.Errorf("storagemigration: unsupported target scheme %q", opts.To) + } + + srcScheme := inst.StorageScheme() + if opts.To == srcScheme { + return nil, fmt.Errorf("storagemigration: instance %s already uses %q as its storage scheme", inst.DomainName(), opts.To) + } + + if opts.To == config.SchemeS3 && !config.HasS3Client() { + return nil, errors.New("storagemigration: cannot migrate to s3: no S3 client is configured") + } + if opts.To == config.SchemeSwift && !config.HasSwiftConnection() { + return nil, errors.New("storagemigration: cannot migrate to swift: no swift connection is configured") + } + + if srcScheme == config.SchemeSwift || srcScheme == config.SchemeSwiftSecure { + if inst.SwiftLayout != 2 { + return nil, fmt.Errorf("storagemigration: source swift layout %d is not supported, only layout 2 (v3) can be migrated", inst.SwiftLayout) + } + } + + if opts.FlagOnly && !opts.Force { + return nil, errors.New("storagemigration: flag-only migration requires Force: any write performed against the source since the previous cutover would be lost") + } + + // Build the SOURCE from the instance's current backend before touching + // anything (the instance already knows how to build it for its current + // scheme). + src := inst.VFS() + srcAv := inst.AvatarFS() + + dst, dstAv, err := buildTarget(inst, opts.To) + if err != nil { + return nil, err + } + + if err := lifecycle.Block(inst, instance.BlockedMoving.Code); err != nil { + return nil, fmt.Errorf("storagemigration: block instance: %w", err) + } + defer func() { + _ = lifecycle.Unblock(inst) + }() + + if opts.FlagOnly { + return flip(inst, opts, srcScheme, &Report{}) + } + + rep, err := CopyContent(inst, src, dst, srcAv, dstAv) + if err != nil { + return rep, err + } + if err := Verify(inst, dst, dstAv, rep); err != nil { + return rep, err + } + + if opts.DryRun { + return rep, nil + } + + return flip(inst, opts, srcScheme, rep) +} + +// buildTarget constructs the VFS + Avatarer pair for the target scheme, +// ensuring the underlying bucket/container exists, without touching the +// CouchDB index (no InitFs: the index is shared with the source and must not +// be reinitialized). +func buildTarget(inst *instance.Instance, to string) (vfs.VFS, vfs.Avatarer, error) { + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migration-target") + + switch to { + case config.SchemeS3: + dst, err := vfss3.New(inst, index, disk, mutex) + if err != nil { + return nil, nil, fmt.Errorf("storagemigration: build s3 target: %w", err) + } + bucket := vfss3.BucketName(inst.GetOrgID(), config.GetS3BucketPrefix()) + if err := s3util.EnsureBucket(context.Background(), config.GetS3Client(), bucket, config.GetS3Region()); err != nil { + return nil, nil, fmt.Errorf("storagemigration: ensure target bucket: %w", err) + } + dstAv := vfss3.NewAvatarFs(config.GetS3Client(), bucket, inst.DBPrefix()+"/") + return dst, dstAv, nil + + case config.SchemeSwift: + dst, err := vfsswift.NewV3(inst, index, disk, mutex) + if err != nil { + return nil, nil, fmt.Errorf("storagemigration: build swift target: %w", err) + } + if cn, ok := dst.(containerNamer); ok { + if container, ok := cn.ContainerNames()["container"]; ok && container != "" { + if err := config.GetSwiftConnection().ContainerCreate(context.Background(), container, nil); err != nil { + return nil, nil, fmt.Errorf("storagemigration: ensure target container: %w", err) + } + } + } + dstAv := vfsswift.NewAvatarFsV3(config.GetSwiftConnection(), inst) + return dst, dstAv, nil + + default: + return nil, nil, fmt.Errorf("storagemigration: unsupported target scheme %q", to) + } +} + +// flip persists the FsScheme change to the target scheme and, if requested, +// purges the source backend's objects on a best-effort basis. It never +// reverts the flip: once the instance points at the target, the target is +// the source of truth for the instance's content. +func flip(inst *instance.Instance, opts Options, srcScheme string, rep *Report) (*Report, error) { + inst.FsScheme = opts.To + if err := instance.Update(inst); err != nil { + return rep, fmt.Errorf("storagemigration: persist storage scheme flip: %w", err) + } + + if opts.PurgeSource { + if err := purgeSource(inst, srcScheme); err != nil { + return rep, fmt.Errorf("storagemigration: purge source after flip: %w", err) + } + } + + return rep, nil +} + +// purgeSource best-effort deletes the source backend's objects after a +// successful flip. The instance already fully serves reads/writes from the +// target at this point, so a purge failure is reported but never reverts the +// flip. +func purgeSource(inst *instance.Instance, srcScheme string) error { + switch srcScheme { + case config.SchemeS3: + bucket := vfss3.BucketName(inst.GetOrgID(), config.GetS3BucketPrefix()) + prefix := inst.DBPrefix() + "/" + return s3util.DeletePrefixObjects(context.Background(), config.GetS3Client(), bucket, prefix) + case config.SchemeSwift, config.SchemeSwiftSecure: + return errors.New("storagemigration: purging a swift source is not implemented") + default: + return fmt.Errorf("storagemigration: purging source scheme %q is not implemented", srcScheme) + } +} diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go index 6c7a4e9a9e1..2375965c2cf 100644 --- a/model/instance/storagemigration/migration_test.go +++ b/model/instance/storagemigration/migration_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/cozy/cozy-stack/model/instance" "github.com/cozy/cozy-stack/model/instance/storagemigration" "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" @@ -260,3 +261,102 @@ func assertFileContentOn(t *testing.T, fs vfs.VFS, doc *vfs.FileDoc, want []byte require.NoError(t, r.Close()) assert.Equal(t, want, got) } + +// setupMigrateInstance creates a real instance (via testutils, on the global +// test backend, "mem") and populates it with a couple of files and an +// avatar, then starts a MinIO test server and wires up the global S3 client +// so config.HasS3Client() is true and Migrate can build an S3 target. +func setupMigrateInstance(t *testing.T) *instance.Instance { + t.Helper() + + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + inst := setup.GetTestInstance() + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "migrate-file1.txt", []byte("hello from migrate file 1")) + createInstanceFile(t, inst, "migrate-file2.txt", []byte("hello from migrate file 2, a bit longer")) + + aw, err := inst.AvatarFS().CreateAvatar("image/png") + require.NoError(t, err) + _, err = aw.Write([]byte("fake png bytes for the migrate avatar")) + require.NoError(t, err) + require.NoError(t, aw.Close()) + + return inst +} + +// createInstanceFile creates a file of the given name/content on the +// instance's current VFS. +func createInstanceFile(t *testing.T, inst *instance.Instance, name string, content []byte) *vfs.FileDoc { + t.Helper() + + doc, err := vfs.NewFileDoc(name, "", int64(len(content)), nil, "text/plain", "text", time.Now(), false, false, false, []string{}) + require.NoError(t, err) + + f, err := inst.VFS().CreateFile(doc, nil) + require.NoError(t, err) + + _, err = io.Copy(f, bytes.NewReader(content)) + require.NoError(t, err) + require.NoError(t, f.Close()) + + got, err := inst.VFS().FileByPath("/" + name) + require.NoError(t, err) + return got +} + +func TestMigrateFlipsSchemeAfterVerify(t *testing.T) { + inst := setupMigrateInstance(t) + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.NotNil(t, rep) + + assert.Equal(t, config.SchemeS3, inst.FsScheme) + assert.Greater(t, rep.Files, 0) + assert.False(t, inst.Blocked, "instance must be unblocked after a successful migration") + + // Reads are now served from S3: build a fresh S3 VFS for the instance + // (mirroring what inst.VFS() would now build) and confirm the migrated + // files are readable from it. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migrate-test-read") + s3fs, err := vfss3.New(inst, index, disk, mutex) + require.NoError(t, err) + + doc1, err := s3fs.FileByPath("/migrate-file1.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc1, []byte("hello from migrate file 1")) + + doc2, err := s3fs.FileByPath("/migrate-file2.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc2, []byte("hello from migrate file 2, a bit longer")) +} + +func TestMigrateDryRunDoesNotFlip(t *testing.T) { + inst := setupMigrateInstance(t) + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, DryRun: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Greater(t, rep.Files, 0) + + assert.Equal(t, "", inst.FsScheme) + assert.False(t, inst.Blocked, "instance must be unblocked after a dry-run migration") +} + +func TestMigrateFlagOnlyRequiresForce(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true}) + require.Error(t, err) + assert.Equal(t, "", inst.FsScheme) +} diff --git a/pkg/config/config/s3.go b/pkg/config/config/s3.go index 506fd57afe2..0cf7a1e729a 100644 --- a/pkg/config/config/s3.go +++ b/pkg/config/config/s3.go @@ -95,6 +95,14 @@ func GetS3Client() *minio.Client { return s3Client } +// HasS3Client reports whether the global S3 client has been initialized, +// without panicking. Callers that need to guard against a missing S3 +// connection (e.g. before attempting a storage migration to S3) should use +// this instead of recovering from GetS3Client's panic. +func HasS3Client() bool { + return s3Client != nil +} + // GetS3BucketPrefix returns the configured bucket prefix. func GetS3BucketPrefix() string { return s3BucketPrefix diff --git a/pkg/config/config/swift.go b/pkg/config/config/swift.go index 5d7a83a23ca..66c336e2ae8 100644 --- a/pkg/config/config/swift.go +++ b/pkg/config/config/swift.go @@ -102,3 +102,12 @@ func GetSwiftConnection() *swift.Connection { } return swiftConn } + +// HasSwiftConnection reports whether the global swift connection has been +// initialized, without panicking. Callers that need to guard against a +// missing swift connection (e.g. before attempting a storage migration to +// swift) should use this instead of recovering from GetSwiftConnection's +// panic. +func HasSwiftConnection() bool { + return swiftConn != nil +} From d15a2a285595673a9f596e84de45091fc72793d8 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 11:25:55 +0200 Subject: [PATCH 19/23] fix(storagemigration): verify FlagOnly target and purge swift source FlagOnly previously flipped FsScheme after buildTarget merely ensured the target bucket/container existed, with no check that it actually held the source's content (buildTarget's EnsureBucket/ContainerCreate create an empty target if none exists). Add sourceReport, which computes the expected file/version counts and avatar presence from the source's CouchDB docs, and run Verify against it before flipping; a failed Verify now returns without touching FsScheme. purgeSource also returned "not implemented" for a swift source after the flip had already persisted, making a successful migration look like a failure. Reuse the same per-instance container Delete() that instance destroy/reset already use to actually purge the swift source. Adds coverage for the flag-only flip (populated vs. empty target) and an end-to-end swift-source purge test using the in-memory swifttest server. --- model/instance/storagemigration/migration.go | 75 ++++++++++- .../storagemigration/migration_test.go | 119 ++++++++++++++++++ 2 files changed, 192 insertions(+), 2 deletions(-) diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go index b96537c1ae4..e74c7f834e1 100644 --- a/model/instance/storagemigration/migration.go +++ b/model/instance/storagemigration/migration.go @@ -117,6 +117,51 @@ func Verify(db prefixer.Prefixer, dst vfs.VFS, dstAv vfs.Avatarer, expected *Rep return nil } +// sourceReport enumerates the instance's CouchDB documents (io.cozy.files +// and io.cozy.files.versions) and checks srcAv for an avatar, to compute the +// Report a FlagOnly flip expects the already-populated target to satisfy. It +// does not read or write any object-storage content itself: it describes +// what the source SHOULD have on the target, for Verify to confirm. +func sourceReport(db prefixer.Prefixer, srcAv vfs.Avatarer) (*Report, error) { + rep := &Report{} + + err := couchdb.ForeachDocs(db, consts.Files, func(_ string, raw json.RawMessage) error { + var doc vfs.FileDoc + if err := json.Unmarshal(raw, &doc); err != nil { + return fmt.Errorf("storagemigration: decode file doc: %w", err) + } + if doc.Type == consts.DirType { + return nil + } + rep.Files++ + return nil + }) + if err != nil { + return nil, err + } + + err = couchdb.ForeachDocs(db, consts.FilesVersions, func(_ string, _ json.RawMessage) error { + rep.Versions++ + return nil + }) + if err != nil { + return nil, err + } + + ar, _, err := srcAv.OpenAvatar() + switch { + case errors.Is(err, os.ErrNotExist): + // No avatar on the source: rep.AvatarCopied stays false. + case err != nil: + return nil, fmt.Errorf("storagemigration: open source avatar: %w", err) + default: + _ = ar.Close() + rep.AvatarCopied = true + } + + return rep, nil +} + // verifyFiles re-enumerates every io.cozy.files document and confirms the // target object for each non-directory file exists with a matching size. func verifyFiles(db prefixer.Prefixer, stater contentStater, got *Report) error { @@ -361,7 +406,20 @@ func Migrate(inst *instance.Instance, opts Options) (*Report, error) { }() if opts.FlagOnly { - return flip(inst, opts, srcScheme, &Report{}) + // FlagOnly does not copy anything, but it must not flip onto a + // target that does not already hold the source's content: an + // unpopulated (or merely-existing, e.g. freshly EnsureBucket'd) + // target would otherwise silently strand the instance on zero + // files. Compute the expected counts from the source and verify + // the target really has them before flipping. + expected, err := sourceReport(inst, srcAv) + if err != nil { + return nil, err + } + if err := Verify(inst, dst, dstAv, expected); err != nil { + return expected, err + } + return flip(inst, opts, srcScheme, expected) } rep, err := CopyContent(inst, src, dst, srcAv, dstAv) @@ -451,7 +509,20 @@ func purgeSource(inst *instance.Instance, srcScheme string) error { prefix := inst.DBPrefix() + "/" return s3util.DeletePrefixObjects(context.Background(), config.GetS3Client(), bucket, prefix) case config.SchemeSwift, config.SchemeSwiftSecure: - return errors.New("storagemigration: purging a swift source is not implemented") + // The v3 swift layout uses a single, per-instance container (see + // vfsswift.NewV3), so purging the source is just deleting that + // container: build the same source VFS instance destroy/reset use + // (see lifecycle.destroy/reset calling inst.VFS().Delete()) and + // reuse its Delete(), which marks the container to-be-deleted and + // removes all its objects before removing the container itself. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migration-purge-source") + src, err := vfsswift.NewV3(inst, index, disk, mutex) + if err != nil { + return fmt.Errorf("storagemigration: build swift source for purge: %w", err) + } + return src.Delete() default: return fmt.Errorf("storagemigration: purging source scheme %q is not implemented", srcScheme) } diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go index 2375965c2cf..fdf3fa2e9dc 100644 --- a/model/instance/storagemigration/migration_test.go +++ b/model/instance/storagemigration/migration_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/md5" + "errors" "io" "net/url" "testing" @@ -14,12 +15,14 @@ import ( "github.com/cozy/cozy-stack/model/vfs" "github.com/cozy/cozy-stack/model/vfs/vfsafero" "github.com/cozy/cozy-stack/model/vfs/vfss3" + "github.com/cozy/cozy-stack/model/vfs/vfsswift" "github.com/cozy/cozy-stack/pkg/config/config" "github.com/cozy/cozy-stack/pkg/consts" "github.com/cozy/cozy-stack/pkg/couchdb" "github.com/cozy/cozy-stack/pkg/utils" "github.com/cozy/cozy-stack/tests/testutils" "github.com/minio/minio-go/v7" + swiftv2 "github.com/ncw/swift/v2" "github.com/spf13/afero" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -360,3 +363,119 @@ func TestMigrateFlagOnlyRequiresForce(t *testing.T) { require.Error(t, err) assert.Equal(t, "", inst.FsScheme) } + +// TestMigrateFlagOnlyFlipsWhenTargetPopulated covers the CRITICAL fix: a +// FlagOnly+Force flip must succeed (and actually flip) once the target +// backend genuinely already holds the source's content. +func TestMigrateFlagOnlyFlipsWhenTargetPopulated(t *testing.T) { + inst := setupMigrateInstance(t) + + // Populate the S3 target for real once, so it already contains the + // instance's full content (2 files + avatar) by the time the flag-only + // flip below relies on it. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Simulate a rollback scenario: the instance is pointed back at its + // (still fully intact, never purged) previous scheme, and we now want + // to flip it back onto the S3 target without recopying anything, since + // that target is already fully populated from the migration above. + inst.FsScheme = "" + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true, Force: true}) + require.NoError(t, err) + require.NotNil(t, rep) + + assert.Equal(t, config.SchemeS3, inst.FsScheme) + assert.Equal(t, 2, rep.Files) + assert.True(t, rep.AvatarCopied) +} + +// TestMigrateFlagOnlyFailsWhenTargetEmpty covers the CRITICAL fix's negative +// path: a FlagOnly+Force flip against a target that only exists (e.g. an +// empty bucket created by buildTarget's EnsureBucket call) but does not +// actually hold the source's content must fail, and must NOT flip +// FsScheme. +func TestMigrateFlagOnlyFailsWhenTargetEmpty(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true, Force: true}) + require.Error(t, err) + assert.Equal(t, "", inst.FsScheme) +} + +// TestMigratePurgeSourceRemovesSourceObjects covers the IMPORTANT fix: a +// swift source must actually be purged (not return a "not implemented" +// error) after a successful flip. It exercises the real swift-source purge +// path end-to-end: an instance is first migrated from mem to a real +// (in-memory swifttest server) swift backend, populating swift for real; +// it is then migrated from swift to S3 with PurgeSource, and the test +// confirms the swift container backing the instance is gone afterward. +func TestMigratePurgeSourceRemovesSourceObjects(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + setup.SetupSwiftTest() + inst := setup.GetTestInstance() + + // GetTestInstance created this instance against the test config's + // default (non-swift) scheme, so it was never assigned a swift layout. + // Migrate requires layout v3 for any swift source (see the SwiftLayout + // guard in Migrate), so set it explicitly here to simulate a real + // swift-scheme instance, as would exist in production. + inst.SwiftLayout = 2 + require.NoError(t, instance.Update(inst)) + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "purge-file1.txt", []byte("hello from purge file 1")) + createInstanceFile(t, inst, "purge-file2.txt", []byte("hello from purge file 2, a bit longer")) + + // Step 1: migrate mem -> swift for real, so the swift container backing + // this instance is genuinely populated. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeSwift}) + require.NoError(t, err) + require.Equal(t, config.SchemeSwift, inst.FsScheme) + + containerName := swiftContainerName(t, inst) + + // Sanity check: the container really exists before the purge. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + require.NoError(t, err, "the swift container must exist after the first migration") + + // Step 2: migrate swift -> S3 with PurgeSource, exercising the swift + // source purge implementation. + _, err = storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, PurgeSource: true}) + require.NoError(t, err) + assert.Equal(t, config.SchemeS3, inst.FsScheme) + + // The swift container must be gone now: purgeSource must have actually + // deleted it, not returned a "not implemented" error after a + // successful (and now unrevertable) flip. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + assert.True(t, errors.Is(err, swiftv2.ContainerNotFound), "expected the swift container to be gone after purge, got err=%v", err) +} + +// swiftContainerName builds the same per-instance swift V3 container that +// buildTarget/purgeSource use, so tests can inspect it directly against the +// swift connection. +func swiftContainerName(t *testing.T, inst *instance.Instance) string { + t.Helper() + + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "storagemigration-test-swift-container-name") + + sfs, err := vfsswift.NewV3(inst, index, disk, mutex) + require.NoError(t, err) + + cn, ok := sfs.(interface{ ContainerNames() map[string]string }) + require.True(t, ok, "vfsswift.NewV3 must expose ContainerNames()") + + return cn.ContainerNames()["container"] +} From 9d6b3baed923446cb6ee508fcbe390453b50cfc1 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 11:34:31 +0200 Subject: [PATCH 20/23] feat(web/instances): add migrate-storage admin endpoint and client --- client/instances.go | 57 +++++++++++++++++++++++++++++++++ web/instances/instances.go | 24 ++++++++++++++ web/instances/instances_test.go | 23 +++++++++++++ 3 files changed, 104 insertions(+) diff --git a/client/instances.go b/client/instances.go index 07833c035aa..793479ed6f4 100644 --- a/client/instances.go +++ b/client/instances.go @@ -282,6 +282,63 @@ func (ac *AdminClient) ModifyInstance(opts *InstanceOptions) (*Instance, error) return readInstance(res) } +// MigrateStorageOptions contains the options for MigrateStorage. It mirrors +// the fields of storagemigration.Options in the model, without importing +// that package: the client stays free of any model/... dependency. +type MigrateStorageOptions struct { + To string + DryRun bool + FlagOnly bool + Force bool + PurgeSource bool +} + +// MigrateStorageReport is the report returned by MigrateStorage. It mirrors +// the fields of storagemigration.Report in the model. +type MigrateStorageReport struct { + Files int `json:"Files"` + Versions int `json:"Versions"` + Bytes int64 `json:"Bytes"` + AvatarCopied bool `json:"AvatarCopied"` +} + +// MigrateStorage moves an instance's object-storage content (files, +// versions, avatar) from its current backend to the target scheme. +func (ac *AdminClient) MigrateStorage(domain string, opts MigrateStorageOptions) (*MigrateStorageReport, error) { + if !validDomain(domain) { + return nil, fmt.Errorf("Invalid domain: %s", domain) + } + q := url.Values{ + "to": {opts.To}, + } + if opts.DryRun { + q.Add("dry_run", "true") + } + if opts.FlagOnly { + q.Add("flag_only", "true") + } + if opts.Force { + q.Add("force", "true") + } + if opts.PurgeSource { + q.Add("purge_source", "true") + } + res, err := ac.Req(&request.Options{ + Method: "POST", + Path: "/instances/" + domain + "/migrate-storage", + Queries: q, + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + rep := &MigrateStorageReport{} + if err := json.NewDecoder(res.Body).Decode(rep); err != nil { + return nil, err + } + return rep, nil +} + // DestroyInstance is used to delete an instance and all its data. func (ac *AdminClient) DestroyInstance(domain string) error { if !validDomain(domain) { diff --git a/web/instances/instances.go b/web/instances/instances.go index 4e716c3aea0..1778ad96a95 100644 --- a/web/instances/instances.go +++ b/web/instances/instances.go @@ -15,6 +15,7 @@ import ( "github.com/cozy/cozy-stack/model/app" "github.com/cozy/cozy-stack/model/instance" "github.com/cozy/cozy-stack/model/instance/lifecycle" + "github.com/cozy/cozy-stack/model/instance/storagemigration" "github.com/cozy/cozy-stack/model/notification" "github.com/cozy/cozy-stack/model/notification/center" "github.com/cozy/cozy-stack/model/oauth" @@ -300,6 +301,26 @@ func deleteHandler(c echo.Context) error { return c.NoContent(http.StatusNoContent) } +func migrateStorageHandler(c echo.Context) error { + domain := c.Param("domain") + inst, err := lifecycle.GetInstance(domain) + if err != nil { + return wrapError(err) + } + opts := storagemigration.Options{ + To: c.QueryParam("to"), + DryRun: c.QueryParam("dry_run") == "true", + FlagOnly: c.QueryParam("flag_only") == "true", + Force: c.QueryParam("force") == "true", + PurgeSource: c.QueryParam("purge_source") == "true", + } + rep, err := storagemigration.Migrate(inst, opts) + if err != nil { + return wrapError(err) + } + return c.JSON(http.StatusOK, rep) +} + func setAuthMode(c echo.Context) error { domain := c.Param("domain") inst, err := lifecycle.GetInstance(domain) @@ -785,6 +806,9 @@ func Routes(router *echo.Group) { router.GET("/contexts/:name", showContext) router.GET("/with-app-version/:slug/:version", appVersion) + // Storage migration + router.POST("/:domain/migrate-storage", migrateStorageHandler) + // Checks router.GET("/:domain/fsck", fsckHandler) router.POST("/:domain/checks/triggers", checkTriggers) diff --git a/web/instances/instances_test.go b/web/instances/instances_test.go index e6bc473bb4c..4b1cb6cd523 100644 --- a/web/instances/instances_test.go +++ b/web/instances/instances_test.go @@ -97,4 +97,27 @@ func TestInstances(t *testing.T) { attrs.HasValue("feature_sets", []string{"71df3022-abd9-11ee-b79b-9cb6d0907fa3", "790789f8-abd9-11ee-ae09-9cb6d0907fa3"}) }) }) + + t.Run("MigrateStorage", func(t *testing.T) { + domain := "migrate-storage.cozy.localhost" + t.Cleanup(func() { _ = lifecycle.Destroy(domain) }) + + e := testutils.CreateTestClient(t, ts.URL) + + e.POST("/instances"). + WithQuery("Domain", domain). + WithQuery("Locale", "en"). + WithQuery("SwiftLayout", "-1"). + WithHeader("Authorization", "Bearer "+token). + Expect().Status(201) + + // An unsupported target scheme is rejected by the Migrate guard + // before touching any storage backend, so this exercises the route + // and error propagation without requiring a live S3/Swift target. + e.POST("/instances/"+domain+"/migrate-storage"). + WithQuery("to", "not-a-scheme"). + WithQuery("dry_run", "true"). + WithHeader("Authorization", "Bearer "+token). + Expect().Status(500) + }) } From 7b7b734bff671a5dbdad5ff99ac3516480bb949d Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 11:38:12 +0200 Subject: [PATCH 21/23] feat(cmd): add instances migrate-storage command --- cmd/instances.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cmd/instances.go b/cmd/instances.go index bf60ab23622..8551929a041 100644 --- a/cmd/instances.go +++ b/cmd/instances.go @@ -45,6 +45,11 @@ var flagPassphrase string var flagForce bool var flagJSON bool var flagSwiftLayout int +var flagMigrateTo string +var flagMigrateDryRun bool +var flagMigrateFlagOnly bool +var flagMigrateForce bool +var flagMigratePurgeSource bool var flagCouchCluster int var flagUUID string var flagOIDCID string @@ -248,6 +253,33 @@ be used as the error message. }, } +var migrateStorageCmd = &cobra.Command{ + Use: "migrate-storage ", + Short: "Migrate an instance's file storage to another backend (e.g. s3)", + Long: `cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given.`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return cmd.Usage() + } + ac := newAdminClient() + rep, err := ac.MigrateStorage(args[0], client.MigrateStorageOptions{ + To: flagMigrateTo, + DryRun: flagMigrateDryRun, + FlagOnly: flagMigrateFlagOnly, + Force: flagMigrateForce, + PurgeSource: flagMigratePurgeSource, + }) + if err != nil { + return err + } + fmt.Printf("migrated: %d files, %d versions, %d bytes, avatar=%v\n", + rep.Files, rep.Versions, rep.Bytes, rep.AvatarCopied) + return nil + }, +} + var modifyInstanceCmd = &cobra.Command{ Use: "modify ", Short: "Modify the instance properties", @@ -1064,6 +1096,7 @@ func init() { instanceCmdGroup.AddCommand(showInstanceCmd) instanceCmdGroup.AddCommand(showDBPrefixInstanceCmd) instanceCmdGroup.AddCommand(addInstanceCmd) + instanceCmdGroup.AddCommand(migrateStorageCmd) instanceCmdGroup.AddCommand(modifyInstanceCmd) instanceCmdGroup.AddCommand(countInstanceCmd) instanceCmdGroup.AddCommand(lsInstanceCmd) @@ -1101,6 +1134,11 @@ func init() { addInstanceCmd.Flags().StringVar(&flagPhone, "phone", "", "The phone number of the owner") addInstanceCmd.Flags().StringVar(&flagSettings, "settings", "", "A list of settings (eg context:foo,offer:premium)") addInstanceCmd.Flags().IntVar(&flagSwiftLayout, "swift-layout", -1, "Specify the layout to use for Swift (from 0 for layout V1 to 2 for layout V3, -1 means the default)") + migrateStorageCmd.Flags().StringVar(&flagMigrateTo, "to", "s3", "Target storage scheme") + migrateStorageCmd.Flags().BoolVar(&flagMigrateDryRun, "dry-run", false, "Report what would be copied without writing or switching") + migrateStorageCmd.Flags().BoolVar(&flagMigrateFlagOnly, "flag-only", false, "Switch the backend pointer without copying (rollback to a retained source)") + migrateStorageCmd.Flags().BoolVar(&flagMigrateForce, "force", false, "Required with --flag-only; writes since cutover are lost") + migrateStorageCmd.Flags().BoolVar(&flagMigratePurgeSource, "purge-source", false, "Delete source objects after a successful switch") addInstanceCmd.Flags().IntVar(&flagCouchCluster, "couch-cluster", -1, "Specify the CouchDB cluster where the instance will be created (-1 means the default)") addInstanceCmd.Flags().StringVar(&flagDiskQuota, "disk-quota", "", "The quota allowed to the instance's VFS") addInstanceCmd.Flags().StringSliceVar(&flagApps, "apps", nil, "Apps to be preinstalled") From 55dd75bf5d90441c09c748ce5ffce31e113c6230 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 11:43:25 +0200 Subject: [PATCH 22/23] docs: document fs.migration_target and instances migrate-storage Documents the fs.migration_target config key that lets an instance be migrated to a new storage backend (e.g. S3) without changing the global fs.url, and adds a step-by-step Swift-to-S3 migration guide covering the migrate-storage command, rollback, and source purge. Also regenerates the CLI reference page for the new instances migrate-storage command. --- docs/cli/cozy-stack_instances.md | 1 + .../cozy-stack_instances_migrate-storage.md | 39 +++++++++ docs/config.md | 28 ++++++ docs/s3.md | 85 +++++++++++++++++++ 4 files changed, 153 insertions(+) create mode 100644 docs/cli/cozy-stack_instances_migrate-storage.md diff --git a/docs/cli/cozy-stack_instances.md b/docs/cli/cozy-stack_instances.md index 8a47bfcb0a8..e17f2a7f39d 100644 --- a/docs/cli/cozy-stack_instances.md +++ b/docs/cli/cozy-stack_instances.md @@ -50,6 +50,7 @@ cozy-stack instances [flags] * [cozy-stack instances fsck](cozy-stack_instances_fsck.md) - Check a vfs * [cozy-stack instances import](cozy-stack_instances_import.md) - Import data from an export link * [cozy-stack instances ls](cozy-stack_instances_ls.md) - List instances +* [cozy-stack instances migrate-storage](cozy-stack_instances_migrate-storage.md) - Migrate an instance's file storage to another backend (e.g. s3) * [cozy-stack instances modify](cozy-stack_instances_modify.md) - Modify the instance properties * [cozy-stack instances refresh-token-oauth](cozy-stack_instances_refresh-token-oauth.md) - Generate a new OAuth refresh token * [cozy-stack instances set-disk-quota](cozy-stack_instances_set-disk-quota.md) - Change the disk-quota of the instance diff --git a/docs/cli/cozy-stack_instances_migrate-storage.md b/docs/cli/cozy-stack_instances_migrate-storage.md new file mode 100644 index 00000000000..8bd3876ab81 --- /dev/null +++ b/docs/cli/cozy-stack_instances_migrate-storage.md @@ -0,0 +1,39 @@ +## cozy-stack instances migrate-storage + +Migrate an instance's file storage to another backend (e.g. s3) + +### Synopsis + +cozy-stack instances migrate-storage copies an instance's files, file +versions and avatar to another storage backend and switches the instance to it. +The source data is kept unless --purge-source is given. + +``` +cozy-stack instances migrate-storage [flags] +``` + +### Options + +``` + --dry-run Report what would be copied without writing or switching + --flag-only Switch the backend pointer without copying (rollback to a retained source) + --force Required with --flag-only; writes since cutover are lost + -h, --help help for migrate-storage + --purge-source Delete source objects after a successful switch + --to string Target storage scheme (default "s3") +``` + +### Options inherited from parent commands + +``` + --admin-host string administration server host (default "localhost") + --admin-port int administration server port (default 6060) + -c, --config string configuration file (default "$HOME/.cozy.yaml") + --host string server host (default "localhost") + -p, --port int server port (default 8080) +``` + +### SEE ALSO + +* [cozy-stack instances](cozy-stack_instances.md) - Manage instances of a stack + diff --git a/docs/config.md b/docs/config.md index e10d2459803..1bd70e19731 100644 --- a/docs/config.md +++ b/docs/config.md @@ -100,6 +100,34 @@ Magick, konnectors and services for example). And they can take several GB for the case of importing a Cozy. If needed, it is possible to configure the directory where they will be created via the `TMPDIR` environment variable. +## Storage backend migration + +The `fs.url` parameter configures the storage backend (`file://`, +`swift://` or `s3://`) used by all instances, as shown in +[cozy.example.yaml](https://github.com/cozy/cozy-stack/blob/master/cozy.example.yaml) +and detailed in the [S3 storage backend](s3.md) documentation. + +To move instances to a different backend one at a time, without changing +the backend used by the rest of the fleet, an optional `fs.migration_target` +key can be set to a second storage URL. Its connection is initialized +alongside the default one at startup, so instances can be migrated to it +while `fs.url` keeps pointing at the previous backend: + +```yaml +fs: + url: swift://openstack/?UserName={{ .Env.OS_USERNAME }}&Password={{ .Env.OS_PASSWORD }} + migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +As with `fs.url`, S3 credentials are passed as `access_key` and `secret_key` +query parameters of the URL. + +Once `fs.migration_target` is set, the +[`cozy-stack instances migrate-storage`](cli/cozy-stack_instances_migrate-storage.md) +command can move individual instances to it. See +[Migrating an instance from Swift to S3](s3.md#migrating-an-instance-from-swift-to-s3) +for the full procedure, including rollback. + ## Multiple CouchDB clusters With a large number of instances, a single CouchDB cluster may not be enough. diff --git a/docs/s3.md b/docs/s3.md index c757f3b4455..9952db6e5ee 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -122,6 +122,91 @@ fs: Note: files uploaded to S3 won't be accessible when using the local filesystem backend, and vice versa. Each backend has its own storage. +## Migrating an instance from Swift to S3 + +Instances can be moved from Swift to S3 one at a time, without changing the +storage backend for the rest of the fleet. This is useful to validate S3 on +a few instances before committing the whole platform to it. + +### 1. Configure the migration target + +Set `fs.migration_target` to the S3 URL, keeping `fs.url` on `swift://`. +Both connections (Swift and S3) are then initialized at startup: + +```yaml +fs: + url: swift://openstack/?UserName=... + migration_target: s3://s3.rbx.io.cloud.ovh.net?access_key=ACCESS&secret_key=SECRET®ion=rbx&bucket_prefix=cozy&use_ssl=true +``` + +See [`fs.migration_target`](config.md#storage-backend-migration) in the +configuration documentation for details on this key. + +### 2. Run the migration + +```bash +cozy-stack instances migrate-storage --to s3 +``` + +It is recommended to run with `--dry-run` first to see what would be copied +without writing anything or switching the instance. + +The command: + +- blocks the instance (read-only, HTTP traffic only) for the duration of + the copy; +- copies the files, file versions, and the user's avatar to the S3 target + (thumbnails and installed apps are not copied; they regenerate on the + target backend); +- verifies the copied objects against the source; +- flips the instance's storage backend to S3 and unblocks the instance. + +By default the Swift source is kept as-is after a successful migration, so +it stays available for a rollback. + +**Known limitation:** blocking only gates HTTP traffic. Background workers +and triggers can still write to the source during the migration window. +Run migrations during low-activity periods until this is addressed. + +### 3. Roll back if needed + +If something looks wrong shortly after the switch, before any real write +has landed on the S3 target, flip the instance back to the retained Swift +source instantly, without copying anything back: + +```bash +cozy-stack instances migrate-storage --to swift --flag-only --force +``` + +`--force` is required with `--flag-only` because any writes made against S3 +since the cutover are lost. + +If real data now lives on S3 and needs to be preserved, run a full +migration back to Swift instead, which copies the data: + +```bash +cozy-stack instances migrate-storage --to swift +``` + +### 4. Reclaim the source + +Once confident the instance is stable on its new backend, delete the +retained source objects: + +```bash +cozy-stack instances migrate-storage --to s3 --purge-source +``` + +(`--purge-source` can also be supplied on the initial migration if no +rollback window is needed.) + +### 5. Switch the global default + +After the whole fleet has been migrated to S3, change the global `fs.url` +to the S3 URL and remove `fs.migration_target`. From then on, the +per-instance backend flag set by earlier migrations simply matches the +global default. + ## Bucket strategy ### Design rationale From 0ca48fb5d10ed233b051e7472e23f177833279a3 Mon Sep 17 00:00:00 2001 From: Crash-- Date: Fri, 17 Jul 2026 11:56:35 +0200 Subject: [PATCH 23/23] fix(storagemigration): support deferred purge-only reclaim and fix flag-only dry-run Migrate rejected opts.To == the instance's current scheme even when PurgeSource was set, making the documented deferred reclaim step (docs/s3.md step 4, run --purge-source after the flip) and any retry of a failed inline purge impossible. Migrate now detects this case and runs a purge-only mode: it deletes the other backend's leftover data for the instance without copying, verifying, or flipping anything, and without blocking the instance. Also fix --flag-only --force --dry-run mutating state: the FlagOnly branch now checks opts.DryRun after Verify and returns without flipping FsScheme when set. Finally, guard model/stack Start against fs.migration_target pointing at S3 while fs.url is already an s3 scheme, which previously silently overwrote the global S3 client with the migration endpoint. --- docs/s3.md | 6 + model/instance/storagemigration/migration.go | 63 +++++++++- .../storagemigration/migration_test.go | 114 ++++++++++++++++++ model/stack/main.go | 6 + 4 files changed, 188 insertions(+), 1 deletion(-) diff --git a/docs/s3.md b/docs/s3.md index 9952db6e5ee..384c21956a0 100644 --- a/docs/s3.md +++ b/docs/s3.md @@ -197,6 +197,12 @@ retained source objects: cozy-stack instances migrate-storage --to s3 --purge-source ``` +Since the instance already uses `s3`, this runs in purge-only mode: nothing +is copied, verified, or flipped, and the previously retained Swift data is +simply deleted. The instance is not blocked for this step. Running the same +command again is safe and is the way to retry a purge that failed right +after an earlier switch. + (`--purge-source` can also be supplied on the initial migration if no rollback window is needed.) diff --git a/model/instance/storagemigration/migration.go b/model/instance/storagemigration/migration.go index e74c7f834e1..e6eef52e0a7 100644 --- a/model/instance/storagemigration/migration.go +++ b/model/instance/storagemigration/migration.go @@ -323,7 +323,8 @@ type Options struct { To string // DryRun copies and verifies the content on the target backend but does // not flip the instance's FsScheme: the instance keeps serving reads and - // writes from its current (source) backend. + // writes from its current (source) backend. Combined with FlagOnly, it + // still verifies the already-populated target but likewise never flips. DryRun bool // FlagOnly switches the instance's FsScheme pointer to an already // populated target backend without copying anything. It is intended for @@ -338,6 +339,13 @@ type Options struct { // flip. It is a best-effort cleanup performed once the instance is // already fully served from the target: a failure here does not revert // the flip. + // + // If To already equals the instance's current storage scheme, + // PurgeSource switches Migrate into purge-only mode: nothing is copied, + // verified, or flipped, and only the OTHER backend's leftover data for + // this instance is deleted. This is what makes a deferred reclaim + // (running --purge-source well after the flip, or retrying an inline + // purge that failed) possible. PurgeSource bool } @@ -358,6 +366,11 @@ type containerNamer interface { // FsScheme is updated ONLY after Verify succeeds (or, for FlagOnly, after the // target backend has been validated); a DryRun or a failed Verify always // leaves FsScheme unchanged. +// +// If opts.To already equals the instance's current storage scheme AND +// opts.PurgeSource is set, Migrate runs in purge-only mode instead: see +// purgeOnly. Without PurgeSource, opts.To == the current scheme is still an +// error. func Migrate(inst *instance.Instance, opts Options) (*Report, error) { switch opts.To { case config.SchemeS3, config.SchemeSwift: @@ -367,6 +380,14 @@ func Migrate(inst *instance.Instance, opts Options) (*Report, error) { srcScheme := inst.StorageScheme() if opts.To == srcScheme { + if opts.PurgeSource { + // Purge-only mode: the instance is already on opts.To (either + // because a previous migration flipped it, or the caller is + // retrying an inline purge that failed after that flip). There + // is nothing to copy, verify, or flip: just reclaim the OTHER + // backend's leftover data for this instance. + return purgeOnly(inst, opts.To) + } return nil, fmt.Errorf("storagemigration: instance %s already uses %q as its storage scheme", inst.DomainName(), opts.To) } @@ -419,6 +440,9 @@ func Migrate(inst *instance.Instance, opts Options) (*Report, error) { if err := Verify(inst, dst, dstAv, expected); err != nil { return expected, err } + if opts.DryRun { + return expected, nil + } return flip(inst, opts, srcScheme, expected) } @@ -527,3 +551,40 @@ func purgeSource(inst *instance.Instance, srcScheme string) error { return fmt.Errorf("storagemigration: purging source scheme %q is not implemented", srcScheme) } } + +// purgeOnly implements Migrate's purge-only mode: opts.To already equals the +// instance's current storage scheme, so there is nothing to copy, verify, or +// flip. It only deletes the OTHER backend's (still-retained) leftover data +// for this instance, which is what makes the documented deferred reclaim +// step (running --purge-source well after the flip) work, and also gives a +// retry path when an inline purge failed after a previous flip. The active +// backend (to) is never touched and the instance is not blocked, since reads +// and writes against it are unaffected. +func purgeOnly(inst *instance.Instance, to string) (*Report, error) { + var otherScheme string + switch to { + case config.SchemeS3: + otherScheme = config.SchemeSwift + case config.SchemeSwift: + otherScheme = config.SchemeS3 + default: + return nil, fmt.Errorf("storagemigration: unsupported target scheme %q", to) + } + + switch otherScheme { + case config.SchemeS3: + if !config.HasS3Client() { + return nil, errors.New("storagemigration: cannot purge s3: no S3 client is configured") + } + case config.SchemeSwift: + if !config.HasSwiftConnection() { + return nil, errors.New("storagemigration: cannot purge swift: no swift connection is configured") + } + } + + if err := purgeSource(inst, otherScheme); err != nil { + return nil, fmt.Errorf("storagemigration: purge-only: %w", err) + } + + return &Report{}, nil +} diff --git a/model/instance/storagemigration/migration_test.go b/model/instance/storagemigration/migration_test.go index fdf3fa2e9dc..d50c5542413 100644 --- a/model/instance/storagemigration/migration_test.go +++ b/model/instance/storagemigration/migration_test.go @@ -479,3 +479,117 @@ func swiftContainerName(t *testing.T, inst *instance.Instance) string { return cn.ContainerNames()["container"] } + +// TestMigratePurgeOnlyReclaimsOtherBackend covers the CRITICAL fix: once an +// instance already sits on its target scheme (a previous migration flipped +// it, and the Swift source was deliberately retained for rollback, as +// docs/s3.md step 4 describes), a later call with PurgeSource and the SAME +// To must not hit the "already uses that scheme" guard. Instead it must run +// in purge-only mode: reclaim the other backend's leftover data without +// copying, verifying, or flipping anything. +func TestMigratePurgeOnlyReclaimsOtherBackend(t *testing.T) { + if testing.Short() { + t.Skip("an instance is required for this test: test skipped due to the use of --short flag") + } + + config.UseTestFile(t) + setup := testutils.NewSetup(t, t.Name()) + setup.SetupSwiftTest() + inst := setup.GetTestInstance() + + // See TestMigratePurgeSourceRemovesSourceObjects: a swift source requires + // layout v3 to be migrated. + inst.SwiftLayout = 2 + require.NoError(t, instance.Update(inst)) + + mf := testutils.StartMinio(t) + require.NoError(t, config.InitS3Connection(config.Fs{URL: mf.FsURL("test")})) + + createInstanceFile(t, inst, "purge-only-file1.txt", []byte("hello from purge-only file 1")) + createInstanceFile(t, inst, "purge-only-file2.txt", []byte("hello from purge-only file 2, a bit longer")) + + // Step 1: migrate mem -> swift for real, so the swift container backing + // this instance is genuinely populated. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeSwift}) + require.NoError(t, err) + require.Equal(t, config.SchemeSwift, inst.FsScheme) + + containerName := swiftContainerName(t, inst) + + // Step 2: migrate swift -> S3 WITHOUT PurgeSource, so the instance ends + // on S3 while the swift source is deliberately retained, exactly as + // docs/s3.md's rollback window describes. + _, err = storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Sanity check: the retained swift container still exists after the + // flip, since PurgeSource was not requested. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + require.NoError(t, err, "the swift container must still exist: PurgeSource was not requested on the flip") + + // Step 3 (the deferred reclaim, run later): call Migrate again with + // To == the instance's CURRENT scheme (s3) and PurgeSource set. This + // must not error out on the "already uses that scheme" guard; it must + // instead purge the other backend (swift) and leave the instance as-is. + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, PurgeSource: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, config.SchemeS3, inst.FsScheme, "purge-only must not change the instance's scheme") + assert.False(t, inst.Blocked, "purge-only must not leave the instance blocked") + + // The swift container must be gone now. + _, _, err = config.GetSwiftConnection().Container(context.Background(), containerName) + assert.True(t, errors.Is(err, swiftv2.ContainerNotFound), "expected the swift container to be gone after purge-only, got err=%v", err) + + // The instance's S3 content (the active backend) must be untouched. + index := vfs.NewCouchdbIndexer(inst) + disk := vfs.DiskThresholder(inst) + mutex := config.Lock().ReadWrite(inst, "vfs-migrate-test-purge-only-read") + s3fs, err := vfss3.New(inst, index, disk, mutex) + require.NoError(t, err) + doc1, err := s3fs.FileByPath("/purge-only-file1.txt") + require.NoError(t, err) + assertFileContentOn(t, s3fs, doc1, []byte("hello from purge-only file 1")) +} + +// TestMigratePurgeOnlyWithoutPurgeFlagStillErrors covers the guard that must +// still hold for a plain re-run against the current scheme without +// PurgeSource: purge-only mode is only entered when PurgeSource is set. +func TestMigratePurgeOnlyWithoutPurgeFlagStillErrors(t *testing.T) { + inst := setupMigrateInstance(t) + + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + _, err = storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.Error(t, err) + assert.Equal(t, config.SchemeS3, inst.FsScheme) +} + +// TestMigrateFlagOnlyDryRunDoesNotFlip covers the IMPORTANT fix: combining +// FlagOnly with DryRun must still verify the target, but must NOT flip +// FsScheme, even with Force set. +func TestMigrateFlagOnlyDryRunDoesNotFlip(t *testing.T) { + inst := setupMigrateInstance(t) + + // Populate the S3 target for real once, so it already contains the + // instance's full content by the time the flag-only dry-run below + // relies on it. + _, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3}) + require.NoError(t, err) + require.Equal(t, config.SchemeS3, inst.FsScheme) + + // Reset the scheme, as a rollback scenario would have it, then attempt a + // flag-only flip back onto S3 as a dry run. + inst.FsScheme = "" + + rep, err := storagemigration.Migrate(inst, storagemigration.Options{To: config.SchemeS3, FlagOnly: true, Force: true, DryRun: true}) + require.NoError(t, err) + require.NotNil(t, rep) + assert.Equal(t, 2, rep.Files) + + assert.Equal(t, "", inst.FsScheme, "a dry-run flag-only migration must not flip FsScheme") + assert.False(t, inst.Blocked, "instance must be unblocked after a dry-run flag-only migration") +} diff --git a/model/stack/main.go b/model/stack/main.go index 861b13456ca..820c36af832 100644 --- a/model/stack/main.go +++ b/model/stack/main.go @@ -93,6 +93,12 @@ security features. Please do not use this binary as your production server. // When a storage migration target is configured (e.g. migrating instances // to S3 while the global default is still Swift), init that connection too. if config.HasS3Target() { + // If fs.url is already an s3 scheme, initializing the migration + // target here would silently overwrite the global S3 client with the + // migration endpoint: refuse to start instead of risking that. + if config.FsURL().Scheme == config.SchemeS3 { + return nil, nil, fmt.Errorf("fs.migration_target must not be set when fs.url is already an s3 scheme") + } if err := config.InitS3Connection(config.Fs{URL: config.MigrationTargetURL()}); err != nil { return nil, nil, fmt.Errorf("failed to init the S3 migration target connection: %w", err) }