From c44685f5fd292e57892d57622473ae492396dea3 Mon Sep 17 00:00:00 2001 From: francisco souza <108725+fsouza@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:25:50 -0400 Subject: [PATCH 1/4] Revert "readme: add note about Windows support being dropped" This reverts commit 095cc2f586cdbe245960d795553239d79c630bcc. --- README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/README.md b/README.md index cf98bb9eba..3b9e203543 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,6 @@ [![Build Status](https://github.com/fsouza/fake-gcs-server/workflows/Build/badge.svg)](https://github.com/fsouza/fake-gcs-server/actions?query=branch:main+workflow:Build) [![GoDoc](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc) -> **Note:** as of version 1.41.0, fake-gcs-server no longer supports Windows -> natively due to weak filesystem guarantees. You can still use fake-gcs-server -> on Windows with the memory backend, but we no longer distribute pre-compiled -> binaries for Windows and while using the filesystem backend _may_ work, it's -> not officially supported. - fake-gcs-server provides an emulator for Google Cloud Storage API. It can be used as a library in Go projects and/or as a standalone binary/Docker image. From 0db894a85cdd6358bfa33b0df8a6497bc30bd06b Mon Sep 17 00:00:00 2001 From: francisco souza <108725+fsouza@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:36:37 -0400 Subject: [PATCH 2/4] Add support for Windows back It's limited: can't atomically rename things, but otherwise it should work! --- .github/workflows/main.yml | 1 + ci/.goreleaser.yml | 5 ++++ go.mod | 1 + go.sum | 2 ++ internal/backend/backend_test.go | 4 +++ internal/backend/fs.go | 35 ++++----------------------- internal/backend/writefile_unix.go | 17 +++++++++++++ internal/backend/writefile_windows.go | 13 ++++++++++ 8 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 internal/backend/writefile_unix.go create mode 100644 internal/backend/writefile_windows.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30d39a15c0..87ee027bab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,6 +18,7 @@ jobs: os: - macos - ubuntu + - windows arch: - 386 - amd64 diff --git a/ci/.goreleaser.yml b/ci/.goreleaser.yml index 60c86ca6b7..17a8d26e73 100644 --- a/ci/.goreleaser.yml +++ b/ci/.goreleaser.yml @@ -9,10 +9,15 @@ builds: goos: - darwin - linux + - windows + ignore: + - goos: windows + goarch: arm64 archives: - replacements: darwin: Darwin linux: Linux + windows: Windows files: - LICENSE - README.md diff --git a/go.mod b/go.mod index 2a8dc6c115..4826d7d396 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/felixge/httpsnoop v1.0.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/renameio/v2 v2.0.0 github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index 3f741347f6..d30ec26ff8 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go index 3700b401ad..a27f39ac35 100644 --- a/internal/backend/backend_test.go +++ b/internal/backend/backend_test.go @@ -104,6 +104,10 @@ func uploadAndCompare(t *testing.T, storage Storage, obj Object) int64 { } func TestObjectCRUD(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("time resolution on Windows makes this test flaky on that platform") + } + const bucketName = "prod-bucket" const objectName = "video/hi-res/best_video_1080p.mp4" content1 := []byte("content1") diff --git a/internal/backend/fs.go b/internal/backend/fs.go index d1a3d6e35a..ebe934af95 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -5,6 +5,7 @@ package backend import ( + "bytes" "encoding/json" "errors" "fmt" @@ -179,32 +180,11 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name)) - tempFile, err := os.CreateTemp(filepath.Dir(path), "fake-gcs-object") - if err != nil { - return StreamingObject{}, err - } - tempFile.Close() - - tempFile, err = os.OpenFile(tempFile.Name(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666) - if err != nil { - return StreamingObject{}, err - } - defer tempFile.Close() - - // The file is renamed below, which causes this to be a no-op. If the - // function returns before the rename, though, the temp file will be - // removed. - defer os.Remove(tempFile.Name()) - - err = os.Chmod(tempFile.Name(), 0o600) - if err != nil { - return StreamingObject{}, err - } - + var buf bytes.Buffer hasher := checksum.NewStreamingHasher() objectContent := io.TeeReader(obj.Content, hasher) - if _, err = io.Copy(tempFile, objectContent); err != nil { + if _, err = io.Copy(&buf, objectContent); err != nil { return StreamingObject{}, err } @@ -218,16 +198,11 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St return StreamingObject{}, err } - if err = s.mh.write(tempFile.Name(), encoded); err != nil { - return StreamingObject{}, err - } - - err = os.Rename(tempFile.Name(), path) - if err != nil { + if err := writeFile(path, buf.Bytes(), 0o600); err != nil { return StreamingObject{}, err } - if err = s.mh.rename(tempFile.Name(), path); err != nil { + if err = s.mh.write(path, encoded); err != nil { return StreamingObject{}, err } diff --git a/internal/backend/writefile_unix.go b/internal/backend/writefile_unix.go new file mode 100644 index 0000000000..2e5e510fbc --- /dev/null +++ b/internal/backend/writefile_unix.go @@ -0,0 +1,17 @@ +// Copyright 2022 Francisco Souza. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows + +package backend + +import ( + "os" + + "github.com/google/renameio/v2" +) + +func writeFile(filename string, data []byte, perm os.FileMode) error { + return renameio.WriteFile(filename, data, perm) +} diff --git a/internal/backend/writefile_windows.go b/internal/backend/writefile_windows.go new file mode 100644 index 0000000000..2d6600c803 --- /dev/null +++ b/internal/backend/writefile_windows.go @@ -0,0 +1,13 @@ +// Copyright 2022 Francisco Souza. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package backend + +import ( + "os" +) + +func writeFile(filename string, data []byte, perm os.FileMode) error { + return os.WriteFile(filename, data, perm) +} From 7c5ad479a49de74439a184c4767b9a1d3a1791bd Mon Sep 17 00:00:00 2001 From: francisco souza <108725+fsouza@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:44:08 -0400 Subject: [PATCH 3/4] backend/metadata_file: use writeFile here too --- internal/backend/metadata_file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/backend/metadata_file.go b/internal/backend/metadata_file.go index 03682b5fa3..94cce654a8 100644 --- a/internal/backend/metadata_file.go +++ b/internal/backend/metadata_file.go @@ -14,7 +14,7 @@ const metadataSuffix = ".metadata" type metadataFile struct{} func (m metadataFile) write(path string, encoded []byte) error { - return os.WriteFile(path+metadataSuffix, encoded, 0o600) + return writeFile(path+metadataSuffix, encoded, 0o600) } func (m metadataFile) read(path string) ([]byte, error) { From 2109a1c4511c729c6151fdbecef5c6d20fa73389 Mon Sep 17 00:00:00 2001 From: francisco souza <108725+fsouza@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:54:24 -0400 Subject: [PATCH 4/4] backend/fs: introduce a lazy implementation of ReadSeekCloser --- internal/backend/fs.go | 10 ++----- internal/backend/lazy_file.go | 53 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 8 deletions(-) create mode 100644 internal/backend/lazy_file.go diff --git a/internal/backend/fs.go b/internal/backend/fs.go index ebe934af95..01f1ccef7d 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -285,18 +285,12 @@ func (s *storageFS) getObject(bucketName, objectName string) (StreamingObject, e } func openObjectAndSetSize(obj *StreamingObject, path string) error { - // file is expected to be closed by the caller by calling obj.Close() - file, err := os.Open(path) + info, err := os.Stat(path) if err != nil { return err } - info, err := file.Stat() - if err != nil { - return err - } - - obj.Content = file + obj.Content = newLazyReader(path) obj.Size = info.Size() return nil diff --git a/internal/backend/lazy_file.go b/internal/backend/lazy_file.go new file mode 100644 index 0000000000..8c30a31492 --- /dev/null +++ b/internal/backend/lazy_file.go @@ -0,0 +1,53 @@ +// Copyright 2022 Francisco Souza. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package backend + +import ( + "io" + "os" + "sync" +) + +type lazyReader struct { + filename string + once *sync.Once + f *os.File + err error +} + +func newLazyReader(filename string) io.ReadSeekCloser { + return &lazyReader{ + filename: filename, + once: &sync.Once{}, + } +} + +func (r *lazyReader) open() { + r.f, r.err = os.Open(r.filename) +} + +func (r *lazyReader) Read(p []byte) (int, error) { + r.once.Do(r.open) + if r.err != nil { + return 0, r.err + } + return r.f.Read(p) +} + +func (r *lazyReader) Seek(offset int64, whence int) (int64, error) { + r.once.Do(r.open) + if r.err != nil { + return 0, r.err + } + return r.f.Seek(offset, whence) +} + +func (r *lazyReader) Close() error { + r.once.Do(r.open) + if r.err != nil { + return r.err + } + return r.f.Close() +}