Add back limited support for Windows

.github/workflows/main.yml

@@ -18,6 +18,7 @@ jobs:
         os:
           - macos
           - ubuntu
+          - windows
         arch:
           - 386
           - amd64

README.md

@@ -3,12 +3,6 @@
 [![Build Status](https://github.com/fsouza/fake-gcs-server/workflows/Build/badge.svg)](https://github.com/fsouza/fake-gcs-server/actions?query=branch:main+workflow:Build)
 [![GoDoc](https://img.shields.io/badge/api-Godoc-blue.svg?style=flat-square)](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc)
 
-> **Note:** as of version 1.41.0, fake-gcs-server no longer supports Windows
-> natively due to weak filesystem guarantees. You can still use fake-gcs-server
-> on Windows with the memory backend, but we no longer distribute pre-compiled
-> binaries for Windows and while using the filesystem backend _may_ work, it's
-> not officially supported.
-
 fake-gcs-server provides an emulator for Google Cloud Storage API. It can be
 used as a library in Go projects and/or as a standalone binary/Docker image.
 

ci/.goreleaser.yml

@@ -9,10 +9,15 @@ builds:
     goos:
       - darwin
       - linux
+      - windows
+    ignore:
+      - goos: windows
+        goarch: arm64
 archives:
   - replacements:
       darwin: Darwin
       linux: Linux
+      windows: Windows
     files:
       - LICENSE
       - README.md

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/felixge/httpsnoop v1.0.2 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/renameio/v2 v2.0.0
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.6.0 // indirect

go.sum

@@ -52,6 +52,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ=
+github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg=
+github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

internal/backend/backend_test.go

@@ -104,6 +104,10 @@ func uploadAndCompare(t *testing.T, storage Storage, obj Object) int64 {
 }
 
 func TestObjectCRUD(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("time resolution on Windows makes this test flaky on that platform")
+	}
+
 	const bucketName = "prod-bucket"
 	const objectName = "video/hi-res/best_video_1080p.mp4"
 	content1 := []byte("content1")

internal/backend/fs.go

@@ -5,6 +5,7 @@
 package backend
 
 import (
+	"bytes"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -179,32 +180,11 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St
 
 	path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name))
 
-	tempFile, err := os.CreateTemp(filepath.Dir(path), "fake-gcs-object")
-	if err != nil {
-		return StreamingObject{}, err
-	}
-	tempFile.Close()
-
-	tempFile, err = os.OpenFile(tempFile.Name(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666)
-	if err != nil {
-		return StreamingObject{}, err
-	}
-	defer tempFile.Close()
-
-	// The file is renamed below, which causes this to be a no-op. If the
-	// function returns before the rename, though, the temp file will be
-	// removed.
-	defer os.Remove(tempFile.Name())
-
-	err = os.Chmod(tempFile.Name(), 0o600)
-	if err != nil {
-		return StreamingObject{}, err
-	}
-
+	var buf bytes.Buffer
 	hasher := checksum.NewStreamingHasher()
 	objectContent := io.TeeReader(obj.Content, hasher)
 
-	if _, err = io.Copy(tempFile, objectContent); err != nil {
+	if _, err = io.Copy(&buf, objectContent); err != nil {
 		return StreamingObject{}, err
 	}
 
@@ -218,16 +198,11 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St
 		return StreamingObject{}, err
 	}
 
-	if err = s.mh.write(tempFile.Name(), encoded); err != nil {
-		return StreamingObject{}, err
-	}
-
-	err = os.Rename(tempFile.Name(), path)
-	if err != nil {
+	if err := writeFile(path, buf.Bytes(), 0o600); err != nil {
 		return StreamingObject{}, err
 	}
 
-	if err = s.mh.rename(tempFile.Name(), path); err != nil {
+	if err = s.mh.write(path, encoded); err != nil {
 		return StreamingObject{}, err
 	}
 
@@ -310,18 +285,12 @@ func (s *storageFS) getObject(bucketName, objectName string) (StreamingObject, e
 }
 
 func openObjectAndSetSize(obj *StreamingObject, path string) error {
-	// file is expected to be closed by the caller by calling obj.Close()
-	file, err := os.Open(path)
-	if err != nil {
-		return err
-	}
-
-	info, err := file.Stat()
+	info, err := os.Stat(path)
 	if err != nil {
 		return err
 	}
 
-	obj.Content = file
+	obj.Content = newLazyReader(path)
 	obj.Size = info.Size()
 
 	return nil

internal/backend/lazy_file.go

@@ -0,0 +1,53 @@
+// Copyright 2022 Francisco Souza. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package backend
+
+import (
+	"io"
+	"os"
+	"sync"
+)
+
+type lazyReader struct {
+	filename string
+	once     *sync.Once
+	f        *os.File
+	err      error
+}
+
+func newLazyReader(filename string) io.ReadSeekCloser {
+	return &lazyReader{
+		filename: filename,
+		once:     &sync.Once{},
+	}
+}
+
+func (r *lazyReader) open() {
+	r.f, r.err = os.Open(r.filename)
+}
+
+func (r *lazyReader) Read(p []byte) (int, error) {
+	r.once.Do(r.open)
+	if r.err != nil {
+		return 0, r.err
+	}
+	return r.f.Read(p)
+}
+
+func (r *lazyReader) Seek(offset int64, whence int) (int64, error) {
+	r.once.Do(r.open)
+	if r.err != nil {
+		return 0, r.err
+	}
+	return r.f.Seek(offset, whence)
+}
+
+func (r *lazyReader) Close() error {
+	r.once.Do(r.open)
+	if r.err != nil {
+		return r.err
+	}
+	return r.f.Close()
+}

internal/backend/metadata_file.go

@@ -14,7 +14,7 @@ const metadataSuffix = ".metadata"
 type metadataFile struct{}
 
 func (m metadataFile) write(path string, encoded []byte) error {
-	return os.WriteFile(path+metadataSuffix, encoded, 0o600)
+	return writeFile(path+metadataSuffix, encoded, 0o600)
 }
 
 func (m metadataFile) read(path string) ([]byte, error) {

internal/backend/writefile_unix.go

@@ -0,0 +1,17 @@
+// Copyright 2022 Francisco Souza. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !windows
+
+package backend
+
+import (
+	"os"
+
+	"github.com/google/renameio/v2"
+)
+
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	return renameio.WriteFile(filename, data, perm)
+}

internal/backend/writefile_windows.go

@@ -0,0 +1,13 @@
+// Copyright 2022 Francisco Souza. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package backend
+
+import (
+	"os"
+)
+
+func writeFile(filename string, data []byte, perm os.FileMode) error {
+	return os.WriteFile(filename, data, perm)
+}