internal/relui: use atomic writes in test code too

In CL 429275 I changed gcsfs to write atomically, but I didn't use gcsfs
in the test infrastructure, which is probably what was causing the
flakes. For whatever reason, the test got much flakier, with timeouts
this time :(

Finish the job. I'm hoping this fixes the timeouts. I can't reproduce
them locally, but I did get some errors where it saw the temporary
files, so also hide them from directory listings.

For golang/go#53972.

Change-Id: I2dff87305f9114b975990cd6649cb4a087fadffd
Reviewed-on: https://go-review.googlesource.com/c/build/+/429536
Run-TryBot: Heschi Kreinick <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Auto-Submit: Heschi Kreinick <[email protected]>

internal/gcsfs/gcsfs.go

@@ -44,7 +44,7 @@ func FromURL(ctx context.Context, client *storage.Client, base string) (fs.FS, e
 }
 
 // Create creates a new file on fsys, which must be a CreateFS.
-func Create(fsys fs.FS, name string) (WriteFile, error) {
+func Create(fsys fs.FS, name string) (WriterFile, error) {
 	cfs, ok := fsys.(CreateFS)
 	if !ok {
 		return nil, &fs.PathError{Op: "create", Path: name, Err: fmt.Errorf("not implemented on type %T", fsys)}
@@ -55,16 +55,29 @@ func Create(fsys fs.FS, name string) (WriteFile, error) {
 // CreateFS is an fs.FS that supports creating writable files.
 type CreateFS interface {
 	fs.FS
-	Create(string) (WriteFile, error)
+	Create(string) (WriterFile, error)
 }
 
-// WriteFile is an fs.File that can be written to.
+// WriterFile is an fs.File that can be written to.
 // The behavior of writing and reading the same file is undefined.
-type WriteFile interface {
+type WriterFile interface {
 	fs.File
 	io.Writer
 }
 
+// WriteFile is like os.WriteFile for CreateFSs.
+func WriteFile(fsys fs.FS, filename string, contents []byte) error {
+	f, err := Create(fsys, filename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	if _, err := f.Write(contents); err != nil {
+		return err
+	}
+	return f.Close()
+}
+
 // gcsFS implements fs.FS for GCS.
 type gcsFS struct {
 	ctx    context.Context
@@ -110,7 +123,7 @@ func (fsys *gcsFS) Open(name string) (fs.File, error) {
 }
 
 // Create creates the named file.
-func (fsys *gcsFS) Create(name string) (WriteFile, error) {
+func (fsys *gcsFS) Create(name string) (WriterFile, error) {
 	f, err := fsys.Open(name)
 	if err != nil {
 		return nil, err

internal/gcsfs/gcsfs_test.go

@@ -9,6 +9,7 @@ import (
 	"flag"
 	"io/fs"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 	"testing/fstest"
@@ -54,6 +55,20 @@ func TestDirFS(t *testing.T) {
 	}
 }
 
+func TestDirFSDotFiles(t *testing.T) {
+	temp := t.TempDir()
+	if err := os.WriteFile(temp+"/.foo", nil, 0777); err != nil {
+		t.Fatal(err)
+	}
+	files, err := fs.ReadDir(DirFS(temp), ".")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != 0 {
+		t.Errorf("ReadDir didn't hide . files: %v", files)
+	}
+}
+
 func TestDirFSWrite(t *testing.T) {
 	temp := t.TempDir()
 	fsys := DirFS(temp)

internal/gcsfs/osfs.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"strings"
 )
 
 var _ = fs.FS((*dirFS)(nil))
@@ -41,7 +42,7 @@ func (dir dirFS) Open(name string) (fs.File, error) {
 	if err != nil {
 		return nil, err // nil fs.File
 	}
-	return f, nil
+	return &atomicWriteFile{f, nil}, nil
 }
 
 func (dir dirFS) Stat(name string) (fs.FileInfo, error) {
@@ -55,7 +56,7 @@ func (dir dirFS) Stat(name string) (fs.FileInfo, error) {
 	return f, nil
 }
 
-func (dir dirFS) Create(name string) (WriteFile, error) {
+func (dir dirFS) Create(name string) (WriterFile, error) {
 	if !fs.ValidPath(name) || runtime.GOOS == "windows" && containsAny(name, `\:`) {
 		return nil, &fs.PathError{Op: "create", Path: name, Err: fs.ErrInvalid}
 	}
@@ -73,15 +74,26 @@ func (dir dirFS) Create(name string) (WriteFile, error) {
 	finalize := func() error {
 		return os.Rename(temp.Name(), fullName)
 	}
-	return &writingFile{temp, finalize}, nil
+	return &atomicWriteFile{temp, finalize}, nil
 }
 
-type writingFile struct {
+type atomicWriteFile struct {
 	*os.File
 	finalize func() error
 }
 
-func (wf *writingFile) Close() error {
+func (wf *atomicWriteFile) ReadDir(n int) ([]fs.DirEntry, error) {
+	unfiltered, err := wf.File.ReadDir(n)
+	var result []fs.DirEntry
+	for _, de := range unfiltered {
+		if !strings.HasPrefix(de.Name(), ".") {
+			result = append(result, de)
+		}
+	}
+	return result, err
+}
+
+func (wf *atomicWriteFile) Close() error {
 	if err := wf.File.Close(); err != nil {
 		return err
 	}

internal/relui/buildrelease_test.go

@@ -13,6 +13,7 @@ import (
 	"crypto/sha256"
 	"fmt"
 	"io"
+	"io/fs"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -34,6 +35,7 @@ import (
 	"golang.org/x/build/buildlet"
 	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal"
+	"golang.org/x/build/internal/gcsfs"
 	"golang.org/x/build/internal/task"
 	"golang.org/x/build/internal/untar"
 	"golang.org/x/build/internal/workflow"
@@ -861,14 +863,15 @@ func fakeSign(ctx context.Context, t *testing.T, dir string) {
 }
 
 func fakeSignOnce(t *testing.T, dir string, seen map[string]bool) error {
-	_, err := os.Stat(filepath.Join(dir, "ready"))
+	dirFS := gcsfs.DirFS(dir)
+	_, err := fs.Stat(dirFS, "ready")
 	if os.IsNotExist(err) {
 		return nil
 	}
 	if err != nil {
 		return err
 	}
-	contents, err := os.ReadDir(dir)
+	contents, err := fs.ReadDir(dirFS, ".")
 	if err != nil {
 		return err
 	}
@@ -892,24 +895,16 @@ func fakeSignOnce(t *testing.T, dir string, seen map[string]bool) error {
 			copy = true
 		}
 
-		if err := os.MkdirAll(filepath.Join(dir, "signed"), 0777); err != nil {
-			t.Fatal(err)
-		}
-
 		writeSignedWithHash := func(filename string, contents []byte) error {
-			path := filepath.Join(dir, "signed", filename)
-			if err := ioutil.WriteFile(path, contents, 0777); err != nil {
+			if err := gcsfs.WriteFile(dirFS, "signed/"+filename, contents); err != nil {
 				return err
 			}
 			hash := fmt.Sprintf("%x", sha256.Sum256(contents))
-			if err := ioutil.WriteFile(path+".sha256", []byte(hash), 0777); err != nil {
-				return err
-			}
-			return nil
+			return gcsfs.WriteFile(dirFS, "signed/"+filename+".sha256", []byte(hash))
 		}
 
 		if copy {
-			bytes, err := ioutil.ReadFile(filepath.Join(dir, fn))
+			bytes, err := fs.ReadFile(dirFS, fn)
 			if err != nil {
 				return err
 			}
@@ -1022,9 +1017,10 @@ func TestFakeSign(t *testing.T) {
 }
 
 func fakeCDNLoad(ctx context.Context, t *testing.T, from, to string) {
+	fromFS, toFS := gcsfs.DirFS(from), gcsfs.DirFS(to)
 	seen := map[string]bool{}
 	periodicallyDo(ctx, t, 100*time.Millisecond, func() error {
-		files, err := os.ReadDir(from)
+		files, err := fs.ReadDir(fromFS, ".")
 		if err != nil {
 			return err
 		}
@@ -1033,11 +1029,11 @@ func fakeCDNLoad(ctx context.Context, t *testing.T, from, to string) {
 				continue
 			}
 			seen[f.Name()] = true
-			contents, err := os.ReadFile(filepath.Join(from, f.Name()))
+			contents, err := fs.ReadFile(fromFS, f.Name())
 			if err != nil {
 				return err
 			}
-			if err := os.WriteFile(filepath.Join(to, f.Name()), contents, 0777); err != nil {
+			if err := gcsfs.WriteFile(toFS, f.Name(), contents); err != nil {
 				return err
 			}
 		}

internal/relui/workflows.go

@@ -904,14 +904,7 @@ func (tasks *BuildReleaseTasks) copyToStaging(ctx *wf.TaskContext, version strin
 			return nil, err
 		}
 	}
-	out, err := gcsfs.Create(scratchFS, path.Join(signingStagingDir(ctx, version), "ready"))
-	if err != nil {
-		return nil, err
-	}
-	if _, err := out.Write([]byte("ready")); err != nil {
-		return nil, err
-	}
-	if err := out.Close(); err != nil {
+	if err := gcsfs.WriteFile(scratchFS, path.Join(signingStagingDir(ctx, version), "ready"), []byte("ready")); err != nil {
 		return nil, err
 	}
 	return stagedArtifacts, nil
@@ -1084,28 +1077,12 @@ func uploadArtifact(scratchFS, servingFS fs.FS, a artifact) error {
 		return err
 	}
 
-	sha256, err := gcsfs.Create(servingFS, a.Filename+".sha256")
-	if err != nil {
-		return err
-	}
-	defer sha256.Close()
-	if _, err := sha256.Write([]byte(a.SHA256)); err != nil {
-		return err
-	}
-	if err := sha256.Close(); err != nil {
+	if err := gcsfs.WriteFile(servingFS, a.Filename+".sha256", []byte(a.SHA256)); err != nil {
 		return err
 	}
 
 	if a.GPGSignature != "" {
-		asc, err := gcsfs.Create(servingFS, a.Filename+".asc")
-		if err != nil {
-			return err
-		}
-		defer asc.Close()
-		if _, err := asc.Write([]byte(a.GPGSignature)); err != nil {
-			return err
-		}
-		if err := asc.Close(); err != nil {
+		if err := gcsfs.WriteFile(servingFS, a.Filename+".asc", []byte(a.GPGSignature)); err != nil {
 			return err
 		}
 	}