tes: added GLOB file type

tes/tes.proto

@@ -82,6 +82,7 @@ message Task {
 enum FileType {
   FILE = 0;
   DIRECTORY = 1;
+  GLOB = 2;
 }
 
 // Input describes Task input files.

tes/utils.go

@@ -33,6 +33,7 @@ const (
 	Full      = TaskView_FULL
 	File      = FileType_FILE
 	Directory = FileType_DIRECTORY
+	Glob      = FileType_GLOB
 )
 
 // GenerateID generates a task ID string.

util/fsutil/glob.go

@@ -0,0 +1,53 @@
+package fsutil
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/mattn/go-zglob"
+)
+
+func adjustRel(newroot string, files []Hostfile) []Hostfile {
+	out := []Hostfile{}
+	for _, f := range files {
+		f.Rel = filepath.Join(newroot, f.Rel)
+		out = append(out, f)
+	}
+	return out
+}
+
+// Glob returns a list of HostFile objects matching pattern or nil if there is no matching file.
+// This method uses the implementation in github.com/mattn/go-zglob.
+func Glob(pattern string) ([]Hostfile, error) {
+	var files []Hostfile
+	matches, err := zglob.Glob(pattern)
+	if err != nil {
+		return nil, err
+	}
+	for _, path := range matches {
+		finfo, err := os.Stat(path)
+		if err != nil {
+			return nil, err
+		}
+		if finfo.IsDir() {
+			dfiles, err := WalkFiles(path)
+			if err != nil {
+				return nil, err
+			}
+			dfiles = adjustRel(path, dfiles)
+			files = append(files, dfiles...)
+			continue
+		}
+		absPath, err := filepath.Abs(path)
+		if err != nil {
+			return nil, err
+		}
+		files = append(files, Hostfile{
+			Rel:          path,
+			Abs:          absPath,
+			Size:         finfo.Size(),
+			LastModified: finfo.ModTime(),
+		})
+	}
+	return files, nil
+}

util/fsutil/walk.go

@@ -33,9 +33,13 @@ func WalkFiles(root string) ([]Hostfile, error) {
 			if err != nil {
 				return err
 			}
+			absPath, err := filepath.Abs(p)
+			if err != nil {
+				return err
+			}
 			files = append(files, Hostfile{
 				Rel:          rel,
-				Abs:          p,
+				Abs:          absPath,
 				Size:         f.Size(),
 				LastModified: f.ModTime(),
 			})

worker/storage.go

@@ -93,18 +93,40 @@ func FlattenOutputs(ctx context.Context, outputs []*tes.Output, store storage.St
 		case tes.Directory:
 			list, err := fsutil.WalkFiles(output.Path)
 			if err != nil {
-				return nil, fmt.Errorf("walking directory: %s", err)
+				return nil, fmt.Errorf("walking directory: %v", err)
 			}
 
 			if len(list) == 0 {
-				ev.Warn("download source directory is empty", "url", output.Url)
+				ev.Warn("output directory is empty", "url", output.Url, "path", output.Path)
 				continue
 			}
 
 			for _, f := range list {
 				u, err := store.Join(output.Url, f.Rel)
 				if err != nil {
-					return nil, fmt.Errorf("joining storage url: %s", err)
+					return nil, fmt.Errorf("joining storage url: %v", err)
+				}
+				flat = append(flat, &tes.Output{
+					Url:  u,
+					Path: f.Abs,
+				})
+			}
+
+		case tes.Glob:
+			list, err := fsutil.Glob(output.Path)
+			if err != nil {
+				return nil, fmt.Errorf("globbing pattern: %v", err)
+			}
+
+			if len(list) == 0 {
+				ev.Warn("glob matched zero files", "url", output.Url, "path", output.Path)
+				continue
+			}
+
+			for _, f := range list {
+				u, err := store.Join(output.Url, f.Rel)
+				if err != nil {
+					return nil, fmt.Errorf("joining storage url: %v", err)
 				}
 				flat = append(flat, &tes.Output{
 					Url:  u,