Add collection and dataobject search APIs

Author: iychoi

Makefile

@@ -17,6 +17,7 @@ examples:
 	CGO_ENABLED=0 GOOS=linux go build -o ./examples/list_dir/list_dir.out ./examples/list_dir/list_dir.go
 	CGO_ENABLED=0 GOOS=linux go build -o ./examples/list_acls/list_acls.out ./examples/list_acls/list_acls.go
 	CGO_ENABLED=0 GOOS=linux go build -o ./examples/list_user/list_user.out ./examples/list_user/list_user.go
+	CGO_ENABLED=0 GOOS=linux go build -o ./examples/search/search.out ./examples/search/search.go
 	CGO_ENABLED=0 GOOS=linux go build -o ./examples/upload/upload.out ./examples/upload/upload.go
 	CGO_ENABLED=0 GOOS=linux go build -o ./examples/upload_parallel/upload_parallel.out ./examples/upload_parallel/upload_parallel.go
 	CGO_ENABLED=0 GOOS=linux go build -o ./examples/download/download.out ./examples/download/download.go

examples/search/account.yml

@@ -0,0 +1,6 @@
+irods_authentication_scheme: "native"
+irods_host: "data.cyverse.org"
+irods_port: 1247
+irods_user_name: ""
+irods_user_password: ""
+irods_zone_name: "iplant"

examples/search/search.go

@@ -0,0 +1,71 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+
+	"github.com/cyverse/go-irodsclient/config"
+	"github.com/cyverse/go-irodsclient/fs"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	logger := log.WithFields(log.Fields{
+		"package":  "main",
+		"function": "main",
+	})
+
+	// Parse cli parameters
+	flag.Parse()
+	args := flag.Args()
+
+	if len(args) != 1 {
+		fmt.Fprintf(os.Stderr, "Give an iRODS path with wildcard!\n")
+		os.Exit(1)
+	}
+
+	inputPath := args[0]
+
+	// Read account configuration from YAML file
+	cfg, err := config.NewConfigFromYAMLFile(config.GetDefaultConfig(), "account.yml")
+	if err != nil {
+		logger.Error(err)
+		panic(err)
+	}
+
+	account := cfg.ToIRODSAccount()
+	logger.Debugf("Account : %v", account.GetRedacted())
+
+	// Create a file system
+	appName := "search"
+	filesystem, err := fs.NewFileSystemWithDefault(account, appName)
+	if err != nil {
+		logger.Error(err)
+		panic(err)
+	}
+
+	defer filesystem.Release()
+
+	entries, err := filesystem.Search(inputPath)
+	if err != nil {
+		logger.Error(err)
+		panic(err)
+	}
+
+	if len(entries) == 0 {
+		fmt.Printf("Found no entries in the directory %q\n", inputPath)
+	} else {
+		fmt.Printf("DIR: %s\n", inputPath)
+		for _, entry := range entries {
+			if entry.Type == fs.FileEntry {
+				fmt.Printf("> FILE:\t%d\t%s\t%d\n", entry.ID, entry.Path, entry.Size)
+			} else {
+				// dir
+				fmt.Printf("> DIRECTORY:\t%d\t%s\n", entry.ID, entry.Path)
+			}
+
+		}
+	}
+}

fs/fs.go

@@ -270,6 +270,78 @@ func (fs *FileSystem) List(path string) ([]*Entry, error) {
 	return fs.listEntries(collection)
 }
 
+func (fs *FileSystem) Search(pathWildcard string) ([]*Entry, error) {
+	conn, err := fs.metadataSession.AcquireConnection()
+	if err != nil {
+		return nil, err
+	}
+	defer fs.metadataSession.ReturnConnection(conn) //nolint
+
+	results := []*Entry{}
+
+	collEntries, err := irods_fs.SearchCollections(conn, pathWildcard)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range collEntries {
+		results = append(results, fs.getEntryFromCollection(entry))
+	}
+
+	objectEntries, err := irods_fs.SearchDataObjectsMasterReplica(conn, pathWildcard)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range objectEntries {
+		results = append(results, fs.getEntryFromDataObject(entry))
+	}
+
+	return results, nil
+}
+
+func (fs *FileSystem) SearchDir(pathWildcard string) ([]*Entry, error) {
+	conn, err := fs.metadataSession.AcquireConnection()
+	if err != nil {
+		return nil, err
+	}
+	defer fs.metadataSession.ReturnConnection(conn) //nolint
+
+	results := []*Entry{}
+
+	collEntries, err := irods_fs.SearchCollections(conn, pathWildcard)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range collEntries {
+		results = append(results, fs.getEntryFromCollection(entry))
+	}
+
+	return results, nil
+}
+
+func (fs *FileSystem) SearchFile(pathWildcard string) ([]*Entry, error) {
+	conn, err := fs.metadataSession.AcquireConnection()
+	if err != nil {
+		return nil, err
+	}
+	defer fs.metadataSession.ReturnConnection(conn) //nolint
+
+	results := []*Entry{}
+
+	objectEntries, err := irods_fs.SearchDataObjectsMasterReplica(conn, pathWildcard)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, entry := range objectEntries {
+		results = append(results, fs.getEntryFromDataObject(entry))
+	}
+
+	return results, nil
+}
+
 // RemoveDir deletes a directory
 func (fs *FileSystem) RemoveDir(path string, recurse bool, force bool) error {
 	irodsPath := util.GetCorrectIRODSPath(path)
@@ -890,6 +962,7 @@ func (fs *FileSystem) listEntries(collection *types.IRODSCollection) ([]*Entry,
 				cachedEntries = append(cachedEntries, cachedEntry)
 			} else {
 				useCached = false
+				break
 			}
 		}
 	}

fs/fs_cache.go

@@ -95,7 +95,7 @@ func (fs *FileSystem) invalidateCacheForDirRemove(path string, recurse bool) {
 		entry = fs.cache.GetEntryCache(path)
 	}
 
-	// we need to expunge all negatie entry caches under irodsDestPath
+	// we need to expunge all negative entry caches under irodsDestPath
 	// since all sub-directories/files are also moved
 	fs.cache.RemoveAllNegativeEntryCacheForPath(path)
 

go.mod

@@ -3,6 +3,7 @@ module github.com/cyverse/go-irodsclient
 go 1.18
 
 require (
+	github.com/dlclark/regexp2 v1.11.5
 	github.com/hashicorp/go-rootcerts v1.0.2
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible

go.sum

@@ -1,6 +1,8 @@
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=