Skip to content

Commit

Permalink
Merge list responses across storage providers
Browse files Browse the repository at this point in the history
  • Loading branch information
@ishank011
ishank011 committed Mar 23, 2021
1 parent a8850b9 commit 2848f9f
Show file tree
Hide file tree
Showing 4 changed files with 55 additions and 77 deletions.
73 changes: 41 additions & 32 deletions internal/grpc/services/gateway/storageprovider.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,6 @@ import (
"github.com/cs3org/reva/pkg/rgrpc/status"
"github.com/cs3org/reva/pkg/rgrpc/todo/pool"
"github.com/cs3org/reva/pkg/storage/utils/etag"
"github.com/cs3org/reva/pkg/storage/utils/templates"
"github.com/cs3org/reva/pkg/user"
"github.com/cs3org/reva/pkg/utils"
"github.com/dgrijalva/jwt-go"
"github.com/pkg/errors"
Expand Down Expand Up @@ -367,6 +365,7 @@ func (s *svc) InitiateFileDownload(ctx context.Context, req *provider.InitiateFi
}

func (s *svc) initiateFileDownload(ctx context.Context, req *provider.InitiateFileDownloadRequest) (*gateway.InitiateFileDownloadResponse, error) {
// TODO(ishank011): enable downloading references spread across storage providers, eg. /eos
c, err := s.find(ctx, req.Ref)
if err != nil {
return &gateway.InitiateFileDownloadResponse{
Expand Down Expand Up @@ -858,6 +857,7 @@ func (s *svc) Delete(ctx context.Context, req *provider.DeleteRequest) (*provide
}

func (s *svc) delete(ctx context.Context, req *provider.DeleteRequest) (*provider.DeleteResponse, error) {
// TODO(ishank011): enable deleting references spread across storage providers, eg. /eos
c, err := s.find(ctx, req.Ref)
if err != nil {
return &provider.DeleteResponse{
Expand Down Expand Up @@ -1009,6 +1009,7 @@ func (s *svc) move(ctx context.Context, req *provider.MoveRequest) (*provider.Mo
}

func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitraryMetadataRequest) (*provider.SetArbitraryMetadataResponse, error) {
// TODO(ishank011): enable for references spread across storage providers, eg. /eos
c, err := s.find(ctx, req.Ref)
if err != nil {
return &provider.SetArbitraryMetadataResponse{
Expand All @@ -1025,6 +1026,7 @@ func (s *svc) SetArbitraryMetadata(ctx context.Context, req *provider.SetArbitra
}

func (s *svc) UnsetArbitraryMetadata(ctx context.Context, req *provider.UnsetArbitraryMetadataRequest) (*provider.UnsetArbitraryMetadataResponse, error) {
// TODO(ishank011): enable for references spread across storage providers, eg. /eos
c, err := s.find(ctx, req.Ref)
if err != nil {
return &provider.UnsetArbitraryMetadataResponse{
Expand Down Expand Up @@ -1144,6 +1146,7 @@ func (s *svc) statSharesFolder(ctx context.Context) (*provider.StatResponse, err
}

func (s *svc) stat(ctx context.Context, req *provider.StatRequest) (*provider.StatResponse, error) {
// TODO(ishank011): enable for references spread across storage providers, eg. /eos
c, err := s.find(ctx, req.Ref)
if err != nil {
return &provider.StatResponse{
Expand Down Expand Up @@ -1456,19 +1459,49 @@ func (s *svc) listSharesFolder(ctx context.Context) (*provider.ListContainerResp
}

func (s *svc) listContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) {
c, err := s.find(ctx, req.Ref)
providers, err := s.findProviders(ctx, req.Ref)
if err != nil {
return &provider.ListContainerResponse{
Status: status.NewStatusFromErrType(ctx, "listContainer ref="+req.Ref.String(), err),
Status: status.NewStatusFromErrType(ctx, "listContainer ref: "+req.Ref.String(), err),
}, nil
}

res, err := c.ListContainer(ctx, req)
if err != nil {
return nil, errors.Wrap(err, "gateway: error calling ListContainer")
var infos []*provider.ResourceInfo

for _, p := range providers {
c, err := s.getStorageProviderClient(ctx, p)
if err != nil {
return &provider.ListContainerResponse{
Status: status.NewInternal(ctx, err, "error connecting to storage provider="+providers[0].Address),
}, nil
}

resPath := req.Ref.GetPath()
if resPath != "" && !strings.HasPrefix(resPath, p.ProviderPath) {
req = &provider.ListContainerRequest{
Ref: &provider.Reference{
Spec: &provider.Reference_Path{
Path: p.ProviderPath,
},
},
}
}
res, err := c.ListContainer(ctx, req)
if err != nil {
return nil, errors.Wrap(err, "gateway: error calling ListContainer")
}
for _, inf := range res.Infos {
if parent := path.Base(inf.Path); resPath != "" && path.Clean(resPath) != parent {
continue
}
infos = append(infos, inf)
}
}

return res, nil
return &provider.ListContainerResponse{
Status: status.NewOK(ctx),
Infos: infos,
}, nil
}

func (s *svc) ListContainer(ctx context.Context, req *provider.ListContainerRequest) (*provider.ListContainerResponse, error) {
Expand Down Expand Up @@ -1908,30 +1941,6 @@ func (s *svc) getStorageProviderClient(_ context.Context, p *registry.ProviderIn
}

func (s *svc) findProviders(ctx context.Context, ref *provider.Reference) ([]*registry.ProviderInfo, error) {
home := s.getHome(ctx)
if strings.HasPrefix(ref.GetPath(), home) && s.c.HomeMapping != "" {
if u, ok := user.ContextGetUser(ctx); ok {
layout := templates.WithUser(u, s.c.HomeMapping)
newRef := &provider.Reference{
Spec: &provider.Reference_Path{
Path: path.Join(layout, strings.TrimPrefix(ref.GetPath(), home)),
},
}
res, err := s.getStorageProviders(ctx, newRef)
if err != nil {
// if we get a NotFound error, default to the original reference
if _, ok := err.(errtypes.IsNotFound); !ok {
return nil, err
}
} else {
return res, nil
}
}
}
return s.getStorageProviders(ctx, ref)
}

func (s *svc) getStorageProviders(ctx context.Context, ref *provider.Reference) ([]*registry.ProviderInfo, error) {
c, err := pool.GetStorageRegistryClient(s.c.StorageRegistryEndpoint)
if err != nil {
return nil, errors.Wrap(err, "gateway: error getting storage registry client")
Expand Down
5 changes: 5 additions & 0 deletions internal/http/services/owncloud/ocs/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type Config struct {
DefaultUploadProtocol string `mapstructure:"default_upload_protocol"`
UserAgentChunkingMap map[string]string `mapstructure:"user_agent_chunking_map"`
SharePrefix string `mapstructure:"share_prefix"`
HomeNamespace string `mapstructure:home_namespace`
}

// Init sets sane defaults
Expand All @@ -48,5 +49,9 @@ func (c *Config) Init() {
c.DefaultUploadProtocol = "tus"
}

if c.HomeNamespace == "" {
c.HomeNamespace = "/home"
}

c.GatewaySvc = sharedconf.GetGatewaySVC(c.GatewaySvc)
}
49 changes: 6 additions & 43 deletions internal/http/services/owncloud/ocs/handlers/apps/sharing/shares/shares.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ type Handler struct {
gatewayAddr string
publicURL string
sharePrefix string
homeNamespace string
userIdentifierCache *ttlcache.Cache
}

Expand All @@ -70,6 +71,7 @@ func (h *Handler) Init(c *config.Config) error {
h.gatewayAddr = c.GatewaySvc
h.publicURL = c.Config.Host
h.sharePrefix = c.SharePrefix
h.homeNamespace = c.HomeNamespace

h.userIdentifierCache = ttlcache.NewCache()
_ = h.userIdentifierCache.SetTTL(60 * time.Second)
Expand Down Expand Up @@ -173,16 +175,9 @@ func (h *Handler) createShare(w http.ResponseWriter, r *http.Request) {
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting grpc gateway client", err)
return
}
// prefix the path with the owners home, because ocs share requests are relative to the home dir
// TODO the path actually depends on the configured webdav_namespace
hRes, err := c.GetHome(ctx, &provider.GetHomeRequest{})
if err != nil {
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc get home request", err)
return
}

prefix := hRes.GetPath()
fn := path.Join(prefix, r.FormValue("path"))
// prefix the path with the owners home, because ocs share requests are relative to the home dir
fn := path.Join(h.homeNamespace, r.FormValue("path"))

statReq := provider.StatRequest{
Ref: &provider.Reference{
Expand Down Expand Up @@ -591,14 +586,7 @@ func (h *Handler) listSharesWithMe(w http.ResponseWriter, r *http.Request) {
// we need to lookup the resource id so we can filter the list of shares later
if p != "" {
// prefix the path with the owners home, because ocs share requests are relative to the home dir
// TODO the path actually depends on the configured webdav_namespace
hRes, err := gwc.GetHome(ctx, &provider.GetHomeRequest{})
if err != nil {
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc get home request", err)
return
}

target := path.Join(hRes.Path, r.FormValue("path"))
target := path.Join(h.homeNamespace, r.FormValue("path"))

statReq := &provider.StatRequest{
Ref: &provider.Reference{
Expand Down Expand Up @@ -728,33 +716,8 @@ func (h *Handler) listSharesWithOthers(w http.ResponseWriter, r *http.Request) {
// shared with others
p := r.URL.Query().Get("path")
if p != "" {
c, err := pool.GetGatewayServiceClient(h.gatewayAddr)
if err != nil {
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error getting storage grpc client", err)
return
}

// prefix the path with the owners home, because ocs share requests are relative to the home dir
// TODO the path actually depends on the configured webdav_namespace
hRes, err := c.GetHome(r.Context(), &provider.GetHomeRequest{})
if err != nil {
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "error sending a grpc get home request", err)
return
}

if hRes.Status.Code != rpc.Code_CODE_OK {
switch hRes.Status.Code {
case rpc.Code_CODE_NOT_FOUND:
response.WriteOCSError(w, r, response.MetaNotFound.StatusCode, "path not found", nil)
case rpc.Code_CODE_PERMISSION_DENIED:
response.WriteOCSError(w, r, response.MetaUnauthorized.StatusCode, "permission denied", nil)
default:
response.WriteOCSError(w, r, response.MetaServerError.StatusCode, "grpc stat request failed", nil)
}
return
}

filters, linkFilters, err = h.addFilters(w, r, hRes.GetPath())
filters, linkFilters, err = h.addFilters(w, r, h.homeNamespace)
if err != nil {
// result has been written as part of addFilters
return
Expand Down
5 changes: 3 additions & 2 deletions pkg/storage/utils/eosfs/eosfs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -659,7 +659,8 @@ func (fs *eosfs) listWithNominalHome(ctx context.Context, p string) (finfos []*p
}
}

if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil {
// Remove the hidden folders in the topmost directory
if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil && finfo.Path != "/" && !strings.HasPrefix(finfo.Path, "/.") {
finfos = append(finfos, finfo)
}
}
Expand Down Expand Up @@ -713,7 +714,7 @@ func (fs *eosfs) listHome(ctx context.Context, home string) ([]*provider.Resourc
}
}

if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil {
if finfo, err := fs.convertToResourceInfo(ctx, eosFileInfo); err == nil && finfo.Path != "/" && !strings.HasPrefix(finfo.Path, "/.") {
finfos = append(finfos, finfo)
}
}
Expand Down

0 comments on commit 2848f9f

Please sign in to comment.