working on listing files

Dockerfile

@@ -59,8 +59,7 @@ RUN go get -u go.uber.org/zap
 # RUN goinstall
 RUN cd /go/src/mort; go build cmd/mort.go; cp mort /go/mort; cp -r /go/src/mort/configuration /go/
 # clean up
-RUN rm -rf /go/src
-RUN rm -rf /go/pkg
+RUN rm -rf /go/src; rm -rf /go/pkg; rm -rf /usr/share/; rm -rf /usr/include/
 
 # Run the outyet command by default when the container starts.
 ENTRYPOINT ["/go/mort"]

README.md

@@ -0,0 +1,7 @@
+# Mort [![Build Status](https://travis-ci.org/aldor007/mort.png)](https://travis-ci.org/aldor007/mort) [![Docker](https://img.shields.io/badge/docker-aldor007/mort-blue.svg)](https://hub.docker.com/r/aldor007/mort/) [![Docker Registry](https://img.shields.io/docker/pulls/aldor007/mort.svg)](https://hub.docker.com/r/aldor007/mort/) [![Go Report Card](http://goreportcard.com/badge/aldor007/mort)](http://goreportcard.com/report/aldor007/mort) ![ImageLayers](https://badge.imagelayers.io/aldor007/mort.svg)
+
+TODO
+LOGO
+
+S3 compatible image processing server written in Go.
+

bitbucket-pipelines.yml

@@ -14,7 +14,7 @@ pipelines:
     - step:
         script: # Modify the commands below to build your repository.
           # Set $DOCKER_HUB_USERNAME and $DOCKER_HUB_PASSWORD as environment variables in repository settings
-          - export IMAGE_NAME=aldor007/mort:$BITBUCKET_COMMIT
+          - export IMAGE_NAME=aldor007/mort:latest
 
           # build the Docker image (this will use the Dockerfile in the root of the repo)
           - docker build -t $IMAGE_NAME .

configuration/config.yml

@@ -78,7 +78,7 @@ buckets:
             transform:
                 kind: "local"
                 rootPath: "/Users/aldor/workspace/mkaciubacom/web"
-    media:
+    media2:
         keys:
           - accessKey: "acc"
             secretAccessKey: "sec"

processor.go

@@ -110,7 +110,7 @@ func handleS3Get(ctx echo.Context, obj *object.FileObject) *response.Response {
 		delimeter = delimeterQuery[0]
 	}
 
-	if prefixQuery, ok := query["query"]; ok {
+	if prefixQuery, ok := query["prefix"]; ok {
 		prefix = prefixQuery[0]
 	}
 

response/response.go

@@ -86,5 +86,7 @@ func (r *Response) CopyBody() ([]byte, error) {
 }
 
 func (r *Response) Close() {
-	r.Stream.Close()
+	if r.Stream != nil {
+		r.Stream.Close()
+	}
 }

s3.go

@@ -125,6 +125,7 @@ func S3AuthMiddleware(mortConfig *config.Config) echo.MiddlewareFunc {
 				validiatonReq.Header.Set(h, req.Header.Get(h))
 			}
 
+			// FIXME: there will be problem with escaped paths
 			validiatonReq.URL = req.URL
 			validiatonReq.Method = req.Method
 			validiatonReq.Body = req.Body

storage/storage.go

@@ -12,10 +12,11 @@ import (
 	s3Storage "github.com/aldor007/stow/s3"
 	httpStorage "mort/storage/http"
 
+	"encoding/xml"
+	"mort/log"
 	"mort/object"
 	"mort/response"
-	"mort/log"
-	"encoding/xml"
+	"strings"
 	"time"
 )
 
@@ -25,7 +26,7 @@ func Get(obj *object.FileObject) *response.Response {
 	key := getKey(obj)
 	client, err := getClient(obj)
 	if err != nil {
-		log.Log().Infow("Storage/Get get client","obj.Key" ,obj.Key, "error", err)
+		log.Log().Infow("Storage/Get get client", "obj.Key", obj.Key, "error", err)
 		return response.NewError(503, err)
 	}
 
@@ -42,7 +43,7 @@ func Get(obj *object.FileObject) *response.Response {
 
 	metadata, err := item.Metadata()
 	if err != nil {
-		log.Log().Warnw("Storage/Get read metadata", "obj.Key", obj.Key,"sc", 500, "error", err)
+		log.Log().Warnw("Storage/Get read metadata", "obj.Key", obj.Key, "sc", 500, "error", err)
 		return response.NewError(500, err)
 	}
 
@@ -65,7 +66,7 @@ func Set(obj *object.FileObject, _ http.Header, contentLen int64, body io.ReadCl
 	_, err = client.Put(getKey(obj), body, contentLen, nil)
 
 	if err != nil {
-		log.Log().Warnw("Storage/Set cannot set" , "obj.Key", obj.Key, "sc", 500, "error", err)
+		log.Log().Warnw("Storage/Set cannot set", "obj.Key", obj.Key, "sc", 500, "error", err)
 		return response.NewError(500, err)
 	}
 
@@ -81,37 +82,80 @@ func List(obj *object.FileObject, maxKeys int, delimeter string, prefix string,
 		return response.NewError(503, err)
 	}
 
-	items, resultMarker, err := client.Items(prefix, "", maxKeys)
+	items, resultMarker, err := client.Items(prefix, marker, maxKeys)
 	if err != nil {
 		return response.NewError(500, err)
 	}
 
 	type contentXml struct {
-		Key   string `xml:"Key"`
-		LastModified time.Time`xml:"LastModified"`
-		ETag         string `xml:"ETag"`
-		Size         int64 `xml:"Size"`
+		Key          string    `xml:"Key"`
+		StorageClass string    `xml::"StorageClass"`
+		LastModified time.Time `xml:"LastModified"`
+		ETag         string    `xml:"ETag"`
+		Size         int64     `xml:"Size"`
 	}
 
+	type commonPrefixXml struct {
+		Prefix string `xml:"Prefix"`
+	}
 
 	type listBucketResult struct {
-		XMLName     xml.Name `xml:"ListBucketResult"`
-		Name        string   `xml:"Name"`
-		Prefix      string   `xml:"Prefix"`
-		Marker      string   `xml:"Marker"`
-		MaxKeys     int      `xml:"MaxKeys"`
-		IsTruncated bool      `xml:"IsTruncated"`
-		Contents   []contentXml`xml:"Contents"`
+		XMLName        xml.Name          `xml:"ListBucketResult"`
+		Name           string            `xml:"Name"`
+		Prefix         string            `xml:"Prefix"`
+		Marker         string            `xml:"Marker"`
+		MaxKeys        int               `xml:"MaxKeys"`
+		IsTruncated    bool              `xml:"IsTruncated"`
+		Contents       []contentXml      `xml:"Contents"`
+		CommonPrefixes []commonPrefixXml `xml:"CommonPrefixes"`
 	}
 
 	result := listBucketResult{Name: obj.Bucket, Prefix: prefix, Marker: resultMarker, MaxKeys: maxKeys, IsTruncated: false}
 
-
+	commonPrefixes := make(map[string]bool, len(items))
 	for _, item := range items {
 		lastMod, _ := item.LastMod()
 		size, _ := item.Size()
 		etag, _ := item.ETag()
-		result.Contents = append(result.Contents, contentXml{Key: item.ID(), LastModified: lastMod, Size: size, ETag: etag})
+		filePath := strings.Split(item.ID(), "/")
+		prefixPath := strings.Split(prefix, "/")
+		var commonPrefix string
+		var key string
+
+
+		if len(filePath) > len(prefixPath) {
+			key = strings.Join(filePath[0:len(prefixPath)], "/")
+
+			_, ok := commonPrefixes[key]
+			if !ok {
+				commonPrefix = key
+				commonPrefixes[commonPrefix] = true
+			} else {
+				commonPrefix = ""
+				key = ""
+			}
+			log.Log().Infof("if key = %s commonprefix = %s %s %s", key, commonPrefix, filePath, prefixPath)
+		} else {
+			key = item.Name()
+			log.Log().Infof("else %s %s %s %s", key, commonPrefix, item.ID(), item.Name())
+			// FIXME: add is dir for others adapters
+			itemMeta, _ := item.Metadata()
+			_, ok := commonPrefixes[key]
+			if itemMeta["is_dir"].(bool)  && !ok{
+				commonPrefix = key
+				commonPrefixes[key] = true
+			}
+		}
+
+		if key != "" {
+			result.Contents = append(result.Contents, contentXml{Key: key, LastModified: lastMod, Size: size, ETag: etag, StorageClass: "STANDARD"})
+		}
+
+
+		if commonPrefix != ""  {
+			result.CommonPrefixes = append(result.CommonPrefixes, commonPrefixXml{commonPrefix})
+		}
+
 	}
 
 	resultXml, err := xml.Marshal(result)
@@ -169,25 +213,24 @@ func getClient(obj *object.FileObject) (stow.Container, error) {
 				return nil, err
 			}
 
-			return container,nil
+			return container, nil
 		}
 
-		return  nil, err
+		return nil, err
 	}
 
 	return container, nil
 }
 
-
-func getKey (obj *object.FileObject) string {
+func getKey(obj *object.FileObject) string {
 	return path.Join(obj.Storage.PathPrefix, obj.Key)
 }
 func prepareResponse(obj *object.FileObject, stream io.ReadCloser, metadata map[string]interface{}) *response.Response {
 	res := response.New(200, stream)
 
 	for k, v := range metadata {
 		switch k {
-		case  "etag", "last-modified":
+		case "etag", "last-modified":
 			res.Set(k, v.(string))
 
 		}

upload.js

@@ -3,6 +3,7 @@ const AWS = require('aws-sdk');
 // const mime = require('mime')
 
 const host = 'localhost:8080'
+// const host = 'mort.mkaciuba.com'
 const accessKeyId = 'acc'
 const secretAccessKey = 'sec'
 
@@ -13,7 +14,7 @@ const s3opts = {
    sslEnabled: false,
    accessKeyId: accessKeyId,
    secretAccessKey: secretAccessKey,
-   signatureVersion: 'v4',
+   signatureVersion: 's3',
    computeChecksums: true
 };
 const body = fs.readFileSync('file.jpeg')
@@ -41,14 +42,27 @@ const options = {
     queueSize: 1
 };
 
-s3.listBuckets({}, function (err, data) {
-	console.info(err, data)
-})
-s3.upload(params, options, function (err, data) {
+// s3.listBuckets({}, function (err, data) {
+// 	console.info(err, data)
+// })
+
+const listParams = {
+    Bucket: 'liip',
+}
+
+s3.listObjects(listParams, function (err, data) {
     if (err) {
-        console.error('Error uploading file', err);
-        return;
+        console.error(err);
+        throw err;
     }
+    console.info('list', data)
+})
+
+// s3.upload(params, options, function (err, data) {
+//     if (err) {
+//         console.error('Error uploading file', err);
+//         return;
+//     }
 
-    console.info('Successful uploaded file');
-});
+//     console.info('Successful uploaded file');
+// });