Merge branch 'master' of ssh://bitbucket.org/aldor007/go-img

Dockerfile

@@ -17,7 +17,6 @@ RUN \
   libwebp-dev libtiff5-dev libgif-dev libexif-dev libxml2-dev libpoppler-glib-dev \
   swig libmagickwand-dev libpango1.0-dev libmatio-dev libopenslide-dev libcfitsio-dev \
   libgsf-1-dev fftw3-dev liborc-0.4-dev librsvg2-dev && \
-
   # Build libvips
   cd /tmp && \
   curl -OL https://github.com/jcupitt/libvips/releases/download/v${LIBVIPS_VERSION}/vips-${LIBVIPS_VERSION}.tar.gz && \
@@ -27,7 +26,6 @@ RUN \
   make && \
   make install && \
   ldconfig && \
-
   # Clean up
   apt-get remove -y curl automake build-essential && \
   apt-get autoremove -y && \
@@ -61,8 +59,7 @@ RUN go get -u go.uber.org/zap
 # RUN goinstall
 RUN cd /go/src/mort; go build cmd/mort.go; cp mort /go/mort; cp -r /go/src/mort/configuration /go/
 # clean up
-RUN rm -rf /go/src
-RUN rm -rf /go/pkg
+RUN rm -rf /go/src; rm -rf /go/pkg; rm -rf /usr/share/; rm -rf /usr/include/
 
 # Run the outyet command by default when the container starts.
 ENTRYPOINT ["/go/mort"]

README.md

@@ -0,0 +1,7 @@
+# Mort [![Build Status](https://travis-ci.org/aldor007/mort.png)](https://travis-ci.org/aldor007/mort) [![Docker](https://img.shields.io/badge/docker-aldor007/mort-blue.svg)](https://hub.docker.com/r/aldor007/mort/) [![Docker Registry](https://img.shields.io/docker/pulls/aldor007/mort.svg)](https://hub.docker.com/r/aldor007/mort/) [![Go Report Card](http://goreportcard.com/badge/aldor007/mort)](http://goreportcard.com/report/aldor007/mort) ![ImageLayers](https://badge.imagelayers.io/aldor007/mort.svg)
+
+TODO
+LOGO
+
+S3 compatible image processing server written in Go.
+

bitbucket-pipelines.yml

@@ -0,0 +1,24 @@
+# This is a sample build configuration for Docker.
+# Check our guides at https://confluence.atlassian.com/x/O1toN for more examples.
+# Only use spaces to indent your .yml configuration.
+# -----
+# You can specify a custom docker image from Docker Hub as your build environment.
+# image: atlassian/default-image:latest
+
+# enable Docker for your repository
+options:
+  docker: true
+
+pipelines:
+  default:
+    - step:
+        script: # Modify the commands below to build your repository.
+          # Set $DOCKER_HUB_USERNAME and $DOCKER_HUB_PASSWORD as environment variables in repository settings
+          - export IMAGE_NAME=aldor007/mort:latest
+
+          # build the Docker image (this will use the Dockerfile in the root of the repo)
+          - docker build -t $IMAGE_NAME .
+          # authenticate with the Docker Hub registry
+          - docker login --username $DOCKER_HUB_USERNAME --password $DOCKER_HUB_PASSWORD
+          # push the new Docker image to the Docker registry
+          - docker push $IMAGE_NAME

configuration/config.yml

@@ -78,7 +78,7 @@ buckets:
             transform:
                 kind: "local"
                 rootPath: "/Users/aldor/workspace/mkaciubacom/web"
-    media:
+    media2:
         keys:
           - accessKey: "acc"
             secretAccessKey: "sec"

processor.go

@@ -14,6 +14,7 @@ import (
 	"mort/storage"
 	"mort/transforms"
 	"mort/log"
+	"strconv"
 )
 const S3_LOCATION_STR = "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LocationConstraint xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">EU</LocationConstraint>"
 
@@ -96,7 +97,29 @@ func handleS3Get(ctx echo.Context, obj *object.FileObject) *response.Response {
 		return response.NewBuf(200, []byte(S3_LOCATION_STR))
 	}
 
-	return response.NewBuf(405, []byte(""))
+	maxKeys := 1000
+	delimeter := ""
+	prefix := ""
+	marker := ""
+
+	if maxKeysQuery, ok := query["max-keys"]; ok {
+		maxKeys, _ = strconv.Atoi(maxKeysQuery[0])
+	}
+
+	if delimeterQuery, ok := query["delimeter"]; ok {
+		delimeter = delimeterQuery[0]
+	}
+
+	if prefixQuery, ok := query["prefix"]; ok {
+		prefix = prefixQuery[0]
+	}
+
+	if markerQuery, ok := query["marker"]; ok {
+		marker = markerQuery[0]
+	}
+
+	return storage.List(obj, maxKeys, delimeter, prefix, marker)
+
 }
 
 func processImage(obj *object.FileObject, parent *response.Response, transforms []transforms.Transforms) *response.Response {

response/response.go

@@ -86,5 +86,7 @@ func (r *Response) CopyBody() ([]byte, error) {
 }
 
 func (r *Response) Close() {
-	r.Stream.Close()
+	if r.Stream != nil {
+		r.Stream.Close()
+	}
 }

s3.go

@@ -105,7 +105,7 @@ func S3AuthMiddleware(mortConfig *config.Config) echo.MiddlewareFunc {
 				return echo.ErrUnauthorized
 			}
 
-			validiatonReq, err := http.NewRequest(req.Method, req.URL.String(), req.Body)
+			validiatonReq, err := http.NewRequest(req.Method, req.RequestURI, req.Body)
 			if err != nil {
 				return echo.ErrUnauthorized
 			}
@@ -116,7 +116,7 @@ func S3AuthMiddleware(mortConfig *config.Config) echo.MiddlewareFunc {
 				}
 
 				switch h {
-				case "Content-Type", "Content-Md5", "Host":
+				case "Content-Type", "Content-Md5", "Host", "Date":
 					validiatonReq.Header.Set(h, v[0])
 				}
 			}
@@ -125,13 +125,27 @@ func S3AuthMiddleware(mortConfig *config.Config) echo.MiddlewareFunc {
 				validiatonReq.Header.Set(h, req.Header.Get(h))
 			}
 
+			// FIXME: there will be problem with escaped paths
 			validiatonReq.URL = req.URL
 			validiatonReq.Method = req.Method
 			validiatonReq.Body = req.Body
 			validiatonReq.Host = req.Host
 
-			if authAlg== "s3" {
+			if authAlg == "s3" {
 				awsauth.SignS3(validiatonReq,  credential)
+				if auth == validiatonReq.Header.Get(echo.HeaderAuthorization) {
+					c.Set("accessKey", accessKey)
+					return next(c)
+				}
+
+				reqDate := validiatonReq.Header.Get("Date")
+				if reqDate != "" {
+					fDate, err := time.Parse(time.RFC1123, reqDate)
+					if err == nil {
+						validiatonReq.Header.Set("Date", fDate.Format(time.RFC1123Z))
+						awsauth.SignS3(validiatonReq,  credential)
+					}
+				}
 
 			} else {
 				awsauth.Sign4ForRegion(validiatonReq, "mort", "s3", credential)
@@ -160,7 +174,7 @@ func S3Middleware(mortConfig *config.Config) echo.MiddlewareFunc {
 	}
 
 	type listAllBucketsResult struct {
-		XMLName     xml.Name `xml:"ListBucketsResult"`
+		XMLName     xml.Name `xml:"ListAllMyBucketsResult"`
 		Owner      struct {
 			ID     string `xml:"ID"`
 			DisplayName string `xml:"DisplayName"`
@@ -170,16 +184,16 @@ func S3Middleware(mortConfig *config.Config) echo.MiddlewareFunc {
 
 	return func(next echo.HandlerFunc) echo.HandlerFunc {
 		return func(c echo.Context) error {
-			accessKey := c.Get("accessKey")
+			accessKeyVal := c.Get("accessKey")
 
 			if c.Request().URL.Path != "/" {
 				return next(c)
 			}
-
-			buckets := mortConfig.BucketsByAccessKey(accessKey.(string))
+			accessKey := accessKeyVal.(string)
+			buckets := mortConfig.BucketsByAccessKey(accessKey)
 			listAllBucketsXML := listAllBucketsResult{}
-			listAllBucketsXML.Owner.DisplayName = "test"
-			listAllBucketsXML.Owner.ID = "test"
+			listAllBucketsXML.Owner.DisplayName = accessKey
+			listAllBucketsXML.Owner.ID = accessKey
 			//listAllBucketsXML.Buckets = make([]bucketXml, len(buckets))
 			for _, bucket := range buckets {
 				b := bucketXml{}

storage/http/item.go

@@ -76,7 +76,7 @@ func (i *item) Open() (io.ReadCloser, error) {
 	if err != nil {
 		return nil, err
 	}
-	
+
 	return response.Body, err
 }
 

storage/storage.go

@@ -12,9 +12,12 @@ import (
 	s3Storage "github.com/aldor007/stow/s3"
 	httpStorage "mort/storage/http"
 
+	"encoding/xml"
+	"mort/log"
 	"mort/object"
 	"mort/response"
-	"mort/log"
+	"strings"
+	"time"
 )
 
 const notFound = "{\"error\":\"item not found\"}"
@@ -23,7 +26,7 @@ func Get(obj *object.FileObject) *response.Response {
 	key := getKey(obj)
 	client, err := getClient(obj)
 	if err != nil {
-		log.Log().Infow("Storage/Get get client","obj.Key" ,obj.Key, "error", err)
+		log.Log().Infow("Storage/Get get client", "obj.Key", obj.Key, "error", err)
 		return response.NewError(503, err)
 	}
 
@@ -40,7 +43,7 @@ func Get(obj *object.FileObject) *response.Response {
 
 	metadata, err := item.Metadata()
 	if err != nil {
-		log.Log().Warnw("Storage/Get read metadata", "obj.Key", obj.Key,"sc", 500, "error", err)
+		log.Log().Warnw("Storage/Get read metadata", "obj.Key", obj.Key, "sc", 500, "error", err)
 		return response.NewError(500, err)
 	}
 
@@ -63,7 +66,7 @@ func Set(obj *object.FileObject, _ http.Header, contentLen int64, body io.ReadCl
 	_, err = client.Put(getKey(obj), body, contentLen, nil)
 
 	if err != nil {
-		log.Log().Warnw("Storage/Set cannot set" , "obj.Key", obj.Key, "sc", 500, "error", err)
+		log.Log().Warnw("Storage/Set cannot set", "obj.Key", obj.Key, "sc", 500, "error", err)
 		return response.NewError(500, err)
 	}
 
@@ -72,6 +75,98 @@ func Set(obj *object.FileObject, _ http.Header, contentLen int64, body io.ReadCl
 	return res
 }
 
+func List(obj *object.FileObject, maxKeys int, delimeter string, prefix string, marker string) *response.Response {
+	client, err := getClient(obj)
+	if err != nil {
+		log.Log().Warnw("Storage/Set create client", "obj.Key", obj.Key, "sc", 503, "error", err)
+		return response.NewError(503, err)
+	}
+
+	items, resultMarker, err := client.Items(prefix, marker, maxKeys)
+	if err != nil {
+		return response.NewError(500, err)
+	}
+
+	type contentXml struct {
+		Key          string    `xml:"Key"`
+		StorageClass string    `xml::"StorageClass"`
+		LastModified time.Time `xml:"LastModified"`
+		ETag         string    `xml:"ETag"`
+		Size         int64     `xml:"Size"`
+	}
+
+	type commonPrefixXml struct {
+		Prefix string `xml:"Prefix"`
+	}
+
+	type listBucketResult struct {
+		XMLName        xml.Name          `xml:"ListBucketResult"`
+		Name           string            `xml:"Name"`
+		Prefix         string            `xml:"Prefix"`
+		Marker         string            `xml:"Marker"`
+		MaxKeys        int               `xml:"MaxKeys"`
+		IsTruncated    bool              `xml:"IsTruncated"`
+		Contents       []contentXml      `xml:"Contents"`
+		CommonPrefixes []commonPrefixXml `xml:"CommonPrefixes"`
+	}
+
+	result := listBucketResult{Name: obj.Bucket, Prefix: prefix, Marker: resultMarker, MaxKeys: maxKeys, IsTruncated: false}
+
+	commonPrefixes := make(map[string]bool, len(items))
+	for _, item := range items {
+		lastMod, _ := item.LastMod()
+		size, _ := item.Size()
+		etag, _ := item.ETag()
+		filePath := strings.Split(item.ID(), "/")
+		prefixPath := strings.Split(prefix, "/")
+		var commonPrefix string
+		var key string
+
+
+		if len(filePath) > len(prefixPath) {
+			key = strings.Join(filePath[0:len(prefixPath)], "/")
+
+			_, ok := commonPrefixes[key]
+			if !ok {
+				commonPrefix = key
+				commonPrefixes[commonPrefix] = true
+			} else {
+				commonPrefix = ""
+			}
+			key = ""
+		} else {
+			key = item.Name()
+			// FIXME: add is dir for others adapters
+			itemMeta, _ := item.Metadata()
+			_, ok := commonPrefixes[key]
+			if itemMeta["is_dir"].(bool)  && !ok{
+				commonPrefix = key
+				commonPrefixes[key] = true
+				key = ""
+			}
+		}
+
+		if key != "" {
+			result.Contents = append(result.Contents, contentXml{Key: key, LastModified: lastMod, Size: size, ETag: etag, StorageClass: "STANDARD"})
+		}
+
+
+		if commonPrefix != ""  {
+			result.CommonPrefixes = append(result.CommonPrefixes, commonPrefixXml{commonPrefix + "/"})
+		}
+
+	}
+
+	resultXml, err := xml.Marshal(result)
+	if err != nil {
+		return response.NewError(500, err)
+	}
+
+	res := response.NewBuf(200, resultXml)
+	res.SetContentType("application/xml")
+	return res
+}
+
 func getClient(obj *object.FileObject) (stow.Container, error) {
 	storageCfg := obj.Storage
 	var config stow.Config
@@ -117,25 +212,24 @@ func getClient(obj *object.FileObject) (stow.Container, error) {
 				return nil, err
 			}
 
-			return container,nil
+			return container, nil
 		}
 
-		return  nil, err
+		return nil, err
 	}
 
 	return container, nil
 }
 
-
-func getKey (obj *object.FileObject) string {
+func getKey(obj *object.FileObject) string {
 	return path.Join(obj.Storage.PathPrefix, obj.Key)
 }
 func prepareResponse(obj *object.FileObject, stream io.ReadCloser, metadata map[string]interface{}) *response.Response {
 	res := response.New(200, stream)
 
 	for k, v := range metadata {
 		switch k {
-		case  "etag", "last-modified":
+		case "etag", "last-modified":
 			res.Set(k, v.(string))
 
 		}