From 5e192e0cd5ee006210a2821827c45319f1e805cf Mon Sep 17 00:00:00 2001 From: Charles-Antoine Mathieu Date: Thu, 22 Oct 2020 21:43:37 +0200 Subject: [PATCH] S3 Server Side Encryption --- README.md | 3 +- go.mod | 11 ++-- go.sum | 62 ++++++++++++++++----- server/data/data.go | 2 +- server/data/file/file.go | 12 ++--- server/data/file/file_test.go | 12 ++--- server/data/s3/s3.go | 50 +++++++++++++---- server/data/s3/sse.go | 83 +++++++++++++++++++++++++++++ server/data/stream/stream.go | 4 +- server/data/stream/stream_test.go | 4 +- server/data/swift/swift.go | 10 ++-- server/data/testing/testing.go | 10 ++-- server/data/testing/testing_test.go | 10 ++-- server/handlers/add_file.go | 2 +- server/handlers/get_file_test.go | 2 +- server/plikd.cfg | 3 ++ server/server/server_test.go | 2 +- testing/minio/plikd.cfg | 19 ++++++- testing/minio/run.sh | 1 + 19 files changed, 238 insertions(+), 64 deletions(-) create mode 100644 server/data/s3/sse.go diff --git a/README.md b/README.md index 41ee6dd9..26272cac 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Plik is a scalable & friendly temporary file upload system ( wetransfer like ) i - User authentication : Local / Google / OVH - Upload restriction : Source IP / Token - Administrator dashboard + - Server side encryption ( with S3 data backend ) - [ShareX](https://getsharex.com/) Uploader : Directly integrated into ShareX - [plikSharp](https://github.com/iss0/plikSharp) : A .NET API client for Plik - [Filelink for Plik](https://gitlab.com/joendres/filelink-plik) : Thunderbird Addon to upload attachments to Plik @@ -143,7 +144,7 @@ Store uploaded files in a local or mounted file system directory. - Openstack Swift databackend : http://docs.openstack.org/developer/swift/ -Openstack Swift is a highly available, distributed, eventually consistent object/blob store. +Openstack Swift is a highly available, distributed, eventually consistent object/blob store which supports Server Side Encryption ### Available metadata backends diff --git a/go.mod b/go.mod index 4dc7b69e..b92ad990 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/fatih/color v1.9.0 // indirect github.com/golang/protobuf v1.3.2-0.20190409050943-e91709a02e0e // indirect github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf + github.com/google/uuid v1.1.2 // indirect github.com/gorilla/mux v1.7.1 github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 // indirect @@ -23,7 +24,7 @@ require ( github.com/mattn/go-colorable v0.1.6 // indirect github.com/mattn/go-runewidth v0.0.5-0.20181218000649-703b5e6b11ae // indirect github.com/mattn/go-sqlite3 v2.0.3+incompatible // indirect - github.com/minio/minio-go/v6 v6.0.49 + github.com/minio/minio-go/v7 v7.0.5 github.com/mitchellh/go-homedir v1.1.0 github.com/ncw/swift v1.0.48-0.20190410202254-753d2090bb62 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d @@ -33,13 +34,15 @@ require ( github.com/root-gg/utils v0.0.0-20151025161626-38f45ede2ce2 github.com/spf13/cobra v0.0.6-0.20200106181057-89c7ffb5129b github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709 + github.com/stretchr/testify v1.4.0 go.opencensus.io v0.20.2 // indirect - golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd + golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 + golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a - golang.org/x/text v0.3.1-0.20190410012825-f4905fbd45b6 // indirect + golang.org/x/sys v0.0.0-20201020230747-6e5568b54d1a // indirect google.golang.org/api v0.3.3-0.20190418015003-33b7e862cd15 google.golang.org/appengine v1.5.0 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/gormigrate.v1 v1.6.0 + gopkg.in/ini.v1 v1.62.0 // indirect ) diff --git a/go.sum b/go.sum index ee9af216..7feaaf86 100644 --- a/go.sum +++ b/go.sum @@ -40,7 +40,6 @@ github.com/dgrijalva/jwt-go v3.2.1-0.20180921172315-3af4c746e1c2+incompatible/go github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815 h1:bWDMxwH3px2JBh6AyO7hdCn/PkvCZXii8TGj7sbtEbQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac h1:opbrjaN/L8gg6Xh5D04Tem+8xVcz6ajZlGCs49mQgyg= github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= @@ -81,8 +80,13 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -119,6 +123,8 @@ github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/ github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -127,10 +133,15 @@ github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uia github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -152,13 +163,21 @@ github.com/mattn/go-sqlite3 v2.0.1+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/minio/minio-go/v6 v6.0.49 h1:bU4kIa/qChTLC1jrWZ8F+8gOiw1MClubddAJVR4gW3w= -github.com/minio/minio-go/v6 v6.0.49/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4= +github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncw/swift v1.0.48-0.20190410202254-753d2090bb62 h1:t2kC2L3N+IG8J8eoaGCZu3yHo3Q/YATUyNqUKC0/ni8= github.com/ncw/swift v1.0.48-0.20190410202254-753d2090bb62/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= @@ -197,10 +216,11 @@ github.com/root-gg/logger v0.0.0-20150501173826-5d9a47a35312 h1:gLYLMWhdoU02+VLb github.com/root-gg/logger v0.0.0-20150501173826-5d9a47a35312/go.mod h1:SZ5sf0POVtIvp+vNCGYwJp1/ZT6/nShOau6aVMGgtGU= github.com/root-gg/utils v0.0.0-20151025161626-38f45ede2ce2 h1:2IC5+HunAKLL0GxcXtqbsQxEOmDcFTf8gn34mxpTs20= github.com/root-gg/utils v0.0.0-20151025161626-38f45ede2ce2/go.mod h1:XF39UJcqum+uYclrw+wB0V1qg9vgglJ8gz8AMrVJ/AQ= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= @@ -220,8 +240,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709 h1:Ko2LQMrRU+Oy/+EDBwX7eZ2jp3C47eDBB8EIhKTun+I= -github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -237,9 +257,13 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20181112202954-3d3f9f413869/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd h1:GGJVjV8waZKRHrgwvtH66z9ZGVurTD1MT0n1Bb+q4aM= golang.org/x/crypto v0.0.0-20191205180655-e7c4368fe9dd/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg= +golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897 h1:pLI5jrR7OSLijeIDcmRxNmw2api+jEfxLoykJVice/E= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= @@ -257,6 +281,10 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA= @@ -274,15 +302,20 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201020230747-6e5568b54d1a h1:e3IU37lwO4aq3uoRKINC7JikojFmE5gO7xhfxs8VC34= +golang.org/x/sys v0.0.0-20201020230747-6e5568b54d1a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20190410012825-f4905fbd45b6 h1:5P9sojHzOkdU7Ag2EKEEn6JkBY2WGJffafizGfacSBI= -golang.org/x/text v0.3.1-0.20190410012825-f4905fbd45b6/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -312,19 +345,24 @@ google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gormigrate.v1 v1.6.0 h1:XpYM6RHQPmzwY7Uyu+t+xxMXc86JYFJn4nEc9HzQjsI= gopkg.in/gormigrate.v1 v1.6.0/go.mod h1:Lf00lQrHqfSYWiTtPcyQabsDdM6ejZaMgV0OU6JMSlw= -gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= -gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= +gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/server/data/data.go b/server/data/data.go index ee615ec2..bd474b90 100644 --- a/server/data/data.go +++ b/server/data/data.go @@ -9,7 +9,7 @@ import ( // Backend interface describes methods that data backend // must implements to be compatible with plik. type Backend interface { - AddFile(file *common.File, reader io.Reader) (backendDetails string, err error) + AddFile(file *common.File, reader io.Reader) (err error) GetFile(file *common.File) (reader io.ReadCloser, err error) RemoveFile(file *common.File) (err error) } diff --git a/server/data/file/file.go b/server/data/file/file.go index 87687e46..6ae3b85a 100644 --- a/server/data/file/file.go +++ b/server/data/file/file.go @@ -61,32 +61,32 @@ func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) { // AddFile implementation for file data backend will creates a new file for the given upload // and save it on filesystem with the given file reader -func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) { +func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) { dir, path, err := b.getPath(file) if err != nil { - return "", err + return err } // Create directory err = os.MkdirAll(dir, 0777) if err != nil { - return "", fmt.Errorf("unable to create upload directory") + return fmt.Errorf("unable to create upload directory") } // Create file out, err := os.Create(path) if err != nil { - return "", fmt.Errorf("unable to create file %s : %s", path, err) + return fmt.Errorf("unable to create file %s : %s", path, err) } // Copy file data from the client request body // to the file system _, err = io.Copy(out, fileReader) if err != nil { - return "", fmt.Errorf("unable to save file %s : %s", path, err) + return fmt.Errorf("unable to save file %s : %s", path, err) } - return "", nil + return nil } // RemoveFile implementation for file data backend will delete the given diff --git a/server/data/file/file_test.go b/server/data/file/file_test.go index 55dba997..7c401a4e 100644 --- a/server/data/file/file_test.go +++ b/server/data/file/file_test.go @@ -41,7 +41,7 @@ func TestAddFileInvalidUploadId(t *testing.T) { upload := &common.Upload{} file := upload.NewFile() - _, err := backend.AddFile(file, &bytes.Buffer{}) + err := backend.AddFile(file, &bytes.Buffer{}) require.Error(t, err, "no error with invalid upload id") } @@ -56,7 +56,7 @@ func TestAddFileImpossibleToCreateDirectory(t *testing.T) { file := upload.NewFile() upload.PrepareInsertForTests() - _, err := backend.AddFile(file, &bytes.Buffer{}) + err := backend.AddFile(file, &bytes.Buffer{}) require.Error(t, err, "unable to create directory") } @@ -69,7 +69,7 @@ func TestAddFileInvalidReader(t *testing.T) { upload.PrepareInsertForTests() reader := common.NewErrorReader(errors.New("io error")) - _, err := backend.AddFile(file, reader) + err := backend.AddFile(file, reader) require.Error(t, err, "unable to create directory") require.Contains(t, err.Error(), "io error", "invalid error") } @@ -83,7 +83,7 @@ func TestAddFile(t *testing.T) { upload.PrepareInsertForTests() reader := bytes.NewBufferString("data") - _, err := backend.AddFile(file, reader) + err := backend.AddFile(file, reader) require.NoError(t, err, "unable to add file") _, path, err := backend.getPathCompat(file) @@ -135,7 +135,7 @@ func TestGetFile(t *testing.T) { upload.PrepareInsertForTests() reader := bytes.NewBufferString("data") - _, err := backend.AddFile(file, reader) + err := backend.AddFile(file, reader) require.NoError(t, err, "unable to add file") fileReader, err := backend.GetFile(file) @@ -215,7 +215,7 @@ func TestRemoveFile(t *testing.T) { upload.PrepareInsertForTests() reader := bytes.NewBufferString("data") - _, err := backend.AddFile(file, reader) + err := backend.AddFile(file, reader) require.NoError(t, err, "unable to add file") _, path, err := backend.getPathCompat(file) diff --git a/server/data/s3/s3.go b/server/data/s3/s3.go index 125b26fd..050e60b0 100644 --- a/server/data/s3/s3.go +++ b/server/data/s3/s3.go @@ -1,10 +1,12 @@ package s3 import ( + "context" "fmt" "io" - "github.com/minio/minio-go/v6" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/root-gg/utils" "github.com/root-gg/plik/server/common" @@ -24,6 +26,7 @@ type Config struct { Prefix string PartSize uint64 UseSSL bool + SSE string } // NewConfig instantiate a new default configuration @@ -60,6 +63,11 @@ func (config *Config) Validate() error { return nil } +// BackendDetails additional backend metadata +type BackendDetails struct { + SSEKey string +} + // Backend object type Backend struct { config *Config @@ -77,20 +85,24 @@ func NewBackend(config *Config) (b *Backend, err error) { return nil, fmt.Errorf("invalid s3 data backend config : %s", err) } - b.client, err = minio.New(config.Endpoint, config.AccessKeyID, config.SecretAccessKey, config.UseSSL) + b.client, err = minio.New(config.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, ""), + //Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, + Secure: config.UseSSL, + }) if err != nil { return nil, err } // Check if bucket exists - exists, err := b.client.BucketExists(config.Bucket) + exists, err := b.client.BucketExists(context.TODO(), config.Bucket) if err != nil { return nil, fmt.Errorf("unable to check if bucket %s exists : %s", config.Bucket, err) } if !exists { // Create bucket - err = b.client.MakeBucket(config.Bucket, config.Location) + err = b.client.MakeBucket(context.TODO(), config.Bucket, minio.MakeBucketOptions{Region: config.Location}) if err != nil { return nil, fmt.Errorf("unable to create bucket %s : %s", config.Bucket, err) } @@ -101,26 +113,44 @@ func NewBackend(config *Config) (b *Backend, err error) { // GetFile implementation for S3 Data Backend func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) { - return b.client.GetObject(b.config.Bucket, b.getObjectName(file.ID), minio.GetObjectOptions{}) + getOpts := minio.GetObjectOptions{} + + // Configure server side encryption + getOpts.ServerSideEncryption, err = b.getServerSideEncryption(file) + if err != nil { + return nil, err + } + + return b.client.GetObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), getOpts) } // AddFile implementation for S3 Data Backend -func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) { +func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) { + putOpts := minio.PutObjectOptions{ContentType: file.Type} + + // Configure server side encryption + putOpts.ServerSideEncryption, err = b.getServerSideEncryption(file) + if err != nil { + return err + } + if file.Size > 0 { - _, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, file.Size, minio.PutObjectOptions{ContentType: file.Type}) + _, err = b.client.PutObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), fileReader, file.Size, putOpts) } else { // https://github.com/minio/minio-go/issues/989 // Minio defaults to 128MB chunks and has to actually allocate a buffer of this size before uploading the chunk // This can lead to very high memory usage when uploading a lot of small files in parallel // We default to 16MB which allow to store files up to 160GB ( 10000 chunks of 16MB ), feel free to adjust this parameter to your needs. - _, err = b.client.PutObject(b.config.Bucket, b.getObjectName(file.ID), fileReader, -1, minio.PutObjectOptions{ContentType: file.Type, PartSize: b.config.PartSize}) + putOpts.PartSize = b.config.PartSize + + _, err = b.client.PutObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), fileReader, -1, putOpts) } - return "", err + return err } // RemoveFile implementation for S3 Data Backend func (b *Backend) RemoveFile(file *common.File) (err error) { - return b.client.RemoveObject(b.config.Bucket, b.getObjectName(file.ID)) + return b.client.RemoveObject(context.TODO(), b.config.Bucket, b.getObjectName(file.ID), minio.RemoveObjectOptions{}) } func (b *Backend) getObjectName(name string) string { diff --git a/server/data/s3/sse.go b/server/data/s3/sse.go new file mode 100644 index 00000000..e3e5df03 --- /dev/null +++ b/server/data/s3/sse.go @@ -0,0 +1,83 @@ +package s3 + +import ( + "encoding/json" + "fmt" + + "github.com/minio/minio-go/v7/pkg/encrypt" + + "github.com/root-gg/plik/server/common" +) + +// Build Server Side Encryption configuration +func (b *Backend) getServerSideEncryption(file *common.File) (sse encrypt.ServerSide, err error) { + switch encrypt.Type(b.config.SSE) { + case "": + return nil, nil + case encrypt.S3: + return encrypt.NewSSE(), nil + case encrypt.SSEC: + key, err := getServerSideEncryptionKey(file) + if err != nil { + return nil, fmt.Errorf("unable to get Server Side Encryption Key : %s", err) + } + return encrypt.NewSSEC([]byte(key)) + case encrypt.KMS: + return nil, fmt.Errorf("KMS server side encryption is not yet implemented") + default: + return nil, fmt.Errorf("invalid SSE type %s", b.config.SSE) + } +} + +// Generate a 32Bytes / 256bits encryption key +func genServerSideEncryptionKey() string { + return common.GenerateRandomID(32) +} + +// Get the SSE Key from the file backend details or generate one and store it in the file backend details +func getServerSideEncryptionKey(file *common.File) (key string, err error) { + // Retrieve the SSE Key from the backend details + if file.BackendDetails != "" { + backendDetails := &BackendDetails{} + err = json.Unmarshal([]byte(file.BackendDetails), backendDetails) + if err != nil { + return "", fmt.Errorf("unable to deserialize backend details : %s", err) + } + + if backendDetails.SSEKey != "" { + return backendDetails.SSEKey, nil + } + } + + key = genServerSideEncryptionKey() + + // Store the SSE Key in the backend details + err = setServerSideEncryptionKey(file, key) + if err != nil { + return "", err + } + + return key, nil +} + +// Add the SSE Key to the file backend details +func setServerSideEncryptionKey(file *common.File, key string) (err error) { + backendDetails := &BackendDetails{} + + if file.BackendDetails != "" { + err = json.Unmarshal([]byte(file.BackendDetails), backendDetails) + if err != nil { + return fmt.Errorf("unable to deserialize backend details : %s", err) + } + } + + backendDetails.SSEKey = key + + backendDetailsJSON, err := json.Marshal(backendDetails) + if err != nil { + return fmt.Errorf("unable to serialize backend details : %s", err) + } + + file.BackendDetails = string(backendDetailsJSON) + return nil +} diff --git a/server/data/stream/stream.go b/server/data/stream/stream.go index 2c094fdb..65161344 100644 --- a/server/data/stream/stream.go +++ b/server/data/stream/stream.go @@ -45,7 +45,7 @@ func (b *Backend) GetFile(file *common.File) (stream io.ReadCloser, err error) { // AddFile implementation for steam data backend will creates a new steam for the given upload // and save it on filesystem with the given steam reader -func (b *Backend) AddFile(file *common.File, stream io.Reader) (backendDetails string, err error) { +func (b *Backend) AddFile(file *common.File, stream io.Reader) (err error) { storeID := file.UploadID + "/" + file.ID pipeReader, pipeWriter := io.Pipe() @@ -61,7 +61,7 @@ func (b *Backend) AddFile(file *common.File, stream io.Reader) (backendDetails s _, err = io.Copy(pipeWriter, stream) _ = pipeWriter.Close() - return "", nil + return nil } // RemoveFile is not implemented diff --git a/server/data/stream/stream_test.go b/server/data/stream/stream_test.go index d0a61146..2aa5bc73 100644 --- a/server/data/stream/stream_test.go +++ b/server/data/stream/stream_test.go @@ -23,9 +23,9 @@ func TestAddGetFile(t *testing.T) { wg.Add(1) go func() { time.Sleep(10 * time.Millisecond) - details, err := backend.AddFile(file, bytes.NewBufferString("data")) + err := backend.AddFile(file, bytes.NewBufferString("data")) require.NoError(t, err, "unable to add file") - require.NotNil(t, details, "invalid nil details") + require.NotNil(t, file.BackendDetails, "invalid nil details") wg.Done() }() diff --git a/server/data/swift/swift.go b/server/data/swift/swift.go index 2bf91c2e..26b977e5 100644 --- a/server/data/swift/swift.go +++ b/server/data/swift/swift.go @@ -65,10 +65,10 @@ func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) { } // AddFile implementation for Swift Data Backend -func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) { +func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) { err = b.auth() if err != nil { - return "", err + return err } objectID := objectID(file) @@ -76,14 +76,14 @@ func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetai _, err = io.Copy(object, fileReader) if err != nil { - return "", err + return err } err = object.Close() if err != nil { - return "", err + return err } - return backendDetails, nil + return nil } // RemoveFile implementation for Swift Data Backend diff --git a/server/data/testing/testing.go b/server/data/testing/testing.go index 99f5a01d..1238d59c 100644 --- a/server/data/testing/testing.go +++ b/server/data/testing/testing.go @@ -48,26 +48,26 @@ func (b *Backend) GetFile(file *common.File) (reader io.ReadCloser, err error) { // AddFile implementation for testing data backend will creates a new file for the given upload // and save it on filesystem with the given file reader -func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (backendDetails string, err error) { +func (b *Backend) AddFile(file *common.File, fileReader io.Reader) (err error) { b.mu.Lock() defer b.mu.Unlock() if b.err != nil { - return "", b.err + return b.err } if _, ok := b.files[file.ID]; ok { - return "", errors.New("file exists") + return errors.New("file exists") } content, err := ioutil.ReadAll(fileReader) if err != nil { - return "", err + return err } b.files[file.ID] = content - return "", nil + return nil } // RemoveFile implementation for testing data backend will delete the given diff --git a/server/data/testing/testing_test.go b/server/data/testing/testing_test.go index 7e5a96bb..ad72b3cb 100644 --- a/server/data/testing/testing_test.go +++ b/server/data/testing/testing_test.go @@ -21,7 +21,7 @@ func TestAddFileError(t *testing.T) { upload := &common.Upload{} file := upload.NewFile() - _, err := backend.AddFile(file, &bytes.Buffer{}) + err := backend.AddFile(file, &bytes.Buffer{}) require.Error(t, err, "missing error") require.Equal(t, "error", err.Error(), "invalid error message") } @@ -33,7 +33,7 @@ func TestAddFileReaderError(t *testing.T) { file := upload.NewFile() reader := common.NewErrorReader(errors.New("io error")) - _, err := backend.AddFile(file, reader) + err := backend.AddFile(file, reader) require.Error(t, err, "missing error") require.Equal(t, "io error", err.Error(), "invalid error message") } @@ -43,7 +43,7 @@ func TestAddFile(t *testing.T) { upload := &common.Upload{} file := upload.NewFile() - _, err := backend.AddFile(file, &bytes.Buffer{}) + err := backend.AddFile(file, &bytes.Buffer{}) require.NoError(t, err, "unable to add file") } @@ -64,7 +64,7 @@ func TestGetFile(t *testing.T) { upload := &common.Upload{} file := upload.NewFile() - _, err := backend.AddFile(file, &bytes.Buffer{}) + err := backend.AddFile(file, &bytes.Buffer{}) require.NoError(t, err, "unable to add file") _, err = backend.GetFile(file) @@ -89,7 +89,7 @@ func TestRemoveFile(t *testing.T) { upload := &common.Upload{} file := upload.NewFile() - _, err := backend.AddFile(file, &bytes.Buffer{}) + err := backend.AddFile(file, &bytes.Buffer{}) require.NoError(t, err, "unable to add file") _, err = backend.GetFile(file) diff --git a/server/handlers/add_file.go b/server/handlers/add_file.go index c06e6fef..f7f6a05a 100644 --- a/server/handlers/add_file.go +++ b/server/handlers/add_file.go @@ -141,7 +141,7 @@ func AddFile(ctx *context.Context, resp http.ResponseWriter, req *http.Request) backend = ctx.GetDataBackend() } - _, err = backend.AddFile(file, preprocessReader) + err = backend.AddFile(file, preprocessReader) if err != nil { // TODO : file status is left to common.FileUploading we should set it to some common.FileUploadError // TODO : or we can set it back to common.FileMissing if we are sure data backends will handle that diff --git a/server/handlers/get_file_test.go b/server/handlers/get_file_test.go index 3c25646b..729895c0 100644 --- a/server/handlers/get_file_test.go +++ b/server/handlers/get_file_test.go @@ -20,7 +20,7 @@ import ( func createTestFile(ctx *context.Context, file *common.File, reader io.Reader) (err error) { dataBackend := ctx.GetDataBackend() - _, err = dataBackend.AddFile(file, reader) + err = dataBackend.AddFile(file, reader) return err } diff --git a/server/plikd.cfg b/server/plikd.cfg index 539ab96d..812b2fcb 100644 --- a/server/plikd.cfg +++ b/server/plikd.cfg @@ -72,6 +72,9 @@ OvhApiEndpoint = "" # OVH api endpoint to use. Defaults to https # UseSSL = true # PartSize = 16000000 // Chunk size when file size is not known. (default to 16MB) # // Multiply by 10000 to get the max upload file size (max upload file size 160GB) +# SSE = "" // the following encryption methods are available : +# // - SSE-C: server-side-encryption with customer provided keys ( managed by Plik ) +# // - S3: server-side-encryption using S3 storage encryption ( managed by the S3 backend ) DataBackend = "file" [DataBackendConfig] diff --git a/server/server/server_test.go b/server/server/server_test.go index 88b78ef7..223fa03d 100644 --- a/server/server/server_test.go +++ b/server/server/server_test.go @@ -112,7 +112,7 @@ func TestClean(t *testing.T) { err := ps.metadataBackend.CreateUpload(upload) require.NoError(t, err, "unable to save upload") - _, err = ps.dataBackend.AddFile(file, bytes.NewBufferString("data data data")) + err = ps.dataBackend.AddFile(file, bytes.NewBufferString("data data data")) require.NoError(t, err, "unable to save file") ps.Clean() diff --git a/testing/minio/plikd.cfg b/testing/minio/plikd.cfg index 39d6f491..af073554 100644 --- a/testing/minio/plikd.cfg +++ b/testing/minio/plikd.cfg @@ -9,11 +9,26 @@ ListenAddress = "0.0.0.0" DataBackend = "s3" [DataBackendConfig] - Endpoint = "127.0.0.1:2604" + Endpoint = "localhost:2604" AccessKeyID = "access_key" SecretAccessKey = "access_key_secret" Prefix = "" PartSize = 5242880 # 5MB + #UseSSL = true + #SSE = "SSE-C" [MetadataBackendConfig] - Path = "plik.db" \ No newline at end of file + Path = "plik.db" + +# +# How to test server side encryption +# +# --> Setup minio with SSL +# --> See https://github.com/minio/minio/issues/6820 +# --> Create a certs directory with a self signed certificate for localhost / 127.0.0.1 +# --> openssl req -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout private.key -out public.crt +# --> Mount this directory as a volume in /root/.minio +# --> add -v $(pwd):/root/.minio \ in run.sh +# --> Uncomment InsecureSkipVerify:true in s3.go +# --> Uncomment settings above +# --> Start plik server with this config \ No newline at end of file diff --git a/testing/minio/run.sh b/testing/minio/run.sh index 9a68a208..d5d48f6d 100755 --- a/testing/minio/run.sh +++ b/testing/minio/run.sh @@ -21,6 +21,7 @@ function start { pull_docker_image echo -e "\n - Starting $DOCKER_NAME\n" + docker run -d -p "$DOCKER_PORT:9000" \ -e MINIO_ACCESS_KEY="access_key" \ -e MINIO_SECRET_KEY="access_key_secret" \