Merge pull request root-gg#283 from camathieu/master

1.2.4

.travis.yml

@@ -3,7 +3,9 @@ language: go
 sudo: required
 
 go:
-  - 1.11.1
+  - 1.11.13
+
+go_import_path: github.com/root-gg/plik
 
 before_install:
   # Update node_js version : https://github.com/travis-ci/travis-ci/issues/7108

Makefile

@@ -24,7 +24,7 @@
 # THE SOFTWARE.
 ###
 
-RELEASE_VERSION="1.2.3"
+RELEASE_VERSION="1.2.4"
 RELEASE_DIR="release/plik-$(RELEASE_VERSION)"
 RELEASE_TARGETS=darwin-386 darwin-amd64 freebsd-386 \
 freebsd-amd64 linux-386 linux-amd64 linux-arm openbsd-386 \
@@ -241,7 +241,7 @@ test:
 			FMT=`gofmt -l $$file` ; \
 			if [ "$$FMT" = "" ] ; then echo "OK" ; else echo "FAIL" && ERR="1" ; fi ; \
 		done; \
-		echo -n "go vet $$directory : "; \
+		echo -n "go tool vet $$directory : "; \
 		for file in $$(find -name "*.go" | grep -v vendor ); do \
 			echo -n " - file $$file : " ; \
 			FMT=`go tool vet $$file` ; \

README.md

@@ -24,16 +24,16 @@ Plik is a scalable & friendly temporary file upload system ( wetransfer like ) i
    - [plikSharp](https://github.com/iss0/plikSharp) : A .NET API client for Plik
 
 ### Version
-1.2.3
+1.2.4
 
 ### Installation
 
 ##### From release
 To run plik, it's very simple :
 ```sh
-$ wget https://github.com/root-gg/plik/releases/download/1.2.2/plik-1.2.2-linux-64bits.tar.gz
-$ tar xzvf plik-1.2.2-linux-64bits.tar.gz
-$ cd plik-1.2.2/server
+$ wget https://github.com/root-gg/plik/releases/download/1.2.4/plik-1.2.4-linux-64bits.tar.gz
+$ tar xzvf plik-1.2.4-linux-64bits.tar.gz
+$ cd plik-1.2.4/server
 $ ./plikd
 ```
 Et voilà ! You now have a fully functional instance of plik running on http://127.0.0.1:8080.  

changelog/1.2.4

@@ -0,0 +1,11 @@
+Hi,
+
+We're happy to release Plik 1.2.4
+
+New :
+    - Metadata exporter to migrate to plik 1.3
+
+The binaires are compiled with go 1.11.13
+
+Faithfully,
+The Plik Team.

client/test.sh

@@ -72,6 +72,7 @@ EOF
 # Run server
 ###
 
+
 echo -n "Start Plik server : "
 
 PLIKD_CONFIG=${PLIKD_CONFIG-../server/plikd.cfg}
@@ -604,13 +605,13 @@ echo "OK"
 
 #---------------------------------------------
 
-echo -n " - openssl custom options : "
-before
-cp $SPECIMEN $TMPDIR/upload/FILE1
-upload -s --secure-options '-a' && download && check
-curl $(cat $CLIENT_LOG | grep "curl" | sed -n 's/^.*"\(.*\)".*$/\1/p') >$TMPDIR/download/ARMORED 2>/dev/null
-file $TMPDIR/download/ARMORED | grep "ASCII text" >/dev/null 2>/dev/null
-echo "OK"
+#echo -n " - openssl custom options : "
+#before
+#cp $SPECIMEN $TMPDIR/upload/FILE1
+#upload -s --secure-options '-a' && download && check
+#curl $(cat $CLIENT_LOG | grep "curl" | sed -n 's/^.*"\(.*\)".*$/\1/p') >$TMPDIR/download/ARMORED 2>/dev/null
+#file $TMPDIR/download/ARMORED | grep "ASCII text" >/dev/null 2>/dev/null
+#echo "OK"
 
 ###
 # PGP

client/test_downgrade.sh

@@ -46,6 +46,7 @@ RELEASES=(
     1.2
     1.2.1
     1.2.2
+    1.2.3
 )
 
 ###

client/test_upgrade.sh

@@ -47,6 +47,7 @@ RELEASES=(
     1.2
     1.2.1
     1.2.2
+    1.2.3
 )
 
 ###

sdk/client.go

@@ -23,7 +23,7 @@ var (
 //  - Add/Remove files to the upload
 //  - Get plik server configuration
 type Client struct {
-	HttpClient *http.Client
+	HTTPClient *http.Client
 	BaseURL    *url.URL
 	serverConf *common.Configuration
 	once       *sync.Once
@@ -33,7 +33,7 @@ type Client struct {
 func NewClient(baseURL string) (c *Client, err error) {
 
 	c = new(Client)
-	c.HttpClient = new(http.Client)
+	c.HTTPClient = new(http.Client)
 	c.BaseURL, err = url.Parse(baseURL)
 	c.once = new(sync.Once)
 	if err != nil {
@@ -74,9 +74,9 @@ func (c *Client) NewUploadWithOptions(opts *UploadOptions) (upload *Upload) {
 }
 
 // GetUpload will get upload from Plik
-func (c *Client) GetUpload(uploadId string) (upload *common.Upload, err error) {
+func (c *Client) GetUpload(uploadID string) (upload *common.Upload, err error) {
 
-	path := fmt.Sprintf("/upload/%s", uploadId)
+	path := fmt.Sprintf("/upload/%s", uploadID)
 	req, err := c.newRequest("GET", path, nil)
 	if err != nil {
 		return nil, err
@@ -184,7 +184,7 @@ func (c *Client) newFileRequest(method, path string, fileName string, file io.Re
 }
 func (c *Client) do(req *http.Request, v interface{}) (*http.Response, error) {
 
-	resp, err := c.HttpClient.Do(req)
+	resp, err := c.HTTPClient.Do(req)
 	if err != nil {
 		return nil, err
 	}

sdk/upload.go

@@ -11,6 +11,7 @@ import (
 	"github.com/root-gg/plik/server/common"
 )
 
+// PlikUpload is an alias for common.Upload
 type PlikUpload = common.Upload
 
 // Upload override from a Plik common.Upload
@@ -21,7 +22,7 @@ type Upload struct {
 	client *Client
 }
 
-// UploadStatus
+// UploadStatus represents the status of a file upload
 type UploadStatus struct {
 	Error error
 	File  *File
@@ -44,6 +45,7 @@ type UploadOptions struct {
 	Yubikey             string
 }
 
+// URL of the upload
 func (u *Upload) URL() string {
 
 	if u.client == nil {

server/metadata/bolt/bolt.go

@@ -33,6 +33,7 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"fmt"
+	"github.com/root-gg/plik/server/metadata/exporter"
 	"log"
 	"time"
 
@@ -136,7 +137,7 @@ func (bmb *MetadataBackend) Create(ctx *juliet.Context, upload *common.Upload) (
 		// Expire date index
 		if upload.TTL > 0 {
 			// Expire index is build as follow :
-			//  - Expire index prefix 2 byte ( "_e" )
+			//  - Expire index pplik.dbrefix 2 byte ( "_e" )
 			//  - The expire timestamp ( 8 bytes )
 			//  - The upload id ( 16 bytes )
 			// Upload id is stored in the key to ensure uniqueness
@@ -621,3 +622,78 @@ func (bmb *MetadataBackend) GetUploadsToRemove(ctx *juliet.Context) (ids []strin
 
 	return
 }
+
+// Export implementation for bolt metadata backend
+func (bmb *MetadataBackend) Export(ctx *juliet.Context, path string) (err error) {
+	fmt.Printf("Exporting metadata from %s to %s\n", bmb.Config.Path, path)
+
+	e, err := exporter.NewExporter(path)
+	if err != nil {
+		return err
+	}
+	defer e.Close()
+
+	err = bmb.db.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte("uploads")).Cursor()
+		c.Seek([]byte{})
+		for {
+			// Scan the bucket
+			k, v := c.Next()
+			if k == nil {
+				break
+			}
+			if bytes.HasPrefix(k, []byte("_")) {
+				// Skip index entries
+				continue
+			}
+
+			// Unserialize metadata from json
+			upload := new(common.Upload)
+			if err = json.Unmarshal(v, upload); err != nil {
+				return fmt.Errorf("Unable to unserialize upload metadata from json \"%s\" : %s", string(v), err)
+			}
+
+			err = e.AddUpload(upload)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	err = bmb.db.View(func(tx *bolt.Tx) error {
+		c := tx.Bucket([]byte("users")).Cursor()
+		c.Seek([]byte{})
+		for {
+			// Scan the bucket
+			k, v := c.Next()
+			if k == nil {
+				break
+			}
+			if !(bytes.HasPrefix(k, []byte("google")) || bytes.HasPrefix(k, []byte("ovh"))) {
+				// Skip index entries
+				continue
+			}
+
+			// Unserialize metadata from json
+			user := new(common.User)
+			if err = json.Unmarshal(v, user); err != nil {
+				return fmt.Errorf("Unable to unserialize user metadata from json \"%s\" : %s", string(v), err)
+			}
+
+			err = e.AddUser(user)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}