diff --git a/clients/storage/blob.go b/clients/storage/blob.go
new file mode 100644
index 000000000000..0c66b493b841
--- /dev/null
+++ b/clients/storage/blob.go
@@ -0,0 +1,677 @@
+package storage
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+type BlobStorageClient struct {
+ client StorageClient
+}
+
+type Container struct {
+ Name string `xml:"Name"`
+ Properties ContainerProperties `xml:"Properties"`
+ // TODO (ahmetalpbalkan) Metadata
+}
+
+type Blob struct {
+ Name string `xml:"Name"`
+ Properties BlobProperties `xml:"Properties"`
+ // TODO (ahmetalpbalkan) Metadata
+}
+
+type ContainerProperties struct {
+ LastModified string `xml:"Last-Modified"`
+ Etag string `xml:"Etag"`
+ LeaseStatus string `xml:"LeaseStatus"`
+ LeaseState string `xml:"LeaseState"`
+ LeaseDuration string `xml:"LeaseDuration"`
+ // TODO (ahmetalpbalkan) remaining fields
+}
+
+type BlobProperties struct {
+ LastModified string `xml:"Last-Modified"`
+ Etag string `xml:"Etag"`
+ ContentMD5 string `xml:"Content-MD5"`
+ ContentLength uint64 `xml:"Content-Length"`
+ ContentType string `xml:"Content-Type"`
+ ContentEncoding string `xml:"Content-Encoding"`
+ SequenceNumber int64 `xml:"x-ms-blob-sequence-number"`
+ CopyId string `xml:"CopyId"`
+ CopyStatus string `xml:"CopyStatus"`
+ CopySource string `xml:"CopySource"`
+ CopyProgress string `xml:"CopyProgress"`
+ CopyCompletionTime string `xml:"CopyCompletionTime"`
+ CopyStatusDescription string `xml:"CopyStatusDescription"`
+}
+
+type BlobListResponse struct {
+ XMLName xml.Name `xml:"EnumerationResults"`
+ Xmlns string `xml:"xmlns,attr"`
+ Prefix string `xml:"Prefix"`
+ Marker string `xml:"Marker"`
+ NextMarker string `xml:"NextMarker"`
+ MaxResults int64 `xml:"MaxResults"`
+ Blobs []Blob `xml:"Blobs>Blob"`
+}
+
+type ContainerListResponse struct {
+ XMLName xml.Name `xml:"EnumerationResults"`
+ Xmlns string `xml:"xmlns,attr"`
+ Prefix string `xml:"Prefix"`
+ Marker string `xml:"Marker"`
+ NextMarker string `xml:"NextMarker"`
+ MaxResults int64 `xml:"MaxResults"`
+ Containers []Container `xml:"Containers>Container"`
+}
+
+type ListContainersParameters struct {
+ Prefix string
+ Marker string
+ Include string
+ MaxResults uint
+ Timeout uint
+}
+
+func (p ListContainersParameters) GetParameters() url.Values {
+ out := url.Values{}
+
+ if p.Prefix != "" {
+ out.Set("prefix", p.Prefix)
+ }
+ if p.Marker != "" {
+ out.Set("marker", p.Marker)
+ }
+ if p.Include != "" {
+ out.Set("include", p.Include)
+ }
+ if p.MaxResults != 0 {
+ out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults))
+ }
+ if p.Timeout != 0 {
+ out.Set("timeout", fmt.Sprintf("%v", p.Timeout))
+ }
+
+ return out
+}
+
+type ListBlobsParameters struct {
+ Prefix string
+ Delimiter string
+ Marker string
+ Include string
+ MaxResults uint
+ Timeout uint
+}
+
+func (p ListBlobsParameters) GetParameters() url.Values {
+ out := url.Values{}
+
+ if p.Prefix != "" {
+ out.Set("prefix", p.Prefix)
+ }
+ if p.Delimiter != "" {
+ out.Set("delimiter", p.Delimiter)
+ }
+ if p.Marker != "" {
+ out.Set("marker", p.Marker)
+ }
+ if p.Include != "" {
+ out.Set("include", p.Include)
+ }
+ if p.MaxResults != 0 {
+ out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults))
+ }
+ if p.Timeout != 0 {
+ out.Set("timeout", fmt.Sprintf("%v", p.Timeout))
+ }
+
+ return out
+}
+
+type BlobType string
+
+const (
+ BlobTypeBlock BlobType = "BlockBlob"
+ BlobTypePage BlobType = "PageBlob"
+)
+
+const (
+ blobCopyStatusPending = "pending"
+ blobCopyStatusSuccess = "success"
+ blobCopyStatusAborted = "aborted"
+ blobCopyStatusFailed = "failed"
+)
+
+type BlockListType string
+
+const (
+ BlockListTypeAll BlockListType = "all"
+ BlockListTypeCommitted BlockListType = "committed"
+ BlockListTypeUncommitted BlockListType = "uncommitted"
+)
+
+type ContainerAccessType string
+
+const (
+ ContainerAccessTypePrivate ContainerAccessType = ""
+ ContainerAccessTypeBlob ContainerAccessType = "blob"
+ ContainerAccessTypeContainer ContainerAccessType = "container"
+)
+
+const MaxBlobBlockSize = 4 * 1024 * 1024
+
+type BlockStatus string
+
+const (
+ BlockStatusUncommitted BlockStatus = "Uncommitted"
+ BlockStatusCommitted BlockStatus = "Committed"
+ BlockStatusLatest BlockStatus = "Latest"
+)
+
+type Block struct {
+ Id string
+ Status BlockStatus
+}
+
+type BlockListResponse struct {
+ XMLName xml.Name `xml:"BlockList"`
+ CommittedBlocks []BlockResponse `xml:"CommittedBlocks>Block"`
+ UncommittedBlocks []BlockResponse `xml:"UncommittedBlocks>Block"`
+}
+
+type BlockResponse struct {
+ Name string `xml:"Name"`
+ Size uint64 `xml:"Size"`
+}
+
+var (
+ ErrNotCreated = errors.New("storage: operation has returned a successful error code other than 201 Created.")
+ ErrNotAccepted = errors.New("storage: operation has returned a successful error code other than 202 Accepted.")
+
+ errBlobCopyAborted = errors.New("storage: blob copy is aborted")
+ errBlobCopyIdMismatch = errors.New("storage: blob copy id is a mismatch")
+)
+
+const errUnexpectedStatus = "storage: was expecting status code: %d, got: %d"
+
+func (b BlobStorageClient) ListContainers(params ListContainersParameters) (ContainerListResponse, error) {
+ q := mergeParams(params.GetParameters(), url.Values{"comp": {"list"}})
+ uri := b.client.getEndpoint(blobServiceName, "", q)
+ headers := b.client.getStandardHeaders()
+
+ var out ContainerListResponse
+ resp, err := b.client.exec("GET", uri, headers, nil)
+ if err != nil {
+ return out, err
+ }
+
+ err = xmlUnmarshal(resp.body, &out)
+ return out, err
+}
+
+func (b BlobStorageClient) CreateContainer(name string, access ContainerAccessType) error {
+ resp, err := b.createContainer(name, access)
+ if err != nil {
+ return err
+ }
+ if resp.statusCode != http.StatusCreated {
+ return ErrNotCreated
+ }
+ return nil
+}
+
+func (b BlobStorageClient) CreateContainerIfNotExists(name string, access ContainerAccessType) (bool, error) {
+ resp, err := b.createContainer(name, access)
+ if resp != nil && (resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict) {
+ return resp.statusCode == http.StatusCreated, nil
+ }
+ return false, err
+}
+
+func (b BlobStorageClient) createContainer(name string, access ContainerAccessType) (*storageResponse, error) {
+ verb := "PUT"
+ uri := b.client.getEndpoint(blobServiceName, name, url.Values{"restype": {"container"}})
+
+ headers := b.client.getStandardHeaders()
+ headers["Content-Length"] = "0"
+ if access != "" {
+ headers["x-ms-blob-public-access"] = string(access)
+ }
+ return b.client.exec(verb, uri, headers, nil)
+}
+
+func (b BlobStorageClient) ContainerExists(container string) (bool, error) {
+ verb := "HEAD"
+ path := fmt.Sprintf("%s", container)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{"restype": {"container"}})
+ headers := b.client.getStandardHeaders()
+
+ resp, err := b.client.exec(verb, uri, headers, nil)
+ if resp != nil && (resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound) {
+ return resp.statusCode == http.StatusOK, nil
+ }
+ return false, err
+}
+
+func (b BlobStorageClient) DeleteContainer(name string) error {
+ resp, err := b.deleteContainer(name)
+ if err != nil {
+ return err
+ }
+ if resp.statusCode != http.StatusAccepted {
+ return ErrNotAccepted
+ }
+ return nil
+}
+
+func (b BlobStorageClient) DeleteContainerIfExists(container string) (bool, error) {
+ resp, err := b.deleteContainer(container)
+ if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) {
+ return resp.statusCode == http.StatusAccepted, nil
+ }
+ return false, err
+}
+
+func (b BlobStorageClient) deleteContainer(name string) (*storageResponse, error) {
+ verb := "DELETE"
+ uri := b.client.getEndpoint(blobServiceName, name, url.Values{"restype": {"container"}})
+
+ headers := b.client.getStandardHeaders()
+ return b.client.exec(verb, uri, headers, nil)
+}
+
+func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameters) (BlobListResponse, error) {
+ q := mergeParams(params.GetParameters(), url.Values{
+ "restype": {"container"},
+ "comp": {"list"}})
+ uri := b.client.getEndpoint(blobServiceName, container, q)
+ headers := b.client.getStandardHeaders()
+
+ var out BlobListResponse
+ resp, err := b.client.exec("GET", uri, headers, nil)
+ if err != nil {
+ return out, err
+ }
+
+ err = xmlUnmarshal(resp.body, &out)
+ return out, err
+}
+
+func (b BlobStorageClient) BlobExists(container, name string) (bool, error) {
+ verb := "HEAD"
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
+
+ headers := b.client.getStandardHeaders()
+ resp, err := b.client.exec(verb, uri, headers, nil)
+ if resp != nil && (resp.statusCode == http.StatusOK || resp.statusCode == http.StatusNotFound) {
+ return resp.statusCode == http.StatusOK, nil
+ }
+ return false, err
+}
+
+func (b BlobStorageClient) GetBlobUrl(container, name string) string {
+ if container == "" {
+ container = "$root"
+ }
+ path := fmt.Sprintf("%s/%s", container, name)
+ return b.client.getEndpoint(blobServiceName, path, url.Values{})
+}
+
+func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error) {
+ resp, err := b.getBlobRange(container, name, "")
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.statusCode != http.StatusOK {
+ return nil, fmt.Errorf(errUnexpectedStatus, http.StatusOK, resp.statusCode)
+ }
+ return resp.body, nil
+}
+
+func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string) (io.ReadCloser, error) {
+ resp, err := b.getBlobRange(container, name, bytesRange)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.statusCode != http.StatusPartialContent {
+ return nil, fmt.Errorf(errUnexpectedStatus, http.StatusPartialContent, resp.statusCode)
+ }
+ return resp.body, nil
+}
+
+func (b BlobStorageClient) getBlobRange(container, name, bytesRange string) (*storageResponse, error) {
+ verb := "GET"
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
+
+ headers := b.client.getStandardHeaders()
+ if bytesRange != "" {
+ headers["Range"] = fmt.Sprintf("bytes=%s", bytesRange)
+ }
+ resp, err := b.client.exec(verb, uri, headers, nil)
+ if err != nil {
+ return nil, err
+ }
+ return resp, err
+}
+
+func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobProperties, error) {
+ verb := "HEAD"
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
+
+ headers := b.client.getStandardHeaders()
+ resp, err := b.client.exec(verb, uri, headers, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ if resp.statusCode != http.StatusOK {
+ return nil, fmt.Errorf(errUnexpectedStatus, http.StatusOK, resp.statusCode)
+ }
+
+ var contentLength uint64
+ contentLengthStr := resp.headers.Get("Content-Length")
+ if contentLengthStr != "" {
+ contentLength, err = strconv.ParseUint(contentLengthStr, 0, 64)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ var sequenceNum int64
+ sequenceNumStr := resp.headers.Get("Content-Length")
+ if sequenceNumStr != "" {
+ sequenceNum, err = strconv.ParseInt(sequenceNumStr, 0, 64)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return &BlobProperties{
+ LastModified: resp.headers.Get("Last-Modified"),
+ Etag: resp.headers.Get("Etag"),
+ ContentMD5: resp.headers.Get("Content-MD5"),
+ ContentLength: contentLength,
+ ContentEncoding: resp.headers.Get("Content-Encodng"),
+ SequenceNumber: sequenceNum,
+ CopyCompletionTime: resp.headers.Get("x-ms-copy-completion-time"),
+ CopyStatusDescription: resp.headers.Get("x-ms-copy-status-description"),
+ CopyId: resp.headers.Get("x-ms-copy-id"),
+ CopyProgress: resp.headers.Get("x-ms-copy-progress"),
+ CopySource: resp.headers.Get("x-ms-copy-source"),
+ CopyStatus: resp.headers.Get("x-ms-copy-status"),
+ }, nil
+}
+
+func (b BlobStorageClient) PutBlockBlob(container, name string, blob io.Reader) error { // TODO (ahmetalpbalkan) consider ReadCloser and closing
+ return b.putBlockBlob(container, name, blob, MaxBlobBlockSize)
+}
+
+func (b BlobStorageClient) putBlockBlob(container, name string, blob io.Reader, chunkSize int) error {
+ if chunkSize <= 0 || chunkSize > MaxBlobBlockSize {
+ chunkSize = MaxBlobBlockSize
+ }
+
+ chunk := make([]byte, chunkSize)
+ n, err := blob.Read(chunk)
+ if err != nil && err != io.EOF {
+ return err
+ }
+
+ if err == io.EOF {
+ // Fits into one block
+ return b.putSingleBlockBlob(container, name, chunk[:n])
+ } else {
+ // Does not fit into one block. Upload block by block then commit the block list
+ blockList := []Block{}
+ blockNum := 0
+
+ // Put blocks
+ for {
+ id := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v", blockNum)))
+ data := chunk[:n]
+ err = b.PutBlock(container, name, id, data)
+ if err != nil {
+ return err
+ }
+ blockList = append(blockList, Block{id, BlockStatusLatest})
+
+ // Read next block
+ n, err = blob.Read(chunk)
+ if err != nil && err != io.EOF {
+ return err
+ }
+ if err == io.EOF {
+ break
+ }
+
+ blockNum++
+ }
+
+ // Commit block list
+ return b.PutBlockList(container, name, blockList)
+ }
+}
+
+func (b BlobStorageClient) putSingleBlockBlob(container, name string, chunk []byte) error {
+ if len(chunk) > MaxBlobBlockSize {
+ return fmt.Errorf("storage: provided chunk (%d bytes) cannot fit into single-block blob (max %d bytes)", len(chunk), MaxBlobBlockSize)
+ }
+
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
+ headers := b.client.getStandardHeaders()
+ headers["x-ms-blob-type"] = string(BlobTypeBlock)
+ headers["Content-Length"] = fmt.Sprintf("%v", len(chunk))
+
+ resp, err := b.client.exec("PUT", uri, headers, bytes.NewReader(chunk))
+ if err != nil {
+ return err
+ }
+ if resp.statusCode != http.StatusCreated {
+ return ErrNotCreated
+ }
+
+ return nil
+}
+
+func (b BlobStorageClient) PutBlock(container, name, blockId string, chunk []byte) error {
+ return b.PutBlockWithLength(container, name, blockId, uint64(len(chunk)), bytes.NewReader(chunk))
+}
+
+func (b BlobStorageClient) PutBlockWithLength(container, name, blockId string, size uint64, blob io.Reader) error {
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"block"}, "blockid": {blockId}})
+ headers := b.client.getStandardHeaders()
+ headers["x-ms-blob-type"] = string(BlobTypeBlock)
+ headers["Content-Length"] = fmt.Sprintf("%v", size)
+
+ resp, err := b.client.exec("PUT", uri, headers, blob)
+ if err != nil {
+ return err
+ }
+ if resp.statusCode != http.StatusCreated {
+ return ErrNotCreated
+ }
+
+ return nil
+}
+
+func (b BlobStorageClient) PutBlockList(container, name string, blocks []Block) error {
+ blockListXml := prepareBlockListRequest(blocks)
+
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"blocklist"}})
+ headers := b.client.getStandardHeaders()
+ headers["Content-Length"] = fmt.Sprintf("%v", len(blockListXml))
+
+ resp, err := b.client.exec("PUT", uri, headers, strings.NewReader(blockListXml))
+ if err != nil {
+ return err
+ }
+ if resp.statusCode != http.StatusCreated {
+ return ErrNotCreated
+ }
+ return nil
+}
+
+func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockListType) (BlockListResponse, error) {
+ params := url.Values{"comp": {"blocklist"}, "blocklisttype": {string(blockType)}}
+ uri := b.client.getEndpoint(blobServiceName, fmt.Sprintf("%s/%s", container, name), params)
+ headers := b.client.getStandardHeaders()
+
+ var out BlockListResponse
+ resp, err := b.client.exec("GET", uri, headers, nil)
+ if err != nil {
+ return out, err
+ }
+
+ err = xmlUnmarshal(resp.body, &out)
+ return out, err
+}
+
+func (b BlobStorageClient) CopyBlob(container, name, sourceBlob string) error {
+ copyId, err := b.startBlobCopy(container, name, sourceBlob)
+ if err != nil {
+ return err
+ }
+
+ return b.waitForBlobCopy(container, name, copyId)
+}
+
+func (b BlobStorageClient) startBlobCopy(container, name, sourceBlob string) (string, error) {
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
+
+ headers := b.client.getStandardHeaders()
+ headers["Content-Length"] = "0"
+ headers["x-ms-copy-source"] = sourceBlob
+
+ resp, err := b.client.exec("PUT", uri, headers, nil)
+ if err != nil {
+ return "", err
+ }
+ if resp.statusCode != http.StatusAccepted && resp.statusCode != http.StatusCreated {
+ return "", fmt.Errorf(errUnexpectedStatus, []int{http.StatusAccepted, http.StatusCreated}, resp.statusCode)
+ }
+
+ copyId := resp.headers.Get("x-ms-copy-id")
+ if copyId == "" {
+ return "", errors.New("Got empty copy id header")
+ }
+ return copyId, nil
+}
+
+func (b BlobStorageClient) waitForBlobCopy(container, name, copyId string) error {
+ for {
+ props, err := b.GetBlobProperties(container, name)
+ if err != nil {
+ return err
+ }
+
+ if props.CopyId != copyId {
+ return errBlobCopyIdMismatch
+ }
+
+ switch props.CopyStatus {
+ case blobCopyStatusSuccess:
+ return nil
+ case blobCopyStatusPending:
+ continue
+ case blobCopyStatusAborted:
+ return errBlobCopyAborted
+ case blobCopyStatusFailed:
+ return fmt.Errorf("storage: blob copy failed. Id=%s Description=%s", props.CopyId, props.CopyStatusDescription)
+ default:
+ return fmt.Errorf("storage: unhandled blob copy status: '%s'", props.CopyStatus)
+ }
+ }
+}
+
+func (b BlobStorageClient) DeleteBlob(container, name string) error {
+ resp, err := b.deleteBlob(container, name)
+ if err != nil {
+ return err
+ }
+ if resp.statusCode != http.StatusAccepted {
+ return ErrNotAccepted
+ }
+ return nil
+}
+
+func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) {
+ var (
+ signedPermissions = permissions
+ blobUrl = b.GetBlobUrl(container, name)
+ )
+ canonicalizedResource, err := b.client.buildCanonicalizedResource(blobUrl)
+ if err != nil {
+ return "", err
+ }
+ signedExpiry := expiry.Format(time.RFC3339)
+ signedResource := "b"
+
+ stringToSign, err := blobSASStringToSign(b.client.apiVersion, canonicalizedResource, signedExpiry, signedPermissions)
+ if err != nil {
+ return "", err
+ }
+
+ sig := b.client.computeHmac256(stringToSign)
+ sasParams := url.Values{
+ "sv": {b.client.apiVersion},
+ "se": {signedExpiry},
+ "sr": {signedResource},
+ "sp": {signedPermissions},
+ "sig": {sig},
+ }
+
+ sasUrl, err := url.Parse(blobUrl)
+ if err != nil {
+ return "", err
+ }
+ sasUrl.RawQuery = sasParams.Encode()
+ return sasUrl.String(), nil
+}
+
+func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, signedPermissions string) (string, error) {
+ var signedStart, signedIdentifier, rscc, rscd, rsce, rscl, rsct string
+
+ // reference: http://msdn.microsoft.com/en-us/library/azure/dn140255.aspx
+ if signedVersion >= "2013-08-15" {
+ return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s", signedPermissions, signedStart, signedExpiry, canonicalizedResource, signedIdentifier, signedVersion, rscc, rscd, rsce, rscl, rsct), nil
+ } else {
+ return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15")
+ }
+}
+
+func (b BlobStorageClient) DeleteBlobIfExists(container, name string) (bool, error) {
+ resp, err := b.deleteBlob(container, name)
+ if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) {
+ return resp.statusCode == http.StatusAccepted, nil
+ }
+ return false, err
+}
+
+func (b BlobStorageClient) deleteBlob(container, name string) (*storageResponse, error) {
+ verb := "DELETE"
+ path := fmt.Sprintf("%s/%s", container, name)
+ uri := b.client.getEndpoint(blobServiceName, path, url.Values{})
+
+ headers := b.client.getStandardHeaders()
+ return b.client.exec(verb, uri, headers, nil)
+}
diff --git a/clients/storage/blob_test.go b/clients/storage/blob_test.go
new file mode 100644
index 000000000000..e017f251c1c0
--- /dev/null
+++ b/clients/storage/blob_test.go
@@ -0,0 +1,882 @@
+package storage
+
+import (
+ "bytes"
+ "crypto/rand"
+ "encoding/base64"
+ "errors"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "os"
+ "reflect"
+ "sort"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+const testContainerPrefix = "zzzztest-"
+
+func TestblobSASStringToSign(t *testing.T) {
+ _, err := blobSASStringToSign("2012-02-12", "CS", "SE", "SP")
+ if err == nil {
+ t.Fatal("Expected error, got nil")
+ }
+
+ out, err := blobSASStringToSign("2013-08-15", "CS", "SE", "SP")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if expected := "SP\n\nSE\nCS\n\n\n2013-08-15\n%s\n%s\n%s\n%s\n%s"; out != expected {
+ t.Errorf("Wrong stringToSign. Expected: '%s', got: '%s'", expected, out)
+ }
+}
+
+func TestGetBlobSASURI(t *testing.T) {
+ api, err := NewClient("foo", "YmFy", DefaultBaseUrl, "2013-08-15", true)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cli := api.GetBlobService()
+ expiry := time.Time{}
+
+ expectedParts := url.URL{
+ Scheme: "https",
+ Host: "foo.blob.core.windows.net",
+ Path: "container/name",
+ RawQuery: url.Values{
+ "sv": {"2013-08-15"},
+ "sig": {"/OXG7rWh08jYwtU03GzJM0DHZtidRGpC6g69rSGm3I0="},
+ "sr": {"b"},
+ "sp": {"r"},
+ "se": {"0001-01-01T00:00:00Z"},
+ }.Encode()}
+
+ u, err := cli.GetBlobSASURI("container", "name", expiry, "r")
+ if err != nil {
+ t.Fatal(err)
+ }
+ sasParts, err := url.Parse(u)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ expectedQuery := expectedParts.Query()
+ sasQuery := sasParts.Query()
+
+ expectedParts.RawQuery = "" // reset
+ sasParts.RawQuery = ""
+
+ if expectedParts.String() != sasParts.String() {
+ t.Fatalf("Base URL wrong for SAS. Expected: '%s', got: '%s'", expectedParts, sasParts)
+ }
+
+ if len(expectedQuery) != len(sasQuery) {
+ t.Fatalf("Query string wrong for SAS URL. Expected: '%d keys', got: '%d keys'", len(expectedQuery), len(sasQuery))
+ }
+
+ for k, v := range expectedQuery {
+ out, ok := sasQuery[k]
+ if !ok {
+ t.Fatalf("Query parameter '%s' not found in generated SAS query. Expected: '%s'", k, v)
+ }
+ if !reflect.DeepEqual(v, out) {
+ t.Fatalf("Wrong value for query parameter '%s'. Expected: '%s', got: '%s'", k, v, out)
+ }
+ }
+}
+
+func TestBlobSASURICorrectness(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+ cnt := randContainer()
+ blob := randString(20)
+ body := []byte(randString(100))
+ expiry := time.Now().UTC().Add(time.Hour)
+ permissions := "r"
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ err = cli.PutBlockBlob(cnt, blob, bytes.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ sasUri, err := cli.GetBlobSASURI(cnt, blob, expiry, permissions)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ resp, err := http.Get(sasUri)
+ if err != nil {
+ t.Logf("SAS URI: %s", sasUri)
+ t.Fatal(err)
+ }
+
+ blobResp, err := ioutil.ReadAll(resp.Body)
+ defer resp.Body.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if resp.StatusCode != http.StatusOK {
+ t.Fatalf("Non-ok status code: %s", resp.Status)
+ }
+
+ if len(blobResp) != len(body) {
+ t.Fatalf("Wrong blob size on SAS URI. Expected: %d, Got: %d", len(body), len(blobResp))
+ }
+}
+
+func TestListContainersPagination(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = deleteTestContainers(cli)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ const n = 5
+ const pageSize = 2
+
+ // Create test containers
+ created := []string{}
+ for i := 0; i < n; i++ {
+ name := randContainer()
+ err := cli.CreateContainer(name, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatalf("Error creating test container: %s", err)
+ }
+ created = append(created, name)
+ }
+ sort.Strings(created)
+
+ // Defer test container deletions
+ defer func() {
+ var wg sync.WaitGroup
+ for _, cnt := range created {
+ wg.Add(1)
+ go func(name string) {
+ err := cli.DeleteContainer(name)
+ if err != nil {
+ t.Logf("Error while deleting test container: %s", err)
+ }
+ wg.Done()
+ }(cnt)
+ }
+ wg.Wait()
+ }()
+
+ // Paginate results
+ seen := []string{}
+ marker := ""
+ for {
+ resp, err := cli.ListContainers(ListContainersParameters{
+ Prefix: testContainerPrefix,
+ MaxResults: pageSize,
+ Marker: marker})
+
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ containers := resp.Containers
+
+ if len(containers) > pageSize {
+ t.Fatalf("Got a bigger page. Expected: %d, got: %d", pageSize, len(containers))
+ }
+
+ for _, c := range containers {
+ seen = append(seen, c.Name)
+ }
+
+ marker = resp.NextMarker
+ if marker == "" || len(containers) == 0 {
+ break
+ }
+ }
+
+ // Compare
+ if !reflect.DeepEqual(created, seen) {
+ t.Fatal("Wrong pagination results:\nExpected:\t\t%v\nGot:\t\t%v", created, seen)
+ }
+}
+
+func TestContainerExists(t *testing.T) {
+ cnt := randContainer()
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ ok, err := cli.ContainerExists(cnt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Fatalf("Non-existing container returned as existing: %s", cnt)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypeBlob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ ok, err = cli.ContainerExists(cnt)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Fatalf("Existing container returned as non-existing: %s", cnt)
+ }
+}
+
+func TestCreateDeleteContainer(t *testing.T) {
+ cnt := randContainer()
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ err = cli.DeleteContainer(cnt)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestCreateContainerIfNotExists(t *testing.T) {
+ cnt := randContainer()
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // First create
+ ok, err := cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if expected := true; ok != expected {
+ t.Fatalf("Wrong creation status. Expected: %v; Got: %v", expected, ok)
+ }
+
+ // Second create, should not give errors
+ ok, err = cli.CreateContainerIfNotExists(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if expected := false; ok != expected {
+ t.Fatalf("Wrong creation status. Expected: %v; Got: %v", expected, ok)
+ }
+
+ defer cli.DeleteContainer(cnt)
+}
+
+func TestDeleteContainerIfExists(t *testing.T) {
+ cnt := randContainer()
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Nonexisting container
+ err = cli.DeleteContainer(cnt)
+ if err == nil {
+ t.Fatal("Expected error, got nil")
+ }
+
+ ok, err := cli.DeleteContainerIfExists(cnt)
+ if err != nil {
+ t.Fatalf("Not supposed to return error, got: %s", err)
+ }
+ if expected := false; ok != expected {
+ t.Fatal("Wrong deletion status. Expected: %v; Got: %v", expected, ok)
+ }
+
+ // Existing container
+ err = cli.CreateContainer(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ok, err = cli.DeleteContainerIfExists(cnt)
+ if err != nil {
+ t.Fatalf("Not supposed to return error, got: %s", err)
+ }
+ if expected := true; ok != expected {
+ t.Fatal("Wrong deletion status. Expected: %v; Got: %v", expected, ok)
+ }
+}
+
+func TestBlobExists(t *testing.T) {
+ cnt := randContainer()
+ blob := randString(20)
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypeBlob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+ err = cli.PutBlockBlob(cnt, blob, strings.NewReader("Hello!"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteBlob(cnt, blob)
+
+ ok, err := cli.BlobExists(cnt, blob+".foo")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if ok {
+ t.Errorf("Non-existing blob returned as existing: %s/%s", cnt, blob)
+ }
+
+ ok, err = cli.BlobExists(cnt, blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !ok {
+ t.Errorf("Existing blob returned as non-existing: %s/%s", cnt, blob)
+ }
+}
+
+func TestGetBlobUrl(t *testing.T) {
+ api, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ cli := api.GetBlobService()
+
+ out := cli.GetBlobUrl("c", "nested/blob")
+ if expected := "https://foo.blob.core.windows.net/c/nested/blob"; out != expected {
+ t.Fatalf("Wrong blob URL. Expected: '%s', got:'%s'", expected, out)
+ }
+
+ out = cli.GetBlobUrl("", "blob")
+ if expected := "https://foo.blob.core.windows.net/$root/blob"; out != expected {
+ t.Fatalf("Wrong blob URL. Expected: '%s', got:'%s'", expected, out)
+ }
+
+ out = cli.GetBlobUrl("", "nested/blob")
+ if expected := "https://foo.blob.core.windows.net/$root/nested/blob"; out != expected {
+ t.Fatalf("Wrong blob URL. Expected: '%s', got:'%s'", expected, out)
+ }
+}
+
+func TestBlobCopy(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping blob copy in short mode, no SLA on async operation")
+ }
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cnt := randContainer()
+ src := randString(20)
+ dst := randString(20)
+ body := []byte(randString(1024))
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.deleteContainer(cnt)
+
+ err = cli.PutBlockBlob(cnt, src, bytes.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteBlob(cnt, src)
+
+ err = cli.CopyBlob(cnt, dst, cli.GetBlobUrl(cnt, src))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteBlob(cnt, dst)
+
+ blobBody, err := cli.GetBlob(cnt, dst)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ b, err := ioutil.ReadAll(blobBody)
+ defer blobBody.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !reflect.DeepEqual(body, b) {
+ t.Fatalf("Copied blob is wrong. Expected: %d bytes, got: %d bytes\n%s\n%s", len(body), len(b), body, b)
+ }
+}
+
+func TestDeleteBlobIfExists(t *testing.T) {
+ cnt := randContainer()
+ blob := randString(20)
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.DeleteBlob(cnt, blob)
+ if err == nil {
+ t.Fatal("Nonexisting blob did not return error")
+ }
+
+ ok, err := cli.DeleteBlobIfExists(cnt, blob)
+ if err != nil {
+ t.Fatalf("Not supposed to return error: %s", err)
+ }
+ if expected := false; ok != expected {
+ t.Fatalf("Wrong deletion status. Expected: %v; Got: %v", expected, ok)
+ }
+}
+
+func TestGetBlobProperies(t *testing.T) {
+ cnt := randContainer()
+ blob := randString(20)
+ contents := randString(64)
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal("Nonexisting blob did not return error")
+ }
+
+ // Nonexisting blob
+ _, err = cli.GetBlobProperties(cnt, blob)
+ if err == nil {
+ t.Fatal("Did not return error for non-existing blob")
+ }
+
+ // Put the blob
+ err = cli.PutBlockBlob(cnt, blob, strings.NewReader(contents))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Get blob properties
+ props, err := cli.GetBlobProperties(cnt, blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if props.ContentLength != uint64(len(contents)) {
+ t.Fatalf("Got wrong Content-Length: '%d', expected: %d", props.ContentLength, len(contents))
+ }
+}
+
+func TestListBlobsPagination(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cnt := randContainer()
+ err = cli.CreateContainer(cnt, ContainerAccessTypePrivate)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ blobs := []string{}
+ const n = 5
+ const pageSize = 2
+ for i := 0; i < n; i++ {
+ name := randString(20)
+ err := cli.PutBlockBlob(cnt, name, strings.NewReader("Hello, world!"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ blobs = append(blobs, name)
+ }
+ sort.Strings(blobs)
+
+ // Paginate
+ seen := []string{}
+ marker := ""
+ for {
+ resp, err := cli.ListBlobs(cnt, ListBlobsParameters{
+ MaxResults: pageSize,
+ Marker: marker})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, v := range resp.Blobs {
+ seen = append(seen, v.Name)
+ }
+
+ marker = resp.NextMarker
+ if marker == "" || len(resp.Blobs) == 0 {
+ break
+ }
+ }
+
+ // Compare
+ if !reflect.DeepEqual(blobs, seen) {
+ t.Fatalf("Got wrong list of blobs. Expected: %s, Got: %s", blobs, seen)
+ }
+
+ err = cli.DeleteContainer(cnt)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPutEmptyBlockBlob(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cnt := randContainer()
+ if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil {
+ t.Fatal(err)
+ }
+ defer cli.deleteContainer(cnt)
+
+ blob := randString(20)
+ err = cli.PutBlockBlob(cnt, blob, bytes.NewReader([]byte{}))
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ props, err := cli.GetBlobProperties(cnt, blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if props.ContentLength != 0 {
+ t.Fatal("Wrong content length for empty blob: %s", props.ContentLength)
+ }
+}
+
+func TestPutSingleBlockBlob(t *testing.T) {
+ cnt := randContainer()
+ blob := randString(20)
+ body := []byte(randString(1024))
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypeBlob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ err = cli.PutBlockBlob(cnt, blob, bytes.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteBlob(cnt, blob)
+
+ resp, err := cli.GetBlob(cnt, blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify contents
+ respBody, err := ioutil.ReadAll(resp)
+ defer resp.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if !reflect.DeepEqual(body, respBody) {
+ t.Fatalf("Wrong blob contents.\nExpected: %d bytes, Got: %d byes", len(body), len(respBody))
+ }
+
+ // Verify block list
+ blocks, err := cli.GetBlockList(cnt, blob, BlockListTypeAll)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if expected := 1; len(blocks.CommittedBlocks) != expected {
+ t.Fatalf("Wrong committed block count. Expected: %d, Got: %d", expected, len(blocks.CommittedBlocks))
+ }
+ if expected := 0; len(blocks.UncommittedBlocks) != expected {
+ t.Fatalf("Wrong unccommitted block count. Expected: %d, Got: %d", expected, len(blocks.UncommittedBlocks))
+ }
+ thatBlock := blocks.CommittedBlocks[0]
+ if expected := base64.StdEncoding.EncodeToString([]byte("0")); thatBlock.Name != expected {
+ t.Fatalf("Wrong block name. Expected: %s, Got: %s", expected, thatBlock.Name)
+ }
+}
+
+func TestGetBlobRange(t *testing.T) {
+ cnt := randContainer()
+ blob := randString(20)
+ body := "0123456789"
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypeBlob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ err = cli.PutBlockBlob(cnt, blob, strings.NewReader(body))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteBlob(cnt, blob)
+
+ // Read 1-3
+ for _, r := range []struct {
+ rangeStr string
+ expected string
+ }{
+ {"0-", body},
+ {"1-3", body[1 : 3+1]},
+ {"3-", body[3:]},
+ } {
+ resp, err := cli.GetBlobRange(cnt, blob, r.rangeStr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ blobBody, err := ioutil.ReadAll(resp)
+ if err != nil {
+ t.Fatal(err)
+ }
+ str := string(blobBody)
+ if str != r.expected {
+ t.Fatalf("Got wrong range. Expected: '%s'; Got:'%s'", r.expected, str)
+ }
+ }
+}
+
+func TestPutBlock(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cnt := randContainer()
+ if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil {
+ t.Fatal(err)
+ }
+ defer cli.deleteContainer(cnt)
+
+ blob := randString(20)
+ chunk := []byte(randString(1024))
+ blockId := base64.StdEncoding.EncodeToString([]byte("foo"))
+ err = cli.PutBlock(cnt, blob, blockId, chunk)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestPutMultiBlockBlob(t *testing.T) {
+ var (
+ cnt = randContainer()
+ blob = randString(20)
+ blockSize = 32 * 1024 // 32 KB
+ body = []byte(randString(blockSize*2 + blockSize/2)) // 3 blocks
+ )
+
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.CreateContainer(cnt, ContainerAccessTypeBlob)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteContainer(cnt)
+
+ err = cli.putBlockBlob(cnt, blob, bytes.NewReader(body), blockSize)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.DeleteBlob(cnt, blob)
+
+ resp, err := cli.GetBlob(cnt, blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Verify contents
+ respBody, err := ioutil.ReadAll(resp)
+ defer resp.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(body, respBody) {
+ t.Fatalf("Wrong blob contents.\nExpected: %d bytes, Got: %d byes", len(body), len(respBody))
+ }
+
+ err = cli.DeleteBlob(cnt, blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ err = cli.DeleteContainer(cnt)
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestGetBlockList_PutBlockList(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cnt := randContainer()
+ if err := cli.CreateContainer(cnt, ContainerAccessTypePrivate); err != nil {
+ t.Fatal(err)
+ }
+ defer cli.deleteContainer(cnt)
+
+ blob := randString(20)
+ chunk := []byte(randString(1024))
+ blockId := base64.StdEncoding.EncodeToString([]byte("foo"))
+
+ // Put one block
+ err = cli.PutBlock(cnt, blob, blockId, chunk)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cli.deleteBlob(cnt, blob)
+
+ // Get committed blocks
+ committed, err := cli.GetBlockList(cnt, blob, BlockListTypeCommitted)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(committed.CommittedBlocks) > 0 {
+ t.Fatal("There are committed blocks")
+ }
+
+ // Get uncommitted blocks
+ uncommitted, err := cli.GetBlockList(cnt, blob, BlockListTypeUncommitted)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if expected := 1; len(uncommitted.UncommittedBlocks) != expected {
+ t.Fatal("Uncommitted blocks wrong. Expected: %d, got: %d", expected, len(uncommitted.UncommittedBlocks))
+ }
+
+ // Commit block list
+ err = cli.PutBlockList(cnt, blob, []Block{{blockId, BlockStatusUncommitted}})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // Get all blocks
+ all, err := cli.GetBlockList(cnt, blob, BlockListTypeAll)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if expected := 1; len(all.CommittedBlocks) != expected {
+ t.Fatalf("Uncommitted blocks wrong. Expected: %d, got: %d", expected, len(uncommitted.CommittedBlocks))
+ }
+ if expected := 0; len(all.UncommittedBlocks) != expected {
+ t.Fatalf("Uncommitted blocks wrong. Expected: %d, got: %d", expected, len(uncommitted.UncommittedBlocks))
+ }
+
+ // Verify the block
+ thatBlock := all.CommittedBlocks[0]
+ if expected := blockId; expected != thatBlock.Name {
+ t.Fatalf("Wrong block name. Expected: %s, got: %s", expected, thatBlock.Name)
+ }
+ if expected := uint64(len(chunk)); expected != thatBlock.Size {
+ t.Fatalf("Wrong block name. Expected: %d, got: %d", expected, thatBlock.Size)
+ }
+}
+
+func deleteTestContainers(cli *BlobStorageClient) error {
+ for {
+ resp, err := cli.ListContainers(ListContainersParameters{Prefix: testContainerPrefix})
+ if err != nil {
+ return err
+ }
+ if len(resp.Containers) == 0 {
+ break
+ }
+ for _, c := range resp.Containers {
+ err = cli.DeleteContainer(c.Name)
+ if err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func getBlobClient() (*BlobStorageClient, error) {
+ name := os.Getenv("ACCOUNT_NAME")
+ if name == "" {
+ return nil, errors.New("ACCOUNT_NAME not set, need an empty storage account to test")
+ }
+ key := os.Getenv("ACCOUNT_KEY")
+ if key == "" {
+ return nil, errors.New("ACCOUNT_KEY not set")
+ }
+ cli, err := NewBasicClient(name, key)
+ if err != nil {
+ return nil, err
+ }
+ return cli.GetBlobService(), nil
+}
+
+func randContainer() string {
+ return testContainerPrefix + randString(32-len(testContainerPrefix))
+}
+
+func randString(n int) string {
+ if n <= 0 {
+ panic("negative number")
+ }
+ const alphanum = "0123456789abcdefghijklmnopqrstuvwxyz"
+ var bytes = make([]byte, n)
+ rand.Read(bytes)
+ for i, b := range bytes {
+ bytes[i] = alphanum[b%byte(len(alphanum))]
+ }
+ return string(bytes)
+}
diff --git a/clients/storage/client.go b/clients/storage/client.go
new file mode 100644
index 000000000000..caa5503f6140
--- /dev/null
+++ b/clients/storage/client.go
@@ -0,0 +1,303 @@
+package storage
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/xml"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "regexp"
+ "sort"
+ "strings"
+)
+
+const (
+ DefaultBaseUrl = "core.windows.net"
+ DefaultApiVersion = "2014-02-14"
+ defaultUseHttps = true
+
+ blobServiceName = "blob"
+ tableServiceName = "table"
+ queueServiceName = "queue"
+)
+
+type StorageClient struct {
+ accountName string
+ accountKey []byte
+ useHttps bool
+ baseUrl string
+ apiVersion string
+}
+
+type storageResponse struct {
+ statusCode int
+ headers http.Header
+ body io.ReadCloser
+}
+
+type StorageServiceError struct {
+ Code string `xml:"Code"`
+ Message string `xml:"Message"`
+ AuthenticationErrorDetail string `xml:"AuthenticationErrorDetail"`
+ QueryParameterName string `xml:"QueryParameterName"`
+ QueryParameterValue string `xml:"QueryParameterValue"`
+ Reason string `xml:"Reason"`
+ StatusCode int
+ RequestId string
+}
+
+func NewBasicClient(accountName, accountKey string) (*StorageClient, error) {
+ return NewClient(accountName, accountKey, DefaultBaseUrl, DefaultApiVersion, defaultUseHttps)
+}
+
+func NewClient(accountName, accountKey, blobServiceBaseUrl, apiVersion string, useHttps bool) (*StorageClient, error) {
+ if accountName == "" {
+ return nil, fmt.Errorf("azure: account name required")
+ } else if accountKey == "" {
+ return nil, fmt.Errorf("azure: account key required")
+ } else if blobServiceBaseUrl == "" {
+ return nil, fmt.Errorf("azure: base storage service url required")
+ }
+
+ key, err := base64.StdEncoding.DecodeString(accountKey)
+ if err != nil {
+ return nil, err
+ }
+
+ return &StorageClient{
+ accountName: accountName,
+ accountKey: key,
+ useHttps: useHttps,
+ baseUrl: blobServiceBaseUrl,
+ apiVersion: apiVersion}, nil
+}
+
+func (c StorageClient) getBaseUrl(service string) string {
+ scheme := "http"
+ if c.useHttps {
+ scheme = "https"
+ }
+
+ host := fmt.Sprintf("%s.%s.%s", c.accountName, service, c.baseUrl)
+
+ u := &url.URL{
+ Scheme: scheme,
+ Host: host}
+ return u.String()
+}
+
+func (c StorageClient) getEndpoint(service, path string, params url.Values) string {
+ u, err := url.Parse(c.getBaseUrl(service))
+ if err != nil {
+ // really should not be happening
+ panic(err)
+ }
+
+ if path == "" {
+ path = "/" // API doesn't accept path segments not starting with '/'
+ }
+
+ u.Path = path
+ u.RawQuery = params.Encode()
+ return u.String()
+}
+
+func (c StorageClient) GetBlobService() *BlobStorageClient {
+ return &BlobStorageClient{c}
+}
+
+func (c StorageClient) createAuthorizationHeader(canonicalizedString string) string {
+ signature := c.computeHmac256(canonicalizedString)
+ return fmt.Sprintf("%s %s:%s", "SharedKey", c.accountName, signature)
+}
+
+func (c StorageClient) getAuthorizationHeader(verb, url string, headers map[string]string) (string, error) {
+ canonicalizedResource, err := c.buildCanonicalizedResource(url)
+ if err != nil {
+ return "", err
+ }
+
+ canonicalizedString := c.buildCanonicalizedString(verb, headers, canonicalizedResource)
+ return c.createAuthorizationHeader(canonicalizedString), nil
+}
+
+func (c StorageClient) getStandardHeaders() map[string]string {
+ return map[string]string{
+ "x-ms-version": c.apiVersion,
+ "x-ms-date": currentTimeRfc1123Formatted(),
+ }
+}
+
+func (c StorageClient) buildCanonicalizedHeader(headers map[string]string) string {
+ cm := make(map[string]string)
+
+ for k, v := range headers {
+ headerName := strings.TrimSpace(strings.ToLower(k))
+ match, _ := regexp.MatchString("x-ms-", headerName)
+ if match {
+ cm[headerName] = v
+ }
+ }
+
+ if len(cm) == 0 {
+ return ""
+ }
+
+ keys := make([]string, 0, len(cm))
+ for key := range cm {
+ keys = append(keys, key)
+ }
+
+ sort.Strings(keys)
+
+ ch := ""
+
+ for i, key := range keys {
+ if i == len(keys)-1 {
+ ch += fmt.Sprintf("%s:%s", key, cm[key])
+ } else {
+ ch += fmt.Sprintf("%s:%s\n", key, cm[key])
+ }
+ }
+ return ch
+}
+
+func (c StorageClient) buildCanonicalizedResource(uri string) (string, error) {
+ errMsg := "buildCanonicalizedResource error: %s"
+ u, err := url.Parse(uri)
+ if err != nil {
+ return "", fmt.Errorf(errMsg, err.Error())
+ }
+
+ cr := "/" + c.accountName
+ if len(u.Path) > 0 {
+ cr += u.Path
+ }
+
+ params, err := url.ParseQuery(u.RawQuery)
+ if err != nil {
+ return "", fmt.Errorf(errMsg, err.Error())
+ }
+
+ if len(params) > 0 {
+ cr += "\n"
+ keys := make([]string, 0, len(params))
+ for key := range params {
+ keys = append(keys, key)
+ }
+
+ sort.Strings(keys)
+
+ for i, key := range keys {
+ if len(params[key]) > 1 {
+ sort.Strings(params[key])
+ }
+
+ if i == len(keys)-1 {
+ cr += fmt.Sprintf("%s:%s", key, strings.Join(params[key], ","))
+ } else {
+ cr += fmt.Sprintf("%s:%s\n", key, strings.Join(params[key], ","))
+ }
+ }
+ }
+ return cr, nil
+}
+
+func (c StorageClient) buildCanonicalizedString(verb string, headers map[string]string, canonicalizedResource string) string {
+ canonicalizedString := fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s",
+ verb,
+ headers["Content-Encoding"],
+ headers["Content-Language"],
+ headers["Content-Length"],
+ headers["Content-MD5"],
+ headers["Content-Type"],
+ headers["Date"],
+ headers["If-Modified-Singe"],
+ headers["If-Match"],
+ headers["If-None-Match"],
+ headers["If-Unmodified-Singe"],
+ headers["Range"],
+ c.buildCanonicalizedHeader(headers),
+ canonicalizedResource)
+
+ return canonicalizedString
+}
+
+func (c StorageClient) exec(verb, url string, headers map[string]string, body io.Reader) (*storageResponse, error) {
+ authHeader, err := c.getAuthorizationHeader(verb, url, headers)
+ if err != nil {
+ return nil, err
+ }
+ headers["Authorization"] = authHeader
+
+ if err != nil {
+ return nil, err
+ }
+
+ req, err := http.NewRequest(verb, url, body)
+ for k, v := range headers {
+ req.Header.Add(k, v)
+ }
+ httpClient := http.Client{}
+ resp, err := httpClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ statusCode := resp.StatusCode
+ if statusCode >= 400 && statusCode <= 505 {
+ var respBody []byte
+ respBody, err = readResponseBody(resp)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(respBody) == 0 {
+ // no error in response body
+ err = fmt.Errorf("storage: service returned without a response body (%s).", resp.Status)
+ } else {
+ // response contains storage service error object, unmarshal
+ storageErr, errIn := serviceErrFromXml(respBody, resp.StatusCode, resp.Header.Get("x-ms-request-id"))
+ if err != nil { // error unmarshaling the error response
+ err = errIn
+ }
+ err = storageErr
+ }
+ return &storageResponse{
+ statusCode: resp.StatusCode,
+ headers: resp.Header,
+ body: ioutil.NopCloser(bytes.NewReader(respBody)), /* restore the body */
+ }, err
+ }
+
+ return &storageResponse{
+ statusCode: resp.StatusCode,
+ headers: resp.Header,
+ body: resp.Body}, nil
+}
+
+func readResponseBody(resp *http.Response) ([]byte, error) {
+ defer resp.Body.Close()
+ out, err := ioutil.ReadAll(resp.Body)
+ if err == io.EOF {
+ err = nil
+ }
+ return out, err
+}
+
+func serviceErrFromXml(body []byte, statusCode int, requestId string) (StorageServiceError, error) {
+ var storageErr StorageServiceError
+ if err := xml.Unmarshal(body, &storageErr); err != nil {
+ return storageErr, err
+ }
+ storageErr.StatusCode = statusCode
+ storageErr.RequestId = requestId
+ return storageErr, nil
+}
+
+func (e StorageServiceError) Error() string {
+ return fmt.Sprintf("storage: remote server returned error. StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", e.StatusCode, e.Code, e.Message, e.RequestId)
+}
diff --git a/clients/storage/client_test.go b/clients/storage/client_test.go
new file mode 100644
index 000000000000..3d340c46338b
--- /dev/null
+++ b/clients/storage/client_test.go
@@ -0,0 +1,203 @@
+package storage
+
+import (
+ "encoding/base64"
+ "net/url"
+ "testing"
+)
+
+func TestGetBaseUrl_Basic_Https(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if cli.apiVersion != DefaultApiVersion {
+ t.Fatalf("Wrong api version. Expected: '%s', got: '%s'", DefaultApiVersion, cli.apiVersion)
+ }
+
+ if err != nil {
+ t.Fatal(err)
+ }
+ output := cli.getBaseUrl("table")
+
+ if expected := "https://foo.table.core.windows.net"; output != expected {
+ t.Fatalf("Wrong base url. Expected: '%s', got: '%s'", expected, output)
+ }
+}
+
+func TestGetBaseUrl_Custom_NoHttps(t *testing.T) {
+ apiVersion := DefaultApiVersion
+ cli, err := NewClient("foo", "YmFy", "core.chinacloudapi.cn", apiVersion, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if cli.apiVersion != apiVersion {
+ t.Fatalf("Wrong api version. Expected: '%s', got: '%s'", apiVersion, cli.apiVersion)
+ }
+
+ output := cli.getBaseUrl("table")
+
+ if expected := "http://foo.table.core.chinacloudapi.cn"; output != expected {
+ t.Fatalf("Wrong base url. Expected: '%s', got: '%s'", expected, output)
+ }
+}
+
+func TestGetEndpoint_None(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ output := cli.getEndpoint(blobServiceName, "", url.Values{})
+
+ if expected := "https://foo.blob.core.windows.net/"; output != expected {
+ t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output)
+ }
+}
+
+func TestGetEndpoint_PathOnly(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ output := cli.getEndpoint(blobServiceName, "path", url.Values{})
+
+ if expected := "https://foo.blob.core.windows.net/path"; output != expected {
+ t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output)
+ }
+}
+
+func TestGetEndpoint_ParamsOnly(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ params := url.Values{}
+ params.Set("a", "b")
+ params.Set("c", "d")
+ output := cli.getEndpoint(blobServiceName, "", params)
+
+ if expected := "https://foo.blob.core.windows.net/?a=b&c=d"; output != expected {
+ t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output)
+ }
+}
+
+func TestGetEndpoint_Mixed(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ params := url.Values{}
+ params.Set("a", "b")
+ params.Set("c", "d")
+ output := cli.getEndpoint(blobServiceName, "path", params)
+
+ if expected := "https://foo.blob.core.windows.net/path?a=b&c=d"; output != expected {
+ t.Fatalf("Wrong endpoint url. Expected: '%s', got: '%s'", expected, output)
+ }
+}
+
+func Test_getStandardHeaders(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ headers := cli.getStandardHeaders()
+ if len(headers) != 2 {
+ t.Fatal("Wrong standard header count")
+ }
+ if v, ok := headers["x-ms-version"]; !ok || v != cli.apiVersion {
+ t.Fatal("Wrong version header")
+ }
+ if _, ok := headers["x-ms-date"]; !ok {
+ t.Fatal("Missing date header")
+ }
+}
+
+func Test_buildCanonicalizedResource(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ type test struct{ url, expected string }
+ tests := []test{
+ {"https://foo.blob.core.windows.net/path?a=b&c=d", "/foo/path\na:b\nc:d"},
+ {"https://foo.blob.core.windows.net/?comp=list", "/foo/\ncomp:list"},
+ {"https://foo.blob.core.windows.net/cnt/blob", "/foo/cnt/blob"},
+ }
+
+ for _, i := range tests {
+ if out, err := cli.buildCanonicalizedResource(i.url); err != nil {
+ t.Fatal(err)
+ } else if out != i.expected {
+ t.Fatalf("Wrong canonicalized resource. Expected:\n'%s', Got:\n'%s'", i.expected, out)
+ }
+ }
+}
+
+func Test_buildCanonicalizedHeader(t *testing.T) {
+ cli, err := NewBasicClient("foo", "YmFy")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ type test struct {
+ headers map[string]string
+ expected string
+ }
+ tests := []test{
+ {map[string]string{}, ""},
+ {map[string]string{"x-ms-foo": "bar"}, "x-ms-foo:bar"},
+ {map[string]string{"foo:": "bar"}, ""},
+ {map[string]string{"foo:": "bar", "x-ms-foo": "bar"}, "x-ms-foo:bar"},
+ {map[string]string{
+ "x-ms-version": "9999-99-99",
+ "x-ms-blob-type": "BlockBlob"}, "x-ms-blob-type:BlockBlob\nx-ms-version:9999-99-99"}}
+
+ for _, i := range tests {
+ if out := cli.buildCanonicalizedHeader(i.headers); out != i.expected {
+ t.Fatalf("Wrong canonicalized resource. Expected:\n'%s', Got:\n'%s'", i.expected, out)
+ }
+ }
+}
+
+func TestReturnsStorageServiceError(t *testing.T) {
+ cli, err := getBlobClient()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ // attempt to delete a nonexisting container
+ _, err = cli.deleteContainer(randContainer())
+ if err == nil {
+ t.Fatal("Service has not returned an error")
+ }
+
+ if v, ok := err.(StorageServiceError); !ok {
+ t.Fatal("Cannot assert to specific error")
+ } else if v.StatusCode != 404 {
+ t.Fatalf("Expected status:%d, got: %d", 404, v.StatusCode)
+ } else if v.Code != "ContainerNotFound" {
+ t.Fatalf("Expected code: %s, got: %s", "ContainerNotFound", v.Code)
+ } else if v.RequestId == "" {
+ t.Fatalf("RequestId does not exist")
+ }
+}
+
+func Test_createAuthorizationHeader(t *testing.T) {
+ key := base64.StdEncoding.EncodeToString([]byte("bar"))
+ cli, err := NewBasicClient("foo", key)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ canonicalizedString := `foobarzoo`
+ expected := `SharedKey foo:h5U0ATVX6SpbFX1H6GNuxIMeXXCILLoIvhflPtuQZ30=`
+
+ if out := cli.createAuthorizationHeader(canonicalizedString); out != expected {
+ t.Fatalf("Wrong authoriztion header. Expected: '%s', Got:'%s'", expected, out)
+ }
+}
diff --git a/clients/storage/util.go b/clients/storage/util.go
new file mode 100644
index 000000000000..a3fa2bd80945
--- /dev/null
+++ b/clients/storage/util.go
@@ -0,0 +1,64 @@
+package storage
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/xml"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "net/url"
+ "time"
+)
+
+func (c StorageClient) computeHmac256(message string) string {
+ h := hmac.New(sha256.New, c.accountKey)
+ h.Write([]byte(message))
+ return base64.StdEncoding.EncodeToString(h.Sum(nil))
+}
+
+func currentTimeRfc1123Formatted() string {
+ const dateLayout = http.TimeFormat // reuse from net/http package
+ return timeRfc1123Formatted(time.Now().UTC())
+}
+
+func timeRfc1123Formatted(t time.Time) string {
+ return t.Format(http.TimeFormat)
+}
+
+func mergeParams(v1, v2 url.Values) url.Values {
+ out := url.Values{}
+ for k, v := range v1 {
+ out[k] = v
+ }
+ for k, v := range v2 {
+ vals, ok := out[k]
+ if ok {
+ vals = append(vals, v...)
+ out[k] = vals
+ } else {
+ out[k] = v
+ }
+ }
+ return out
+}
+
+func prepareBlockListRequest(blocks []Block) string {
+ s := ``
+ for _, v := range blocks {
+ s += fmt.Sprintf("<%s>%s%s>", v.Status, v.Id, v.Status)
+ }
+ s += ``
+ return s
+}
+
+func xmlUnmarshal(body io.ReadCloser, v interface{}) error {
+ data, err := ioutil.ReadAll(body)
+ if err != nil {
+ return err
+ }
+ defer body.Close()
+ return xml.Unmarshal(data, v)
+}
diff --git a/clients/storage/util_test.go b/clients/storage/util_test.go
new file mode 100644
index 000000000000..91af973fdef2
--- /dev/null
+++ b/clients/storage/util_test.go
@@ -0,0 +1,80 @@
+package storage
+
+import (
+ "io/ioutil"
+ "net/url"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+)
+
+func Test_timeRfc1123Formatted(t *testing.T) {
+ now := time.Now().UTC()
+
+ expectedLayout := "Mon, 02 Jan 2006 15:04:05 GMT"
+ expected := now.Format(expectedLayout)
+
+ if output := timeRfc1123Formatted(now); output != expected {
+ t.Errorf("Expected: %s, got: %s", expected, output)
+ }
+}
+
+func Test_mergeParams(t *testing.T) {
+ v1 := url.Values{
+ "k1": {"v1"},
+ "k2": {"v2"}}
+ v2 := url.Values{
+ "k1": {"v11"},
+ "k3": {"v3"}}
+
+ out := mergeParams(v1, v2)
+ if v := out.Get("k1"); v != "v1" {
+ t.Errorf("Wrong value for k1: %s", v)
+ }
+
+ if v := out.Get("k2"); v != "v2" {
+ t.Errorf("Wrong value for k2: %s", v)
+ }
+
+ if v := out.Get("k3"); v != "v3" {
+ t.Errorf("Wrong value for k3: %s", v)
+ }
+
+ if v := out["k1"]; !reflect.DeepEqual(v, []string{"v1", "v11"}) {
+ t.Errorf("Wrong multi-value for k1: %s", v)
+ }
+}
+
+func Test_prepareBlockListRequest(t *testing.T) {
+ empty := []Block{}
+ expected := ``
+ if out := prepareBlockListRequest(empty); expected != out {
+ t.Error("Wrong block list. Expected: '%s', got: '%s'", expected, out)
+ }
+
+ blocks := []Block{{"foo", BlockStatusLatest}, {"bar", BlockStatusUncommitted}}
+ expected = `foobar`
+ if out := prepareBlockListRequest(blocks); expected != out {
+ t.Error("Wrong block list. Expected: '%s', got: '%s'", expected, out)
+ }
+}
+
+func Test_xmlUnmarshal(t *testing.T) {
+ xml := `
+
+ myblob
+ `
+
+ body := ioutil.NopCloser(strings.NewReader(xml))
+
+ var blob Blob
+ err := xmlUnmarshal(body, &blob)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if blob.Name != "myblob" {
+ t.Fatal("Got wrong value")
+ }
+}