diff --git a/clients/storage/blob.go b/clients/storage/blob.go index 83919ee797ef..baabcae6484f 100644 --- a/clients/storage/blob.go +++ b/clients/storage/blob.go @@ -18,18 +18,22 @@ type BlobStorageClient struct { client StorageClient } +// A Container is an entry in ContainerListResponse. type Container struct { Name string `xml:"Name"` Properties ContainerProperties `xml:"Properties"` // TODO (ahmetalpbalkan) Metadata } +// A Blob is an entry in BlobListResponse. type Blob struct { Name string `xml:"Name"` Properties BlobProperties `xml:"Properties"` // TODO (ahmetalpbalkan) Metadata } +// ContainerProperties contains various properties of a +// container returned from various endpoints like ListContainers. type ContainerProperties struct { LastModified string `xml:"Last-Modified"` Etag string `xml:"Etag"` @@ -39,6 +43,8 @@ type ContainerProperties struct { // TODO (ahmetalpbalkan) remaining fields } +// BlobProperties contains various properties of a blob +// returned in various endpoints like ListBlobs or GetBlobProperties. type BlobProperties struct { LastModified string `xml:"Last-Modified"` Etag string `xml:"Etag"` @@ -55,6 +61,8 @@ type BlobProperties struct { CopyStatusDescription string `xml:"CopyStatusDescription"` } +// ContainerListResponse contains the reponse fields from +// ListBlobs call. https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx type BlobListResponse struct { XMLName xml.Name `xml:"EnumerationResults"` Xmlns string `xml:"xmlns,attr"` @@ -65,6 +73,8 @@ type BlobListResponse struct { Blobs []Blob `xml:"Blobs>Blob"` } +// ContainerListResponse contains the reponse fields from +// ListContainers call. https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx type ContainerListResponse struct { XMLName xml.Name `xml:"EnumerationResults"` Xmlns string `xml:"xmlns,attr"` @@ -75,6 +85,8 @@ type ContainerListResponse struct { Containers []Container `xml:"Containers>Container"` } +// ListContainersParameters defines the set of customizable +// parameters to make a List Containers call. https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx type ListContainersParameters struct { Prefix string Marker string @@ -83,7 +95,7 @@ type ListContainersParameters struct { Timeout uint } -func (p ListContainersParameters) GetParameters() url.Values { +func (p ListContainersParameters) getParameters() url.Values { out := url.Values{} if p.Prefix != "" { @@ -105,6 +117,8 @@ func (p ListContainersParameters) GetParameters() url.Values { return out } +// ListBlobsParameters defines the set of customizable +// parameters to make a List Blobs call. https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx type ListBlobsParameters struct { Prefix string Delimiter string @@ -114,7 +128,7 @@ type ListBlobsParameters struct { Timeout uint } -func (p ListBlobsParameters) GetParameters() url.Values { +func (p ListBlobsParameters) getParameters() url.Values { out := url.Values{} if p.Prefix != "" { @@ -139,6 +153,7 @@ func (p ListBlobsParameters) GetParameters() url.Values { return out } +// BlobType defines the type of the Azure Blob. type BlobType string const ( @@ -153,6 +168,10 @@ const ( blobCopyStatusFailed = "failed" ) +// BlockListType is used to filter out types of blocks +// in a Get Blocks List call for a block blob. See +// https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx +// for all block types. type BlockListType string const ( @@ -161,6 +180,9 @@ const ( BlockListTypeUncommitted BlockListType = "uncommitted" ) +// ContainerAccessType defines the access level to the container +// from a public request. See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx +// and "x-ms-blob-public-access" header. type ContainerAccessType string const ( @@ -171,6 +193,8 @@ const ( const MaxBlobBlockSize = 4 * 1024 * 1024 +// BlockStatus defines states a block for a block blob can +// be in. type BlockStatus string const ( @@ -179,17 +203,23 @@ const ( BlockStatusLatest BlockStatus = "Latest" ) +// Block is used to create Block entities for Put Block List +// call. type Block struct { Id string Status BlockStatus } +// BlockListResponse contains the reponse fields from +// Get Block List call. https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx type BlockListResponse struct { XMLName xml.Name `xml:"BlockList"` CommittedBlocks []BlockResponse `xml:"CommittedBlocks>Block"` UncommittedBlocks []BlockResponse `xml:"UncommittedBlocks>Block"` } +// BlockResponse contains the block information returned +// in the GetBlockListCall. type BlockResponse struct { Name string `xml:"Name"` Size uint64 `xml:"Size"` @@ -205,8 +235,10 @@ var ( const errUnexpectedStatus = "storage: was expecting status code: %d, got: %d" +// ListContainers returns the list of containers in a storage account along with +// pagination token and other response details. See https://msdn.microsoft.com/en-us/library/azure/dd179352.aspx func (b BlobStorageClient) ListContainers(params ListContainersParameters) (ContainerListResponse, error) { - q := mergeParams(params.GetParameters(), url.Values{"comp": {"list"}}) + q := mergeParams(params.getParameters(), url.Values{"comp": {"list"}}) uri := b.client.getEndpoint(blobServiceName, "", q) headers := b.client.getStandardHeaders() @@ -220,6 +252,9 @@ func (b BlobStorageClient) ListContainers(params ListContainersParameters) (Cont return out, err } +// CreateContainer creates a blob container within the storage account +// with given name and access level. See https://msdn.microsoft.com/en-us/library/azure/dd179468.aspx +// Returns error if container already exists. func (b BlobStorageClient) CreateContainer(name string, access ContainerAccessType) error { resp, err := b.createContainer(name, access) if err != nil { @@ -231,6 +266,8 @@ func (b BlobStorageClient) CreateContainer(name string, access ContainerAccessTy return nil } +// CreateContainerIfNotExists creates a blob container if it does not exist. Returns +// true if container is newly created or false if container already exists. func (b BlobStorageClient) CreateContainerIfNotExists(name string, access ContainerAccessType) (bool, error) { resp, err := b.createContainer(name, access) if resp != nil && (resp.statusCode == http.StatusCreated || resp.statusCode == http.StatusConflict) { @@ -251,6 +288,8 @@ func (b BlobStorageClient) createContainer(name string, access ContainerAccessTy return b.client.exec(verb, uri, headers, nil) } +// ContainerExists returns true if a container with given name exists +// on the storage account, otherwise returns false. func (b BlobStorageClient) ContainerExists(container string) (bool, error) { verb := "HEAD" path := fmt.Sprintf("%s", container) @@ -264,6 +303,9 @@ func (b BlobStorageClient) ContainerExists(container string) (bool, error) { return false, err } +// DeleteContainer deletes the container with given name on the storage +// account. See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx +// If the container does not exist returns error. func (b BlobStorageClient) DeleteContainer(name string) error { resp, err := b.deleteContainer(name) if err != nil { @@ -275,6 +317,10 @@ func (b BlobStorageClient) DeleteContainer(name string) error { return nil } +// DeleteContainer deletes the container with given name on the storage +// account if it exists. See https://msdn.microsoft.com/en-us/library/azure/dd179408.aspx +// Returns true if container is deleted with this call, or false +// if the container did not exist at the time of the Delete Container operation. func (b BlobStorageClient) DeleteContainerIfExists(container string) (bool, error) { resp, err := b.deleteContainer(container) if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) { @@ -291,8 +337,11 @@ func (b BlobStorageClient) deleteContainer(name string) (*storageResponse, error return b.client.exec(verb, uri, headers, nil) } +// ListBlobs returns an object that contains list of blobs in the container, +// pagination token and other information in the response of List Blobs call. +// See https://msdn.microsoft.com/en-us/library/azure/dd135734.aspx func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameters) (BlobListResponse, error) { - q := mergeParams(params.GetParameters(), url.Values{ + q := mergeParams(params.getParameters(), url.Values{ "restype": {"container"}, "comp": {"list"}}) uri := b.client.getEndpoint(blobServiceName, container, q) @@ -308,6 +357,8 @@ func (b BlobStorageClient) ListBlobs(container string, params ListBlobsParameter return out, err } +// BlobExists returns true if a blob with given name exists on the +// specified container of the storage account. func (b BlobStorageClient) BlobExists(container, name string) (bool, error) { verb := "HEAD" path := fmt.Sprintf("%s/%s", container, name) @@ -321,6 +372,10 @@ func (b BlobStorageClient) BlobExists(container, name string) (bool, error) { return false, err } +// GetBlobUrl gets the canonical URL to the blob with the specified +// name in the specified container. This method does not create a +// publicly accessible URL if the blob or container is private and this +// method does not check if the blob exists. func (b BlobStorageClient) GetBlobUrl(container, name string) string { if container == "" { container = "$root" @@ -329,6 +384,7 @@ func (b BlobStorageClient) GetBlobUrl(container, name string) string { return b.client.getEndpoint(blobServiceName, path, url.Values{}) } +// GetBlob downloads a blob to a stream. See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error) { resp, err := b.getBlobRange(container, name, "") if err != nil { @@ -341,6 +397,9 @@ func (b BlobStorageClient) GetBlob(container, name string) (io.ReadCloser, error return resp.body, nil } +// GetBlobRange reads the specified range of a blob to a stream. +// The bytesRange string must be in a format like "0-", "10-100" +// as defined in HTTP 1.1 spec. See https://msdn.microsoft.com/en-us/library/azure/dd179440.aspx func (b BlobStorageClient) GetBlobRange(container, name, bytesRange string) (io.ReadCloser, error) { resp, err := b.getBlobRange(container, name, bytesRange) if err != nil { @@ -369,6 +428,8 @@ func (b BlobStorageClient) getBlobRange(container, name, bytesRange string) (*st return resp, err } +// GetBlobProperties provides various information about the specified +// blob. See https://msdn.microsoft.com/en-us/library/azure/dd179394.aspx func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobProperties, error) { verb := "HEAD" path := fmt.Sprintf("%s/%s", container, name) @@ -418,6 +479,10 @@ func (b BlobStorageClient) GetBlobProperties(container, name string) (*BlobPrope }, nil } +// PutBlockBlob uploads given stream into a block blob by splitting +// data stream into chunks and uploading as blocks. Commits the block +// list at the end. This is a helper method built on top of PutBlock +// and PutBlockList methods with sequential block ID counting logic. func (b BlobStorageClient) PutBlockBlob(container, name string, blob io.Reader) error { // TODO (ahmetalpbalkan) consider ReadCloser and closing return b.putBlockBlob(container, name, blob, MaxBlobBlockSize) } @@ -490,10 +555,16 @@ func (b BlobStorageClient) putSingleBlockBlob(container, name string, chunk []by return nil } +// PutBlock saves the given data chunk to the specified block blob with +// given ID. See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx func (b BlobStorageClient) PutBlock(container, name, blockId string, chunk []byte) error { return b.PutBlockWithLength(container, name, blockId, uint64(len(chunk)), bytes.NewReader(chunk)) } +// PutBlockWithLength saves the given data stream of exactly specified size to the block blob +// with given ID. See https://msdn.microsoft.com/en-us/library/azure/dd135726.aspx +// It is an alternative to PutBlocks where data comes as stream but the length is +// known in advance. func (b BlobStorageClient) PutBlockWithLength(container, name, blockId string, size uint64, blob io.Reader) error { path := fmt.Sprintf("%s/%s", container, name) uri := b.client.getEndpoint(blobServiceName, path, url.Values{"comp": {"block"}, "blockid": {blockId}}) @@ -512,6 +583,8 @@ func (b BlobStorageClient) PutBlockWithLength(container, name, blockId string, s return nil } +// PutBlockList saves list of blocks to the specified block blob. See +// https://msdn.microsoft.com/en-us/library/azure/dd179467.aspx func (b BlobStorageClient) PutBlockList(container, name string, blocks []Block) error { blockListXml := prepareBlockListRequest(blocks) @@ -530,6 +603,8 @@ func (b BlobStorageClient) PutBlockList(container, name string, blocks []Block) return nil } +// GetBlockList retrieves list of blocks in the specified block blob. See +// https://msdn.microsoft.com/en-us/library/azure/dd179400.aspx func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockListType) (BlockListResponse, error) { params := url.Values{"comp": {"blocklist"}, "blocklisttype": {string(blockType)}} uri := b.client.getEndpoint(blobServiceName, fmt.Sprintf("%s/%s", container, name), params) @@ -545,6 +620,10 @@ func (b BlobStorageClient) GetBlockList(container, name string, blockType BlockL return out, err } +// CopyBlob starts a blob copy operation and waits for the operation to complete. +// sourceBlob parameter must be a canonical URL to the blob (can be obtained using +// GetBlobURL method.) There is no SLA on blob copy and therefore this helper +// method works faster on smaller files. See https://msdn.microsoft.com/en-us/library/azure/dd894037.aspx func (b BlobStorageClient) CopyBlob(container, name, sourceBlob string) error { copyId, err := b.startBlobCopy(container, name, sourceBlob) if err != nil { @@ -603,6 +682,9 @@ func (b BlobStorageClient) waitForBlobCopy(container, name, copyId string) error } } +// DeleteBlob deletes the given blob from the specified container. +// If the blob does not exists at the time of the Delete Blob operation, it +// returns error. See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx func (b BlobStorageClient) DeleteBlob(container, name string) error { resp, err := b.deleteBlob(container, name) if err != nil { @@ -614,6 +696,28 @@ func (b BlobStorageClient) DeleteBlob(container, name string) error { return nil } +// DeleteBlobIfExists deletes the given blob from the specified container +// If the blob is deleted with this call, returns true. Otherwise returns +// false. See https://msdn.microsoft.com/en-us/library/azure/dd179413.aspx +func (b BlobStorageClient) DeleteBlobIfExists(container, name string) (bool, error) { + resp, err := b.deleteBlob(container, name) + if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) { + return resp.statusCode == http.StatusAccepted, nil + } + return false, err +} + +func (b BlobStorageClient) deleteBlob(container, name string) (*storageResponse, error) { + verb := "DELETE" + path := fmt.Sprintf("%s/%s", container, name) + uri := b.client.getEndpoint(blobServiceName, path, url.Values{}) + + headers := b.client.getStandardHeaders() + return b.client.exec(verb, uri, headers, nil) +} + +// GetBlobSASURI creates an URL to the specified blob which contains the Shared Access Signature +// with specified permissions and expiration time. See https://msdn.microsoft.com/en-us/library/azure/ee395415.aspx func (b BlobStorageClient) GetBlobSASURI(container, name string, expiry time.Time, permissions string) (string, error) { var ( signedPermissions = permissions @@ -658,20 +762,3 @@ func blobSASStringToSign(signedVersion, canonicalizedResource, signedExpiry, sig return "", errors.New("storage: not implemented SAS for versions earlier than 2013-08-15") } } - -func (b BlobStorageClient) DeleteBlobIfExists(container, name string) (bool, error) { - resp, err := b.deleteBlob(container, name) - if resp != nil && (resp.statusCode == http.StatusAccepted || resp.statusCode == http.StatusNotFound) { - return resp.statusCode == http.StatusAccepted, nil - } - return false, err -} - -func (b BlobStorageClient) deleteBlob(container, name string) (*storageResponse, error) { - verb := "DELETE" - path := fmt.Sprintf("%s/%s", container, name) - uri := b.client.getEndpoint(blobServiceName, path, url.Values{}) - - headers := b.client.getStandardHeaders() - return b.client.exec(verb, uri, headers, nil) -} diff --git a/clients/storage/client.go b/clients/storage/client.go index caa5503f6140..1cbabaee3eb2 100644 --- a/clients/storage/client.go +++ b/clients/storage/client.go @@ -24,6 +24,8 @@ const ( queueServiceName = "queue" ) +// StorageClient is the object that needs to be constructed +// to perform operations on the storage account. type StorageClient struct { accountName string accountKey []byte @@ -38,6 +40,9 @@ type storageResponse struct { body io.ReadCloser } +// StorageServiceError contains fields of the error response from +// Azure Storage Service REST API. See https://msdn.microsoft.com/en-us/library/azure/dd179382.aspx +// Some fields might be specific to certain calls. type StorageServiceError struct { Code string `xml:"Code"` Message string `xml:"Message"` @@ -49,10 +54,15 @@ type StorageServiceError struct { RequestId string } +// NewBasicClient constructs a StorageClient with given storage service name +// and key. func NewBasicClient(accountName, accountKey string) (*StorageClient, error) { return NewClient(accountName, accountKey, DefaultBaseUrl, DefaultApiVersion, defaultUseHttps) } +// NewClient constructs a StorageClient. This should be used if the caller +// wants to specify whether to use HTTPS, a specific REST API version or a +// custom storage endpoint than Azure Public Cloud. func NewClient(accountName, accountKey, blobServiceBaseUrl, apiVersion string, useHttps bool) (*StorageClient, error) { if accountName == "" { return nil, fmt.Errorf("azure: account name required") @@ -105,6 +115,8 @@ func (c StorageClient) getEndpoint(service, path string, params url.Values) stri return u.String() } +// GetBlobService returns a BlobStorageClient which can operate on the +// blob service of the storage account. func (c StorageClient) GetBlobService() *BlobStorageClient { return &BlobStorageClient{c} }