Feature Work for Storage (#4584)

* Storage SAS implementation (#4404)

* SAS implementation

* Fixed some minor formatting issues

* Fixed checkstyle problems and test issue

* Remove RawClients from Blobs (#4375)

Removes RawClients from Storage Blobs

* Add deleteContainer to StorageClient and getBlobClient with Snapshot to ContainerClient (#4376)

* Removed raw clients

* Added deleteContainer to StorageClient

* Added getAppendBlob with snapshot to ContainerClient

* Storage queue linting, builder refactor, tests (#4383)

* Initial check in for storage queue

* Initial checkin for Storage file (#4414)

* Finished the restructure, refactor builder. Added sleep in record feature. Linting

* Merge Storage Blob Client Builders (#4468)

Merges AppendBlobClientBuilder, BlobClientBuilder, BlockBlobClientBuilder, and PageBlobClientBuilder into a single builder class BlobClientBuilder. Additionally, JavaDoc comments for the other builder classes, ContainerClientBuilder and StorageAccountClientBuilder, were cleaned up and the way the endpoint is handled in builders was changed.

Author: alzimmermsft

eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml

@@ -368,6 +368,7 @@
   <Match>
     <Or>
       <Package name="com.microsoft.azure.storage.blob"/>
+      <Package name="com.azure.storage.file.models"/>
       <Class name="com.azure.storage.blob.HTTPGetterInfo"/>
     </Or>
     <Bug pattern="NM_CONFUSING"/>
@@ -439,6 +440,7 @@
     <Or>
       <Class name="com.azure.storage.blob.BlobInputStream"/>
       <Class name="com.azure.storage.blob.BlobOutputStream"/>
+      <Class name="com.azure.storage.queue.QueueServiceClient"/>
     </Or>
     <Bug pattern="NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"/>
   </Match>

eng/jacoco-test-coverage/pom.xml

@@ -32,6 +32,7 @@
     <azure-keyvault.version>4.0.0-preview.1</azure-keyvault.version>
     <azure-messaging-eventhubs.version>5.0.0-preview.2</azure-messaging-eventhubs.version>
     <azure-storage-blob.version>12.0.0-preview.1</azure-storage-blob.version>
+    <azure-storage-queue.version>12.0.0-preview.1</azure-storage-queue.version>
   </properties>
 
   <distributionManagement>
@@ -93,7 +94,11 @@
 <!--      <artifactId>azure-storage-blob</artifactId>-->
 <!--      <version>${azure-storage-blob.version}</version>-->
 <!--    </dependency>-->
-
+    <dependency>
+      <groupId>com.azure</groupId>
+      <artifactId>azure-storage-queue</artifactId>
+      <version>${azure-storage-queue.version}</version>
+    </dependency>
     <!-- Tracing will be built and released separately. Removing tracing dependency
      until we finalize dependency composition -->
     <!--    <dependency>-->

eng/spotbugs-aggregate-report/pom.xml

@@ -30,6 +30,8 @@
     <azure-keyvault.version>4.0.0-preview.1</azure-keyvault.version>
     <azure-messaging-eventhubs.version>5.0.0-preview.2</azure-messaging-eventhubs.version>
     <azure-storage.version>12.0.0-preview.1</azure-storage.version>
+    <azure-storage-file.version>12.0.0-preview.1</azure-storage-file.version>
+    <azure-storage-queue.version>12.0.0-preview.1</azure-storage-queue.version>
   </properties>
   <distributionManagement>
     <site>
@@ -183,6 +185,16 @@
 <!--          <version>${azure-storage.version}</version>-->
 <!--        </dependency>-->
 
+        <dependency>
+          <groupId>com.azure</groupId>
+          <artifactId>azure-storage-file</artifactId>
+          <version>${azure-storage-file.version}</version>
+        </dependency>
+        <dependency>
+          <groupId>com.azure</groupId>
+          <artifactId>azure-storage-queue</artifactId>
+          <version>${azure-storage-queue.version}</version>
+        </dependency>
         <!-- Added this dependency to include necessary annotations used by reactor core.
         Without this dependency, javadoc throws a warning as it cannot find enum When.MAYBE
         which is used in @Nullable annotation in reactor core classes -->

pom.client.xml

@@ -393,6 +393,14 @@
                 <title>Azure Storage - Blobs</title>
                 <packages>com.azure.storage.blob*</packages>
               </group>
+              <group>
+                <title>Azure Storage - Files</title>
+                <packages>com.azure.storage.file*</packages>
+              </group>
+              <group>
+                <title>Azure Storage - Queues</title>
+                <packages>com.azure.storage.queue*</packages>
+              </group>
             </groups>
             <links>
               <link>https://docs.oracle.com/javase/8/docs/api/</link>
@@ -537,7 +545,8 @@
                 -snippetpath ${project.basedir}/sdk/eventhubs/azure-eventhubs/src/samples/java
                 -snippetpath ${project.basedir}/sdk/keyvault/azure-keyvault-keys/src/samples/java
                 -snippetpath ${project.basedir}/sdk/keyvault/azure-keyvault-secrets/src/samples/java
-<!--                -snippetpath ${project.basedir}/storage/client/blob/src/samples/java-->
+                <!-- -snippetpath ${project.basedir}/storage/client/blob/src/samples/java-->
+                -snippetpath ${project.basedir}/storage/client/queue/src/samples/java
               </additionalOptions>
             </configuration>
           </reportSet>
@@ -736,5 +745,7 @@
     <!--    <module>./sdk/tracing</module>-->
     <module>./sdk/identity/azure-identity</module>
     <module>./storage/client/blob</module>
+    <module>./storage/client/file</module>
+    <module>./storage/client/queue</module>
   </modules>
 </project>

sdk/keyvault/azure-keyvault-secrets/src/main/java/com/azure/security/keyvault/secrets/KeyVaultErrorCodeStrings.java

@@ -45,4 +45,3 @@ private static synchronized void loadProperties() {
 }
 
 
-

storage/client/blob/src/main/java/com/azure/storage/blob/AppendBlobAsyncClient.java

@@ -5,6 +5,7 @@
 
 import com.azure.core.http.rest.Response;
 import com.azure.core.http.rest.SimpleResponse;
+import com.azure.core.util.Context;
 import com.azure.storage.blob.implementation.AzureBlobStorageBuilder;
 import com.azure.storage.blob.models.AppendBlobAccessConditions;
 import com.azure.storage.blob.models.AppendBlobItem;
@@ -13,17 +14,18 @@
 import com.azure.storage.blob.models.BlobRange;
 import com.azure.storage.blob.models.Metadata;
 import com.azure.storage.blob.models.SourceModifiedAccessConditions;
-import io.netty.buffer.Unpooled;
+import io.netty.buffer.ByteBuf;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 import java.net.URL;
-import java.nio.ByteBuffer;
+
+import static com.azure.storage.blob.Utility.postProcessResponse;
 
 
 /**
- * Client to an append blob. It may only be instantiated through a {@link AppendBlobClientBuilder#buildAsyncClient()}, via
- * the method {@link BlobAsyncClient#asAppendBlobAsyncClient()}, or via the method
+ * Client to an append blob. It may only be instantiated through a {@link BlobClientBuilder#buildAppendBlobAsyncClient()},
+ * via the method {@link BlobAsyncClient#asAppendBlobAsyncClient()}, or via the method
  * {@link ContainerAsyncClient#getAppendBlobAsyncClient(String)}. This class does not hold
  * any state about a particular blob, but is instead a convenient way of sending appropriate
  * requests to the resource on the service.
@@ -45,8 +47,6 @@
  * object through {@link Mono#toFuture()}.
  */
 public final class AppendBlobAsyncClient extends BlobAsyncClient {
-    final AppendBlobAsyncRawClient appendBlobAsyncRawClient;
-
     /**
      * Indicates the maximum number of bytes that can be sent in a call to appendBlock.
      */
@@ -58,12 +58,11 @@ public final class AppendBlobAsyncClient extends BlobAsyncClient {
     public static final int MAX_BLOCKS = 50000;
 
     /**
-     * Package-private constructor for use by {@link AppendBlobClientBuilder}.
+     * Package-private constructor for use by {@link BlobClientBuilder}.
      * @param azureBlobStorageBuilder the API client builder for blob storage API
      */
     AppendBlobAsyncClient(AzureBlobStorageBuilder azureBlobStorageBuilder, String snapshot) {
         super(azureBlobStorageBuilder, snapshot);
-        appendBlobAsyncRawClient = new AppendBlobAsyncRawClient(azureBlobStorageBuilder.build());
     }
 
     /**
@@ -90,8 +89,13 @@ public Mono<Response<AppendBlobItem>> create() {
      *      A reactive response containing the information of the created appended blob.
      */
     public Mono<Response<AppendBlobItem>> create(BlobHTTPHeaders headers, Metadata metadata, BlobAccessConditions accessConditions) {
-        return appendBlobAsyncRawClient
-            .create(headers, metadata, accessConditions)
+        metadata = (metadata == null) ? new Metadata() : metadata;
+        accessConditions = (accessConditions == null) ? new BlobAccessConditions() : accessConditions;
+
+        return postProcessResponse(this.azureBlobStorage.appendBlobs().createWithRestResponseAsync(null,
+            null, 0, null, metadata, null, null,
+            null, null, headers, accessConditions.leaseAccessConditions(),
+            accessConditions.modifiedAccessConditions(), Context.NONE))
             .map(rb -> new SimpleResponse<>(rb, new AppendBlobItem(rb.deserializedHeaders())));
     }
 
@@ -111,7 +115,7 @@ public Mono<Response<AppendBlobItem>> create(BlobHTTPHeaders headers, Metadata m
      * @return
      *      A reactive response containing the information of the append blob operation.
      */
-    public Mono<Response<AppendBlobItem>> appendBlock(Flux<ByteBuffer> data, long length) {
+    public Mono<Response<AppendBlobItem>> appendBlock(Flux<ByteBuf> data, long length) {
         return this.appendBlock(data, length, null);
     }
 
@@ -133,10 +137,17 @@ public Mono<Response<AppendBlobItem>> appendBlock(Flux<ByteBuffer> data, long le
      * @return
      *      A reactive response containing the information of the append blob operation.
      */
-    public Mono<Response<AppendBlobItem>> appendBlock(Flux<ByteBuffer> data, long length,
-                                                           AppendBlobAccessConditions appendBlobAccessConditions) {
-        return appendBlobAsyncRawClient
-            .appendBlock(data.map(Unpooled::wrappedBuffer), length, appendBlobAccessConditions)
+    public Mono<Response<AppendBlobItem>> appendBlock(Flux<ByteBuf> data, long length,
+                                                      AppendBlobAccessConditions appendBlobAccessConditions) {
+        appendBlobAccessConditions = appendBlobAccessConditions == null ? new AppendBlobAccessConditions()
+            : appendBlobAccessConditions;
+
+        return postProcessResponse(this.azureBlobStorage.appendBlobs().appendBlockWithRestResponseAsync(
+            null, null, data, length, null, null,
+            null, null, null, null,
+            appendBlobAccessConditions.leaseAccessConditions(),
+            appendBlobAccessConditions.appendPositionAccessConditions(),
+            appendBlobAccessConditions.modifiedAccessConditions(), Context.NONE))
             .map(rb -> new SimpleResponse<>(rb, new AppendBlobItem(rb.deserializedHeaders())));
     }
 
@@ -183,8 +194,16 @@ public Mono<Response<AppendBlobItem>> appendBlockFromUrl(URL sourceURL, BlobRang
     public Mono<Response<AppendBlobItem>> appendBlockFromUrl(URL sourceURL, BlobRange sourceRange,
             byte[] sourceContentMD5, AppendBlobAccessConditions destAccessConditions,
             SourceModifiedAccessConditions sourceAccessConditions) {
-        return appendBlobAsyncRawClient
-            .appendBlockFromUrl(sourceURL, sourceRange, sourceContentMD5, destAccessConditions, sourceAccessConditions)
+        sourceRange = sourceRange == null ? new BlobRange(0) : sourceRange;
+        destAccessConditions = destAccessConditions == null
+            ? new AppendBlobAccessConditions() : destAccessConditions;
+
+        return postProcessResponse(
+            this.azureBlobStorage.appendBlobs().appendBlockFromUrlWithRestResponseAsync(null, null,
+                sourceURL, 0, sourceRange.toString(), sourceContentMD5, null, null,
+                destAccessConditions.leaseAccessConditions(),
+                destAccessConditions.appendPositionAccessConditions(),
+                destAccessConditions.modifiedAccessConditions(), sourceAccessConditions, Context.NONE))
             .map(rb -> new SimpleResponse<>(rb, new AppendBlobItem(rb.deserializedHeaders())));
     }
 }