Merge pull request #415 from WindowsAzure/master

merge code from public master to dev

ChangeLog.txt

@@ -1,3 +1,9 @@
+2013.09.30 Version 0.4.6
+ * Allow users to set the client-request-id for better tracking/debugging of storage requests. This is set on the OperationContext.
+ * Prevent a potential arithmetic overflow while calculating the exponential retry back-off interval in storage.
+ * Retry on IOException even when it is wrapped within an XMLStreamException in storage.
+ * Throw a more meaningful exception when connection is reset while parsing a request response in storage.
+
 2013.08.26 Version 0.4.5
  * Added support for managing affinity groups
  * Added support for Media Services job notification

microsoft-azure-api/pom.xml

@@ -17,7 +17,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.microsoft.windowsazure</groupId>
   <artifactId>microsoft-windowsazure-api</artifactId>
-  <version>0.4.5</version>
+  <version>0.4.6</version>
   <packaging>jar</packaging>
 
   <name>Microsoft Windows Azure Client API</name>

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/Constants.java

@@ -272,6 +272,11 @@ public static class HeaderConstants {
          */
         public static final String REQUEST_ID_HEADER = PREFIX_FOR_STORAGE_HEADER + "request-id";
 
+        /**
+         * The header that indicates the client request ID.
+         */
+        public static final String CLIENT_REQUEST_ID_HEADER = PREFIX_FOR_STORAGE_HEADER + "client-request-id";
+
         /**
          * The header for the If-Match condition.
          */

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/OperationContext.java

@@ -38,9 +38,7 @@ public final class OperationContext {
     /**
      * The UUID representing the client side trace ID.
      */
-    // V2 expose when logging is available.
-    @SuppressWarnings("unused")
-    private final String clientTraceID;
+    private String clientRequestID;
 
     /**
      * The Logger object associated with this operation.
@@ -98,10 +96,17 @@ public final class OperationContext {
      * Creates an instance of the <code>OperationContext</code> class.
      */
     public OperationContext() {
-        this.clientTraceID = UUID.randomUUID().toString();
+        this.clientRequestID = UUID.randomUUID().toString();
         this.requestResults = new ArrayList<RequestResult>();
     }
 
+    /**
+     * @return the clientRequestID
+     */
+    public String getClientRequestID() {
+        return this.clientRequestID;
+    }
+
     /**
      * @return the clientTimeInMs
      */
@@ -201,6 +206,14 @@ public void initialize() {
         this.setCurrentRequestObject(null);
     }
 
+    /**
+     * @param clientRequestID
+     *            the clientRequestID to set
+     */
+    public void setClientRequestID(final String clientRequestID) {
+        this.clientRequestID = clientRequestID;
+    }
+
     /**
      * @param clientTimeInMs
      *            the clientTimeInMs to set

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/RetryExponentialRetry.java

@@ -126,14 +126,15 @@ public RetryResult shouldRetry(final int currentRetryCount, final int statusCode
         if (currentRetryCount < this.maximumAttempts) {
             // Calculate backoff Interval between 80% and 120% of the desired
             // backoff, multiply by 2^n -1 for exponential
-            int incrementDelta = (int) (Math.pow(2, currentRetryCount) - 1);
+            double incrementDelta = (Math.pow(2, currentRetryCount) - 1);
             final int boundedRandDelta = (int) (this.deltaBackoffIntervalInMs * 0.8)
                     + this.randRef.nextInt((int) (this.deltaBackoffIntervalInMs * 1.2)
                             - (int) (this.deltaBackoffIntervalInMs * 0.8));
             incrementDelta *= boundedRandDelta;
 
             // Enforce max / min backoffs
-            return new RetryResult(Math.min(this.resolvedMinBackoff + incrementDelta, this.resolvedMaxBackoff), true);
+            return new RetryResult((int) Math.round(Math.min(this.resolvedMinBackoff + incrementDelta,
+                    this.resolvedMaxBackoff)), true);
         }
         else {
             return new RetryResult(-1, false);

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/StorageException.java

@@ -16,6 +16,7 @@
 
 import java.io.IOException;
 import java.net.HttpURLConnection;
+import java.net.SocketException;
 
 import javax.xml.stream.XMLStreamException;
 
@@ -89,7 +90,15 @@ public static StorageException translateException(final HttpURLConnection reques
         int responseCode = 0;
         try {
             responseCode = request.getResponseCode();
-            responseMessage = request.getResponseMessage();
+
+            // When the exception is expected(IfNotExists) or we have already parsed the exception, we pass null as
+            // the cause. In such cases, we should just try to get the message from the request.
+            if (cause == null || cause instanceof SocketException) {
+                responseMessage = request.getResponseMessage();
+            }
+            else if (cause != null) {
+                responseMessage = cause.getMessage();
+            }
         }
         catch (final IOException e) {
             // ignore errors

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/BaseRequest.java

@@ -200,6 +200,8 @@ public static HttpURLConnection createURLConnection(final URI uri, final int tim
         retConnection.setRequestProperty(Constants.HeaderConstants.STORAGE_VERSION_HEADER,
                 Constants.HeaderConstants.TARGET_STORAGE_VERSION);
         retConnection.setRequestProperty(Constants.HeaderConstants.USER_AGENT, getUserAgent());
+        retConnection.setRequestProperty(Constants.HeaderConstants.CLIENT_REQUEST_ID_HEADER,
+                opContext.getClientRequestID());
 
         // Java6 TODO remove me, this has to be manually set or it will
         // sometimes default to application/x-www-form-urlencoded without us

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/core/storage/utils/implementation/ExecutionEngine.java

@@ -18,6 +18,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
+import java.net.SocketException;
 import java.net.SocketTimeoutException;
 import java.net.URISyntaxException;
 import java.security.InvalidKeyException;
@@ -34,6 +35,7 @@
 import com.microsoft.windowsazure.services.core.storage.RetryPolicyFactory;
 import com.microsoft.windowsazure.services.core.storage.RetryResult;
 import com.microsoft.windowsazure.services.core.storage.SendingRequestEvent;
+import com.microsoft.windowsazure.services.core.storage.StorageErrorCode;
 import com.microsoft.windowsazure.services.core.storage.StorageErrorCodeStrings;
 import com.microsoft.windowsazure.services.core.storage.StorageException;
 import com.microsoft.windowsazure.services.core.storage.utils.Utility;
@@ -168,11 +170,23 @@ public static <CLIENT_TYPE, PARENT_TYPE, RESULT_TYPE> RESULT_TYPE executeWithRet
                 task.getResult().setException(translatedException);
             }
             catch (final XMLStreamException e) {
-                // Non Retryable, just throw
-                translatedException = StorageException
-                        .translateException(getLastRequestObject(opContext), e, opContext);
+                // Non Retryable except when the inner exception is actually an IOException
+
+                // Only in the case of xml exceptions that are due to connection issues.
+                if (e.getCause() instanceof SocketException) {
+                    translatedException = new StorageException(StorageErrorCode.SERVICE_INTERNAL_ERROR.toString(),
+                            "An unknown failure occurred : ".concat(e.getCause().getMessage()),
+                            HttpURLConnection.HTTP_INTERNAL_ERROR, null, e);
+                }
+                else {
+                    translatedException = StorageException.translateException(getLastRequestObject(opContext), e,
+                            opContext);
+                }
+
                 task.getResult().setException(translatedException);
-                throw translatedException;
+                if (!(e.getCause() instanceof IOException) && !(e.getCause() instanceof SocketException)) {
+                    throw translatedException;
+                }
             }
             catch (final InvalidKeyException e) {
                 // Non Retryable, just throw

microsoft-azure-api/src/main/java/com/microsoft/windowsazure/services/management/implementation/ManagementRestProxy.java

@@ -26,7 +26,7 @@
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
-import com.microsoft.windowsazure.services.blob.implementation.RFC1123DateConverter;
+import com.microsoft.windowsazure.services.core.RFC1123DateConverter;
 import com.microsoft.windowsazure.services.core.ServiceFilter;
 import com.microsoft.windowsazure.services.core.UserAgentFilter;
 import com.microsoft.windowsazure.services.core.utils.pipeline.ClientFilterAdapter;

microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/blob/client/CloudBlobContainerTests.java

@@ -399,6 +399,32 @@ public void testBlobLeaseBreak() throws URISyntaxException, StorageException, IO
         Assert.assertTrue(operationContext.getLastResult().getStatusCode() == HttpURLConnection.HTTP_ACCEPTED);
     }
 
+    @Test
+    public void testListBlobs() throws URISyntaxException, StorageException, IOException, InterruptedException {
+        final int length = 128;
+        final Random randGenerator = new Random();
+        final byte[] buff = new byte[length];
+        randGenerator.nextBytes(buff);
+
+        String blobName = "testBlob" + Integer.toString(randGenerator.nextInt(50000));
+        blobName = blobName.replace('-', '_');
+
+        final CloudBlobContainer existingContainer = bClient.getContainerReference(testSuiteContainerName);
+        final CloudBlob blobRef = existingContainer.getBlockBlobReference(blobName);
+        final BlobRequestOptions options = new BlobRequestOptions();
+        OperationContext context = new OperationContext();
+        context.setClientRequestID("My-Helpful-TraceId");
+
+        // Subscribe to sending request event once we move that. Currently, the sendingrequest event is fired 
+        // after the connection is initiated and we don't have access to the request headers at that time.
+
+        blobRef.upload(new ByteArrayInputStream(buff), -1, null, options, context);
+
+        for (ListBlobItem blob : existingContainer.listBlobs()) {
+            Assert.assertEquals(blobRef.getClass(), blob.getClass());
+        }
+    }
+
     @Test
     public void testBlobLeaseRenew() throws URISyntaxException, StorageException, IOException, InterruptedException {
         final int length = 128;

microsoft-azure-api/src/test/java/com/microsoft/windowsazure/services/table/client/TableClientTests.java

@@ -34,7 +34,12 @@
 import org.junit.Test;
 
 import com.microsoft.windowsazure.services.core.storage.AuthenticationScheme;
+import com.microsoft.windowsazure.services.core.storage.OperationContext;
 import com.microsoft.windowsazure.services.core.storage.ResultSegment;
+import com.microsoft.windowsazure.services.core.storage.RetryExponentialRetry;
+import com.microsoft.windowsazure.services.core.storage.RetryLinearRetry;
+import com.microsoft.windowsazure.services.core.storage.RetryPolicy;
+import com.microsoft.windowsazure.services.core.storage.RetryResult;
 import com.microsoft.windowsazure.services.core.storage.StorageCredentialsSharedAccessSignature;
 import com.microsoft.windowsazure.services.core.storage.StorageException;
 
@@ -672,4 +677,31 @@ private CloudTableClient getTableForSas(CloudTable table, SharedAccessTablePolic
                 .generateSharedAccessSignature(policy, accessIdentifier, startPk, startRk, endPk, endRk);
         return new CloudTableClient(tClient.getEndpoint(), new StorageCredentialsSharedAccessSignature(sasString));
     }
+
+    @Test
+    public void VerifyBackoffTimeOverflow() {
+        RetryExponentialRetry exponentialRetry = new RetryExponentialRetry(4000, 100000);
+        VerifyBackoffTimeOverflow(exponentialRetry, 100000);
+
+        RetryLinearRetry linearRetry = new RetryLinearRetry(4000, 100000);
+        VerifyBackoffTimeOverflow(linearRetry, 100000);
+    }
+
+    private void VerifyBackoffTimeOverflow(RetryPolicy retryPolicy, int maxAttempts) {
+        Exception e = new Exception();
+        OperationContext context = new OperationContext();
+        int previousRetryInterval = 1000; // larger than zero to ensure we never get zero back
+
+        for (int i = 0; i < maxAttempts; i++) {
+            RetryResult result = retryPolicy.shouldRetry(i, HttpURLConnection.HTTP_INTERNAL_ERROR, e, context);
+            int retryInterval = result.getBackOffIntervalInMs();
+            Assert.assertTrue(String.format("Attempt: '%d'", i), result.isShouldRetry());
+            Assert.assertTrue(String.format("Retry Interval: '%d', Previous Retry Interval: '%d', Attempt: '%d'",
+                    retryInterval, previousRetryInterval, i), retryInterval >= previousRetryInterval);
+            previousRetryInterval = retryInterval;
+        }
+
+        Assert.assertFalse(retryPolicy.shouldRetry(maxAttempts, HttpURLConnection.HTTP_INTERNAL_ERROR, e, context)
+                .isShouldRetry());
+    }
 }