Issue 87 - Handle some RequestRateTooLarge (429) errors with retries

RELEASENOTES.md

@@ -1,5 +1,20 @@
 # Release Notes
 
+## Unreleased
+
+- Added `-RetryOnRequestRateTooLarge` parameter to handle Response Code 429 errors in `Invoke-CosmosDbRequest`, `Get-CosmosDbDocument` and `Invoke-CosmosDbStoredProcedure`. This resolves some of [Issue #87](https://github.com/PlagueHO/CosmosDB/issues/87)
+- Added RU Output to Verbose Output Stream in `Invoke-CosmosDbRequest` with `-UseWebRequest`
+  ```
+  VERBOSE: Request Charge: 1570.5 RUs
+  ```
+- Improved ScriptLogResults when using `Invoke-CosmosDbStoredProcedure -Debug -Verbose`
+  ```
+  VERBOSE: ScriptLogResults:
+  enter my procedure
+  some console.log() messages
+  my procedure complete
+  ```
+
 ## What is New in CosmosDB 2.0.16.465
 
 June 20, 2018

src/lib/documents.ps1

@@ -129,7 +129,11 @@ function Get-CosmosDbDocument
 
         [Parameter()]
         [ref]
-        $ResultHeaders
+        $ResultHeaders,
+
+        [Parameter()]
+        [Switch]
+        $RetryOnRequestRateTooLarge
     )
 
     $null = $PSBoundParameters.Remove('Id')

src/lib/storedprocedures.ps1

@@ -162,7 +162,11 @@ function Invoke-CosmosDbStoredProcedure
         [Parameter()]
         [ValidateNotNullOrEmpty()]
         [System.Object[]]
-        $StoredProcedureParameter
+        $StoredProcedureParameter,
+
+        [Parameter()]
+        [Switch]
+        $RetryOnRequestRateTooLarge
     )
 
     $null = $PSBoundParameters.Remove('CollectionId')
@@ -209,7 +213,7 @@ function Invoke-CosmosDbStoredProcedure
 
     if ($result.Headers.'x-ms-documentdb-script-log-results')
     {
-        Write-Verbose -Message "Script Log Results: $($result.Headers.'x-ms-documentdb-script-log-results')"
+        Write-Verbose -Message "ScriptLogResults:`n$([Uri]::UnescapeDataString($result.Headers.'x-ms-documentdb-script-log-results').Trim())"
     }
 
     if ($result.Content)

src/lib/utils.ps1

@@ -323,7 +323,11 @@ function Invoke-CosmosDbRequest
 
         [Parameter()]
         [System.String]
-        $ContentType = 'application/json'
+        $ContentType = 'application/json',
+
+        [Parameter()]
+        [Switch]
+        $RetryOnRequestRateTooLarge
     )
 
     if ($PSCmdlet.ParameterSetName -eq 'Account')
@@ -502,41 +506,61 @@ function Invoke-CosmosDbRequest
             Body = $Body
         }
     }
-
-    try
-    {
-        if ($UseWebRequest)
+    do {
+        try
         {
-            $restResult = Invoke-WebRequest -UseBasicParsing @invokeRestMethodParameters
+            if ($UseWebRequest)
+            {
+                $restResult = Invoke-WebRequest -UseBasicParsing @invokeRestMethodParameters
+            }
+            else
+            {
+                $restResult = Invoke-RestMethod @invokeRestMethodParameters
+            }
+            $done = $true;
         }
-        else
+        catch [Microsoft.PowerShell.Commands.HttpResponseException]
         {
-            $restResult = Invoke-RestMethod @invokeRestMethodParameters
+            if($_.Exception.Response.StatusCode -eq 429 -and $RetryOnRequestRateTooLarge)
+            {
+                [int] $delay = [int](($_.Exception.Response.Headers | Where-Object Key -eq 'x-ms-retry-after-ms').Value[0])
+                Write-Verbose -Message "Retry Caught, delaying $delay ms"
+                Start-Sleep -Milliseconds $delay
+            }
+            else
+            {
+                Throw $_
+            }
         }
-    }
-    catch [System.Net.WebException]
-    {
-        <#
-            Write out additional exception information into the verbose stream
-            In a future version a custom exception type for CosmosDB that
-            contains this additional information.
-        #>
-        if ($_.Exception.Response)
+        catch [System.Net.WebException]
         {
-            $exceptionStream = $_.Exception.Response.GetResponseStream()
-            $streamReader = New-Object -TypeName System.IO.StreamReader -ArgumentList $exceptionStream
-            $exceptionResponse = $streamReader.ReadToEnd()
-            if ($exceptionResponse)
+            <#
+                Write out additional exception information into the verbose stream
+                In a future version a custom exception type for CosmosDB that
+                contains this additional information.
+            #>
+            if ($_.Exception.Response)
             {
-                Write-Verbose -Message $exceptionResponse
+                $exceptionStream = $_.Exception.Response.GetResponseStream()
+                $streamReader = New-Object -TypeName System.IO.StreamReader -ArgumentList $exceptionStream
+                $exceptionResponse = $streamReader.ReadToEnd()
+                if ($exceptionResponse)
+                {
+                    Write-Verbose -Message $exceptionResponse
+                }
             }
+
+            Throw $_
         }
+        catch
+        {
+            Throw $_
+        }
+    } while ($done -ne $true)
 
-        Throw $_
-    }
-    catch
+    if($restResult.Headers.'x-ms-request-charge')
     {
-        Throw $_
+        Write-Verbose -Message "Request Charge: $($restResult.Headers.'x-ms-request-charge') RUs"
     }
 
     return $restResult