Add live test coverage for Cosmos endpoints and mitigate issues

sdk/core/Azure.Core.TestFramework/src/TestEnvironment.cs

@@ -132,7 +132,7 @@ public TokenCredential Credential
                         throw new InvalidOperationException("Azure.Identity must be referenced to use Credential in Live environment.");
                     }
 
-                    _credential = (TokenCredential) Activator.CreateInstance(
+                    _credential = (TokenCredential)Activator.CreateInstance(
                         type,
                         GetVariable("TENANT_ID"),
                         GetVariable("CLIENT_ID"),
@@ -173,8 +173,7 @@ protected string GetRecordedVariable(string name)
         }
 
         /// <summary>
-        /// Returns an environment variable value.
-        /// Throws when variable is not found.
+        /// Returns an environment variable value or null when variable is not found.
         /// </summary>
         protected string GetOptionalVariable(string name)
         {

sdk/tables/Azure.Data.Tables/api/Azure.Data.Tables.netstandard2.0.cs

@@ -9,6 +9,7 @@ protected TableClient() { }
         public virtual System.Threading.Tasks.Task<Azure.Response<System.Collections.ObjectModel.ReadOnlyDictionary<string, object>>> CreateEntityAsync(System.Collections.Generic.IDictionary<string, object> entity, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public virtual System.Threading.Tasks.Task<Azure.Response<T>> CreateEntityAsync<T>(T entity, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) where T : Azure.Data.Tables.TableEntity, new() { throw null; }
         public virtual Azure.Response<T> CreateEntity<T>(T entity, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) where T : Azure.Data.Tables.TableEntity, new() { throw null; }
+        public static string CreateFilter<T>(System.Linq.Expressions.Expression<System.Func<T, bool>> filter) { throw null; }
         public virtual Azure.Response Delete(string partitionKey, string rowKey, string eTag = "*", System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public virtual System.Threading.Tasks.Task<Azure.Response> DeleteAsync(string partitionKey, string rowKey, string eTag = "*", System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public virtual Azure.Response<System.Collections.Generic.IReadOnlyList<Azure.Data.Tables.Models.SignedIdentifier>> GetAccessPolicy(int? timeout = default(int?), string requestId = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
@@ -232,13 +233,6 @@ internal TableServiceStats() { }
         public Azure.Data.Tables.Models.GeoReplication GeoReplication { get { throw null; } }
     }
 }
-namespace Azure.Data.Tables.Queryable
-{
-    public static partial class TableClientExtensions
-    {
-        public static string CreateFilter<T>(this Azure.Data.Tables.TableClient client, System.Linq.Expressions.Expression<System.Func<T, bool>> filter) { throw null; }
-    }
-}
 namespace Azure.Data.Tables.Sas
 {
     [System.Runtime.InteropServices.StructLayoutAttribute(System.Runtime.InteropServices.LayoutKind.Sequential)]

sdk/tables/Azure.Data.Tables/src/TableAcceptHeaderPipelinePolicy.cs

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using Azure.Core;
+using Azure.Core.Pipeline;
+
+namespace Azure.Data.Tables
+{
+    /// <summary>
+    /// HttpPipelinePolicy to add an Accept header to deal with service inconsistencies between Cosmos and Storage endpoints.
+    /// https://github.com/Azure/azure-sdk-for-net/issues/13559
+    /// </summary>
+    internal sealed class TableAcceptHeaderPipelinePolicy : HttpPipelineSynchronousPolicy
+    {
+        /// <summary>
+        /// Add Accept headers.
+        /// </summary>
+        /// <param name="message">The message.</param>
+        public override void OnSendingRequest(HttpMessage message)
+        {
+            base.OnSendingRequest(message);
+
+            // Both storage and cosmos can be inconsistent about requiring an Accept header, so add one here for all requests
+            if (message.Request.Headers.TryGetValue(TableConstants.HeaderNames.Content, out var contentType))
+            {
+                switch (contentType)
+                {
+                    case TableConstants.MimeType.ApplicationXml:
+                        message.Request.Headers.SetValue(TableConstants.HeaderNames.Accept, TableConstants.MimeType.ApplicationXml);
+                        break;
+                    default:
+                        message.Request.Headers.SetValue(TableConstants.HeaderNames.Accept, TableConstants.MimeType.ApplicationJson);
+                        break;
+                }
+            }
+            else if (!message.Request.Uri.Query.Contains("comp="))
+            {
+                // The default Accept header should be application/json, however all requests using the comp= query string are application/xml
+                // These requests don't always set a Content-Type header as a clue for the logic above.
+                message.Request.Headers.SetValue(TableConstants.HeaderNames.Accept, TableConstants.MimeType.ApplicationJson);
+            }
+        }
+    }
+}

sdk/tables/Azure.Data.Tables/src/TableClient.cs

@@ -993,20 +993,15 @@ public virtual Response SetAccessPolicy(IEnumerable<SignedIdentifier> tableAcl,
             }
         }
 
-        internal ExpressionParser GetExpressionParser()
-        {
-            if (_isPremiumEndpoint)
-            {
-                //TODO: Port TableExtensionExpressionParser
-                throw new NotImplementedException();
-            }
-            else
-            {
-                return new ExpressionParser();
-            }
-        }
+        /// <summary>
+        /// Creates an Odata filter query string from the provided expression.
+        /// </summary>
+        /// <typeparam name="T">The type of the entity being queried. Typically this will be derrived from <see cref="TableEntity"/> or <see cref="Dictionary{String, Object}"/>.</typeparam>
+        /// <param name="filter">A filter expresssion.</param>
+        /// <returns>The string representation of the filter expression.</returns>
+        public static string CreateFilter<T>(Expression<Func<T, bool>> filter) => Bind(filter);
 
-        internal string Bind(Expression expression)
+        internal static string Bind(Expression expression)
         {
             Argument.AssertNotNull(expression, nameof(expression));
 
@@ -1019,7 +1014,7 @@ internal string Bind(Expression expression)
             Expression normalizedExpression = ExpressionNormalizer.Normalize(partialEvaluatedExpression, normalizerRewrites);
 
             // Parse the Bound expression into sub components, i.e. take count, filter, select columns, request options, opcontext, etc.
-            ExpressionParser parser = GetExpressionParser();
+            ExpressionParser parser = new ExpressionParser();
             parser.Translate(normalizedExpression);
 
             // Return the FilterString.

sdk/tables/Azure.Data.Tables/src/TableConstants.cs

@@ -14,6 +14,14 @@ internal static class HeaderNames
             public const string SharedKey = "SharedKeyLite";
             public const string Authorization = "Authorization";
             public const string IfMatch = "If-Match";
+            public const string Accept = "Accept";
+            public const string Content = "Content-Type";
+        }
+
+        internal static class MimeType
+        {
+            internal const string ApplicationJson = "application/json";
+            internal const string ApplicationXml = "application/xml";
         }
 
         internal static class QueryParameterNames

sdk/tables/Azure.Data.Tables/src/TableServiceClient.cs

@@ -55,14 +55,15 @@ internal TableServiceClient(Uri endpoint, TableSharedKeyPipelinePolicy policy, T
             var endpointString = endpoint.ToString();
             var secondaryEndpoint = endpointString.Insert(endpointString.IndexOf('.'), "-secondary");
             HttpPipeline pipeline;
+            var acceptPolicy = new TableAcceptHeaderPipelinePolicy();
 
             if (policy == default)
             {
-                pipeline = HttpPipelineBuilder.Build(options);
+                pipeline = HttpPipelineBuilder.Build(options, acceptPolicy);
             }
             else
             {
-                pipeline = HttpPipelineBuilder.Build(options, policy);
+                pipeline = HttpPipelineBuilder.Build(options, policy, acceptPolicy);
             }
 
             _diagnostics = new ClientDiagnostics(options);

sdk/tables/Azure.Data.Tables/src/TableSharedKeyPipelinePolicy.cs

@@ -18,10 +18,10 @@ namespace Azure.Data.Tables
     /// </summary>
     internal sealed class TableSharedKeyPipelinePolicy : HttpPipelineSynchronousPolicy
     {
-        private class InternalStorageCredential: TableSharedKeyCredential
+        private class InternalStorageCredential : TableSharedKeyCredential
         {
             public static InternalStorageCredential Instance = new InternalStorageCredential();
-            public InternalStorageCredential(): base(string.Empty, string.Empty)
+            public InternalStorageCredential() : base(string.Empty, string.Empty)
             {
             }
 
@@ -94,6 +94,10 @@ private string BuildCanonicalizedResource(Uri resource)
             {
                 foreach (var name in parameters.Keys.OrderBy(key => key, StringComparer.Ordinal))
                 {
+                    // If the request URI addresses a component of the resource, append the appropriate query string.
+                    // The query string should include the question mark and the comp parameter (for example, ?comp=metadata).
+                    // No other parameters should be included on the query string.
+                    // https://docs.microsoft.com/en-us/rest/api/storageservices/authorize-with-shared-key#shared-key-lite-and-table-service-format-for-2009-09-19-and-later
                     if (name == "comp")
                     {
 #pragma warning disable CA1308 // Normalize strings to uppercase

sdk/tables/Azure.Data.Tables/test-resources.json

@@ -10,7 +10,6 @@
         "mgmtApiVersion": "2019-04-01",
         "location": "[resourceGroup().location]",
         "primaryAccountName": "[concat(parameters('baseName'), 'prim')]",
-        "location": "[resourceGroup().location]",
         "encryption": {
             "services": {
                 "file": {
@@ -46,16 +45,66 @@
                 "encryption": "[variables('encryption')]",
                 "accessTier": "Cool"
             }
+        },
+        {
+            "type": "Microsoft.DocumentDB/databaseAccounts",
+            "apiVersion": "2020-04-01",
+            "name": "[variables('primaryAccountName')]",
+            "location": "[variables('location')]",
+            "tags": {
+                "defaultExperience": "Azure Table",
+                "hidden-cosmos-mmspecial": "",
+                "CosmosAccountType": "Non-Production"
+            },
+            "kind": "GlobalDocumentDB",
+            "properties": {
+                "publicNetworkAccess": "Enabled",
+                "enableAutomaticFailover": false,
+                "enableMultipleWriteLocations": false,
+                "isVirtualNetworkFilterEnabled": false,
+                "virtualNetworkRules": [],
+                "disableKeyBasedMetadataWriteAccess": false,
+                "enableFreeTier": false,
+                "enableAnalyticalStorage": false,
+                "databaseAccountOfferType": "Standard",
+                "consistencyPolicy": {
+                    "defaultConsistencyLevel": "BoundedStaleness",
+                    "maxIntervalInSeconds": 86400,
+                    "maxStalenessPrefix": 1000000
+                },
+                "locations": [
+                    {
+                        "locationName": "[variables('location')]",
+                        "provisioningState": "Succeeded",
+                        "failoverPriority": 0,
+                        "isZoneRedundant": false
+                    }
+                ],
+                "capabilities": [
+                    {
+                        "name": "EnableTable"
+                    }
+                ],
+                "ipRules": []
+            }
         }
     ],
     "outputs": {
-      "TABLES_STORAGE_ACCOUNT_NAME": {
-        "type": "string",
-        "value": "[variables('primaryAccountName')]"
-      },
-      "TABLES_PRIMARY_STORAGE_ACCOUNT_KEY": {
-        "type": "string",
-        "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('primaryAccountName')), variables('mgmtApiVersion')).keys[0].value]"
-      }
+        "TABLES_STORAGE_ACCOUNT_NAME": {
+            "type": "string",
+            "value": "[variables('primaryAccountName')]"
+        },
+        "TABLES_PRIMARY_STORAGE_ACCOUNT_KEY": {
+            "type": "string",
+            "value": "[listKeys(resourceId('Microsoft.Storage/storageAccounts', variables('primaryAccountName')), variables('mgmtApiVersion')).keys[0].value]"
+        },
+        "TABLES_COSMOS_ACCOUNT_NAME": {
+            "type": "string",
+            "value": "[variables('primaryAccountName')]"
+        },
+        "TABLES_PRIMARY_COSMOS_ACCOUNT_KEY": {
+            "type": "string",
+            "value": "[listKeys(resourceId('Microsoft.DocumentDB/databaseAccounts', variables('primaryAccountName')), '2020-04-01').primaryMasterKey]"
+        }
     }
 }