Search SDK: Fix data corruption when using non-nullable types in mode…

src/Search/Search.Management.Tests/Utilities/SearchServiceFixture.cs

@@ -93,7 +93,7 @@ private string EnsureSearchService(SearchManagementClient client)
                 // wait several minutes, we fail fast here.
                 TimeSpan maxDelay = 
                     (HttpMockServer.Mode == HttpRecorderMode.Record) ? 
-                        TimeSpan.FromMinutes(5) : TimeSpan.FromSeconds(15);
+                        TimeSpan.FromMinutes(10) : TimeSpan.FromSeconds(15);
 
                 if (SearchTestUtilities.WaitForSearchServiceDns(searchServiceName, maxDelay))
                 {

src/Search/Search.Tests/Search.Tests.csproj

@@ -26,6 +26,8 @@
     <Compile Include="Tests\LookupTests.cs" />
     <Compile Include="Tests\Models\Book.cs" />
     <Compile Include="Tests\Models\Hotel.cs" />
+    <Compile Include="Tests\Models\ModelWithInt.cs" />
+    <Compile Include="Tests\Models\ModelWithNullableInt.cs" />
     <Compile Include="Tests\PostSearchTests.cs" />
     <Compile Include="Tests\PostSuggestTests.cs" />
     <Compile Include="Tests\QueryTests.cs" />
@@ -36,6 +38,7 @@
     <Compile Include="Tests\Serialization\GeographyPointConverterTests.cs" />
     <Compile Include="Tests\Serialization\SearchParametersTests.cs" />
     <Compile Include="Tests\Serialization\SuggestParametersTests.cs" />
+    <Compile Include="Tests\Serialization\ValueTypePreservingContractResolverTests.cs" />
     <Compile Include="Tests\SoftDeleteColumnDeletionDetectionPolicyTests.cs" />
     <Compile Include="Tests\SuggestTests.cs" />
     <Compile Include="Utilities\DocumentsFixture.cs" />
@@ -89,9 +92,15 @@
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\CanFilter.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\CanFilterNonNullableType.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\CanGetResultCountInSearch.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\CanRoundTripNonNullableValueTypes.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\CanSearchDynamicDocuments.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -125,6 +134,9 @@
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\DefaultSearchModeIsAny.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\NullCannotBeConvertedToValueType.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.GetSearchTests\OrderByProgressivelyBreaksTies.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -323,9 +335,15 @@
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\CanFilter.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\CanFilterNonNullableType.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\CanGetResultCountInSearch.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\CanRoundTripNonNullableValueTypes.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\CanSearchDynamicDocuments.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -359,6 +377,9 @@
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\DefaultSearchModeIsAny.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\NullCannotBeConvertedToValueType.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Include="SessionRecords\Microsoft.Azure.Search.Tests.PostSearchTests\OrderByProgressivelyBreaksTies.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>

src/Search/Search.Tests/Tests/GetSearchTests.cs

@@ -133,6 +133,24 @@ public void CanSearchWithDateTimeInStaticModel()
             Run(TestCanSearchWithDateTimeInStaticModel);
         }
 
+        [Fact]
+        public void CanRoundTripNonNullableValueTypes()
+        {
+            Run(TestCanRoundTripNonNullableValueTypes);
+        }
+
+        [Fact]
+        public void NullCannotBeConvertedToValueType()
+        {
+            Run(TestNullCannotBeConvertedToValueType);
+        }
+
+        [Fact]
+        public void CanFilterNonNullableType()
+        {
+            Run(TestCanFilterNonNullableType);
+        }
+
         protected override SearchIndexClient GetClient()
         {
             SearchIndexClient client = base.GetClient();

src/Search/Search.Tests/Tests/Models/ModelWithInt.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for
+// license information.
+
+namespace Microsoft.Azure.Search.Tests
+{
+    internal class ModelWithInt
+    {
+        public string Key { get; set; }
+
+        public int IntValue { get; set; }
+    }
+}

src/Search/Search.Tests/Tests/Models/ModelWithNullableInt.cs

@@ -0,0 +1,13 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for
+// license information.
+
+namespace Microsoft.Azure.Search.Tests
+{
+    internal class ModelWithNullableInt
+    {
+        public string Key { get; set; }
+
+        public int? IntValue { get; set; }
+    }
+}

src/Search/Search.Tests/Tests/PostSearchTests.cs

@@ -132,5 +132,23 @@ public void CanSearchWithDateTimeInStaticModel()
         {
             Run(TestCanSearchWithDateTimeInStaticModel);
         }
+
+        [Fact]
+        public void CanRoundTripNonNullableValueTypes()
+        {
+            Run(TestCanRoundTripNonNullableValueTypes);
+        }
+
+        [Fact]
+        public void NullCannotBeConvertedToValueType()
+        {
+            Run(TestNullCannotBeConvertedToValueType);
+        }
+
+        [Fact]
+        public void CanFilterNonNullableType()
+        {
+            Run(TestCanFilterNonNullableType);
+        }
     }
 }

src/Search/Search.Tests/Tests/SearchTests.cs

@@ -13,6 +13,7 @@ namespace Microsoft.Azure.Search.Tests
     using Microsoft.Rest.Azure;
     using Microsoft.Rest.ClientRuntime.Azure.TestFramework;
     using Microsoft.Spatial;
+    using Newtonsoft.Json;
     using Xunit;
 
     // MAINTENANCE NOTE: Test methods (those marked with [Fact]) need to be in the derived classes in order for
@@ -509,6 +510,128 @@ protected void TestCanSearchWithDateTimeInStaticModel()
             Assert.Equal(doc2, response.Results[0].Document);
         }
 
+        protected void TestCanRoundTripNonNullableValueTypes()
+        {
+            SearchServiceClient serviceClient = Data.GetSearchServiceClient();
+
+            Index index = new Index()
+            {
+                Name = TestUtilities.GenerateName(),
+                Fields = new[]
+                {
+                    new Field("Key", DataType.String) { IsKey = true },
+                    new Field("Rating", DataType.Int32),
+                    new Field("Count", DataType.Int64),
+                    new Field("IsEnabled", DataType.Boolean),
+                    new Field("Ratio", DataType.Double),
+                    new Field("StartDate", DataType.DateTimeOffset),
+                    new Field("EndDate", DataType.DateTimeOffset)
+                }
+            };
+
+            serviceClient.Indexes.Create(index);
+            SearchIndexClient indexClient = Data.GetSearchIndexClient(index.Name);
+
+            DateTimeOffset startDate = new DateTimeOffset(2015, 11, 24, 14, 01, 00, TimeSpan.FromHours(-8));
+            DateTime endDate = startDate.UtcDateTime + TimeSpan.FromDays(15);
+
+            var doc1 = new NonNullableModel() 
+            { 
+                Key = "123", 
+                Count = 3, 
+                EndDate = endDate, 
+                IsEnabled = true, 
+                Rating = 5, 
+                Ratio = 3.14, 
+                StartDate = startDate
+            };
+
+            var doc2 = new NonNullableModel()
+            {
+                Key = "456",
+                Count = default(long),
+                EndDate = default(DateTime),
+                IsEnabled = default(bool),
+                Rating = default(int),
+                Ratio = default(double),
+                StartDate = default(DateTimeOffset)
+            };
+
+            var batch = IndexBatch.Create(IndexAction.Create(doc1), IndexAction.Create(doc2));
+
+            indexClient.Documents.Index(batch);
+            SearchTestUtilities.WaitForIndexing();
+
+            DocumentSearchResult<NonNullableModel> response = indexClient.Documents.Search<NonNullableModel>("*");
+
+            Assert.Equal(2, response.Results.Count);
+            Assert.Equal(doc1, response.Results[0].Document);
+            Assert.Equal(doc2, response.Results[1].Document);
+        }
+
+        protected void TestNullCannotBeConvertedToValueType()
+        {
+            SearchServiceClient serviceClient = Data.GetSearchServiceClient();
+
+            Index index = new Index()
+            {
+                Name = TestUtilities.GenerateName(),
+                Fields = new[]
+                {
+                    new Field("Key", DataType.String) { IsKey = true },
+                    new Field("IntValue", DataType.Int32)
+                }
+            };
+
+            serviceClient.Indexes.Create(index);
+            SearchIndexClient indexClient = Data.GetSearchIndexClient(index.Name);
+
+            var doc = new ModelWithNullableInt()
+            {
+                Key = "123",
+                IntValue = null
+            };
+
+            var batch = IndexBatch.Create(IndexAction.Create(doc));
+
+            indexClient.Documents.Index(batch);
+            SearchTestUtilities.WaitForIndexing();
+
+            JsonSerializationException e =
+                Assert.Throws<JsonSerializationException>(() => indexClient.Documents.Search<ModelWithInt>("*"));
+            Assert.Contains("Error converting value {null} to type 'System.Int32'. Path 'IntValue'.", e.Message);
+        }
+
+        protected void TestCanFilterNonNullableType()
+        {
+            SearchServiceClient serviceClient = Data.GetSearchServiceClient();
+
+            Index index = new Index()
+            {
+                Name = TestUtilities.GenerateName(),
+                Fields = new[]
+                {
+                    new Field("Key", DataType.String) { IsKey = true },
+                    new Field("IntValue", DataType.Int32) { IsFilterable = true }
+                }
+            };
+
+            serviceClient.Indexes.Create(index);
+            SearchIndexClient indexClient = Data.GetSearchIndexClient(index.Name);
+
+            var doc = new ModelWithInt() { Key = "123", IntValue = 0 };
+            var batch = IndexBatch.Create(IndexAction.Create(doc));
+
+            indexClient.Documents.Index(batch);
+            SearchTestUtilities.WaitForIndexing();
+
+            var parameters = new SearchParameters() { Filter = "IntValue eq 0" };
+            DocumentSearchResult<ModelWithInt> response = indexClient.Documents.Search<ModelWithInt>("*", parameters);
+
+            Assert.Equal(1, response.Results.Count);
+            Assert.Equal(doc.IntValue, response.Results[0].Document.IntValue);
+        }
+
         private IEnumerable<string> IndexDocuments(SearchIndexClient client, int totalDocCount)
         {
             int existingDocumentCount = Data.TestDocuments.Length;
@@ -607,5 +730,46 @@ private void AssertValueFacetsEqual<T>(ValueFacetResult<T>[] actualFacets, param
                 Assert.Equal(expectedFacets[i].Count, actualFacets[i].Count);
             }
         }
+
+        private class NonNullableModel
+        {
+            public string Key { get; set; }
+
+            public int Rating { get; set; }
+
+            public long Count { get; set; }
+
+            public bool IsEnabled { get; set; }
+
+            public double Ratio { get; set; }
+
+            public DateTimeOffset StartDate { get; set; }
+
+            public DateTime EndDate { get; set; }
+
+            public override bool Equals(object obj)
+            {
+                NonNullableModel other = obj as NonNullableModel;
+
+                if (other == null)
+                {
+                    return false;
+                }
+
+                return
+                    this.Count == other.Count &&
+                    this.EndDate == other.EndDate &&
+                    this.IsEnabled == other.IsEnabled &&
+                    this.Key == other.Key &&
+                    this.Rating == other.Rating &&
+                    this.Ratio == other.Ratio &&
+                    this.StartDate == other.StartDate;
+            }
+
+            public override int GetHashCode()
+            {
+                return (this.Key != null) ? this.Key.GetHashCode() : 0;
+            }
+        }
     }
 }

src/Search/Search.Tests/Tests/Serialization/ValueTypePreservingContractResolverTests.cs

@@ -0,0 +1,52 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for
+// license information.
+
+namespace Microsoft.Azure.Search.Tests
+{
+    using System;
+    using System.Linq;
+    using Newtonsoft.Json;
+    using Newtonsoft.Json.Serialization;
+    using Xunit;
+
+    public sealed class ValueTypePreservingContractResolverTests
+    {
+        private ValueTypePreservingContractResolver _resolver;
+
+        public ValueTypePreservingContractResolverTests()
+        {
+            this._resolver = new ValueTypePreservingContractResolver(new DefaultContractResolver());
+        }
+
+        [Fact]
+        public void NullablePropertiesHaveNoDefaultValueHandlingSet()
+        {
+            JsonObjectContract contract = Resolve<ModelWithNullableInt>();
+
+            Assert.Equal(2, contract.Properties.Count);
+            Assert.True(contract.Properties.All(p => !p.DefaultValueHandling.HasValue));
+        }
+
+        [Fact]
+        public void NonNullablePropertiesSetToDefaultValueInclude()
+        {
+            JsonObjectContract contract = Resolve<ModelWithInt>();
+
+            Assert.Equal(2, contract.Properties.Count);
+
+            JsonProperty key = contract.Properties.GetProperty("Key", StringComparison.Ordinal);
+            JsonProperty intValue = contract.Properties.GetProperty("IntValue", StringComparison.Ordinal);
+
+            Assert.False(key.DefaultValueHandling.HasValue);
+            Assert.True(intValue.DefaultValueHandling.HasValue);
+            Assert.Equal(DefaultValueHandling.Include, intValue.DefaultValueHandling.Value);
+        }
+
+        private JsonObjectContract Resolve<T>()
+        {
+            JsonContract contract = this._resolver.ResolveContract(typeof(T));
+            return Assert.IsType<JsonObjectContract>(contract);
+        }
+    }
+}