diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000000..831615c2554 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,3 @@ +language: csharp +solution: src/Elasticsearch.sln +script: ./build.sh diff --git a/build.sh b/build.sh index bca610409ba..9a4b4e0b599 100755 --- a/build.sh +++ b/build.sh @@ -1,5 +1,5 @@ NUGET="build/tools/nuget/nuget.exe" -FAKE="build/tools/FAKE/tools/Fake.exe" +FAKE="build/tools/FAKE/tools/FAKE.exe" NUNIT="build/tools/NUnit.Runners/tools/nunit-console.exe" FSHARPCLI="build/tools/Fsharp.Formatting.CommandTool/Fsharp.Formatting.CommandTool.nupkg" SOURCELINK="build/tools/SourceLink.Fake/SourceLink.Fake.nupkg" @@ -8,7 +8,7 @@ SOURCELINK="build/tools/SourceLink.Fake/SourceLink.Fake.nupkg" if [[ ! -f "$NUGET" ]]; then echo NUGET not found.. Download... mkdir -p build/tools/nuget - curl -o $NUGET http://build.nuget.org/drops/client/master/NuGet.exe + curl -o $NUGET https://nugetbuild.cloudapp.net/drops/client/master/NuGet.exe fi #we need FAKE to process our build scripts @@ -40,6 +40,9 @@ libdir=$PWD/build/tools/FAKE/tools/ $FSHARPI --lib:$libdir $@ EOF chmod +x fsharpi -mono --runtime=v4.0 "$FAKE" build/build.fsx $@ +mono --runtime=v4.0 "$FAKE" build/build.fsx "skiptests=1" $@ +MONOEXIT=$? rm fsharpi -#"build\tools\FAKE\tools\Fake.exe" "build\build.fsx" "target=%TARGET%" "version=%VERSION%" +#FORCE exit code to be that of calling fake not the last rm action +exit $MONOEXIT +#"build\tools\FAKE\tools\Fake.exe" "build\build.fsx" "target=%TARGET%" "version=%VERSION%" "skiptests=1" diff --git a/build/build.fsx b/build/build.fsx index 4c17d13c068..5e706c7e2f8 100644 --- a/build/build.fsx +++ b/build/build.fsx @@ -1,5 +1,5 @@ // include Fake lib -#r @"tools/FAKE/tools/FakeLib.dll" +#r @"FakeLib.dll" #load @"InheritDoc.fsx" open Fake open System @@ -31,7 +31,7 @@ Target "BuildApp" (fun _ -> //Override the prebuild event because it just calls a fake task BuildApp depends on anyways let msbuildProperties = [ ("Configuration","Release"); - ("PreBuildEvent","ECHO"); + ("PreBuildEvent","echo"); ] //Compile each csproj and output it seperately in build/output/PROJECTNAME @@ -73,7 +73,7 @@ Target "CreateKeysIfAbsent" (fun _ -> ) let getFileVersion = fun _ -> - let assemblyFileContents = ReadFileAsString @"src\NEST\Properties\AssemblyInfo.cs" + let assemblyFileContents = ReadFileAsString @"src/Nest/Properties/AssemblyInfo.cs" let re = @"\[assembly\: AssemblyFileVersionAttribute\(""([^""]+)""\)\]" let matches = Regex.Matches(assemblyFileContents,re) let defaultVersion = regex_replace re "$1" (matches.Item(0).Captures.Item(0).Value) @@ -231,7 +231,7 @@ Target "Nightly" (fun _ -> ==> "CreateKeysIfAbsent" =?> ("Version", hasBuildParam "version") ==> "BuildApp" - ==> "Test" + =?> ("Test", (not (hasBuildParam "skiptests"))) ==> "Build" "CreateKeysIfAbsent" @@ -248,4 +248,4 @@ Target "Nightly" (fun _ -> "CreateKeysIfAbsent" "Version" // start build -RunTargetOrDefault "Build" \ No newline at end of file +RunTargetOrDefault "Build" diff --git a/dep/repositories.config b/dep/repositories.config index 8a535b8985d..cf4469a86fe 100644 --- a/dep/repositories.config +++ b/dep/repositories.config @@ -7,6 +7,7 @@ + diff --git a/src/Nest/ExposedInternals/NestSerializer.cs b/src/Nest/ExposedInternals/NestSerializer.cs index 4c8f9aed089..94ca3bfca11 100644 --- a/src/Nest/ExposedInternals/NestSerializer.cs +++ b/src/Nest/ExposedInternals/NestSerializer.cs @@ -1,317 +1,316 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics.PerformanceData; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Elasticsearch.Net; -using Elasticsearch.Net.Serialization; -using Nest.Resolvers; -using Newtonsoft.Json; - -namespace Nest -{ - public class NestSerializer : INestSerializer - { - private readonly IConnectionSettingsValues _settings; - private readonly JsonSerializerSettings _serializationSettings; - private readonly ElasticInferrer _infer; - - public NestSerializer(IConnectionSettingsValues settings) - { - this._settings = settings; - this._serializationSettings = this.CreateSettings(); - this._infer = new ElasticInferrer(this._settings); - } - - public virtual byte[] Serialize(object data, SerializationFormatting formatting = SerializationFormatting.Indented) - { - var format = formatting == SerializationFormatting.None ? Formatting.None : Formatting.Indented; - var serialized = JsonConvert.SerializeObject(data, format, this._serializationSettings); - return serialized.Utf8Bytes(); - } - - public string Stringify(object valueType) - { - if (valueType == null) - return null; - var s = valueType as string; - if (s != null) - return s; - var ss = valueType as string[]; - if (ss != null) - return string.Join(",", ss); - - var pns = valueType as IEnumerable; - if (pns != null) - return string.Join(",", pns.Select( - oo => - { - if (oo is PropertyNameMarker) - return this._infer.PropertyName(oo as PropertyNameMarker); - if (oo is PropertyPathMarker) - return this._infer.PropertyPath(oo as PropertyPathMarker); - return oo.ToString(); - }) - ); - - var e = valueType as Enum; - if (e != null) return KnownEnums.Resolve(e); - if (valueType is bool) - return ((bool)valueType) ? "true" : "false"; - - var pn = valueType as PropertyNameMarker; - if (pn != null) - return this._infer.PropertyName(pn); - - var pp = valueType as PropertyPathMarker; - if (pp != null) - return this._infer.PropertyPath(pp); - - return valueType.ToString(); - } - - /// - /// Deserialize an object - /// - /// The type you want to deserialize too - /// The stream to deserialize off - public virtual T Deserialize(Stream stream) - { - if (stream == null) return default(T); - - var settings = this._serializationSettings; - - return DeserializeUsingSettings(stream, settings); - } - - - - /// - /// Deserialize to type T bypassing checks for custom deserialization state and or BaseResponse return types. - /// - public T DeserializeInternal(Stream stream, JsonConverter converter) - { - if (stream == null) return default(T); - if (converter == null) return this.Deserialize(stream); - - var settings = this.CreateSettings(converter); - return DeserializeUsingSettings(stream, settings); - } - - private T DeserializeUsingSettings(Stream stream, JsonSerializerSettings settings = null) - { - if (stream == null) return default(T); - settings = settings ?? _serializationSettings; - var serializer = JsonSerializer.Create(settings); - var jsonTextReader = new JsonTextReader(new StreamReader(stream)); - var t = (T)serializer.Deserialize(jsonTextReader, typeof(T)); - return t; - } - - public virtual Task DeserializeAsync(Stream stream) - { - //TODO sadly json .net async does not read the stream async so - //figure out wheter reading the stream async on our own might be beneficial - //over memory possible memory usage - var tcs = new TaskCompletionSource(); - if (stream == null) - { - tcs.SetResult(default(T)); - return tcs.Task; - } - var r = this.Deserialize(stream); - tcs.SetResult(r); - return tcs.Task; - } - - internal JsonSerializerSettings CreateSettings(JsonConverter piggyBackJsonConverter = null) - { - - var piggyBackState = new JsonConverterPiggyBackState { ActualJsonConverter = piggyBackJsonConverter }; - var settings = new JsonSerializerSettings() - { - ContractResolver = new ElasticContractResolver(this._settings), - DefaultValueHandling = DefaultValueHandling.Include, - NullValueHandling = NullValueHandling.Ignore - }; - - if (_settings.ModifyJsonSerializerSettings != null) - _settings.ModifyJsonSerializerSettings(settings); - - settings.ContractResolver = new SettingsContractResolver(settings.ContractResolver, this._settings) { PiggyBackState = piggyBackState }; - - return settings; - } - - public string SerializeBulkDescriptor(IBulkRequest bulkRequest) - { - bulkRequest.ThrowIfNull("bulkRequest"); - bulkRequest.Operations.ThrowIfEmpty("Bulk request does not define any operations"); - var sb = new StringBuilder(); - var inferrer = new ElasticInferrer(this._settings); - - foreach (var operation in bulkRequest.Operations) - { - var command = operation.Operation; - var index = operation.Index - ?? inferrer.IndexName(bulkRequest.Index) - ?? inferrer.IndexName(operation.ClrType); - var typeName = operation.Type - ?? inferrer.TypeName(bulkRequest.Type) - ?? inferrer.TypeName(operation.ClrType); - - var id = operation.GetIdForOperation(inferrer); - if (index.EqualsMarker(bulkRequest.Index)) - { - operation.Index = null; - } - else - { - operation.Index = index; - } - - operation.Type = typeName; - operation.Id = id; - - var opJson = this.Serialize(operation, SerializationFormatting.None).Utf8String(); - - var action = "{{ \"{0}\" : {1} }}\n".F(command, opJson); - sb.Append(action); - var body = operation.GetBody(); - if (body == null) - continue; - var jsonCommand = this.Serialize(body, SerializationFormatting.None).Utf8String(); - sb.Append(jsonCommand + "\n"); - } - var json = sb.ToString(); - return json; - } - - private class PercolateHeader - { - public IndexNameMarker index { get; set; } - public TypeNameMarker type { get; set; } - public string id { get; set; } - public string percolate_index { get; set; } - public string percolate_type { get; set; } - public string[] routing { get; set; } - public string preference { get; set; } - public string percolate_routing { get; set; } - public string percolate_preference { get; set; } - public long? version { get; set; } - } - - public string SerializeMultiPercolate(IMultiPercolateRequest multiPercolateRequest) - { - multiPercolateRequest.ThrowIfNull("multiPercolateRequest"); - multiPercolateRequest.Percolations.ThrowIfEmpty("multiPercolateRequest does not define any percolations"); - var sb = new StringBuilder(); - - foreach (var p in multiPercolateRequest.Percolations) - { - var operation = "percolate"; - object body = p; - var op = new PercolateHeader(); - var countParams = p.GetRequestParameters() ?? new PercolateCountRequestParameters(); - op.percolate_index = countParams.GetQueryStringValue("percolate_index"); - op.percolate_type = countParams.GetQueryStringValue("percolate_type"); - op.routing = countParams.GetQueryStringValue("routing"); - op.preference = countParams.GetQueryStringValue("preference"); - op.percolate_routing = countParams.GetQueryStringValue("percolate_routing"); - op.percolate_preference = countParams.GetQueryStringValue("percolate_preference"); - op.version = countParams.GetQueryStringValue("version"); - - - var count = p as IIndexTypePath; - var percolate = p as IIndexTypePath; - if (count != null) - { - operation = "count"; - if (multiPercolateRequest.Index != null && multiPercolateRequest.Index.EqualsMarker(count.Index)) - { - count.Index = null; - } - op.index = count.Index; - op.type = count.Type; - op.id = p.Id; - } - else if (percolate != null) - { - if (multiPercolateRequest.Index != null && multiPercolateRequest.Index.EqualsMarker(percolate.Index)) - { - percolate.Index = null; - } - op.index = percolate.Index; - op.type = percolate.Type; - op.id = p.Id; - } - else continue; - - var opJson = this.Serialize(op, SerializationFormatting.None).Utf8String(); - var action = "{{\"{0}\":{1}}}\n".F(operation, opJson); - sb.Append(action); - if (body == null) - { - sb.Append("{}\n"); - continue; - } - var jsonCommand = this.Serialize(body, SerializationFormatting.None).Utf8String(); - sb.Append(jsonCommand + "\n"); - - - } - return sb.ToString(); - } - - /// - /// _msearch needs a specialized json format in the body - /// - public string SerializeMultiSearch(IMultiSearchRequest multiSearchRequest) - { - var sb = new StringBuilder(); - var inferrer = new ElasticInferrer(this._settings); - var indexName = inferrer.IndexName(multiSearchRequest.Index); - - foreach (var operation in multiSearchRequest.Operations.Values) - { - var path = operation.ToPathInfo(this._settings); - if (path.Index == indexName) - { - path.Index = null; - } - - var op = new - { - index = path.Index, - type = path.Type, - search_type = this.GetSearchType(operation, multiSearchRequest), - preference = operation.Preference, - routing = operation.Routing, - ignore_unavailable = operation.IgnoreUnavalable - }; - var opJson = this.Serialize(op, SerializationFormatting.None).Utf8String(); - - var action = "{0}\n".F(opJson); - sb.Append(action); - var searchJson = this.Serialize(operation, SerializationFormatting.None).Utf8String(); - sb.Append(searchJson + "\n"); - - } - var json = sb.ToString(); - return json; - } - - - protected string GetSearchType(ISearchRequest descriptor, IMultiSearchRequest multiSearchRequest) - { - if (descriptor.SearchType != null) - { - return descriptor.SearchType.Value.GetStringValue(); - } - return multiSearchRequest.RequestParameters.GetQueryStringValue("search_type"); - } - - } +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Elasticsearch.Net; +using Elasticsearch.Net.Serialization; +using Nest.Resolvers; +using Newtonsoft.Json; + +namespace Nest +{ + public class NestSerializer : INestSerializer + { + private readonly IConnectionSettingsValues _settings; + private readonly JsonSerializerSettings _serializationSettings; + private readonly ElasticInferrer _infer; + + public NestSerializer(IConnectionSettingsValues settings) + { + this._settings = settings; + this._serializationSettings = this.CreateSettings(); + this._infer = new ElasticInferrer(this._settings); + } + + public virtual byte[] Serialize(object data, SerializationFormatting formatting = SerializationFormatting.Indented) + { + var format = formatting == SerializationFormatting.None ? Formatting.None : Formatting.Indented; + var serialized = JsonConvert.SerializeObject(data, format, this._serializationSettings); + return serialized.Utf8Bytes(); + } + + public string Stringify(object valueType) + { + if (valueType == null) + return null; + var s = valueType as string; + if (s != null) + return s; + var ss = valueType as string[]; + if (ss != null) + return string.Join(",", ss); + + var pns = valueType as IEnumerable; + if (pns != null) + return string.Join(",", pns.Select( + oo => + { + if (oo is PropertyNameMarker) + return this._infer.PropertyName(oo as PropertyNameMarker); + if (oo is PropertyPathMarker) + return this._infer.PropertyPath(oo as PropertyPathMarker); + return oo.ToString(); + }) + ); + + var e = valueType as Enum; + if (e != null) return KnownEnums.Resolve(e); + if (valueType is bool) + return ((bool)valueType) ? "true" : "false"; + + var pn = valueType as PropertyNameMarker; + if (pn != null) + return this._infer.PropertyName(pn); + + var pp = valueType as PropertyPathMarker; + if (pp != null) + return this._infer.PropertyPath(pp); + + return valueType.ToString(); + } + + /// + /// Deserialize an object + /// + /// The type you want to deserialize too + /// The stream to deserialize off + public virtual T Deserialize(Stream stream) + { + if (stream == null) return default(T); + + var settings = this._serializationSettings; + + return DeserializeUsingSettings(stream, settings); + } + + + + /// + /// Deserialize to type T bypassing checks for custom deserialization state and or BaseResponse return types. + /// + public T DeserializeInternal(Stream stream, JsonConverter converter) + { + if (stream == null) return default(T); + if (converter == null) return this.Deserialize(stream); + + var settings = this.CreateSettings(converter); + return DeserializeUsingSettings(stream, settings); + } + + private T DeserializeUsingSettings(Stream stream, JsonSerializerSettings settings = null) + { + if (stream == null) return default(T); + settings = settings ?? _serializationSettings; + var serializer = JsonSerializer.Create(settings); + var jsonTextReader = new JsonTextReader(new StreamReader(stream)); + var t = (T)serializer.Deserialize(jsonTextReader, typeof(T)); + return t; + } + + public virtual Task DeserializeAsync(Stream stream) + { + //TODO sadly json .net async does not read the stream async so + //figure out wheter reading the stream async on our own might be beneficial + //over memory possible memory usage + var tcs = new TaskCompletionSource(); + if (stream == null) + { + tcs.SetResult(default(T)); + return tcs.Task; + } + var r = this.Deserialize(stream); + tcs.SetResult(r); + return tcs.Task; + } + + internal JsonSerializerSettings CreateSettings(JsonConverter piggyBackJsonConverter = null) + { + + var piggyBackState = new JsonConverterPiggyBackState { ActualJsonConverter = piggyBackJsonConverter }; + var settings = new JsonSerializerSettings() + { + ContractResolver = new ElasticContractResolver(this._settings), + DefaultValueHandling = DefaultValueHandling.Include, + NullValueHandling = NullValueHandling.Ignore + }; + + if (_settings.ModifyJsonSerializerSettings != null) + _settings.ModifyJsonSerializerSettings(settings); + + settings.ContractResolver = new SettingsContractResolver(settings.ContractResolver, this._settings) { PiggyBackState = piggyBackState }; + + return settings; + } + + public string SerializeBulkDescriptor(IBulkRequest bulkRequest) + { + bulkRequest.ThrowIfNull("bulkRequest"); + bulkRequest.Operations.ThrowIfEmpty("Bulk request does not define any operations"); + var sb = new StringBuilder(); + var inferrer = new ElasticInferrer(this._settings); + + foreach (var operation in bulkRequest.Operations) + { + var command = operation.Operation; + var index = operation.Index + ?? inferrer.IndexName(bulkRequest.Index) + ?? inferrer.IndexName(operation.ClrType); + var typeName = operation.Type + ?? inferrer.TypeName(bulkRequest.Type) + ?? inferrer.TypeName(operation.ClrType); + + var id = operation.GetIdForOperation(inferrer); + if (index.EqualsMarker(bulkRequest.Index)) + { + operation.Index = null; + } + else + { + operation.Index = index; + } + + operation.Type = typeName; + operation.Id = id; + + var opJson = this.Serialize(operation, SerializationFormatting.None).Utf8String(); + + var action = "{{ \"{0}\" : {1} }}\n".F(command, opJson); + sb.Append(action); + var body = operation.GetBody(); + if (body == null) + continue; + var jsonCommand = this.Serialize(body, SerializationFormatting.None).Utf8String(); + sb.Append(jsonCommand + "\n"); + } + var json = sb.ToString(); + return json; + } + + private class PercolateHeader + { + public IndexNameMarker index { get; set; } + public TypeNameMarker type { get; set; } + public string id { get; set; } + public string percolate_index { get; set; } + public string percolate_type { get; set; } + public string[] routing { get; set; } + public string preference { get; set; } + public string percolate_routing { get; set; } + public string percolate_preference { get; set; } + public long? version { get; set; } + } + + public string SerializeMultiPercolate(IMultiPercolateRequest multiPercolateRequest) + { + multiPercolateRequest.ThrowIfNull("multiPercolateRequest"); + multiPercolateRequest.Percolations.ThrowIfEmpty("multiPercolateRequest does not define any percolations"); + var sb = new StringBuilder(); + + foreach (var p in multiPercolateRequest.Percolations) + { + var operation = "percolate"; + object body = p; + var op = new PercolateHeader(); + var countParams = p.GetRequestParameters() ?? new PercolateCountRequestParameters(); + op.percolate_index = countParams.GetQueryStringValue("percolate_index"); + op.percolate_type = countParams.GetQueryStringValue("percolate_type"); + op.routing = countParams.GetQueryStringValue("routing"); + op.preference = countParams.GetQueryStringValue("preference"); + op.percolate_routing = countParams.GetQueryStringValue("percolate_routing"); + op.percolate_preference = countParams.GetQueryStringValue("percolate_preference"); + op.version = countParams.GetQueryStringValue("version"); + + + var count = p as IIndexTypePath; + var percolate = p as IIndexTypePath; + if (count != null) + { + operation = "count"; + if (multiPercolateRequest.Index != null && multiPercolateRequest.Index.EqualsMarker(count.Index)) + { + count.Index = null; + } + op.index = count.Index; + op.type = count.Type; + op.id = p.Id; + } + else if (percolate != null) + { + if (multiPercolateRequest.Index != null && multiPercolateRequest.Index.EqualsMarker(percolate.Index)) + { + percolate.Index = null; + } + op.index = percolate.Index; + op.type = percolate.Type; + op.id = p.Id; + } + else continue; + + var opJson = this.Serialize(op, SerializationFormatting.None).Utf8String(); + var action = "{{\"{0}\":{1}}}\n".F(operation, opJson); + sb.Append(action); + if (body == null) + { + sb.Append("{}\n"); + continue; + } + var jsonCommand = this.Serialize(body, SerializationFormatting.None).Utf8String(); + sb.Append(jsonCommand + "\n"); + + + } + return sb.ToString(); + } + + /// + /// _msearch needs a specialized json format in the body + /// + public string SerializeMultiSearch(IMultiSearchRequest multiSearchRequest) + { + var sb = new StringBuilder(); + var inferrer = new ElasticInferrer(this._settings); + var indexName = inferrer.IndexName(multiSearchRequest.Index); + + foreach (var operation in multiSearchRequest.Operations.Values) + { + var path = operation.ToPathInfo(this._settings); + if (path.Index == indexName) + { + path.Index = null; + } + + var op = new + { + index = path.Index, + type = path.Type, + search_type = this.GetSearchType(operation, multiSearchRequest), + preference = operation.Preference, + routing = operation.Routing, + ignore_unavailable = operation.IgnoreUnavalable + }; + var opJson = this.Serialize(op, SerializationFormatting.None).Utf8String(); + + var action = "{0}\n".F(opJson); + sb.Append(action); + var searchJson = this.Serialize(operation, SerializationFormatting.None).Utf8String(); + sb.Append(searchJson + "\n"); + + } + var json = sb.ToString(); + return json; + } + + + protected string GetSearchType(ISearchRequest descriptor, IMultiSearchRequest multiSearchRequest) + { + if (descriptor.SearchType != null) + { + return descriptor.SearchType.Value.GetStringValue(); + } + return multiSearchRequest.RequestParameters.GetQueryStringValue("search_type"); + } + + } } \ No newline at end of file diff --git a/src/Nest/Nest.csproj b/src/Nest/Nest.csproj index 8153924bb62..7c56f3ba148 100644 --- a/src/Nest/Nest.csproj +++ b/src/Nest/Nest.csproj @@ -298,7 +298,7 @@ - + @@ -830,7 +830,7 @@ - + @@ -940,7 +940,7 @@ - + @@ -1025,7 +1025,7 @@ - + @@ -1095,4 +1095,4 @@ --> - \ No newline at end of file + diff --git a/src/Tests/Nest.Tests.Unit/Nest.Tests.Unit.csproj b/src/Tests/Nest.Tests.Unit/Nest.Tests.Unit.csproj index 57933dc8412..4887af3b640 100644 --- a/src/Tests/Nest.Tests.Unit/Nest.Tests.Unit.csproj +++ b/src/Tests/Nest.Tests.Unit/Nest.Tests.Unit.csproj @@ -541,9 +541,9 @@ - - - + + + @@ -1317,4 +1317,4 @@ --> - \ No newline at end of file +