diff --git a/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java b/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java index ba13033db15a2..4030eca515136 100644 --- a/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java +++ b/src/main/java/org/apache/lucene/queryparser/classic/QueryParserSettings.java @@ -66,6 +66,11 @@ public class QueryParserSettings { float tieBreaker = 0.0f; boolean useDisMax = true; + public boolean isCacheable() { + // a hack for now :) to determine if a query string is cacheable + return !queryString.contains("now"); + } + public String queryString() { return queryString; } diff --git a/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java b/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java index 780be2c76e90f..e96ef8bb10463 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/segments/IndicesSegmentResponse.java @@ -122,14 +122,20 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.startObject(Fields.SEGMENTS); for (Segment segment : shardSegments) { - builder.startObject(segment.name()); - builder.field(Fields.GENERATION, segment.generation()); - builder.field(Fields.NUM_DOCS, segment.numDocs()); - builder.field(Fields.DELETED_DOCS, segment.deletedDocs()); - builder.field(Fields.SIZE, segment.size().toString()); - builder.field(Fields.SIZE_IN_BYTES, segment.sizeInBytes()); - builder.field(Fields.COMMITTED, segment.committed()); - builder.field(Fields.SEARCH, segment.search()); + builder.startObject(segment.getName()); + builder.field(Fields.GENERATION, segment.getGeneration()); + builder.field(Fields.NUM_DOCS, segment.getNumDocs()); + builder.field(Fields.DELETED_DOCS, segment.getDeletedDocs()); + builder.field(Fields.SIZE, segment.getSize().toString()); + builder.field(Fields.SIZE_IN_BYTES, segment.getSizeInBytes()); + builder.field(Fields.COMMITTED, segment.isCommitted()); + builder.field(Fields.SEARCH, segment.isSearch()); + if (segment.getVersion() != null) { + builder.field(Fields.VERSION, segment.getVersion()); + } + if (segment.isCompound() != null) { + builder.field(Fields.COMPOUND, segment.isCompound()); + } builder.endObject(); } builder.endObject(); @@ -166,5 +172,7 @@ static final class Fields { static final XContentBuilderString SIZE_IN_BYTES = new XContentBuilderString("size_in_bytes"); static final XContentBuilderString COMMITTED = new XContentBuilderString("committed"); static final XContentBuilderString SEARCH = new XContentBuilderString("search"); + static final XContentBuilderString VERSION = new XContentBuilderString("version"); + static final XContentBuilderString COMPOUND = new XContentBuilderString("compound"); } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java b/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java index 176757d9edfc9..3fe2e41a0d210 100644 --- a/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java +++ b/src/main/java/org/elasticsearch/action/admin/indices/segments/ShardSegments.java @@ -64,7 +64,7 @@ public List getSegments() { public int getNumberOfCommitted() { int count = 0; for (Segment segment : segments) { - if (segment.committed()) { + if (segment.isCommitted()) { count++; } } @@ -74,7 +74,7 @@ public int getNumberOfCommitted() { public int getNumberOfSearch() { int count = 0; for (Segment segment : segments) { - if (segment.search()) { + if (segment.isSearch()) { count++; } } diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java index 9de3240b3228c..850fef392a8e2 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamInput.java @@ -252,6 +252,17 @@ public final boolean readBoolean() throws IOException { return readByte() != 0; } + @Nullable + public final Boolean readOptionalBoolean() throws IOException { + byte val = readByte(); + if (val == 2) { + return null; + } + if (val == 1) { + return true; + } + return false; + } /** * Resets the stream. diff --git a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java index b0d59e700833b..70884098a3a97 100644 --- a/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java +++ b/src/main/java/org/elasticsearch/common/io/stream/StreamOutput.java @@ -236,6 +236,7 @@ public void writeDouble(double v) throws IOException { private static byte ZERO = 0; private static byte ONE = 1; + private static byte TWO = 2; /** * Writes a boolean. @@ -244,6 +245,14 @@ public void writeBoolean(boolean b) throws IOException { writeByte(b ? ONE : ZERO); } + public void writeOptionalBoolean(@Nullable Boolean b) throws IOException { + if (b == null) { + writeByte(TWO); + } else { + writeByte(b ? ONE : ZERO); + } + } + /** * Forces any buffered output to be written. */ diff --git a/src/main/java/org/elasticsearch/http/HttpServer.java b/src/main/java/org/elasticsearch/http/HttpServer.java index a21b228861824..3c38029700802 100644 --- a/src/main/java/org/elasticsearch/http/HttpServer.java +++ b/src/main/java/org/elasticsearch/http/HttpServer.java @@ -25,9 +25,13 @@ import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.env.Environment; import org.elasticsearch.node.service.NodeService; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.plugins.PluginsService; import org.elasticsearch.rest.*; +import org.elasticsearch.rest.action.support.RestXContentBuilder; import java.io.File; import java.io.IOException; @@ -49,6 +53,8 @@ public class HttpServer extends AbstractLifecycleComponent { private final NodeService nodeService; + private final PluginsService pluginsService; + private final boolean disableSites; private final PluginSiteFilter pluginSiteFilter = new PluginSiteFilter(); @@ -56,12 +62,13 @@ public class HttpServer extends AbstractLifecycleComponent { @Inject public HttpServer(Settings settings, Environment environment, HttpServerTransport transport, RestController restController, - NodeService nodeService) { + NodeService nodeService, PluginsService pluginsService) { super(settings); this.environment = environment; this.transport = transport; this.restController = restController; this.nodeService = nodeService; + this.pluginsService = pluginsService; nodeService.setHttpServer(this); this.disableSites = componentSettings.getAsBoolean("disable_sites", false); @@ -150,6 +157,13 @@ void handlePluginSite(HttpRequest request, HttpChannel channel) { int i1 = path.indexOf('/'); String pluginName; String sitePath; + + + if (path.length() == 0) { + handlePluginList(request, channel); + return; + } + if (i1 == -1) { pluginName = path; sitePath = null; @@ -191,6 +205,29 @@ void handlePluginSite(HttpRequest request, HttpChannel channel) { } } + private void handlePluginList(HttpRequest request, HttpChannel channel) { + ImmutableMap plugins = pluginsService.plugins(); + try { + + XContentBuilder builder = RestXContentBuilder.restContentBuilder(request); + if (request.paramAsBoolean("pretty", false)) + builder.prettyPrint(); + + builder.startObject(); + builder.startArray("plugins"); + + for (String pluginName : plugins.keySet()) { + builder.value(pluginName); + } + builder.endArray(); + builder.endObject(); + + channel.sendResponse(new XContentRestResponse(request, OK, builder)); + } catch (IOException e) { + channel.sendResponse(new StringRestResponse(INTERNAL_SERVER_ERROR)); + } + } + // TODO: Don't respond with a mime type that violates the request's Accept header private String guessMimeType(String path) { diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCache.java b/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCache.java index 9df1dd1dae75c..560efe92a764f 100644 --- a/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCache.java +++ b/src/main/java/org/elasticsearch/index/cache/query/parser/QueryParserCache.java @@ -25,7 +25,8 @@ import org.elasticsearch.index.IndexComponent; /** - * + * The main benefit of the query parser cache is to not parse the same query string on different shards. + * Less about long running query strings. */ public interface QueryParserCache extends IndexComponent, CloseableComponent { diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java b/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java index d572347f65371..71a2f7ed4738e 100644 --- a/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java +++ b/src/main/java/org/elasticsearch/index/cache/query/parser/resident/ResidentQueryParserCache.java @@ -21,7 +21,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; - import org.apache.lucene.queryparser.classic.QueryParserSettings; import org.apache.lucene.search.Query; import org.elasticsearch.ElasticSearchException; @@ -50,8 +49,8 @@ public class ResidentQueryParserCache extends AbstractIndexComponent implements public ResidentQueryParserCache(Index index, @IndexSettings Settings indexSettings) { super(index, indexSettings); - this.maxSize = indexSettings.getAsInt("index.cache.field.max_size", componentSettings.getAsInt("max_size", 100)); - this.expire = indexSettings.getAsTime("index.cache.field.expire", componentSettings.getAsTime("expire", null)); + this.maxSize = componentSettings.getAsInt("max_size", 100); + this.expire = componentSettings.getAsTime("expire", null); logger.debug("using [resident] query cache with max_size [{}], expire [{}]", maxSize, expire); CacheBuilder cacheBuilder = CacheBuilder.newBuilder().maximumSize(maxSize); @@ -69,7 +68,9 @@ public Query get(QueryParserSettings queryString) { @Override public void put(QueryParserSettings queryString, Query query) { - cache.put(queryString, query); + if (queryString.isCacheable()) { + cache.put(queryString, query); + } } @Override diff --git a/src/main/java/org/elasticsearch/index/cache/query/parser/support/AbstractJvmQueryParserCache.java b/src/main/java/org/elasticsearch/index/cache/query/parser/support/AbstractJvmQueryParserCache.java deleted file mode 100644 index 9f3924162d427..0000000000000 --- a/src/main/java/org/elasticsearch/index/cache/query/parser/support/AbstractJvmQueryParserCache.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to ElasticSearch and Shay Banon under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. ElasticSearch licenses this - * file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.index.cache.query.parser.support; - -import org.apache.lucene.queryparser.classic.QueryParserSettings; -import org.apache.lucene.search.Query; -import org.elasticsearch.ElasticSearchException; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.index.AbstractIndexComponent; -import org.elasticsearch.index.Index; -import org.elasticsearch.index.cache.query.parser.QueryParserCache; -import org.elasticsearch.index.settings.IndexSettings; - -import java.util.concurrent.ConcurrentMap; - -/** - * - */ -public class AbstractJvmQueryParserCache extends AbstractIndexComponent implements QueryParserCache { - - final ConcurrentMap cache; - - protected AbstractJvmQueryParserCache(Index index, @IndexSettings Settings indexSettings, ConcurrentMap cache) { - super(index, indexSettings); - this.cache = cache; - } - - @Override - public void close() throws ElasticSearchException { - clear(); - } - - @Override - public void clear() { - cache.clear(); - } - - @Override - public Query get(QueryParserSettings queryString) { - return cache.get(queryString); - } - - @Override - public void put(QueryParserSettings queryString, Query query) { - cache.put(queryString, query); - } -} diff --git a/src/main/java/org/elasticsearch/index/engine/Segment.java b/src/main/java/org/elasticsearch/index/engine/Segment.java index 63b1da32cbca9..4bb2bebc9a0ad 100644 --- a/src/main/java/org/elasticsearch/index/engine/Segment.java +++ b/src/main/java/org/elasticsearch/index/engine/Segment.java @@ -19,6 +19,7 @@ package org.elasticsearch.index.engine; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Streamable; @@ -35,6 +36,8 @@ public class Segment implements Streamable { public long sizeInBytes = -1; public int docCount = -1; public int delDocCount = -1; + public String version = null; + public Boolean compound = null; Segment() { } @@ -44,68 +47,45 @@ public Segment(String name) { this.generation = Long.parseLong(name.substring(1), Character.MAX_RADIX); } - public String name() { - return this.name; - } - public String getName() { - return name(); - } - - public long generation() { - return this.generation; + return this.name; } public long getGeneration() { return this.generation; } - public boolean committed() { - return this.committed; - } - public boolean isCommitted() { return this.committed; } - public boolean search() { - return this.search; - } - public boolean isSearch() { return this.search; } - public int numDocs() { - return this.docCount; - } - public int getNumDocs() { return this.docCount; } - public int deletedDocs() { - return this.delDocCount; - } - public int getDeletedDocs() { return this.delDocCount; } - public ByteSizeValue size() { + public ByteSizeValue getSize() { return new ByteSizeValue(sizeInBytes); } - public ByteSizeValue getSize() { - return size(); + public long getSizeInBytes() { + return this.sizeInBytes; } - public long sizeInBytes() { - return sizeInBytes; + public String getVersion() { + return version; } - public long getSizeInBytes() { - return sizeInBytes(); + @Nullable + public Boolean isCompound() { + return compound; } @Override @@ -140,6 +120,8 @@ public void readFrom(StreamInput in) throws IOException { docCount = in.readInt(); delDocCount = in.readInt(); sizeInBytes = in.readLong(); + version = in.readOptionalString(); + compound = in.readOptionalBoolean(); } @Override @@ -150,5 +132,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeInt(docCount); out.writeInt(delDocCount); out.writeLong(sizeInBytes); + out.writeOptionalString(version); + out.writeOptionalBoolean(compound); } } \ No newline at end of file diff --git a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java index ec9466f1f7c8f..e30339ce17a53 100644 --- a/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java +++ b/src/main/java/org/elasticsearch/index/engine/robin/RobinEngine.java @@ -21,13 +21,11 @@ import com.google.common.collect.Lists; import org.apache.lucene.index.*; -import org.apache.lucene.index.SegmentInfos.FindSegmentsFile; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.SearcherFactory; import org.apache.lucene.search.SearcherManager; import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.Directory; import org.elasticsearch.ElasticSearchException; import org.elasticsearch.ElasticSearchIllegalStateException; import org.elasticsearch.cluster.routing.operation.hash.djb.DjbHashFunction; @@ -65,7 +63,6 @@ import org.elasticsearch.indices.warmer.InternalIndicesWarmer; import org.elasticsearch.threadpool.ThreadPool; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; import java.util.concurrent.ConcurrentMap; @@ -1176,6 +1173,8 @@ public List segments() { segment.search = true; segment.docCount = reader.reader().numDocs(); segment.delDocCount = reader.reader().numDeletedDocs(); + segment.version = info.info.getVersion(); + segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { @@ -1198,6 +1197,8 @@ public List segments() { segment.committed = true; segment.docCount = info.info.getDocCount(); segment.delDocCount = info.getDelCount(); + segment.version = info.info.getVersion(); + segment.compound = info.info.getUseCompoundFile(); try { segment.sizeInBytes = info.sizeInBytes(); } catch (IOException e) { @@ -1214,7 +1215,7 @@ public List segments() { Arrays.sort(segmentsArr, new Comparator() { @Override public int compare(Segment o1, Segment o2) { - return (int) (o1.generation() - o2.generation()); + return (int) (o1.getGeneration() - o2.getGeneration()); } }); @@ -1341,7 +1342,7 @@ private IndexWriter createWriter() throws IOException { } return indexWriter; } - + public static final String INDEX_TERM_INDEX_INTERVAL = "index.term_index_interval"; public static final String INDEX_TERM_INDEX_DIVISOR = "index.term_index_divisor"; public static final String INDEX_INDEX_CONCURRENCY = "index.index_concurrency"; diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java index eb05eea9205c1..b409708472837 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparator.java @@ -40,13 +40,14 @@ public class GeoDistanceComparator extends FieldComparator { protected final DistanceUnit unit; protected final GeoDistance geoDistance; protected final GeoDistance.FixedSourceDistance fixedSourceDistance; + protected final SortMode sortMode; private final double[] values; private double bottom; - private GeoPointValues readerValues; + private GeoDistanceValues geoDistanceValues; - public GeoDistanceComparator(int numHits, IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance) { + public GeoDistanceComparator(int numHits, IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance, SortMode sortMode) { this.values = new double[numHits]; this.indexFieldData = indexFieldData; this.lat = lat; @@ -54,11 +55,17 @@ public GeoDistanceComparator(int numHits, IndexGeoPointFieldData indexFieldDa this.unit = unit; this.geoDistance = geoDistance; this.fixedSourceDistance = geoDistance.fixedSourceDistance(lat, lon, unit); + this.sortMode = sortMode; } @Override public FieldComparator setNextReader(AtomicReaderContext context) throws IOException { - this.readerValues = indexFieldData.load(context).getGeoPointValues(); + GeoPointValues readerValues = indexFieldData.load(context).getGeoPointValues(); + if (readerValues.isMultiValued()) { + geoDistanceValues = new MV(readerValues, fixedSourceDistance, sortMode); + } else { + geoDistanceValues = new SV(readerValues, fixedSourceDistance); + } return this; } @@ -77,15 +84,7 @@ public int compare(int slot1, int slot2) { @Override public int compareBottom(int doc) { - double distance; - GeoPoint geoPoint = readerValues.getValue(doc); - if (geoPoint == null) { - // is this true? push this to the "end" - distance = Double.MAX_VALUE; - } else { - distance = fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon()); - } - final double v2 = distance; + final double v2 = geoDistanceValues.computeDistance(doc); if (bottom > v2) { return 1; } else if (bottom < v2) { @@ -97,14 +96,7 @@ public int compareBottom(int doc) { @Override public int compareDocToValue(int doc, Double distance2) throws IOException { - double distance1; - GeoPoint geoPoint = readerValues.getValue(doc); - if (geoPoint == null) { - // is this true? push this to the "end" - distance1 = Double.MAX_VALUE; - } else { - distance1 = fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon()); - } + double distance1 = geoDistanceValues.computeDistance(doc); if (distance1 < distance2) { return -1; } else if (distance1 == distance2) { @@ -116,15 +108,7 @@ public int compareDocToValue(int doc, Double distance2) throws IOException { @Override public void copy(int slot, int doc) { - double distance; - GeoPoint geoPoint = readerValues.getValue(doc); - if (geoPoint == null) { - // is this true? push this to the "end" - distance = Double.MAX_VALUE; - } else { - distance = fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon()); - } - values[slot] = distance; + values[slot] = geoDistanceValues.computeDistance(doc); } @Override @@ -136,4 +120,81 @@ public void setBottom(final int bottom) { public Double value(int slot) { return values[slot]; } + + // Computes the distance based on geo points. + // Due to this abstractions the geo distance comparator doesn't need to deal with whether fields have one + // or multiple geo points per document. + private static abstract class GeoDistanceValues { + + protected final GeoPointValues readerValues; + protected final GeoDistance.FixedSourceDistance fixedSourceDistance; + + protected GeoDistanceValues(GeoPointValues readerValues, GeoDistance.FixedSourceDistance fixedSourceDistance) { + this.readerValues = readerValues; + this.fixedSourceDistance = fixedSourceDistance; + } + + public abstract double computeDistance(int doc); + + } + + // Deals with one geo point per document + private static final class SV extends GeoDistanceValues { + + SV(GeoPointValues readerValues, GeoDistance.FixedSourceDistance fixedSourceDistance) { + super(readerValues, fixedSourceDistance); + } + + @Override + public double computeDistance(int doc) { + GeoPoint geoPoint = readerValues.getValue(doc); + if (geoPoint == null) { + // is this true? push this to the "end" + return Double.MAX_VALUE; + } else { + return fixedSourceDistance.calculate(geoPoint.lat(), geoPoint.lon()); + } + } + } + + // Deals with more than one geo point per document + private static final class MV extends GeoDistanceValues { + + private final SortMode sortMode; + + MV(GeoPointValues readerValues, GeoDistance.FixedSourceDistance fixedSourceDistance, SortMode sortMode) { + super(readerValues, fixedSourceDistance); + this.sortMode = sortMode; + } + + @Override + public double computeDistance(int doc) { + GeoPointValues.Iter iter = readerValues.getIter(doc); + if (!iter.hasNext()) { + return Double.MAX_VALUE; + } + + GeoPoint point = iter.next(); + double distance = fixedSourceDistance.calculate(point.lat(), point.lon()); + while (iter.hasNext()) { + point = iter.next(); + double newDistance = fixedSourceDistance.calculate(point.lat(), point.lon()); + switch (sortMode) { + case MIN: + if (distance > newDistance) { + distance = newDistance; + } + break; + case MAX: + if (distance < newDistance) { + distance = newDistance; + } + break; + } + } + return distance; + } + + } + } diff --git a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java index 3641d360d69bd..3b1297db9ed7e 100644 --- a/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java +++ b/src/main/java/org/elasticsearch/index/fielddata/fieldcomparator/GeoDistanceComparatorSource.java @@ -37,13 +37,15 @@ public class GeoDistanceComparatorSource extends IndexFieldData.XFieldComparator private final double lon; private final DistanceUnit unit; private final GeoDistance geoDistance; + private final SortMode sortMode; - public GeoDistanceComparatorSource(IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance) { + public GeoDistanceComparatorSource(IndexGeoPointFieldData indexFieldData, double lat, double lon, DistanceUnit unit, GeoDistance geoDistance, SortMode sortMode) { this.indexFieldData = indexFieldData; this.lat = lat; this.lon = lon; this.unit = unit; this.geoDistance = geoDistance; + this.sortMode = sortMode; } @Override @@ -54,7 +56,6 @@ public SortField.Type reducedType() { @Override public FieldComparator newComparator(String fieldname, int numHits, int sortPos, boolean reversed) throws IOException { assert indexFieldData.getFieldNames().indexName().equals(fieldname); - // TODO support multi value? - return new GeoDistanceComparator(numHits, indexFieldData, lat, lon, unit, geoDistance); + return new GeoDistanceComparator(numHits, indexFieldData, lat, lon, unit, geoDistance, sortMode); } } diff --git a/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java b/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java index 69f78957aac94..1919dfc4c1ba3 100644 --- a/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java +++ b/src/main/java/org/elasticsearch/index/store/distributor/AbstractDistributor.java @@ -20,6 +20,8 @@ package org.elasticsearch.index.store.distributor; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RateLimitedFSDirectory; import org.elasticsearch.index.store.DirectoryService; import java.io.IOException; @@ -50,6 +52,16 @@ public Directory any() { } } + protected long getUsableSpace(Directory directory) { + if (directory instanceof RateLimitedFSDirectory) { + return ((RateLimitedFSDirectory) directory).wrappedDirectory().getDirectory().getUsableSpace(); + } else if (directory instanceof FSDirectory) { + return ((FSDirectory) directory).getDirectory().getUsableSpace(); + } else { + return 0; + } + } + protected abstract Directory doAny(); } diff --git a/src/main/java/org/elasticsearch/index/store/distributor/LeastUsedDistributor.java b/src/main/java/org/elasticsearch/index/store/distributor/LeastUsedDistributor.java index 1da7edca0015c..651c72b185e4f 100644 --- a/src/main/java/org/elasticsearch/index/store/distributor/LeastUsedDistributor.java +++ b/src/main/java/org/elasticsearch/index/store/distributor/LeastUsedDistributor.java @@ -21,7 +21,6 @@ import jsr166y.ThreadLocalRandom; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.store.DirectoryService; @@ -41,20 +40,22 @@ public LeastUsedDistributor(DirectoryService directoryService) throws IOExceptio public Directory doAny() { Directory directory = null; long size = Long.MIN_VALUE; + int sameSize = 0; for (Directory delegate : delegates) { - if (delegate instanceof FSDirectory) { - long currentSize = ((FSDirectory) delegate).getDirectory().getUsableSpace(); - if (currentSize > size) { - size = currentSize; + long currentSize = getUsableSpace(delegate); + if (currentSize > size) { + size = currentSize; + directory = delegate; + sameSize = 1; + } else if (currentSize == size) { + sameSize++; + // Ensure uniform distribution between all directories with the same size + if (ThreadLocalRandom.current().nextDouble() < 1.0 / sameSize) { directory = delegate; - } else if (currentSize == size && ThreadLocalRandom.current().nextBoolean()) { - directory = delegate; - } else { } - } else { - directory = delegate; // really, make sense to have multiple directories for FS } } + return directory; } diff --git a/src/main/java/org/elasticsearch/index/store/distributor/RandomWeightedDistributor.java b/src/main/java/org/elasticsearch/index/store/distributor/RandomWeightedDistributor.java index a74ef520da918..a92ceba077fd2 100644 --- a/src/main/java/org/elasticsearch/index/store/distributor/RandomWeightedDistributor.java +++ b/src/main/java/org/elasticsearch/index/store/distributor/RandomWeightedDistributor.java @@ -21,7 +21,6 @@ import jsr166y.ThreadLocalRandom; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.index.store.DirectoryService; @@ -44,12 +43,7 @@ public Directory doAny() { long size = 0; for (int i = 0; i < delegates.length; i++) { - Directory delegate = delegates[i]; - if (delegate instanceof FSDirectory) { - size += ((FSDirectory) delegate).getDirectory().getUsableSpace(); - } else { - // makes little sense to use multiple non fs directories - } + size += getUsableSpace(delegates[i]); usableSpace[i] = size; } diff --git a/src/main/java/org/elasticsearch/search/facet/FacetPhase.java b/src/main/java/org/elasticsearch/search/facet/FacetPhase.java index b9ba627a8b906..d8afafc7f372e 100644 --- a/src/main/java/org/elasticsearch/search/facet/FacetPhase.java +++ b/src/main/java/org/elasticsearch/search/facet/FacetPhase.java @@ -178,9 +178,9 @@ public void execute(SearchContext context) throws ElasticSearchException { for (Map.Entry> entry : filtersByCollector.entrySet()) { Filter filter = entry.getKey(); Query query = new XConstantScoreQuery(filter); - Filter searchFilter = context.mapperService().searchFilter(context.types()); + Filter searchFilter = context.searchFilter(context.types()); if (searchFilter != null) { - query = new XFilteredQuery(query, context.filterCache().cache(searchFilter)); + query = new XFilteredQuery(query, searchFilter); } try { context.searcher().search(query, MultiCollector.wrap(entry.getValue().toArray(new Collector[entry.getValue().size()]))); diff --git a/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java b/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java index 0af8ef18bfdfb..4c9d9b9ecb228 100644 --- a/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java +++ b/src/main/java/org/elasticsearch/search/internal/ContextIndexSearcher.java @@ -19,12 +19,10 @@ package org.elasticsearch.search.internal; -import com.google.common.collect.ImmutableList; import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.*; import org.elasticsearch.common.lucene.MinimumScoreCollector; import org.elasticsearch.common.lucene.MultiCollector; -import org.elasticsearch.common.lucene.search.AndFilter; import org.elasticsearch.common.lucene.search.FilteredCollector; import org.elasticsearch.common.lucene.search.XCollector; import org.elasticsearch.common.lucene.search.XFilteredQuery; @@ -126,35 +124,6 @@ public Weight createNormalizedWeight(Query query) throws IOException { return super.createNormalizedWeight(query); } - private Filter combinedFilter(Filter filter) { - Filter combinedFilter; - if (filter == null) { - combinedFilter = searchContext.aliasFilter(); - } else { - if (searchContext.aliasFilter() != null) { - combinedFilter = new AndFilter(ImmutableList.of(filter, searchContext.aliasFilter())); - } else { - combinedFilter = filter; - } - } - return combinedFilter; - } - - @Override - public void search(Query query, Collector results) throws IOException { - Filter filter = combinedFilter(null); - if (filter != null) { - super.search(wrapFilter(query, filter), results); - } else { - super.search(query, results); - } - } - - @Override - public TopDocs search(Query query, Filter filter, int n) throws IOException { - return super.search(query, combinedFilter(filter), n); - } - @Override public void search(List leaves, Weight weight, Collector collector) throws IOException { if (searchContext.timeoutInMillis() != -1) { diff --git a/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/src/main/java/org/elasticsearch/search/internal/SearchContext.java index a9004cc5800fe..fa0fee30999c2 100644 --- a/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -28,6 +28,7 @@ import org.elasticsearch.action.search.SearchType; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lease.Releasable; +import org.elasticsearch.common.lucene.search.AndFilter; import org.elasticsearch.common.lucene.search.Queries; import org.elasticsearch.common.lucene.search.XConstantScoreQuery; import org.elasticsearch.common.lucene.search.XFilteredQuery; @@ -228,18 +229,32 @@ public void preProcess() { if (queryBoost() != 1.0f) { parsedQuery(new ParsedQuery(new FunctionScoreQuery(query(), new BoostScoreFunction(queryBoost)), parsedQuery())); } - Filter searchFilter = mapperService().searchFilter(types()); + Filter searchFilter = searchFilter(types()); if (searchFilter != null) { if (Queries.isConstantMatchAllQuery(query())) { - Query q = new XConstantScoreQuery(filterCache().cache(searchFilter)); + Query q = new XConstantScoreQuery(searchFilter); q.setBoost(query().getBoost()); parsedQuery(new ParsedQuery(q, parsedQuery())); } else { - parsedQuery(new ParsedQuery(new XFilteredQuery(query(), filterCache().cache(searchFilter)), parsedQuery())); + parsedQuery(new ParsedQuery(new XFilteredQuery(query(), searchFilter), parsedQuery())); } } } + public Filter searchFilter(String[] types) { + Filter filter = mapperService().searchFilter(types); + if (filter == null) { + return aliasFilter; + } else { + filter = filterCache().cache(filter); + if (aliasFilter != null) { + return new AndFilter(ImmutableList.of(filter, aliasFilter)); + } + return filter; + } + } + + public long id() { return this.id; } diff --git a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java index 1f4b51707cc10..f9ebf43c1cfaf 100644 --- a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java +++ b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortBuilder.java @@ -39,6 +39,7 @@ public class GeoDistanceSortBuilder extends SortBuilder { private GeoDistance geoDistance; private DistanceUnit unit; private SortOrder order; + private String sortMode; /** * Constructs a new distance based sort on a geo point like field. @@ -102,6 +103,15 @@ public SortBuilder missing(Object missing) { return this; } + /** + * Defines which distance to use for sorting in the case a document contains multiple geo points. + * Possible values: min and max + */ + public SortBuilder sortMode(String sortMode) { + this.sortMode = sortMode; + return this; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("_geo_distance"); @@ -121,6 +131,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws if (order == SortOrder.DESC) { builder.field("reverse", true); } + if (sortMode != null) { + builder.field("mode", sortMode); + } builder.endObject(); return builder; diff --git a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java index 0140da913bef8..e1c3b034c097a 100644 --- a/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java +++ b/src/main/java/org/elasticsearch/search/sort/GeoDistanceSortParser.java @@ -29,6 +29,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.fielddata.IndexGeoPointFieldData; import org.elasticsearch.index.fielddata.fieldcomparator.GeoDistanceComparatorSource; +import org.elasticsearch.index.fielddata.fieldcomparator.SortMode; import org.elasticsearch.index.mapper.FieldMapper; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import org.elasticsearch.search.internal.SearchContext; @@ -50,6 +51,7 @@ public SortField parse(XContentParser parser, SearchContext context) throws Exce DistanceUnit unit = DistanceUnit.KILOMETERS; GeoDistance geoDistance = GeoDistance.ARC; boolean reverse = false; + SortMode sortMode = null; boolean normalizeLon = true; boolean normalizeLat = true; @@ -96,6 +98,8 @@ public SortField parse(XContentParser parser, SearchContext context) throws Exce } else if ("normalize".equals(currentName)) { normalizeLat = parser.booleanValue(); normalizeLon = parser.booleanValue(); + } else if ("mode".equals(currentName)) { + sortMode = SortMode.fromString(parser.text()); } else { point.resetFromString(parser.text()); fieldName = currentName; @@ -107,12 +111,16 @@ public SortField parse(XContentParser parser, SearchContext context) throws Exce GeoUtils.normalizePoint(point, normalizeLat, normalizeLon); } + if (sortMode == null) { + sortMode = reverse ? SortMode.MAX : SortMode.MIN; + } + FieldMapper mapper = context.smartNameFieldMapper(fieldName); if (mapper == null) { throw new ElasticSearchIllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort"); } IndexGeoPointFieldData indexFieldData = context.fieldData().getForField(mapper); - return new SortField(fieldName, new GeoDistanceComparatorSource(indexFieldData, point.lat(), point.lon(), unit, geoDistance), reverse); + return new SortField(fieldName, new GeoDistanceComparatorSource(indexFieldData, point.lat(), point.lon(), unit, geoDistance, sortMode), reverse); } } diff --git a/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java b/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java index 69c34be42d6dc..e8eacf6ddc90b 100644 --- a/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java +++ b/src/main/java/org/elasticsearch/search/suggest/phrase/CandidateScorer.java @@ -36,6 +36,9 @@ public CandidateScorer(WordScorer scorer, int maxNumCorrections, int gramSize) { public Correction[] findBestCandiates(CandidateSet[] sets, float errorFraction, double cutoffScore) throws IOException { + if (sets.length == 0) { + return Correction.EMPTY; + } PriorityQueue corrections = new PriorityQueue(maxNumCorrections) { @Override protected boolean lessThan(Correction a, Correction b) { diff --git a/src/main/java/org/elasticsearch/search/suggest/phrase/Correction.java b/src/main/java/org/elasticsearch/search/suggest/phrase/Correction.java index 66c0ce6503920..cde05a08a0a4d 100644 --- a/src/main/java/org/elasticsearch/search/suggest/phrase/Correction.java +++ b/src/main/java/org/elasticsearch/search/suggest/phrase/Correction.java @@ -26,6 +26,7 @@ //TODO public for tests public final class Correction { + public static final Correction[] EMPTY = new Correction[0]; public double score; public final Candidate[] candidates; diff --git a/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java b/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java index dc0419285fa1b..21a28ae770c68 100644 --- a/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java +++ b/src/main/java/org/elasticsearch/search/suggest/phrase/NoisyChannelSpellChecker.java @@ -104,6 +104,10 @@ public void end() { } }); + if (candidateSetsList.isEmpty()) { + return Correction.EMPTY; + } + for (CandidateSet candidateSet : candidateSetsList) { generator.drawCandidates(candidateSet); } diff --git a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java index e2dafc0c65f4d..5555322d35e04 100644 --- a/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java +++ b/src/test/java/org/elasticsearch/test/integration/aliases/IndexAliasesTests.java @@ -38,6 +38,9 @@ import org.elasticsearch.node.internal.InternalNode; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.facet.FacetBuilders; +import org.elasticsearch.search.facet.terms.TermsFacet; +import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; @@ -229,6 +232,28 @@ public void testSearchingFilteringAliasesSingleIndex() throws Exception { searchResponse = client1.prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(); assertHits(searchResponse.getHits(), "1", "2", "3"); + logger.info("--> checking single filtering alias search with sort"); + searchResponse = client1.prepareSearch("tests").setQuery(QueryBuilders.matchAllQuery()).addSort("_uid", SortOrder.ASC).execute().actionGet(); + assertHits(searchResponse.getHits(), "1", "2", "3"); + + logger.info("--> checking single filtering alias search with global facets"); + searchResponse = client1.prepareSearch("tests").setQuery(QueryBuilders.matchQuery("name", "bar")) + .addFacet(FacetBuilders.termsFacet("test").field("name").global(true)) + .execute().actionGet(); + assertThat(((TermsFacet) searchResponse.getFacets().facet("test")).getEntries().size(), equalTo(4)); + + logger.info("--> checking single filtering alias search with global facets and sort"); + searchResponse = client1.prepareSearch("tests").setQuery(QueryBuilders.matchQuery("name", "bar")) + .addFacet(FacetBuilders.termsFacet("test").field("name").global(true)) + .addSort("_uid", SortOrder.ASC).execute().actionGet(); + assertThat(((TermsFacet) searchResponse.getFacets().facet("test")).getEntries().size(), equalTo(4)); + + logger.info("--> checking single filtering alias search with non-global facets"); + searchResponse = client1.prepareSearch("tests").setQuery(QueryBuilders.matchQuery("name", "bar")) + .addFacet(FacetBuilders.termsFacet("test").field("name").global(false)) + .addSort("_uid", SortOrder.ASC).execute().actionGet(); + assertThat(((TermsFacet) searchResponse.getFacets().facet("test")).getEntries().size(), equalTo(2)); + searchResponse = client1.prepareSearch("foos", "bars").setQuery(QueryBuilders.matchAllQuery()).execute().actionGet(); assertHits(searchResponse.getHits(), "1", "2"); diff --git a/src/test/java/org/elasticsearch/test/integration/indices/store/SimpleDistributorTests.java b/src/test/java/org/elasticsearch/test/integration/indices/store/SimpleDistributorTests.java new file mode 100644 index 0000000000000..4e05d0022a0ee --- /dev/null +++ b/src/test/java/org/elasticsearch/test/integration/indices/store/SimpleDistributorTests.java @@ -0,0 +1,79 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.indices.store; + +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.env.Environment; +import org.elasticsearch.indices.IndexMissingException; +import org.elasticsearch.node.internal.InternalNode; +import org.elasticsearch.test.integration.AbstractNodesTests; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +import java.io.File; + +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + +/** + * + */ +public class SimpleDistributorTests extends AbstractNodesTests { + protected Environment environment; + + @BeforeClass + public void getTestEnvironment() { + environment = ((InternalNode) startNode("node0")).injector().getInstance(Environment.class); + closeNode("node0"); + } + + @AfterClass + public void closeNodes() { + closeAllNodes(); + } + + public final static String[] STORE_TYPES = {"fs", "simplefs", "niofs", "mmapfs"}; + + @Test + public void testAvailableSpaceDetection() { + File dataRoot = environment.dataFiles()[0]; + startNode("node1", settingsBuilder().putArray("path.data", new File(dataRoot, "data1").getAbsolutePath(), new File(dataRoot, "data2").getAbsolutePath())); + for (String store : STORE_TYPES) { + try { + client("node1").admin().indices().prepareDelete("test").execute().actionGet(); + } catch (IndexMissingException ex) { + // Ignore + } + client("node1").admin().indices().prepareCreate("test") + .setSettings(settingsBuilder() + .put("index.store.distributor", StrictDistributor.class.getCanonicalName()) + .put("index.store.type", store) + .put("index.number_of_replicas", 0) + .put("index.number_of_shards", 1) + ) + .execute().actionGet(); + assertThat(client("node1").admin().cluster().prepareHealth("test").setWaitForGreenStatus() + .setTimeout(TimeValue.timeValueSeconds(5)).execute().actionGet().isTimedOut(), equalTo(false)); + } + } + +} diff --git a/src/test/java/org/elasticsearch/test/integration/indices/store/StrictDistributor.java b/src/test/java/org/elasticsearch/test/integration/indices/store/StrictDistributor.java new file mode 100644 index 0000000000000..17a2463e6d49a --- /dev/null +++ b/src/test/java/org/elasticsearch/test/integration/indices/store/StrictDistributor.java @@ -0,0 +1,49 @@ +/* + * Licensed to ElasticSearch and Shay Banon under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. ElasticSearch licenses this + * file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.test.integration.indices.store; + +import org.apache.lucene.store.Directory; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.index.store.DirectoryService; +import org.elasticsearch.index.store.distributor.AbstractDistributor; + +import java.io.IOException; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; + +/** + * + */ +public class StrictDistributor extends AbstractDistributor { + + @Inject + public StrictDistributor(DirectoryService directoryService) throws IOException { + super(directoryService); + } + + @Override + public Directory doAny() { + for (Directory delegate : delegates) { + assertThat(getUsableSpace(delegate), greaterThan(0L)); + } + return primary(); + } +} \ No newline at end of file diff --git a/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java b/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java index 675ba8bef4e5c..45e990b763bb7 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/geo/GeoDistanceTests.java @@ -29,19 +29,18 @@ import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.elasticsearch.test.integration.AbstractNodesTests; - import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; +import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.FilterBuilders.geoDistanceFilter; import static org.elasticsearch.index.query.FilterBuilders.geoDistanceRangeFilter; import static org.elasticsearch.index.query.QueryBuilders.filteredQuery; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.*; /** */ @@ -244,7 +243,123 @@ public void simpleDistanceTests() throws Exception { assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); assertThat(searchResponse.getHits().getAt(0).id(), equalTo("7")); } - + + @Test + public void testDistanceSortingMVFields() throws Exception { + client.admin().indices().prepareDelete().execute().actionGet(); + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") + .startObject("properties").startObject("locations").field("type", "geo_point").field("lat_lon", true).endObject().endObject() + .endObject().endObject().string(); + + client.admin().indices().prepareCreate("test") + .setSettings(settingsBuilder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)) + .addMapping("type1", mapping) + .execute().actionGet(); + client.admin().cluster().prepareHealth("test").setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); + + client.prepareIndex("test", "type1", "1").setSource(jsonBuilder().startObject() + .field("names", "New York") + .startObject("locations").field("lat", 40.7143528).field("lon", -74.0059731).endObject() + .endObject()).execute().actionGet(); + + client.prepareIndex("test", "type1", "2").setSource(jsonBuilder().startObject() + .field("names", "Times Square", "Tribeca") + .startArray("locations") + // to NY: 5.286 km + .startObject().field("lat", 40.759011).field("lon", -73.9844722).endObject() + // to NY: 0.4621 km + .startObject().field("lat", 40.718266).field("lon", -74.007819).endObject() + .endArray() + .endObject()).execute().actionGet(); + + client.prepareIndex("test", "type1", "3").setSource(jsonBuilder().startObject() + .field("names", "Wall Street", "Soho") + .startArray("locations") + // to NY: 1.055 km + .startObject().field("lat", 40.7051157).field("lon", -74.0088305).endObject() + // to NY: 1.258 km + .startObject().field("lat", 40.7247222).field("lon", -74).endObject() + .endArray() + .endObject()).execute().actionGet(); + + + client.prepareIndex("test", "type1", "4").setSource(jsonBuilder().startObject() + .field("names", "Greenwich Village", "Brooklyn") + .startArray("locations") + // to NY: 2.029 km + .startObject().field("lat", 40.731033).field("lon", -73.9962255).endObject() + // to NY: 8.572 km + .startObject().field("lat", 40.65).field("lon", -73.95).endObject() + .endArray() + .endObject()).execute().actionGet(); + + client.admin().indices().prepareRefresh().execute().actionGet(); + + // Order: Asc + SearchResponse searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.geoDistanceSort("locations").point(40.7143528, -74.0059731).order(SortOrder.ASC)) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(4l)); + assertThat(searchResponse.getHits().hits().length, equalTo(4)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(0d)); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), closeTo(0.4621d, 0.01d)); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), closeTo(1.055d, 0.01d)); + assertThat(searchResponse.getHits().getAt(3).id(), equalTo("4")); + assertThat(((Number) searchResponse.getHits().getAt(3).sortValues()[0]).doubleValue(), closeTo(2.029d, 0.01d)); + + // Order: Asc, Mode: max + searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.geoDistanceSort("locations").point(40.7143528, -74.0059731).order(SortOrder.ASC).sortMode("max")) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(4l)); + assertThat(searchResponse.getHits().hits().length, equalTo(4)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("1")); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), equalTo(0d)); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), closeTo(1.258d, 0.01d)); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), closeTo(5.286d, 0.01d)); + assertThat(searchResponse.getHits().getAt(3).id(), equalTo("4")); + assertThat(((Number) searchResponse.getHits().getAt(3).sortValues()[0]).doubleValue(), closeTo(8.572d, 0.01d)); + + // Order: Desc + searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.geoDistanceSort("locations").point(40.7143528, -74.0059731).order(SortOrder.DESC)) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(4l)); + assertThat(searchResponse.getHits().hits().length, equalTo(4)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("4")); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), closeTo(8.572d, 0.01d)); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("2")); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), closeTo(5.286d, 0.01d)); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("3")); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), closeTo(1.258d, 0.01d)); + assertThat(searchResponse.getHits().getAt(3).id(), equalTo("1")); + assertThat(((Number) searchResponse.getHits().getAt(3).sortValues()[0]).doubleValue(), equalTo(0d)); + + // Order: Desc, Mode: min + searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()) + .addSort(SortBuilders.geoDistanceSort("locations").point(40.7143528, -74.0059731).order(SortOrder.DESC).sortMode("min")) + .execute().actionGet(); + + assertThat(searchResponse.getHits().getTotalHits(), equalTo(4l)); + assertThat(searchResponse.getHits().hits().length, equalTo(4)); + assertThat(searchResponse.getHits().getAt(0).id(), equalTo("4")); + assertThat(((Number) searchResponse.getHits().getAt(0).sortValues()[0]).doubleValue(), closeTo(2.029d, 0.01d)); + assertThat(searchResponse.getHits().getAt(1).id(), equalTo("3")); + assertThat(((Number) searchResponse.getHits().getAt(1).sortValues()[0]).doubleValue(), closeTo(1.055d, 0.01d)); + assertThat(searchResponse.getHits().getAt(2).id(), equalTo("2")); + assertThat(((Number) searchResponse.getHits().getAt(2).sortValues()[0]).doubleValue(), closeTo(0.4621d, 0.01d)); + assertThat(searchResponse.getHits().getAt(3).id(), equalTo("1")); + assertThat(((Number) searchResponse.getHits().getAt(3).sortValues()[0]).doubleValue(), equalTo(0d)); + } + @Test public void distanceScriptTests() throws Exception { try { @@ -252,12 +367,12 @@ public void distanceScriptTests() throws Exception { } catch (Exception e) { // ignore } - + double source_lat = 32.798; double source_long = -117.151; double target_lat = 32.81; double target_long = -117.21; - + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type1") .startObject("properties").startObject("location").field("type", "geo_point").field("lat_lon", true).endObject().endObject() .endObject().endObject().string(); @@ -271,27 +386,27 @@ public void distanceScriptTests() throws Exception { client.admin().indices().prepareRefresh().execute().actionGet(); - SearchResponse searchResponse1 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistance("+target_lat+","+target_long+")").execute().actionGet(); + SearchResponse searchResponse1 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistance(" + target_lat + "," + target_long + ")").execute().actionGet(); Double resultDistance1 = searchResponse1.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance1, equalTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES))); - - SearchResponse searchResponse2 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].distance("+target_lat+","+target_long+")").execute().actionGet(); + + SearchResponse searchResponse2 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].distance(" + target_lat + "," + target_long + ")").execute().actionGet(); Double resultDistance2 = searchResponse2.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance2, equalTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.MILES))); - SearchResponse searchResponse3 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistanceInKm("+target_lat+","+target_long+")").execute().actionGet(); + SearchResponse searchResponse3 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistanceInKm(" + target_lat + "," + target_long + ")").execute().actionGet(); Double resultArcDistance3 = searchResponse3.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance3, equalTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS))); - - SearchResponse searchResponse4 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].distanceInKm("+target_lat+","+target_long+")").execute().actionGet(); + + SearchResponse searchResponse4 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].distanceInKm(" + target_lat + "," + target_long + ")").execute().actionGet(); Double resultDistance4 = searchResponse4.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultDistance4, equalTo(GeoDistance.PLANE.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS))); - SearchResponse searchResponse5 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistanceInKm("+(target_lat)+","+(target_long+360)+")").execute().actionGet(); + SearchResponse searchResponse5 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistanceInKm(" + (target_lat) + "," + (target_long + 360) + ")").execute().actionGet(); Double resultArcDistance5 = searchResponse5.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance5, equalTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS))); - - SearchResponse searchResponse6 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistanceInKm("+(target_lat+360)+","+(target_long)+")").execute().actionGet(); + + SearchResponse searchResponse6 = client.prepareSearch().addField("_source").addScriptField("distance", "doc['location'].arcDistanceInKm(" + (target_lat + 360) + "," + (target_long) + ")").execute().actionGet(); Double resultArcDistance6 = searchResponse6.getHits().getHits()[0].getFields().get("distance").getValue(); assertThat(resultArcDistance6, equalTo(GeoDistance.ARC.calculate(source_lat, source_long, target_lat, target_long, DistanceUnit.KILOMETERS))); } diff --git a/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java b/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java index f397326ae19a2..b65d4bb569c7d 100644 --- a/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java +++ b/src/test/java/org/elasticsearch/test/integration/search/suggest/SuggestSearchTests.java @@ -405,6 +405,29 @@ public void testSizeAndSort() throws Exception { // assertThat(search.suggest().suggestions().get(3).getSuggestedWords().get("prefix_abcd").get(4).getTerm(), equalTo("prefix_accd")); } + @Test // see #2817 + public void testStopwordsOnlyPhraseSuggest() throws ElasticSearchException, IOException { + client.admin().indices().prepareDelete().execute().actionGet(); + Builder builder = ImmutableSettings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0); + client.admin().indices().prepareCreate("test").setSettings(builder.build()).execute().actionGet(); + client.admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet(); + client.prepareIndex("test", "type1") + .setSource(XContentFactory.jsonBuilder().startObject().field("body", "this is a test").endObject()).execute().actionGet(); + client.admin().indices().prepareRefresh().execute().actionGet(); + + Suggest searchSuggest = searchSuggest( + client, + "a an the", + phraseSuggestion("simple_phrase").field("body").gramSize(1) + .addCandidateGenerator(PhraseSuggestionBuilder.candidateGenerator("body").minWordLength(1).suggestMode("always")) + .size(1)); + assertThat(searchSuggest, notNullValue()); + assertThat(searchSuggest.size(), equalTo(1)); + assertThat(searchSuggest.getSuggestion("simple_phrase").getName(), equalTo("simple_phrase")); + assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().size(), equalTo(1)); + assertThat(searchSuggest.getSuggestion("simple_phrase").getEntries().get(0).getOptions().size(), equalTo(0)); + } + @Test public void testMarvelHerosPhraseSuggest() throws ElasticSearchException, IOException { diff --git a/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java b/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java index cc2dee36d7be3..e40127f17d98d 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/engine/AbstractSimpleEngineTests.java @@ -185,19 +185,19 @@ public void testSegments() throws Exception { segments = engine.segments(); assertThat(segments.size(), equalTo(1)); - assertThat(segments.get(0).committed(), equalTo(false)); - assertThat(segments.get(0).search(), equalTo(true)); - assertThat(segments.get(0).numDocs(), equalTo(2)); - assertThat(segments.get(0).deletedDocs(), equalTo(0)); + assertThat(segments.get(0).isCommitted(), equalTo(false)); + assertThat(segments.get(0).isSearch(), equalTo(true)); + assertThat(segments.get(0).getNumDocs(), equalTo(2)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); engine.flush(new Engine.Flush()); segments = engine.segments(); assertThat(segments.size(), equalTo(1)); - assertThat(segments.get(0).committed(), equalTo(true)); - assertThat(segments.get(0).search(), equalTo(true)); - assertThat(segments.get(0).numDocs(), equalTo(2)); - assertThat(segments.get(0).deletedDocs(), equalTo(0)); + assertThat(segments.get(0).isCommitted(), equalTo(true)); + assertThat(segments.get(0).isSearch(), equalTo(true)); + assertThat(segments.get(0).getNumDocs(), equalTo(2)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); ParsedDocument doc3 = testParsedDocument("3", "3", "test", null, -1, -1, testDocumentWithTextField(), Lucene.STANDARD_ANALYZER, B_3, false); @@ -206,32 +206,32 @@ public void testSegments() throws Exception { segments = engine.segments(); assertThat(segments.size(), equalTo(2)); - assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true)); - assertThat(segments.get(0).committed(), equalTo(true)); - assertThat(segments.get(0).search(), equalTo(true)); - assertThat(segments.get(0).numDocs(), equalTo(2)); - assertThat(segments.get(0).deletedDocs(), equalTo(0)); + assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); + assertThat(segments.get(0).isCommitted(), equalTo(true)); + assertThat(segments.get(0).isSearch(), equalTo(true)); + assertThat(segments.get(0).getNumDocs(), equalTo(2)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertThat(segments.get(1).committed(), equalTo(false)); - assertThat(segments.get(1).search(), equalTo(true)); - assertThat(segments.get(1).numDocs(), equalTo(1)); - assertThat(segments.get(1).deletedDocs(), equalTo(0)); + assertThat(segments.get(1).isCommitted(), equalTo(false)); + assertThat(segments.get(1).isSearch(), equalTo(true)); + assertThat(segments.get(1).getNumDocs(), equalTo(1)); + assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); engine.delete(new Engine.Delete("test", "1", newUid("1"))); engine.refresh(new Engine.Refresh(true)); segments = engine.segments(); assertThat(segments.size(), equalTo(2)); - assertThat(segments.get(0).generation() < segments.get(1).generation(), equalTo(true)); - assertThat(segments.get(0).committed(), equalTo(true)); - assertThat(segments.get(0).search(), equalTo(true)); - assertThat(segments.get(0).numDocs(), equalTo(1)); - assertThat(segments.get(0).deletedDocs(), equalTo(1)); - - assertThat(segments.get(1).committed(), equalTo(false)); - assertThat(segments.get(1).search(), equalTo(true)); - assertThat(segments.get(1).numDocs(), equalTo(1)); - assertThat(segments.get(1).deletedDocs(), equalTo(0)); + assertThat(segments.get(0).getGeneration() < segments.get(1).getGeneration(), equalTo(true)); + assertThat(segments.get(0).isCommitted(), equalTo(true)); + assertThat(segments.get(0).isSearch(), equalTo(true)); + assertThat(segments.get(0).getNumDocs(), equalTo(1)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(1)); + + assertThat(segments.get(1).isCommitted(), equalTo(false)); + assertThat(segments.get(1).isSearch(), equalTo(true)); + assertThat(segments.get(1).getNumDocs(), equalTo(1)); + assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); } @Test diff --git a/src/test/java/org/elasticsearch/test/unit/index/store/distributor/DistributorTests.java b/src/test/java/org/elasticsearch/test/unit/index/store/distributor/DistributorTests.java index 17c5223e98314..7c27aae973841 100644 --- a/src/test/java/org/elasticsearch/test/unit/index/store/distributor/DistributorTests.java +++ b/src/test/java/org/elasticsearch/test/unit/index/store/distributor/DistributorTests.java @@ -36,7 +36,7 @@ public class DistributorTests { @Test - public void testEmptyFirstDistributor() throws Exception { + public void testLeastUsedDistributor() throws Exception { FakeFsDirectory[] directories = new FakeFsDirectory[]{ new FakeFsDirectory("dir0", 10L), new FakeFsDirectory("dir1", 20L), @@ -60,6 +60,34 @@ public void testEmptyFirstDistributor() throws Exception { } + directories[0].setUsableSpace(10L); + directories[1].setUsableSpace(20L); + directories[2].setUsableSpace(20L); + for (FakeFsDirectory directory : directories) { + directory.resetAllocationCount(); + } + for (int i = 0; i < 10000; i++) { + ((FakeFsDirectory) distributor.any()).incrementAllocationCount(); + } + assertThat(directories[0].getAllocationCount(), equalTo(0)); + assertThat((double) directories[1].getAllocationCount() / directories[2].getAllocationCount(), closeTo(1.0, 0.5)); + + // Test failover scenario + for (FakeFsDirectory directory : directories) { + directory.resetAllocationCount(); + } + directories[0].setUsableSpace(0L); + directories[1].setUsableSpace(0L); + directories[2].setUsableSpace(0L); + for (int i = 0; i < 10000; i++) { + ((FakeFsDirectory) distributor.any()).incrementAllocationCount(); + } + for (FakeFsDirectory directory : directories) { + assertThat(directory.getAllocationCount(), greaterThan(0)); + } + assertThat((double) directories[0].getAllocationCount() / directories[2].getAllocationCount(), closeTo(1.0, 0.5)); + assertThat((double) directories[1].getAllocationCount() / directories[2].getAllocationCount(), closeTo(1.0, 0.5)); + } @Test