First step to upgrade Elastic to elasticsearch 5.0

In this first step, all tests are checked which are not compatible with elasticsearch 5.0. Either the tests are directly fixed or they are temporarely skipped and have to be cleaned up later. In general it seems as most of the things will just keep working with elasticsearch 5.0 except for the things that were changed / removed in elasticsearch 5.0 anyways (like mapping changes from string to text / keyword).

* Replace remove count search type by query_then_fetch
* Replace optimize by forcemerge as optimize was removed
* Rename getMock() to createMock() to be compatible with newer phpunit versions
* Convert string mapping to text
* Change delete-by-query implementation to use 5.0 api
* Remove delete-by-query plugin checks as now installed by default
* Change script implementation to new 5.0 implementation
* Simplify update-document
* Update test environment to use self built elasticsearch instance
* Improve node info fetching to use new format
* Skip plugin tests if no plugin available
* Fix list of tests
* Apply liniting
* Fix script tests for new script structure
* Remove search_type from scroll as not needed anymore
* Update README.md with new dependency

Part of #1184

CHANGELOG.md

@@ -4,7 +4,10 @@ All notable changes to this project will be documented in this file based on the
 ## [Unreleased](https://github.com/ruflin/Elastica/compare/3.2.3...HEAD)
 
 ### Backward Compatibility Fixes
+- Update elasticsearch dependency to 5.0
 - Composer installations will no longer include tests and other development files. 
+- Replace flush refresh param with a options array
+- Rename Mapping::setFields to Mapping::setStoredFields
 
 ### Bugfixes
 

README.md

@@ -17,7 +17,7 @@ All library issues should go to the [issue tracker from github](https://github.c
 
 Compatibility
 -------------
-This release is compatible with all elasticsearch 2.x releases. It was tested with version 2.4.0
+This release is compatible with all elasticsearch 5.x releases. It was tested with version 5.0.0-rc1
 
 
 Contributing
@@ -29,5 +29,4 @@ Dependencies
 ------------
 | Project | Version | Required |
 |---------|---------|----------|
-|[Elasticsearch](https://github.com/elasticsearch/elasticsearch/tree/v2.4.0)|2.4.0|yes|
-|[Elasticsearch image plugin](https://github.com/Jmoati/elasticsearch-image/releases/tag/1.7.1)|1.7.1|no|
+|[Elasticsearch](https://github.com/elasticsearch/elasticsearch/tree/5.0)|5.0|yes|

docker-compose.yml

@@ -14,6 +14,8 @@ elasticsearch:
   #image: ruflin/elasticsearch-elastica
   volumes_from:
     - data
+  command: elasticsearch -Ehttp.host=0.0.0.0
+
 nginx:
   build: ./env/nginx/
   #image: ruflin/nginx-elastica

env/elasticsearch/Dockerfile

@@ -1,26 +1,54 @@
-FROM elasticsearch:2.4.0
+FROM java:8-jre
 MAINTAINER Nicolas Ruflin <[email protected]>
 
-# Dependencies
-ENV ELASTICSEARCH_VERSION 2.4.0
-ENV ES_IMAGE_PLUGIN_VER 1.7.1
-ENV ES_PLUGIN_BIN /usr/share/elasticsearch/bin/plugin
+ENV VERSION 5.0.0-rc1
 
-# Install Plugins
-RUN ${ES_PLUGIN_BIN} install mapper-attachments
-RUN ${ES_PLUGIN_BIN} install delete-by-query
-#RUN ${ES_PLUGIN_BIN} install image --url https://github.com/Jmoati/elasticsearch-image/releases/download/${ES_IMAGE_PLUGIN_VER}/elasticsearch-image-${ES_IMAGE_PLUGIN_VER}.zip
+ENV URL https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${VERSION}.tar.gz
+
+# Cache variable can be set during building to invalidate the build cache with `--build-arg CACHE=$(date +%s) .`
+ARG CACHE=1
+
+ENV ESHOME /opt/elasticsearch-${VERSION}
+ENV ES_PLUGIN_BIN ${ESHOME}/bin/elasticsearch-plugin
+
+# grab gosu for easy step-down from root
+RUN gpg --keyserver ha.pool.sks-keyservers.net --recv-keys B42F6819007F00F88E364FD4036A9C25BF357DD4
+RUN arch="$(dpkg --print-architecture)" \
+	&& set -x \
+	&& curl -o /usr/local/bin/gosu -fSL "https://github.com/tianon/gosu/releases/download/1.3/gosu-$arch" \
+	&& curl -o /usr/local/bin/gosu.asc -fSL "https://github.com/tianon/gosu/releases/download/1.3/gosu-$arch.asc" \
+	&& gpg --verify /usr/local/bin/gosu.asc \
+	&& rm /usr/local/bin/gosu.asc \
+	&& chmod +x /usr/local/bin/gosu
+
+RUN groupadd -r elasticsearch && useradd -r -m -g elasticsearch elasticsearch
+
+RUN set -x && \
+	cd /opt && \
+	wget -qO elasticsearch.tar.gz "$URL?${CACHE}" && \
+	tar xzvf elasticsearch.tar.gz && \
+	chown -R elasticsearch:elasticsearch ${ESHOME}
+
+ENV PATH ${ESHOME}/bin:$PATH
 
-# Debug interface
-# RUN ${ES_PLUGIN_BIN} install mobz/elasticsearch-head
+VOLUME ${ESHOME}/data
+
+ENV ES_JAVA_OPTS="-Xms512m -Xmx512m"
+
+RUN ${ES_PLUGIN_BIN} install mapper-attachments
 
 # Copy config files
 COPY *.yml /usr/share/elasticsearch/config/
 COPY scripts/* /usr/share/elasticsearch/config/scripts/
 
+COPY docker-entrypoint.sh /
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+
 
 RUN mkdir -p /tmp/backups/backup1
 RUN mkdir -p /tmp/backups/backup2
 
-# Expose standard ports
 EXPOSE 9200 9300
+
+CMD ["elasticsearch"]

env/elasticsearch/docker-entrypoint.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+# Add elasticsearch as command if needed
+if [ "${1:0:1}" = '-' ]; then
+	set -- elasticsearch "$@"
+fi
+
+# Drop root privileges if we are running elasticsearch
+if [ "$1" = 'elasticsearch' ]; then
+	# Change the ownership of /usr/share/elasticsearch/data to elasticsearch
+	chown -R elasticsearch:elasticsearch ${ESHOME-/usr/share/elasticsearch}/data
+	exec gosu elasticsearch "$@"
+fi
+
+# As argument is not related to elasticsearch,
+# then assume that user wants to run his own process,
+# for example a `bash` shell to explore this image
+exec "$@"

env/elasticsearch/elasticsearch.yml

@@ -1,21 +1,18 @@
 
-index.number_of_shards: 2
-index.number_of_replicas: 0
+#index.number_of_shards: 2
+#index.number_of_replicas: 0
 
 # Required plugins
-plugin.mandatory: mapper-attachments, delete-by-query #, image
+plugin.mandatory: mapper-attachments
 
 # For script tests
 script.inline: on
-script.indexed: on
+#script.indexed: on
 
 script.engine.groovy.file: on
 
-# Disable dynamic memory allocation
-bootstrap.mlockall: true
-
-# Accept any connection
-network.host: "0.0.0.0"
+# Accept any connection - disable checks
+http.host: 0.0.0.0
 
 # Limit threadpool by set number of available processors to 1
 # Without this, travis builds will be failed with OutOfMemory error
@@ -31,5 +28,5 @@ transport.tcp.port: 9300
 # Added for snapshot tests
 path.repo: ["/tmp/backups"]
 
-discovery.zen.ping.unicast.enabled: true
-discovery.zen.ping.multicast.enabled: true
+#discovery.zen.ping.unicast.enabled: true
+#discovery.zen.ping.multicast.enabled: true

lib/Elastica/Index.php

@@ -198,13 +198,30 @@ public function deleteDocuments(array $docs)
      *
      * @param array $args OPTIONAL Additional arguments
      *
-     * @return \Elastica\Response Server response
+     * @return array Server response
      *
+     * @deprecated Replaced by forcemerge
      * @link https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-optimize.html
      */
     public function optimize($args = [])
     {
-        return $this->request('_optimize', Request::POST, [], $args);
+        return $this->forcemerge($args);
+    }
+
+    /**
+     * Force merges index.
+     *
+     * Detailed arguments can be found here in the link
+     *
+     * @param array $args OPTIONAL Additional arguments
+     *
+     * @return array Server response
+     *
+     * @link https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-forcemerge.html
+     */
+    public function forcemerge($args = [])
+    {
+        return $this->request('_forcemerge', Request::POST, [], $args);
     }
 
     /**
@@ -480,11 +497,11 @@ public function clearCache()
      *
      * @link https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-flush.html
      */
-    public function flush($refresh = false)
+    public function flush($options = [])
     {
         $path = '_flush';
 
-        return $this->request($path, Request::POST, [], ['refresh' => $refresh]);
+        return $this->request($path, Request::POST, [], $options);
     }
 
     /**

lib/Elastica/IndexTemplate.php

@@ -7,7 +7,7 @@
  * Elastica index template object.
  *
  * @author Dmitry Balabka <[email protected]>
- * 
+ *
  * @link https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-templates.html
  */
 class IndexTemplate
@@ -113,7 +113,7 @@ public function getClient()
      */
     public function request($method, $data = [])
     {
-        $path = '/_template/'.$this->getName();
+        $path = '_template/'.$this->getName();
 
         return $this->getClient()->request($path, $method, $data);
     }

lib/Elastica/Node/Info.php

@@ -211,9 +211,9 @@ public function refresh(array $params = [])
         $path = '_nodes/'.$this->getNode()->getId();
 
         if (!empty($params)) {
-            $path .= '?';
+            $path .= '/';
             foreach ($params as $param) {
-                $path .= $param.'=true&';
+                $path .= $param.',';
             }
         }
 

lib/Elastica/Query.php

@@ -283,9 +283,9 @@ public function setVersion($version = true)
      *
      * @link https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-fields.html
      */
-    public function setFields(array $fields)
+    public function setStoredFields(array $fields)
     {
-        return $this->setParam('fields', $fields);
+        return $this->setParam('stored_fields', $fields);
     }
 
     /**

lib/Elastica/Script/Script.php

@@ -119,11 +119,11 @@ public static function create($data)
      */
     protected static function _createFromArray(array $data)
     {
-        if (!isset($data['script'])) {
-            throw new InvalidException("\$data['script'] is required");
+        if (!isset($data['inline'])) {
+            throw new InvalidException("\$data['inline'] is required");
         }
 
-        $script = new self($data['script']);
+        $script = new self($data['inline']);
 
         if (isset($data['lang'])) {
             $script->setLang($data['lang']);
@@ -145,7 +145,7 @@ protected static function _createFromArray(array $data)
     public function toArray()
     {
         $array = [
-            'script' => $this->_script,
+            'inline' => $this->_script,
         ];
 
         if (!empty($this->_params)) {
@@ -156,6 +156,6 @@ public function toArray()
             $array['lang'] = $this->_lang;
         }
 
-        return $array;
+        return ['script' => $array];
     }
 }

lib/Elastica/Scroll.php

@@ -74,7 +74,6 @@ public function next()
 
         $this->_search->setOption(Search::OPTION_SCROLL, $this->expiryTime);
         $this->_search->setOption(Search::OPTION_SCROLL_ID, $this->_nextScrollId);
-        $this->_search->setOption(Search::OPTION_SEARCH_TYPE, Search::OPTION_SEARCH_TYPE_SCROLL);
         $this->_setScrollId($this->_search->search());
 
         $this->_revertOptions();
@@ -123,7 +122,6 @@ public function rewind()
 
         $this->_search->setOption(Search::OPTION_SCROLL, $this->expiryTime);
         $this->_search->setOption(Search::OPTION_SCROLL_ID, null);
-        $this->_search->setOption(Search::OPTION_SEARCH_TYPE, null);
         $this->_setScrollId($this->_search->search());
 
         $this->_revertOptions();
@@ -156,10 +154,6 @@ protected function _saveOptions()
         if ($this->_search->hasOption(Search::OPTION_SCROLL_ID)) {
             $this->_options[1] = $this->_search->getOption(Search::OPTION_SCROLL_ID);
         }
-
-        if ($this->_search->hasOption(Search::OPTION_SEARCH_TYPE)) {
-            $this->_options[2] = $this->_search->getOption(Search::OPTION_SEARCH_TYPE);
-        }
     }
 
     /**
@@ -169,6 +163,5 @@ protected function _revertOptions()
     {
         $this->_search->setOption(Search::OPTION_SCROLL, $this->_options[0]);
         $this->_search->setOption(Search::OPTION_SCROLL_ID, $this->_options[1]);
-        $this->_search->setOption(Search::OPTION_SEARCH_TYPE, $this->_options[2]);
     }
 }

lib/Elastica/Search.php

@@ -482,13 +482,14 @@ public function count($query = '', $fullResult = false)
         $this->setOptionsAndQuery(null, $query);
 
         $query = $this->getQuery();
+        $query->setSize(0);
         $path = $this->getPath();
 
         $response = $this->getClient()->request(
             $path,
             Request::GET,
             $query->toArray(),
-            [self::OPTION_SEARCH_TYPE => self::OPTION_SEARCH_TYPE_COUNT]
+            [self::OPTION_SEARCH_TYPE => self::OPTION_SEARCH_TYPE_QUERY_THEN_FETCH]
         );
         $resultSet = $this->_builder->buildResultSet($response, $query);
 

lib/Elastica/Tool/CrossIndex.php

@@ -4,7 +4,7 @@
 use Elastica\Bulk;
 use Elastica\Index;
 use Elastica\Query\MatchAll;
-use Elastica\ScanAndScroll;
+use Elastica\Scroll;
 use Elastica\Search;
 use Elastica\Type;
 
@@ -83,10 +83,9 @@ public static function reindex(
         $search->setQuery($options[self::OPTION_QUERY]);
 
         // search on old index and bulk insert in new index
-        $scanAndScroll = new ScanAndScroll(
+        $scanAndScroll = new Scroll(
             $search,
-            $options[self::OPTION_EXPIRY_TIME],
-            $options[self::OPTION_SIZE_PER_SHARD]
+            $options[self::OPTION_EXPIRY_TIME]
         );
         foreach ($scanAndScroll as $resultSet) {
             $bulk = new Bulk($newIndex->getClient());

lib/Elastica/Type.php

@@ -479,15 +479,9 @@ public function deleteIds(array $ids, $routing = false)
      */
     public function deleteByQuery($query, array $options = [])
     {
-        if (is_string($query)) {
-            // query_string queries are not supported for delete by query operations
-            $options['q'] = $query;
-
-            return $this->request('_query', Request::DELETE, [], $options);
-        }
         $query = Query::create($query);
 
-        return $this->request('_query', Request::DELETE, ['query' => is_array($query) ? $query : $query->toArray()], $options);
+        return $this->request('_delete_by_query', Request::POST, is_array($query) ? $query : $query->toArray(), $options);
     }
 
     /**

test/lib/Elastica/Test/Aggregation/CardinalityTest.php

@@ -4,13 +4,21 @@
 use Elastica\Aggregation\Cardinality;
 use Elastica\Document;
 use Elastica\Query;
+use Elastica\Type\Mapping;
 
 class CardinalityTest extends BaseAggregationTest
 {
     protected function _getIndexForTest()
     {
         $index = $this->_createIndex();
 
+        $mapping = new Mapping($index->getType('test'), [
+            'color' => [
+                'type' => 'keyword',
+            ],
+        ]);
+        $index->getType('test')->setMapping($mapping);
+
         $index->getType('test')->addDocuments([
             new Document(1, ['color' => 'blue']),
             new Document(2, ['color' => 'blue']),