Fix out of memory

ansible/provision.sh

@@ -39,17 +39,47 @@ fi
 
 # ----------------------------------------------------------------------------
 
-echo 'Waiting for elasticsearch server ready'
-elasticsearch_ready() {
-    http_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:9200")
-    return $(test $http_code = "200")
+all_nodes_available() {
+    curl -m 5 -s -o /dev/null "http://localhost:9200" &&
+    curl -m 5 -s -o /dev/null "http://localhost:9201"
+    return $?
 }
-while ! elasticsearch_ready; do
-    echo -n '.'
-    sleep 1s
-done
 
-# ----------------------------------------------------------------------------
-# Say bye
+check_cluster() {
+    restarts_left=$1
+    seconds_left=$2
+
+    if [ $seconds_left -eq 0 ]; then
+        if [ $restarts_left -eq 0 ]; then
+            echo "Cluster was restarted 10 times, but still not ready for phpunit. Build failed!"
+            return 1
+        else
+            echo "Restart cluster. Restarts left: $restarts_left"
+            sudo service elasticsearch restart
+            check_cluster $(( $restarts_left - 1 )) 30
+            return $?
+        fi
+    else
+        if all_nodes_available; then
+            echo "All nodes available. Sleep 30 seconds and try to check them again..."
+            if sleep 10s && all_nodes_available; then
+                echo "All nodes still available - cluster ready."
+                return 0
+            else
+                echo "Some nodes were stopped during sleep. Trying cluster restart..."
+                check_cluster $restarts_left 0
+                return $?
+            fi
+        else
+            echo "Some nodes still unavailable. Sleep 1 second and try to check them again. Seconds to start left: $seconds_left"
+            sleep 1s && check_cluster $restarts_left $(( $seconds_left - 1 ))
+            return $?
+        fi
+    fi
+}
 
-echo 'Done'
+echo "Wait cluster start..."
+if ! check_cluster 10 30; then
+    cat /var/log/elasticsearch/*.log
+    exit 1
+fi

ansible/roles/elasticsearch/tasks/main.yml

@@ -80,7 +80,6 @@
   with_items:
     - config-0.yml
     - config-1.yml
-    - config-2.yml
     - logging.yml
   notify: restart elasticsearch
 

ansible/roles/elasticsearch/templates/config-default.yml

@@ -1,30 +1,33 @@
 {% block default_config %}
 
-#################################### Index ####################################
-
 index.number_of_shards: 2
 index.number_of_replicas: 0
 
-#################################### Plugin ###################################
+# Dont write data to hdd in tests
+index.store.type: memory
 
+# Required plugins
 plugin.mandatory: mapper-attachments, geocluster-facet, transport-thrift, transport-memcached
 
-################################## UDP ########################################
-
+# For bulk tests
 bulk.udp.enabled: true
 bulk.udp.bulk_actions: 5
 
-################################## Scripting ##################################
-
+# For script tests
 script.disable_dynamic: false
 
-################################### Memory ####################################
-
+# Disable dynamic memory allocation
 bootstrap.mlockall: true
 
+# Dont accept connections not from localhost
+network.host: "127.0.0.1"
+
+# Limit threadpool by set number of available processors to 1
+# Without this, travis builds will be failed with OutOfMemory error
+processors: 1
+
 {% endblock %}
 
-################################### Node specific #############################
 {% block config %}
-
+# Node specific config should be overwritten in child template
 {% endblock %}

ansible/roles/elasticsearch/templates/elasticsearch.service

@@ -76,7 +76,7 @@ export ES_JAVA_OPTS
 MAX_OPEN_FILES="65535"
 
 # Maximum amount of locked memory
-MAX_LOCKED_MEMORY="$ES_HEAP_SIZE"
+MAX_LOCKED_MEMORY="unlimited"
 
 # Elasticsearch log directory
 LOG_DIR="/var/log/$NAME"
@@ -90,9 +90,6 @@ WORK_DIR="/tmp/$NAME"
 # Elasticsearch configuration directory
 CONF_DIR="/etc/$NAME"
 
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-MAX_MAP_COUNT="262144"
-
 # Define other required variables
 DAEMON="$ES_HOME/bin/elasticsearch"
 
@@ -135,11 +132,9 @@ case "$1" in
         ulimit -l $MAX_LOCKED_MEMORY
     fi
 
-    if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ];then
-        sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
-    fi
+    ulimit -s 1024
 
-    for node in 0 1 2; do
+    for node in 0 1; do
         log_daemon_msg "Starting elasticsearch node #$node"
 
         PID_FILE="/var/run/$NAME-$node.pid"
@@ -184,7 +179,7 @@ case "$1" in
     done
     ;;      
   stop)
-    for node in 0 1 2; do
+    for node in 0 1; do
         log_daemon_msg "Stopping elasticsearch node #$node"
 
         PID_FILE="/var/run/$NAME-$node.pid"
@@ -209,14 +204,14 @@ case "$1" in
     log_end_msg 0
     ;;
   status)
-    for node in 0 1 2; do
+    for node in 0 1; do
         PID_FILE="/var/run/$NAME-$node.pid"
         status_of_proc -p $PID_FILE "Elasticsearch node #$node" "Elasticsearch node #$node"
     done
     exit 0
     ;;
   restart|force-reload)
-    for node in 0 1 2; do
+    for node in 0 1; do
         PID_FILE="/var/run/$NAME-$node.pid"
         if [ -f "$PID_FILE" ]; then
             $0 stop

changes.txt

@@ -1,5 +1,8 @@
 CHANGES
 
+2015-02-11
+- Fixed issue with OutOfMemory exception in travis builds
+
 2015-01-27
 - Exception\ElasticsearchException now can be catched like all other exceptions as Exception\ExceptionInterface
 

test/lib/Elastica/Test/Aggregation/AvgTest.php

@@ -11,7 +11,7 @@ class AvgTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('avg');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/BaseAggregationTest.php

@@ -11,17 +11,4 @@ abstract class BaseAggregationTest extends Base
      * @var Index
      */
     protected $_index;
-
-    protected function tearDown()
-    {
-        parent::tearDown();
-        if ($this->_index instanceof Index) {
-            $this->_index->delete();
-        }
-    }
-
-    protected function _createIndex($name = 'test', $delete = true, $shards = 1)
-    {
-        return parent::_createIndex('test_aggregation_'.$name, $delete, $shards);
-    }
 }

test/lib/Elastica/Test/Aggregation/CardinalityTest.php

@@ -11,7 +11,7 @@ class CardinalityTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("cardinality");
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document("1", array("color" => "blue")),
             new Document("2", array("color" => "blue")),

test/lib/Elastica/Test/Aggregation/DateHistogramTest.php

@@ -12,7 +12,7 @@ class DateHistogramTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("date_histogram");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "created" => array("type" => "date"),

test/lib/Elastica/Test/Aggregation/DateRangeTest.php

@@ -12,7 +12,7 @@ class DateRangeTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("date_range");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "created" => array("type" => "date"),

test/lib/Elastica/Test/Aggregation/ExtendedStatsTest.php

@@ -11,7 +11,7 @@ class ExtendedStatsTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("extended_stats");
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document("1", array("price" => 5)),
             new Document("2", array("price" => 8)),

test/lib/Elastica/Test/Aggregation/FilterTest.php

@@ -14,7 +14,7 @@ class FilterTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("filter");
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document("1", array("price" => 5, "color" => "blue")),
             new Document("2", array("price" => 8, "color" => "blue")),

test/lib/Elastica/Test/Aggregation/GeoDistanceTest.php

@@ -12,7 +12,7 @@ class GeoDistanceTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("geo_distance");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "location" => array("type" => "geo_point"),

test/lib/Elastica/Test/Aggregation/GeohashGridTest.php

@@ -12,7 +12,7 @@ class GeohashGridTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("geohash_grid");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "location" => array("type" => "geo_point"),

test/lib/Elastica/Test/Aggregation/HistogramTest.php

@@ -11,7 +11,7 @@ class HistogramTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("histogram");
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document("1", array("price" => 5, "color" => "blue")),
             new Document("2", array("price" => 8, "color" => "blue")),

test/lib/Elastica/Test/Aggregation/IpRangeTest.php

@@ -12,7 +12,7 @@ class IpRangeTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("ip_range");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "address" => array("type" => "ip"),

test/lib/Elastica/Test/Aggregation/MaxTest.php

@@ -12,7 +12,7 @@ class MaxTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('max');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/MinTest.php

@@ -11,7 +11,7 @@ class MinTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('min');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/MissingTest.php

@@ -11,7 +11,7 @@ class MissingTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('missing');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5, "color" => "blue")),
             new Document('2', array('price' => 8, "color" => "blue")),

test/lib/Elastica/Test/Aggregation/NestedTest.php

@@ -13,7 +13,7 @@ class NestedTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("nested");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "resellers" => array(

test/lib/Elastica/Test/Aggregation/RangeTest.php

@@ -11,7 +11,7 @@ class RangeTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('range');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/ReverseNestedTest.php

@@ -14,7 +14,7 @@ class ReverseNestedTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("nested");
+        $this->_index = $this->_createIndex();
         $mapping = new Mapping();
         $mapping->setProperties(array(
             "comments" => array(

test/lib/Elastica/Test/Aggregation/ScriptTest.php

@@ -12,7 +12,7 @@ class ScriptTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('script');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/StatsTest.php

@@ -11,7 +11,7 @@ class StatsTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('stats');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/SumTest.php

@@ -11,7 +11,7 @@ class SumTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('sum');
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document('1', array('price' => 5)),
             new Document('2', array('price' => 8)),

test/lib/Elastica/Test/Aggregation/TermsTest.php

@@ -11,7 +11,7 @@ class TermsTest extends BaseAggregationTest
     protected function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex("terms");
+        $this->_index = $this->_createIndex();
         $docs = array(
             new Document("1", array("color" => "blue")),
             new Document("2", array("color" => "blue")),

test/lib/Elastica/Test/Aggregation/TopHitsTest.php

@@ -18,7 +18,7 @@ class TopHitsTest extends BaseAggregationTest
     public function setUp()
     {
         parent::setUp();
-        $this->_index = $this->_createIndex('top_hits_test', true);
+        $this->_index = $this->_createIndex();
 
         $docs = array(
             new Document(1, array(