Increase the fields limit (#3275)

Elasticsearch has a default limit of 1000 fields starting with 5.x.
Metricbeat already defines a bit over that in it's current mapping. In most
deployments, these fields won't be actually used all at the same time, so
the effects of increasing the limit shouldn't be that large (can still affect
the cluster state processing).

Longer term we'll need a better solution for this.

CHANGELOG.asciidoc

@@ -51,6 +51,7 @@ https://github.com/elastic/beats/compare/v5.1.1...master[Check the HEAD diff]
 - Add support for passing list and dictionary settings via -E flag.
 - Support for parsing list and dictionary setting from environment variables.
 - Added new flags to import_dashboards (-cacert, -cert, -key, -insecure). {pull}3139[3139] {pull}3163[3163]
+- The limit for the number of fields is increased via the mapping template. {pull}3275[3275]
 
 *Metricbeat*
 - Kafka module broker matching enhancements. {pull}3129[3129]

filebeat/filebeat.template.json

@@ -320,6 +320,7 @@
   },
   "order": 0,
   "settings": {
+    "index.mapping.total_fields.limit": 10000,
     "index.refresh_interval": "5s"
   },
   "template": "filebeat-*"

heartbeat/heartbeat.template.json

@@ -184,6 +184,7 @@
   },
   "order": 0,
   "settings": {
+    "index.mapping.total_fields.limit": 10000,
     "index.refresh_interval": "5s"
   },
   "template": "heartbeat-*"

libbeat/scripts/generate_template.py

@@ -46,7 +46,7 @@ def fields_to_es_template(args, input, output, index, version):
         "template": index,
         "order": 0,
         "settings": {
-            "index.refresh_interval": "5s"
+            "index.refresh_interval": "5s",
         },
         "mappings": {
             "_default_": {
@@ -66,6 +66,11 @@ def fields_to_es_template(args, input, output, index, version):
         template["mappings"]["_default_"]["_all"]["norms"] = {
             "enabled": False
         }
+    else:
+        # For ES 5.x, increase the limit on the max number of fields.
+        # In a typical scenario, most fields are not used, so increasing the
+        # limit shouldn't be that bad.
+        template["settings"]["index.mapping.total_fields.limit"] = 10000
 
     properties = {}
     dynamic_templates = []

metricbeat/metricbeat.template.json

@@ -3696,6 +3696,7 @@
   },
   "order": 0,
   "settings": {
+    "index.mapping.total_fields.limit": 10000,
     "index.refresh_interval": "5s"
   },
   "template": "metricbeat-*"

packetbeat/packetbeat.template.json

@@ -1359,6 +1359,7 @@
   },
   "order": 0,
   "settings": {
+    "index.mapping.total_fields.limit": 10000,
     "index.refresh_interval": "5s"
   },
   "template": "packetbeat-*"

winlogbeat/winlogbeat.template.json

@@ -171,6 +171,7 @@
   },
   "order": 0,
   "settings": {
+    "index.mapping.total_fields.limit": 10000,
     "index.refresh_interval": "5s"
   },
   "template": "winlogbeat-*"