Allow dots in field names.

graylog2-server/src/main/java/org/graylog2/inputs/codecs/GelfCodec.java

@@ -45,7 +45,10 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import javax.inject.Inject;
+
+import java.util.ArrayList;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 
 @Codec(name = "gelf", displayName = "GELF")
@@ -205,7 +208,15 @@ public Message decode(@Nonnull final RawMessage rawMessage) {
             final JsonNode value = entry.getValue();
 
             final Object fieldValue;
-            if (value.isContainerNode()) {
+            if (value.isArray()) {
+                List<String> fieldArray = new ArrayList<>();
+
+                for (final JsonNode objNode : value) {
+                    fieldArray.add(objNode.asText());
+                }
+                fieldValue = fieldArray;
+            }
+            else if (value.isContainerNode()) {
                 fieldValue = value.toString();
             } else if (value.isFloatingPointNumber()) {
                 fieldValue = value.asDouble();

graylog2-server/src/main/java/org/graylog2/inputs/extractors/JsonExtractor.java

@@ -85,7 +85,7 @@ public JsonExtractor(final MetricRegistry metricRegistry,
 
         this.flatten = firstNonNull((Boolean) extractorConfig.get(CK_FLATTEN), false);
         this.listSeparator = firstNonNull((String) extractorConfig.get(CK_LIST_SEPARATOR), ", ");
-        this.keySeparator = firstNonNull((String) extractorConfig.get(CK_KEY_SEPARATOR), "_");
+        this.keySeparator = firstNonNull((String) extractorConfig.get(CK_KEY_SEPARATOR), ".");
         this.kvSeparator = firstNonNull((String) extractorConfig.get(CK_KV_SEPARATOR), "=");
         this.replaceKeyWhitespace = firstNonNull((Boolean) extractorConfig.get(CK_REPLACE_KEY_WHITESPACE), false);
         this.keyWhitespaceReplacement = firstNonNull((String) extractorConfig.get(CK_KEY_WHITESPACE_REPLACEMENT), "_");

graylog2-server/src/main/java/org/graylog2/plugin/Message.java

@@ -247,32 +247,7 @@ public Map<String, Object> toElasticSearchObject(@Nonnull final Meter invalidTim
             }
 
             final Object value = entry.getValue();
-            // Elasticsearch does not allow "." characters in keys since version 2.0.
-            // See: https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
-            if (key.contains(".")) {
-                final String newKey = key.replace('.', KEY_REPLACEMENT_CHAR);
-
-                // If the message already contains the transformed key, we skip the field and emit a warning.
-                // This is still not optimal but better than implementing expensive logic with multiple replacement
-                // character options. Conflicts should be rare...
-                if (!obj.containsKey(newKey)) {
-                    obj.put(newKey, value);
-                } else {
-                    LOG.warn("Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"",
-                        key, value, getId(), KEY_REPLACEMENT_CHAR, newKey, obj.get(newKey));
-                    LOG.debug("Full message with \".\" in message key: {}", this);
-                }
-            } else {
-                if (obj.containsKey(key)) {
-                    final String newKey = key.replace(KEY_REPLACEMENT_CHAR, '.');
-                    // Deliberate warning duplicates because the key with the "." might be transformed before reaching
-                    // the duplicate original key with a "_". Otherwise we would silently overwrite the transformed key.
-                    LOG.warn("Keys must not contain a \".\" character! Ignoring field \"{}\"=\"{}\" in message [{}] - Unable to replace \".\" with a \"{}\" because of key conflict: \"{}\"=\"{}\"",
-                        newKey, fields.get(newKey), getId(), KEY_REPLACEMENT_CHAR, key, value);
-                    LOG.debug("Full message with \".\" in message key: {}", this);
-                }
-                obj.put(key, value);
-            }
+            obj.put(key, value);
         }
 
         obj.put(FIELD_MESSAGE, getMessage());

graylog2-server/src/test/java/org/graylog2/inputs/codecs/GelfCodecTest.java

@@ -34,6 +34,7 @@
 import java.nio.charset.StandardCharsets;
 import java.time.DateTimeException;
 import java.util.Collections;
+import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
@@ -408,4 +409,20 @@ public void decodeSucceedsWithValidTimestampAsStringIssue4027() throws Exception
         assertThat(message).isNotNull();
         assertThat(message.getTimestamp()).isEqualTo(DateTime.parse("2017-07-21T14:23:00.661Z"));
     }
+
+    @Test
+    public void decodeSucceedsWithCustomFieldAsArray() throws Exception {
+        final String json = "{"
+                + "\"version\": \"1.1\","
+                + "\"short_message\": \"A short message that helps you identify what is going on\","
+                + "\"host\": \"example.org\","
+                + "\"timestamp\": \"1500646980.661\","
+                + "\"tags\": [\"tag1\",\"tag2\"]"
+                + "}";
+        final RawMessage rawMessage = new RawMessage(json.getBytes(StandardCharsets.UTF_8));
+
+        final Message message = codec.decode(rawMessage);
+        assertThat("tag2").isEqualTo((message.getFieldAs(List.class, "tags")).get(1));
+
+    }
 }

graylog2-server/src/test/java/org/graylog2/inputs/extractors/JsonExtractorTest.java

@@ -94,10 +94,10 @@ public void testRunWithObject() throws Exception {
         final Extractor.Result[] results = jsonExtractor.run(value);
 
         assertThat(results).contains(
-                new Extractor.Result("foobar", "object_text", -1, -1),
-                new Extractor.Result(1234.5678, "object_number", -1, -1),
-                new Extractor.Result(true, "object_bool", -1, -1),
-                new Extractor.Result("foobar", "object_nested_text", -1, -1)
+                new Extractor.Result("foobar", "object.text", -1, -1),
+                new Extractor.Result(1234.5678, "object.number", -1, -1),
+                new Extractor.Result(true, "object.bool", -1, -1),
+                new Extractor.Result("foobar", "object.nested.text", -1, -1)
         );
     }
 
@@ -158,7 +158,7 @@ public void issue2375() throws Exception {
 
         assertThat(results).contains(
             new Extractor.Result("Myserver#DS", "Source", -1, -1),
-            new Extractor.Result("Purge Vector", "Target_Data_Attribute Name", -1, -1)
+            new Extractor.Result("Purge Vector", "Target.Data.Attribute Name", -1, -1)
         );
     }
 
@@ -213,7 +213,7 @@ public void testRunWithWhitespaceInNestedKey() throws Exception {
                 "");
 
         assertThat(jsonExtractor.run(value)).containsOnly(
-                new Extractor.Result(42, "foo_b-a-r_b-a-z", -1, -1)
+                new Extractor.Result(42, "foo.b-a-r.b-a-z", -1, -1)
         );
     }
 

graylog2-server/src/test/java/org/graylog2/plugin/MessageTest.java

@@ -409,9 +409,9 @@ public void testToElasticSearchObjectWithInvalidKey() throws Exception {
 
         final Map<String, Object> object = message.toElasticSearchObject(invalidTimestampMeter);
 
-        // Elasticsearch >=2.0 does not allow "." in keys. Make sure we replace them before writing the message.
-        assertEquals("#toElasticsearchObject() should replace \".\" in keys with a \"_\"",
-                "dot", object.get("field_3"));
+        // Elasticsearch >=2.0 and < 5.0 does not allow "." in keys. Make sure we replace them before writing the message.
+        assertEquals("#toElasticsearchObject() should not replace \".\" in keys with a \"_\"",
+                "dot", object.get("field.3"));
 
         assertEquals("foo", object.get("message"));
         assertEquals("bar", object.get("source"));