Add OR/AND/NOT to the condition associated with the processors (#1983)

CHANGELOG.asciidoc

@@ -51,6 +51,7 @@ https://github.com/elastic/beats/compare/v5.0.0-alpha4...master[Check the HEAD d
 - Periodically log internal metrics. {pull}1955[1955]
 - Add enable-setting to all output modules. {pull}1987[1987]
 - Command line flag -c can be used multiple times. {pull}1985[1985]
+- Add OR/AND/NOT to the condition associated with the processors. {pull}1983[1983]
 
 *Metricbeat*
 

filebeat/tests/system/test_processors.py

@@ -79,3 +79,30 @@ def test_drop_event(self):
         assert "beat.name" in output
         assert "message" in output
         assert "test" in output["message"]
+
+    def test_condition(self):
+        """
+        Check condition in processors
+        """
+        self.render_config_template(
+            path=os.path.abspath(self.working_dir) + "/test*.log",
+            drop_event={
+                "condition": "not.contains.source: test",
+            },
+        )
+        with open(self.working_dir + "/test1.log", "w") as f:
+            f.write("test1 message\n")
+
+        with open(self.working_dir + "/test2.log", "w") as f:
+            f.write("test2 message\n")
+
+        filebeat = self.start_beat()
+        self.wait_until(lambda: self.output_has(lines=2))
+        filebeat.check_kill_and_wait()
+
+        output = self.read_output(
+            required_fields=["@timestamp", "type"],
+        )[0]
+        assert "beat.name" in output
+        assert "message" in output
+        assert "test" in output["message"]

libbeat/docs/processors-config.asciidoc

@@ -15,36 +15,23 @@
 
 //TODO: Remove was Filters from the above title and remove extra sections that show the alpha4 configuration
 
-coming[5.0.0-beta1,Filters are being renamed to "processors" to better reflect the capabilities they provide. If you are using 5.0.0-alpha4 or earlier, these are still called "filters" even though the documentation refers to "processors"]
+include::../../libbeat/docs/processors.asciidoc[]
 
-You can define a set of `processors` in the +{beatname_lc}.yml+ config file to reduce the number
-of fields that are exported by the Beat. 
-
-If multiple processors are defined, they are executed in the order they are defined. The initial event is passed to the
-first processor and what results from it is passed to the second processor until all processors are applied. The
-condition is checked against the event that is received as input and it might differ from the original event.
-
-[source,yaml]
--------
-event -> processor 1 -> event1 -> processor 2 -> event2 ...
--------
-
-See <<exported-fields>> for the full list of possible fields.
-
-Each processor receives a condition and optionally a set of arguments. The action is executed only if the condition
+Each processor has associated an action with a set of parameters and optionally a condition. If the condition is
+present, then the action is executed only if the condition
 is fulfilled. If no condition is passed then the action is always executed.
 
-deprecated[5.0.0-alpha4,The `filters` section is being renamed to `processors` in 5.0.0-beta1. Therefore the following configuration is deprecated]
+deprecated[5.0.0-alpha4,The `filters` configuration option is being renamed to `processors` in 5.0.0-beta1. Therefore the following configuration is deprecated]
 
 [source,yaml]
 ------
 filters:
- - action1:
-     condition1
-         [arguments]
- - action2:
-     condition2
-         [arguments]
+ - <action>:
+     <condition>
+     <parameters>
+ - <action>:
+     <condition>
+     <parameters>
 ...
 ------
 
@@ -53,18 +40,21 @@ coming[5.0.0-beta1,Begin using the following configuration starting with 5.0.0-b
 [source,yaml]
 ------
 processors:
- - action1:
+ - <action>:
      when:
-        condition1
-           [arguments]
- - action2:
+        <condition>
+     <parameters>
+ - <action>:
      when:
-        condition2
-           [arguments]
+        <condition>
+     <parameters>
 ...
 
 ------
 
+where <action> can be a way to select the fields that are exported or a way to add meta data to the event , <condition> contains the definition of the condition.
+and <parameters> is the list of parameters passed along the <action>. 
+
 See <<filtering-and-enhancing-data>> for specific {beatname_uc} examples.
 
 [[filtering-condition]]
@@ -75,20 +65,17 @@ them. You can see a list of the <<exported-fields,`exported fields`>>.
 
 For each field, you can specify a simple field name or a nested map, for example `dns.question.name`.
 
-[source,yaml]
-----
-condition:
-  field1: value1
-  [field2: value2]
-  ...
-----
 
-Supported conditions are:
+A condition can be:
 
 * <<condition-equals,`equals`>>
 * <<condition-contains,`contains`>>
 * <<condition-regexp,`regexp`>>
 * <<condition-range, `range`>>
+* <<condition-or, `or`>>
+* <<condition-and, `and`>>
+* <<condition-not, `not`>>
+
 
 
 [[condition-equals]]
@@ -126,7 +113,7 @@ The `regexp` condition checks the field against a regular expression. The condit
 
 For example, the following condition checks if the process name contains `foo`:
 
-[source,yaml]]
+[source,yaml]
 -----
 reqexp:
   proc.name: "foo.*"
@@ -162,6 +149,97 @@ range:
     cpu.user_p: 0.8
 ------
 
+
+coming[5.0.0-beta1, You can combine multiple conditions with the `or`, `and` or `not` operators]
+
+[[condition-or]]
+===== OR
+
+The `or` operator receives a list of conditions. 
+
+[source,yaml]
+-------
+or:
+  - <condition1>
+  - <condition2>
+  - <condition3>
+  ...
+
+-------
+
+For example the condition `http.code = 304 OR http.code = 404` translates to: 
+
+[source,yaml]
+------
+or:
+  - equals:
+      http.code: 304
+  - equals:
+      http.code: 404
+------
+
+
+[[condition-and]]
+===== AND
+
+The `and` operator receives a list of conditions. 
+
+[source,yaml]
+-------
+and:
+  - <condition1>
+  - <condition2>
+  - <condition3>
+  ...
+
+-------
+
+For example the condition `http.code = 200 AND status = OK` translates to: 
+
+[source,yaml]
+------
+and:
+  - equals:
+      http.code: 200
+  - equals:
+      status: OK
+------
+
+To configure a condition like `<condition1> OR <condition2> AND <condition3>`:
+
+[source,yaml]
+------
+or:
+ - <condition1>
+ - and:
+    - <condition2>
+    - <condition3>
+
+------
+
+[[condition-not]]
+===== NOT
+
+The `not` operator receives the condition to negate.
+
+[source,yaml]
+-------
+not:
+  <condition>
+
+-------
+
+For example the condition `NOT status = OK` translates to: 
+
+[source,yaml]
+------
+not:
+  equals:
+    status: OK
+------
+
+
+
 ==== Actions
 
 The supported filter actions are:
@@ -170,6 +248,7 @@ The supported filter actions are:
  * <<drop-fields,`drop_fields`>>
  * <<drop-event,`drop_event`>>
 
+See <<exported-fields>> for the full list of possible fields.
 
 [[include-fields]]
 ===== include_fields

libbeat/docs/processors.asciidoc

@@ -17,4 +17,9 @@ enhancing events with additional metadata. Each processor receives an event, app
 and returns the event. If you define a list of processors, they are executed in the order they are defined in the
 configuration file.
 
+[source,yaml]
+-------
+event -> processor 1 -> event1 -> processor 2 -> event2 ...
+-------
+
 The processors are defined in the {beatname_uc} configuration file.

libbeat/processors/condition.go

@@ -28,6 +28,9 @@ type Condition struct {
 	contains map[string]string
 	regexp   map[string]*regexp.Regexp
 	rangexp  map[string]RangeValue
+	or       []Condition
+	and      []Condition
+	not      *Condition
 }
 
 func NewCondition(config *ConditionConfig) (*Condition, error) {
@@ -55,10 +58,33 @@ func NewCondition(config *ConditionConfig) (*Condition, error) {
 		if err := c.setRange(config.Range); err != nil {
 			return nil, err
 		}
+	} else if len(config.OR) > 0 {
+		for _, cond_config := range config.OR {
+			cond, err := NewCondition(&cond_config)
+			if err != nil {
+				return nil, err
+			}
+			c.or = append(c.or, *cond)
+		}
+	} else if len(config.AND) > 0 {
+		for _, cond_config := range config.AND {
+			cond, err := NewCondition(&cond_config)
+			if err != nil {
+				return nil, err
+			}
+			c.and = append(c.and, *cond)
+		}
+	} else if config.NOT != nil {
+		cond, err := NewCondition(config.NOT)
+		if err != nil {
+			return nil, err
+		}
+		c.not = cond
 	} else {
 		return nil, fmt.Errorf("missing condition")
 	}
 
+	logp.Debug("processors", "New condition %s", c)
 	return &c, nil
 }
 
@@ -166,6 +192,18 @@ func (c *Condition) setRange(cfg *ConditionFields) error {
 
 func (c *Condition) Check(event common.MapStr) bool {
 
+	if len(c.or) > 0 {
+		return c.checkOR(event)
+	}
+
+	if len(c.and) > 0 {
+		return c.checkAND(event)
+	}
+
+	if c.not != nil {
+		return c.checkNOT(event)
+	}
+
 	if !c.checkEquals(event) {
 		return false
 	}
@@ -323,6 +361,34 @@ func (c *Condition) checkRange(event common.MapStr) bool {
 	return true
 }
 
+func (c *Condition) checkOR(event common.MapStr) bool {
+
+	for _, cond := range c.or {
+		if cond.Check(event) {
+			return true
+		}
+	}
+	return false
+}
+
+func (c *Condition) checkAND(event common.MapStr) bool {
+
+	for _, cond := range c.and {
+		if !cond.Check(event) {
+			return false
+		}
+	}
+	return true
+}
+
+func (c *Condition) checkNOT(event common.MapStr) bool {
+
+	if c.not.Check(event) {
+		return false
+	}
+	return true
+}
+
 func (c Condition) String() string {
 
 	s := ""
@@ -339,6 +405,22 @@ func (c Condition) String() string {
 	if len(c.rangexp) > 0 {
 		s = s + fmt.Sprintf("range: %v", c.rangexp)
 	}
+	if len(c.or) > 0 {
+		for _, cond := range c.or {
+			s = s + cond.String() + " or "
+		}
+		s = s[:len(s)-len(" or ")] //delete the last or
+	}
+	if len(c.and) > 0 {
+		for _, cond := range c.and {
+			s = s + cond.String() + " and "
+		}
+		s = s[:len(s)-len(" and ")] //delete the last and
+	}
+	if c.not != nil {
+		s = s + "not " + c.not.String()
+	}
+
 	return s
 }