add unit tests and python engine support

eql/engine.py

@@ -983,11 +983,16 @@ def _convert_sequence_term(self, subquery, position, size, lookups, next_pipe=No
         get_join_value = self._convert_key(subquery.join_values, scoped=True)
         last_position = size - 1
         fork = bool(subquery.params.kv.get('fork', Boolean(False)).value)
+        is_negated = bool(subquery.params.kv.get('is_negated', Boolean(False)).value)
 
         if position == 0:
             @self.event_callback(subquery.query.event_type)
             def start_sequence_callback(event):  # type: (Event) -> None
-                if check_event(event):
+                condition = (
+                    (check_event(event) and not is_negated) or
+                    (not check_event(event) and is_negated)
+                )
+                if condition:
                     join_value = get_join_value(event)
                     sequence = [event]
                     lookups[1][join_value] = sequence
@@ -997,7 +1002,12 @@ def start_sequence_callback(event):  # type: (Event) -> None
 
             @self.event_callback(subquery.query.event_type)
             def continue_sequence_callback(event):  # type: (Event) -> None
-                if len(lookups[position]) and check_event(event):
+                condition = (
+                    len(lookups[position]) and
+                    ((check_event(event) and not is_negated) or
+                     (not check_event(event) and is_negated))
+                )
+                if condition:
                     join_value = get_join_value(event)
                     if join_value in lookups[position]:
                         if fork:
@@ -1010,13 +1020,20 @@ def continue_sequence_callback(event):  # type: (Event) -> None
         else:
             @self.event_callback(subquery.query.event_type)
             def finish_sequence(event):  # type: (Event) -> None
-                if len(lookups[position]) and check_event(event):
+                condition = (
+                    len(lookups[position]) and
+                    ((check_event(event) and not is_negated) or
+                     (not check_event(event) and is_negated))
+                )
+                if condition:
                     join_value = get_join_value(event)
                     if join_value in lookups[position]:
                         if fork:
                             sequence = list(lookups[position].get(join_value))
                         else:
                             sequence = lookups[position].pop(join_value)
+                        if is_negated:
+                            pass
                         sequence.append(event)
                         next_pipe(sequence)
 

tests/test_python_engine.py

@@ -6,7 +6,7 @@
 from eql import *  # noqa: F403
 from eql.ast import *  # noqa: F403
 from eql.engine import Scope
-from eql.parser import ignore_missing_functions, allow_sample, elasticsearch_syntax
+from eql.parser import ignore_missing_functions, allow_sample, elasticsearch_syntax, allow_negation
 from eql.schema import EVENT_TYPE_GENERIC
 from eql.tests.base import TestEngine
 
@@ -526,3 +526,56 @@ def evaluate(expr, event):
         self.assertEqual(evaluate("`a.b`", {"a.b": 1}), 1)
         self.assertEqual(evaluate("a.`b.c`[0]", {"a": {"b.c": [1]}}), 1)
         self.assertEqual(evaluate("`!@#$%^&*().`", {"!@#$%^&*().": 1}), 1)
+
+    def test_missing_events(self):
+        """Test the missing event feature."""
+        config = {'flatten': True}
+        events = [Event.from_data(d) for d in [
+            {
+                "event_type": "process",
+                "process_name": "malicious.exe",
+                "unique_pid": "host1-1",
+                "@timestamp": "2023-10-23T00:00:00"
+            },
+            {
+                "event_type": "process",
+                "process_name": "missing.exe",
+                "unique_pid": "host1-1",
+                "@timestamp": "2023-10-23T00:00:00"
+            },
+            {
+                "event_type": "file",
+                "file_name": "suspicious.txt",
+                "unique_pid": "host1-1",
+                "@timestamp": "2023-10-23T00:01:00"
+            }
+        ]]
+
+        # Should return no results since the malicious2.exe event is not missing
+        query = '''
+        sequence by unique_pid with maxspan=1m
+            [ process where process_name == "malicious.exe" ]
+            ![ process where process_name == "missing.exe" ]
+            [ file where file_name == "suspicious.txt" ]
+        '''
+        with elasticsearch_syntax, allow_negation:
+            parsed_query = parse_query(query)
+
+        output = self.get_output(queries=[parsed_query], config=config, events=events)
+
+        self.assertEqual(len(output), 0, "Missing or extra results")
+
+        # Should return results since the second malicious.exe event is missing
+        query = '''
+        sequence by unique_pid with maxspan=1m
+            [ process where process_name == "malicious.exe" ]
+            ![ process where process_name == "malicious.exe" ]
+            [ file where file_name == "suspicious.txt" ]
+        '''
+        with elasticsearch_syntax, allow_negation:
+            parsed_query = parse_query(query)
+
+        output = self.get_output(queries=[parsed_query], config=config, events=events)
+
+        self.assertEqual(len(output), 3, "Missing or extra results")
+