feat: Improve directive validation (#336)

* add ANY handler for custom data * move directive validation to its own file * add directive validation file * add directive validation test * add ANY check on custom data checking * add test for any and reference for custom data * add lock for allbacklogs * fix inconsistent test result for backlog with custom data * remove mutex call from backlog counting * remove unnecessary bp control * add more test cases for custom data check * update test directive * replace found check with uint32 atomic * add test for event with empty custom data Co-authored-by: RikiSyahputra <[email protected]>
defenxor · Jun 27, 2021 · a12dca1 · a12dca1
1 parent a51a6c0
commit a12dca1
Show file tree

Hide file tree

Showing 9 changed files with 997 additions and 158 deletions.
diff --git a/internal/pkg/dsiem/rule/rule.go b/internal/pkg/dsiem/rule/rule.go
@@ -180,13 +180,13 @@ func taxonomyRuleCheck(e event.NormalizedEvent, r DirectiveRule, s *StickyDiffDa
 func customDataCheck(e event.NormalizedEvent, r DirectiveRule, s *StickyDiffData, connID uint64) (ret bool) {
 
 	var r1, r2, r3 = true, true, true
-	if r.CustomData1 != "" {
+	if r.CustomData1 != "" && r.CustomData1 != "ANY" {
 		r1 = isStrMatchCSVRule(r.CustomData1, e.CustomData1, false)
 	}
-	if r.CustomData2 != "" {
+	if r.CustomData2 != "" && r.CustomData2 != "ANY" {
 		r2 = isStrMatchCSVRule(r.CustomData2, e.CustomData2, false)
 	}
-	if r.CustomData3 != "" {
+	if r.CustomData3 != "" && r.CustomData3 != "ANY" {
 		r3 = isStrMatchCSVRule(r.CustomData3, e.CustomData3, false)
 	}
 	switch {

diff --git a/internal/pkg/dsiem/rule/rule_test.go b/internal/pkg/dsiem/rule/rule_test.go
@@ -346,6 +346,52 @@ func TestRule(t *testing.T) {
 	e1.CustomData3 = "custom"
 	rs12.StickyDiff = "CUSTOM_DATA3"
 
+	rAny1 := r1
+	rAny1.CustomData1 = "ANY"
+
+	rAny2 := rAny1
+	rAny2.CustomData1 = ""
+
+	rAny3 := rAny1
+	rAny3.CustomData1 = "quas"
+
+	rAny4 := rAny1
+	rAny4.CustomData1 = "foo"
+
+	rAny5 := r1
+	rAny5.CustomData2 = "ANY"
+
+	rAny6 := rAny5
+	rAny6.CustomData2 = ""
+
+	rAny7 := rAny5
+	rAny7.CustomData2 = "quas"
+
+	rAny8 := rAny5
+	rAny8.CustomData2 = "bar"
+
+	rAny9 := r1
+	rAny9.CustomData3 = "ANY"
+
+	rAny10 := rAny9
+	rAny10.CustomData3 = ""
+
+	rAny11 := rAny9
+	rAny11.CustomData3 = "quas"
+
+	rAny12 := rAny9
+	rAny12.CustomData3 = "qux"
+
+	eAny1 := e1
+	eAny1.CustomData1 = "foo"
+	eAny1.CustomData2 = "bar"
+	eAny1.CustomData3 = "qux"
+
+	eAny2 := eAny1
+	eAny2.CustomData1 = ""
+	eAny2.CustomData2 = ""
+	eAny2.CustomData3 = ""
+
 	var tbl = []ruleTests{
 		{1, e1, r1, s1, true}, {2, e1, r2, s1, true}, {3, e1, r3, s1, false}, {4, e1, r4, s1, false},
 		{5, e1, r5, s1, false}, {6, e1, r6, s1, false}, {7, e1, r7, s1, true}, {8, e1, r8, s1, false},
@@ -368,12 +414,38 @@ func TestRule(t *testing.T) {
 		{106, e1, rs6, nil, true}, {107, e1, rs7, nil, true},
 		{108, e1, rs8, s3, true}, {109, e1, rs9, s3, true},
 		{110, e1, rs10, s4, true}, {111, e1, rs11, s4, true}, {112, e1, rs12, s4, false},
+
+		{113, eAny1, rAny1, nil, true},
+		{114, eAny1, rAny2, nil, true},
+		{115, eAny1, rAny3, nil, false},
+		{116, eAny1, rAny4, nil, true},
+		{117, eAny1, rAny5, nil, true},
+		{118, eAny1, rAny6, nil, true},
+		{119, eAny1, rAny7, nil, false},
+		{120, eAny1, rAny8, nil, true},
+		{121, eAny1, rAny9, nil, true},
+		{122, eAny1, rAny10, nil, true},
+		{123, eAny1, rAny11, nil, false},
+		{124, eAny1, rAny12, nil, true},
+
+		{125, eAny2, rAny1, nil, true},
+		{126, eAny2, rAny2, nil, true},
+		{127, eAny2, rAny3, nil, false},
+		{128, eAny2, rAny4, nil, false},
+		{129, eAny2, rAny5, nil, true},
+		{130, eAny2, rAny6, nil, true},
+		{131, eAny2, rAny7, nil, false},
+		{132, eAny2, rAny8, nil, false},
+		{133, eAny2, rAny9, nil, true},
+		{134, eAny2, rAny10, nil, true},
+		{135, eAny2, rAny11, nil, false},
+		{136, eAny2, rAny12, nil, false},
 	}
 
 	for _, tt := range tbl {
 		actual := DoesEventMatch(tt.e, tt.r, tt.s, 0)
 		if actual != tt.expected {
-			t.Fatalf("Rule %d actual %t != expected %t. Event: %v, Rule: %v, Sticky: %v",
+			t.Fatalf("Rule %d actual %t != expected %t. Event: %#v, Rule: %#v, Sticky: %#v",
 				tt.n, actual, tt.expected, tt.e, tt.r, tt.s)
 		}
 	}

diff --git a/internal/pkg/dsiem/siem/backlogmgr.go b/internal/pkg/dsiem/siem/backlogmgr.go
@@ -18,6 +18,7 @@ package siem
 
 import (
 	"strconv"
+	"sync/atomic"
 
 	"github.com/defenxor/dsiem/internal/pkg/dsiem/alarm"
 	"github.com/defenxor/dsiem/internal/pkg/dsiem/event"
@@ -42,6 +43,9 @@ type backlogs struct {
 }
 
 var (
+	// protects allBacklogs
+	allBacklogsMu sync.RWMutex
+
 	allBacklogs []backlogs
 	fWriter     fs.FileWriter
 )
@@ -107,6 +111,9 @@ func initBpTicker(bpChan chan<- bool, holdDuration int) {
 }
 
 func merge() <-chan bool {
+	allBacklogsMu.RLock()
+	defer allBacklogsMu.RUnlock()
+
 	out := make(chan bool)
 	for _, v := range allBacklogs {
 		go func(ch chan bool) {
@@ -121,6 +128,7 @@ func merge() <-chan bool {
 
 // CountBackLogs returns the number of active backlogs
 func CountBackLogs() (sum int, activeDirectives int, ttlDirectives int) {
+
 	ttlDirectives = len(allBacklogs)
 	for i := range allBacklogs {
 		l := allBacklogs[i].RLock()
@@ -184,7 +192,9 @@ mainLoop:
 			}
 		}
 
-		found := false
+		// found := false
+		// zero means false
+		var found uint32
 		l := blogs.RLock() // to prevent concurrent r/w with delete()
 
 		wg := &sync.WaitGroup{}
@@ -212,7 +222,8 @@ mainLoop:
 					// wait for the result
 					case f := <-blogs.bl[k].chFound:
 						if f {
-							found = true
+							// found = true
+							atomic.AddUint32(&found, 1)
 						}
 					}
 				}
@@ -222,7 +233,7 @@ mainLoop:
 		wg.Wait()
 		l.Unlock()
 
-		if found {
+		if found > 0 {
 			if apm.Enabled() && tx != nil {
 				tx.Result("Event consumed by backlog")
 				tx.End()
@@ -417,17 +428,35 @@ func initBackLogRules(d *Directive, e event.NormalizedEvent) {
 		// add reference for custom datas.
 		r = d.Rules[i].CustomData1
 		if v, ok := str.RefToDigit(r); ok {
-			d.Rules[i].CustomData1 = d.Rules[v-1].CustomData1
+			vmin1 := v - 1
+			ref := d.Rules[vmin1].CustomData1
+			if ref != "ANY" {
+				d.Rules[i].CustomData1 = ref
+			} else {
+				d.Rules[i].CustomData1 = e.CustomData1
+			}
 		}
 
 		r = d.Rules[i].CustomData2
 		if v, ok := str.RefToDigit(r); ok {
-			d.Rules[i].CustomData2 = d.Rules[v-1].CustomData2
+			vmin1 := v - 1
+			ref := d.Rules[vmin1].CustomData2
+			if ref != "ANY" {
+				d.Rules[i].CustomData2 = ref
+			} else {
+				d.Rules[i].CustomData2 = e.CustomData2
+			}
 		}
 
 		r = d.Rules[i].CustomData3
 		if v, ok := str.RefToDigit(r); ok {
-			d.Rules[i].CustomData3 = d.Rules[v-1].CustomData3
+			vmin1 := v - 1
+			ref := d.Rules[vmin1].CustomData3
+			if ref != "ANY" {
+				d.Rules[i].CustomData3 = ref
+			} else {
+				d.Rules[i].CustomData3 = e.CustomData3
+			}
 		}
 	}
 }