diff --git a/internal/pkg/dsiem/rule/rule.go b/internal/pkg/dsiem/rule/rule.go index 7c4c151e..4c6507d5 100644 --- a/internal/pkg/dsiem/rule/rule.go +++ b/internal/pkg/dsiem/rule/rule.go @@ -180,13 +180,13 @@ func taxonomyRuleCheck(e event.NormalizedEvent, r DirectiveRule, s *StickyDiffDa func customDataCheck(e event.NormalizedEvent, r DirectiveRule, s *StickyDiffData, connID uint64) (ret bool) { var r1, r2, r3 = true, true, true - if r.CustomData1 != "" { + if r.CustomData1 != "" && r.CustomData1 != "ANY" { r1 = isStrMatchCSVRule(r.CustomData1, e.CustomData1, false) } - if r.CustomData2 != "" { + if r.CustomData2 != "" && r.CustomData2 != "ANY" { r2 = isStrMatchCSVRule(r.CustomData2, e.CustomData2, false) } - if r.CustomData3 != "" { + if r.CustomData3 != "" && r.CustomData3 != "ANY" { r3 = isStrMatchCSVRule(r.CustomData3, e.CustomData3, false) } switch { diff --git a/internal/pkg/dsiem/rule/rule_test.go b/internal/pkg/dsiem/rule/rule_test.go index 653187f3..419db740 100644 --- a/internal/pkg/dsiem/rule/rule_test.go +++ b/internal/pkg/dsiem/rule/rule_test.go @@ -346,6 +346,52 @@ func TestRule(t *testing.T) { e1.CustomData3 = "custom" rs12.StickyDiff = "CUSTOM_DATA3" + rAny1 := r1 + rAny1.CustomData1 = "ANY" + + rAny2 := rAny1 + rAny2.CustomData1 = "" + + rAny3 := rAny1 + rAny3.CustomData1 = "quas" + + rAny4 := rAny1 + rAny4.CustomData1 = "foo" + + rAny5 := r1 + rAny5.CustomData2 = "ANY" + + rAny6 := rAny5 + rAny6.CustomData2 = "" + + rAny7 := rAny5 + rAny7.CustomData2 = "quas" + + rAny8 := rAny5 + rAny8.CustomData2 = "bar" + + rAny9 := r1 + rAny9.CustomData3 = "ANY" + + rAny10 := rAny9 + rAny10.CustomData3 = "" + + rAny11 := rAny9 + rAny11.CustomData3 = "quas" + + rAny12 := rAny9 + rAny12.CustomData3 = "qux" + + eAny1 := e1 + eAny1.CustomData1 = "foo" + eAny1.CustomData2 = "bar" + eAny1.CustomData3 = "qux" + + eAny2 := eAny1 + eAny2.CustomData1 = "" + eAny2.CustomData2 = "" + eAny2.CustomData3 = "" + var tbl = []ruleTests{ {1, e1, r1, s1, true}, {2, e1, r2, s1, true}, {3, e1, r3, s1, false}, {4, e1, r4, s1, false}, {5, e1, r5, s1, false}, {6, e1, r6, s1, false}, {7, e1, r7, s1, true}, {8, e1, r8, s1, false}, @@ -368,12 +414,38 @@ func TestRule(t *testing.T) { {106, e1, rs6, nil, true}, {107, e1, rs7, nil, true}, {108, e1, rs8, s3, true}, {109, e1, rs9, s3, true}, {110, e1, rs10, s4, true}, {111, e1, rs11, s4, true}, {112, e1, rs12, s4, false}, + + {113, eAny1, rAny1, nil, true}, + {114, eAny1, rAny2, nil, true}, + {115, eAny1, rAny3, nil, false}, + {116, eAny1, rAny4, nil, true}, + {117, eAny1, rAny5, nil, true}, + {118, eAny1, rAny6, nil, true}, + {119, eAny1, rAny7, nil, false}, + {120, eAny1, rAny8, nil, true}, + {121, eAny1, rAny9, nil, true}, + {122, eAny1, rAny10, nil, true}, + {123, eAny1, rAny11, nil, false}, + {124, eAny1, rAny12, nil, true}, + + {125, eAny2, rAny1, nil, true}, + {126, eAny2, rAny2, nil, true}, + {127, eAny2, rAny3, nil, false}, + {128, eAny2, rAny4, nil, false}, + {129, eAny2, rAny5, nil, true}, + {130, eAny2, rAny6, nil, true}, + {131, eAny2, rAny7, nil, false}, + {132, eAny2, rAny8, nil, false}, + {133, eAny2, rAny9, nil, true}, + {134, eAny2, rAny10, nil, true}, + {135, eAny2, rAny11, nil, false}, + {136, eAny2, rAny12, nil, false}, } for _, tt := range tbl { actual := DoesEventMatch(tt.e, tt.r, tt.s, 0) if actual != tt.expected { - t.Fatalf("Rule %d actual %t != expected %t. Event: %v, Rule: %v, Sticky: %v", + t.Fatalf("Rule %d actual %t != expected %t. Event: %#v, Rule: %#v, Sticky: %#v", tt.n, actual, tt.expected, tt.e, tt.r, tt.s) } } diff --git a/internal/pkg/dsiem/siem/backlogmgr.go b/internal/pkg/dsiem/siem/backlogmgr.go index ba9281b3..cb18533f 100644 --- a/internal/pkg/dsiem/siem/backlogmgr.go +++ b/internal/pkg/dsiem/siem/backlogmgr.go @@ -18,6 +18,7 @@ package siem import ( "strconv" + "sync/atomic" "github.com/defenxor/dsiem/internal/pkg/dsiem/alarm" "github.com/defenxor/dsiem/internal/pkg/dsiem/event" @@ -42,6 +43,9 @@ type backlogs struct { } var ( + // protects allBacklogs + allBacklogsMu sync.RWMutex + allBacklogs []backlogs fWriter fs.FileWriter ) @@ -107,6 +111,9 @@ func initBpTicker(bpChan chan<- bool, holdDuration int) { } func merge() <-chan bool { + allBacklogsMu.RLock() + defer allBacklogsMu.RUnlock() + out := make(chan bool) for _, v := range allBacklogs { go func(ch chan bool) { @@ -121,6 +128,7 @@ func merge() <-chan bool { // CountBackLogs returns the number of active backlogs func CountBackLogs() (sum int, activeDirectives int, ttlDirectives int) { + ttlDirectives = len(allBacklogs) for i := range allBacklogs { l := allBacklogs[i].RLock() @@ -184,7 +192,9 @@ mainLoop: } } - found := false + // found := false + // zero means false + var found uint32 l := blogs.RLock() // to prevent concurrent r/w with delete() wg := &sync.WaitGroup{} @@ -212,7 +222,8 @@ mainLoop: // wait for the result case f := <-blogs.bl[k].chFound: if f { - found = true + // found = true + atomic.AddUint32(&found, 1) } } } @@ -222,7 +233,7 @@ mainLoop: wg.Wait() l.Unlock() - if found { + if found > 0 { if apm.Enabled() && tx != nil { tx.Result("Event consumed by backlog") tx.End() @@ -417,17 +428,35 @@ func initBackLogRules(d *Directive, e event.NormalizedEvent) { // add reference for custom datas. r = d.Rules[i].CustomData1 if v, ok := str.RefToDigit(r); ok { - d.Rules[i].CustomData1 = d.Rules[v-1].CustomData1 + vmin1 := v - 1 + ref := d.Rules[vmin1].CustomData1 + if ref != "ANY" { + d.Rules[i].CustomData1 = ref + } else { + d.Rules[i].CustomData1 = e.CustomData1 + } } r = d.Rules[i].CustomData2 if v, ok := str.RefToDigit(r); ok { - d.Rules[i].CustomData2 = d.Rules[v-1].CustomData2 + vmin1 := v - 1 + ref := d.Rules[vmin1].CustomData2 + if ref != "ANY" { + d.Rules[i].CustomData2 = ref + } else { + d.Rules[i].CustomData2 = e.CustomData2 + } } r = d.Rules[i].CustomData3 if v, ok := str.RefToDigit(r); ok { - d.Rules[i].CustomData3 = d.Rules[v-1].CustomData3 + vmin1 := v - 1 + ref := d.Rules[vmin1].CustomData3 + if ref != "ANY" { + d.Rules[i].CustomData3 = ref + } else { + d.Rules[i].CustomData3 = e.CustomData3 + } } } } diff --git a/internal/pkg/dsiem/siem/backlogmgr_test.go b/internal/pkg/dsiem/siem/backlogmgr_test.go index 3de2b879..735746ea 100644 --- a/internal/pkg/dsiem/siem/backlogmgr_test.go +++ b/internal/pkg/dsiem/siem/backlogmgr_test.go @@ -49,7 +49,9 @@ func TestBacklogMgr(t *testing.T) { fmt.Println("Starting TestBackLogMgr.") + allBacklogsMu.Lock() allBacklogs = []backlogs{} + allBacklogsMu.Unlock() setTestDir(t) t.Logf("Using base dir %s", testDir) @@ -295,3 +297,282 @@ func verifyFuncOutput(t *testing.T, f func(), expected string, expectMatch bool) fmt.Println("OK") } } + +func TestBacklogManagerCustomData(t *testing.T) { + waitTime := 100 * time.Millisecond + fmt.Println("Starting TestBackLogMgr.") + allBacklogsMu.Lock() + allBacklogs = make([]backlogs, 0) + allBacklogsMu.Unlock() + setTestDir(t) + + t.Logf("Using base dir %s", testDir) + if !log.TestMode { + t.Logf("Enabling log test mode") + log.EnableTestingMode() + } + + fDir := path.Join(testDir, "internal", "pkg", "dsiem", "siem", "fixtures") + apm.Enable(true) + + tmpLog := path.Join(os.TempDir(), "siem_alarm_events.log") + fWriter.Init(tmpLog, 10) + + cleanUp := func() { + _ = os.Remove(tmpLog) + } + + defer cleanUp() + initAlarm(t) + initAsset(t) + + dirs, _, err := LoadDirectivesFromFile(path.Join(fDir, "directive5"), directiveFileGlob, false) + if err != nil { + t.Fatal(err) + } + + if len(dirs.Dirs) != 1 { + t.Fatalf("expected only 1 directive to be loaded, but got %d", len(dirs.Dirs)) + } + + blogs := &backlogs{ + DRWMutex: drwmutex.New(), + id: 1, + bpCh: make(chan bool), + bl: make(map[string]*backLog), + } + + if err = InitBackLogManager(tmpLog, nil, 4); err != nil { + t.Fatal(err) + } + + testDirective := dirs.Dirs[0] + testEvent := event.NormalizedEvent{ + EventID: "1", + ConnID: 1, + Sensor: "test-sensor", + SrcIP: "1.1.1.1", + DstIP: "2.2.2.2", + Title: "Test Event", + Protocol: "TEST", + PluginID: 1337, + PluginSID: 1, + CustomLabel1: "fsoo", + CustomData1: "bar", + Timestamp: time.Now().Add(time.Second * -300).UTC().Format(time.RFC3339), + } + + input := make(chan event.NormalizedEvent) + go blogs.manager(testDirective, input, 0) + + // first event + input <- testEvent + time.Sleep(waitTime) + + var testBl *backLog + blogs.Lock() + if len(blogs.bl) != 1 { + t.Fatalf("expected 1 backlog, but got %d", len(blogs.bl)) + blogs.Unlock() + } + + for _, v := range blogs.bl { + testBl = v + break + } + blogs.Unlock() + + testBl.Lock() + if testBl.CurrentStage != 2 { + t.Errorf("expected current stage to be 2 but got %d", testBl.CurrentStage) + } + testBl.Unlock() + // 2nd event + testEvent.ConnID = 2 + testEvent.EventID = "2" + testEvent.CustomLabel2 = "foo2" + testEvent.CustomData2 = "bar2" + + input <- testEvent + time.Sleep(waitTime) + + testBl.Lock() + if testBl.CurrentStage != 2 { + t.Errorf("expected current stage to be 2 but got %d", testBl.CurrentStage) + } + testBl.Unlock() + + // 3rd event + testEvent.ConnID = 3 + testEvent.EventID = "3" + + input <- testEvent + time.Sleep(waitTime) + + testBl.Lock() + if testBl.CurrentStage != 3 { + t.Errorf("expected current stage to be 3 but got %d", testBl.CurrentStage) + } + testBl.Unlock() + + // 4th event + testEvent.ConnID = 4 + testEvent.EventID = "4" + + testEvent.CustomData3 = "bar3" + + input <- testEvent + time.Sleep(waitTime) + + testBl.Lock() + if testBl.CurrentStage != 3 { + t.Errorf("expected current stage to be 3 but got %d", testBl.CurrentStage) + } + testBl.Unlock() + + // 5th event -> different custom data, creates a new backlog + testEvent2 := testEvent + testEvent2.ConnID = 5 + testEvent2.EventID = "5" + testEvent2.CustomLabel1 = "aaa" + testEvent2.CustomData1 = "bbb" + + input <- testEvent2 + time.Sleep(waitTime) + + // expected 2 backlogs now + var testBl2 *backLog + blogs.Lock() + if len(blogs.bl) != 2 { + blogs.Unlock() + t.Fatalf("expected 2 backlog, but got %d", len(blogs.bl)) + + } + + for _, v := range blogs.bl { + if testBl == v { + continue + } + + testBl2 = v + break + } + blogs.Unlock() + + testBl2.Lock() + if testBl2.CurrentStage != 2 { + t.Errorf("expected current stage to be 2 but got %d", testBl2.CurrentStage) + } + testBl2.Unlock() + + // 6th event + testEvent2.ConnID = 6 + testEvent2.EventID = "6" + + input <- testEvent2 + time.Sleep(waitTime) + + testBl2.Lock() + if testBl2.CurrentStage != 2 { + t.Errorf("expected current stage to be 2 but got %d", testBl2.CurrentStage) + } + testBl2.Unlock() + + // 7th event -> stage increased for second backlog + testEvent2.ConnID = 7 + testEvent2.EventID = "7" + + input <- testEvent2 + time.Sleep(waitTime) + + testBl2.Lock() + if testBl2.CurrentStage != 3 { + t.Errorf("expected current stage to be 3 but got %d", testBl2.CurrentStage) + } + testBl2.Unlock() + + // 8th event -> this event has no custom data, therefore new backlog should created + testEvent3 := event.NormalizedEvent{ + EventID: "8", + ConnID: 8, + Sensor: "test-sensor", + SrcIP: "1.1.1.1", + DstIP: "2.2.2.2", + Title: "Test Event", + Protocol: "TEST", + PluginID: 1337, + PluginSID: 1, + CustomLabel1: "", + CustomData1: "", + Timestamp: time.Now().Add(time.Second * -300).UTC().Format(time.RFC3339), + } + + input <- testEvent3 + time.Sleep(waitTime) + + var testBl3 *backLog + blogs.Lock() + if len(blogs.bl) != 3 { + blogs.Unlock() + t.Fatalf("expected 3 backlog, but got %d", len(blogs.bl)) + } + + for _, blog := range blogs.bl { + if testBl == blog || testBl2 == blog { + continue + } + + testBl3 = blog + break + } + blogs.Unlock() + + if testBl3 == nil { + t.Fatal("expected third backlog to exist") + } + + testBl3.Lock() + if testBl3.CurrentStage != 2 { + t.Errorf("expected current stage of third backlog to be 2 but got %d", testBl3.CurrentStage) + } + + if testBl3.LastEvent.EventID != testEvent3.EventID { + t.Errorf("expected last event id for third backlog to be %s but got %s", testEvent3.EventID, testBl3.LastEvent.EventID) + } + testBl3.Unlock() + + testEvent3.EventID = "9" + testEvent3.ConnID = 9 + + input <- testEvent3 + time.Sleep(waitTime) + + testBl3.Lock() + if testBl3.CurrentStage != 2 { + t.Errorf("expected current stage of third backlog to be 2 but got %d", testBl3.CurrentStage) + } + + if testBl3.LastEvent.EventID != testEvent3.EventID { + t.Errorf("expected last event id for third backlog to be %s but got %s", testEvent3.EventID, testBl3.LastEvent.EventID) + } + testBl3.Unlock() + + // 10th event, identical to 5th event, should increase second backlog stage instead of the third one + testEvent5 := testEvent2 + testEvent5.EventID = "10" + testEvent5.ConnID = 10 + + input <- testEvent5 + time.Sleep(waitTime) + + testBl2.Lock() + if testBl2.CurrentStage != 3 { + t.Errorf("expected current stage of third backlog to be 3 but got %d", testBl2.CurrentStage) + } + + if testBl2.LastEvent.EventID != testEvent5.EventID { + t.Errorf("expected last event id for third backlog to be %s but got %s", testEvent5.EventID, testBl2.LastEvent.EventID) + } + testBl2.Unlock() + +} diff --git a/internal/pkg/dsiem/siem/directive.go b/internal/pkg/dsiem/siem/directive.go index cd058502..fcbfa9a8 100644 --- a/internal/pkg/dsiem/siem/directive.go +++ b/internal/pkg/dsiem/siem/directive.go @@ -20,7 +20,6 @@ import ( "encoding/json" "errors" "io/ioutil" - "net" "os" "path" "path/filepath" @@ -35,7 +34,6 @@ import ( "github.com/defenxor/dsiem/internal/pkg/shared/apm" log "github.com/defenxor/dsiem/internal/pkg/shared/logger" - "github.com/defenxor/dsiem/internal/pkg/shared/str" "github.com/jonhoo/drwmutex" ) @@ -170,7 +168,7 @@ func LoadDirectivesFromFile(confDir string, namePattern string, includeDisabled strconv.Itoa(d.Dirs[j].ID)}) continue } - err = validateDirective(&d.Dirs[j], &res) + err = ValidateDirective(&d.Dirs[j], &res) if err != nil { log.Warn(log.M{Msg: "Skipping directive ID " + strconv.Itoa(d.Dirs[j].ID) + @@ -186,151 +184,6 @@ func LoadDirectivesFromFile(confDir string, namePattern string, includeDisabled return } -func validateDirective(d *Directive, res *Directives) (err error) { - for _, v := range res.Dirs { - if v.ID == d.ID { - return errors.New(strconv.Itoa(d.ID) + " is already used as an ID by other directive") - } - } - if d.Name == "" || d.Kingdom == "" || d.Category == "" { - return errors.New("Name, Kingdom, and Category cannot be empty") - } - if d.Priority < 1 || d.Priority > 5 { - // return errors.New("Priority must be between 1 - 5") - log.Warn(log.M{Msg: "Directive " + strconv.Itoa(d.ID) + - " has wrong priority set (" + strconv.Itoa(d.Priority) + "), configuring it to 1"}) - d.Priority = 1 - } - if len(d.Rules) <= 1 { - return errors.New(strconv.Itoa(d.ID) + " has no rule therefore has no effect, or only 1 rule and therefore will never expire") - } - - stages := []int{} - for j, v := range d.Rules { - if v.Stage == 0 { - return errors.New("rule stage should start from 1, cannot use 0") - } - for i := range stages { - if stages[i] == v.Stage { - return errors.New("duplicate rule stage " + strconv.Itoa(v.Stage) + " found.") - } - } - if v.Stage == 1 { - if v.Occurrence != 1 { - // return errors.New("Stage 1 rule occurrence is configured to " + strconv.Itoa(v.Occurrence) + ". It must be set to 1") - log.Warn(log.M{Msg: "Directive " + strconv.Itoa(d.ID) + " rule " + strconv.Itoa(v.Stage) + - " has wrong occurrence set (" + strconv.Itoa(v.Occurrence) + "), configuring it to 1"}) - d.Rules[j].Occurrence = 1 - } - } - if v.Type != "PluginRule" && v.Type != "TaxonomyRule" { - return errors.New("Rule Type must be PluginRule or TaxonomyRule") - } - if v.Type == "PluginRule" { - if v.PluginID < 1 { - return errors.New("PluginRule requires PluginID to be 1 or higher") - } - if len(v.PluginSID) == 0 { - return errors.New("PluginRule requires PluginSID to be defined") - } - for i := range v.PluginSID { - if v.PluginSID[i] < 1 { - return errors.New("PluginRule requires PluginSID to be 1 or higher") - } - } - } - if v.Type == "TaxonomyRule" { - if len(v.Product) == 0 { - return errors.New("TaxonomyRule requires Product to be defined") - } - if v.Category == "" { - return errors.New("TaxonomyRule requires Category to be defined") - } - } - // reliability maybe 0 for the first rule! - if v.Reliability < 0 { - log.Warn(log.M{Msg: "Directive " + strconv.Itoa(d.ID) + " rule " + strconv.Itoa(v.Stage) + - " has wrong reliability set (" + strconv.Itoa(v.Reliability) + "), configuring it to 0"}) - d.Rules[j].Reliability = 0 - } - if v.Reliability > 10 { - log.Warn(log.M{Msg: "Directive " + strconv.Itoa(d.ID) + " rule " + strconv.Itoa(v.Stage) + - " has wrong reliability set (" + strconv.Itoa(v.Reliability) + "), configuring it to 10"}) - d.Rules[j].Reliability = 10 - } - isFirstStage := v.Stage == 1 - if err := validateFromTo(v.From, isFirstStage); err != nil { - return err - } - if err := validateFromTo(v.To, isFirstStage); err != nil { - return err - } - if err := validatePort(v.PortFrom); err != nil { - return err - } - if err := validatePort(v.PortTo); err != nil { - return err - } - stages = append(stages, v.Stage) - } - return nil -} - -func validatePort(s string) error { - if s == "ANY" { - return nil - } - if _, ok := str.RefToDigit(s); ok { - return nil - } - sSlice := str.CsvToSlice(s) - for _, v := range sSlice { - isInverse := strings.HasPrefix(v, "!") - if isInverse { - v = str.TrimLeftChar(v) - } - n, err := strconv.Atoi(v) - if err != nil { - return err - } - if n <= 1 || n >= 65535 { - return errors.New(v + " is not a valid TCP/IP port number") - } - } - return nil -} - -func validateFromTo(s string, isFirstRule bool) (err error) { - - if s == "" { - return errors.New("From/To cannot be empty") - } - - if s == "ANY" || s == "HOME_NET" || s == "!HOME_NET" { - return nil - } - if !isFirstRule { - if _, ok := str.RefToDigit(s); ok { - return nil - } - } - // covers r.To == "IP", r.To == "IP1, IP2, !IP3", r.To == CIDR-netaddr, r.To == "CIDR1, CIDR2, !CIDR3" - sSlice := str.CsvToSlice(s) - for i, v := range sSlice { - if !strings.Contains(v, "/") { - v = v + "/32" - } - isInverse := strings.HasPrefix(v, "!") - if isInverse { - v = str.TrimLeftChar(v) - } - if _, _, err := net.ParseCIDR(v); err != nil { - return errors.New(sSlice[i] + " is not a valid IPv4 address or CIDR") - } - } - return nil -} - func copyDirective(dst *Directive, src Directive, e event.NormalizedEvent) { dst.ID = src.ID dst.Priority = src.Priority diff --git a/internal/pkg/dsiem/siem/directive_test.go b/internal/pkg/dsiem/siem/directive_test.go index 10e8de93..a07837b0 100644 --- a/internal/pkg/dsiem/siem/directive_test.go +++ b/internal/pkg/dsiem/siem/directive_test.go @@ -29,7 +29,9 @@ import ( func TestInitDirective(t *testing.T) { + allBacklogsMu.Lock() allBacklogs = []backlogs{} + allBacklogsMu.Unlock() fmt.Println("Starting TestInitDirective.") diff --git a/internal/pkg/dsiem/siem/fixtures/directive5/directives_dsiem-backend-0_testing1.json b/internal/pkg/dsiem/siem/fixtures/directive5/directives_dsiem-backend-0_testing1.json new file mode 100644 index 00000000..c8406b21 --- /dev/null +++ b/internal/pkg/dsiem/siem/fixtures/directive5/directives_dsiem-backend-0_testing1.json @@ -0,0 +1,101 @@ +{ + "directives": [ + { + "id": 1, + "name": "Valid directive, testing custom data with ANY and reference", + "category": "foo", + "kingdom": "Environmental Awareness", + "priority": 3, + "all_rules_always_active": false, + "disabled": false, + "rules": [ + { + "name": "test-custom-data", + "type": "PluginRule", + "stage": 1, + "plugin_id": 1337, + "plugin_sid": [1], + "occurrence": 1, + "from": "ANY", + "to": "ANY", + "port_from": "ANY", + "port_to": "ANY", + "protocol": "ANY", + "reliability": 1, + "timeout": 0, + "custom_label1": "ANY", + "custom_data1": "ANY" + }, + { + "stage": 2, + "occurrence": 2, + "reliability": 1, + "port_from": "ANY", + "port_to": "ANY", + "timeout": 3600, + "name": "test-custom-data", + "type": "PluginRule", + "from": "ANY", + "to": ":1", + "protocol": "ANY", + "plugin_id": 1337, + "custom_label1": ":1", + "custom_data1": ":1", + "custom_label2": "ANY", + "custom_data2": "ANY", + "custom_label3": "ANY", + "custom_data3": "ANY", + "plugin_sid": [ + 1 + ] + }, + { + "stage": 3, + "occurrence": 3, + "reliability": 10, + "port_from": "ANY", + "port_to": "ANY", + "timeout": 21600, + "name": "test-custom-data", + "type": "PluginRule", + "from": "ANY", + "to": ":1", + "protocol": "ANY", + "plugin_id": 1337, + "custom_data1": ":1", + "custom_label1": ":1", + "custom_label2": ":2", + "custom_data2": ":2", + "custom_label3": ":2", + "custom_data3": ":2", + "plugin_sid": [ + 1 + ] + }, + { + "stage": 4, + "occurrence": 100, + "reliability": 10, + "port_from": "ANY", + "port_to": "ANY", + "timeout": 21600, + "name": "test-custom-data", + "type": "PluginRule", + "from": "ANY", + "to": ":1", + "protocol": "ANY", + "plugin_id": 1337, + "custom_data1": ":1", + "custom_label1": ":1", + "custom_label2": ":2", + "custom_data2": ":2", + "custom_label3": ":2", + "custom_data3": ":2", + "plugin_sid": [ + 1 + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/internal/pkg/dsiem/siem/validate.go b/internal/pkg/dsiem/siem/validate.go new file mode 100644 index 00000000..676cbc35 --- /dev/null +++ b/internal/pkg/dsiem/siem/validate.go @@ -0,0 +1,246 @@ +package siem + +import ( + "errors" + "fmt" + "net" + "regexp" + "strconv" + "strings" + + log "github.com/defenxor/dsiem/internal/pkg/shared/logger" + "github.com/defenxor/dsiem/internal/pkg/shared/str" +) + +var ( + validRuleType = []string{ + "PluginRule", + "TaxonomyRule", + } +) + +var ( + refRegexp = regexp.MustCompile("^:[1-9][0-9]?$") + ErrZeroStage = errors.New("can not use 0 as rule stage") + ErrInvalidRuleType = fmt.Errorf("invalid rule type, valid types: %v", validRuleType) + ErrInvalidPluginID = errors.New("PluginRule requires PluginID to be 1 or higher") + ErrNoPluginSID = errors.New("PluginRule requires PluginSID to be defined") + ErrInvalidPluginSID = errors.New("PluginRule requires PluginSID to be 1 or higher") + ErrNoProduct = errors.New("TaxonomyRule requires Product to be defined") + ErrNoCategory = errors.New("TaxonomyRule requires Category to be defined") + ErrNoDirectiveName = errors.New("Directive name cannot be empty") + ErrNoDirectiveKingdom = errors.New("Directive kingdom cannot be empty") + ErrNoDirectiveCategory = errors.New("Directive category cannot be empty") + ErrReferenceOnFirstRule = errors.New("first rule cannot contain reference") + ErrInvalidReference = errors.New("invalid reference number, must be larger than 0 and less than the rule count") + ErrEmptyFromTo = errors.New("rule From/To cannot be empty") +) + +func ValidateDirective(d *Directive, res *Directives) (err error) { + for _, v := range res.Dirs { + if v.ID == d.ID { + return fmt.Errorf("id '%d' is already used as an ID by other directive", d.ID) + } + } + + if d.Name == "" { + return ErrNoDirectiveName + } + + if d.Kingdom == "" { + return ErrNoDirectiveKingdom + } + + if d.Category == "" { + return ErrNoDirectiveCategory + } + + if d.Priority < 1 || d.Priority > 5 { + log.Warn(log.M{Msg: fmt.Sprintf("directive %d has wrong priority set (%d), configuring it to 1", d.ID, d.Priority)}) + d.Priority = 1 + } + + if len(d.Rules) <= 1 { + return fmt.Errorf("directive %d has no rule therefore has no effect, or only 1 rule and therefore will never expire", d.ID) + } + + if err := ValidateRules(d); err != nil { + return fmt.Errorf("directive %d contains invalid rule, %s", d.ID, err.Error()) + } + + return nil +} + +func ValidateRules(d *Directive) error { + stages := make([]int, 0) + for idx := range d.Rules { + if d.Rules[idx].Stage == 0 { + return ErrZeroStage + } + + for i := range stages { + if stages[i] == d.Rules[idx].Stage { + return fmt.Errorf("duplicate rule stage found (%d)", d.Rules[idx].Stage) + } + } + + if d.Rules[idx].Stage == 1 && d.Rules[idx].Occurrence != 1 { + log.Warn(log.M{Msg: fmt.Sprintf("Directive '%d' rule %d has wrong occurence set (%d), configuring it to 1", d.ID, d.Rules[idx].Stage, d.Rules[idx].Occurrence)}) + d.Rules[idx].Occurrence = 1 + } + + if !isValidRuleType(d.Rules[idx].Type) { + return ErrInvalidRuleType + } + + if d.Rules[idx].Type == "PluginRule" { + if d.Rules[idx].PluginID < 1 { + return ErrInvalidPluginID + } + + if len(d.Rules[idx].PluginSID) == 0 { + return ErrNoPluginSID + } + + for i := range d.Rules[idx].PluginSID { + if d.Rules[idx].PluginSID[i] < 1 { + return ErrInvalidPluginSID + } + } + } + + if d.Rules[idx].Type == "TaxonomyRule" { + if len(d.Rules[idx].Product) == 0 { + return ErrNoProduct + } + if d.Rules[idx].Category == "" { + return ErrNoCategory + } + } + + // reliability maybe 0 for the first rule! + if d.Rules[idx].Reliability < 0 { + log.Warn(log.M{Msg: fmt.Sprintf("Directive %d rule %d has wrong reliability set (%d), configuring it to 0", d.ID, d.Rules[idx].Stage, d.Rules[idx].Reliability)}) + d.Rules[idx].Reliability = 0 + } + + if d.Rules[idx].Reliability > 10 { + log.Warn(log.M{Msg: fmt.Sprintf("Directive %d rule %d has wrong reliability set (%d), configuring it to 10", d.ID, d.Rules[idx].Stage, d.Rules[idx].Reliability)}) + d.Rules[idx].Reliability = 10 + } + + isFirstStage := d.Rules[idx].Stage == 1 + ruleCount := len(d.Rules) + if err := validateFromTo(d.Rules[idx].From, isFirstStage, ruleCount); err != nil { + return err + } + + if err := validateFromTo(d.Rules[idx].To, isFirstStage, ruleCount); err != nil { + return err + } + + if err := validatePort(d.Rules[idx].PortFrom, isFirstStage, ruleCount); err != nil { + return err + } + + if err := validatePort(d.Rules[idx].PortTo, isFirstStage, ruleCount); err != nil { + return err + } + + stages = append(stages, d.Rules[idx].Stage) + } + + return nil +} + +func validatePort(s string, isFirstRule bool, ruleCount int) error { + if s == "ANY" { + return nil + } + + if isReference(s) { + if isFirstRule { + return ErrReferenceOnFirstRule + } + + return validateReference(s, int64(ruleCount)) + } + + sSlice := str.CsvToSlice(s) + for _, v := range sSlice { + isInverse := strings.HasPrefix(v, "!") + if isInverse { + v = str.TrimLeftChar(v) + } + n, err := strconv.Atoi(v) + if err != nil { + return err + } + if n <= 1 || n >= 65535 { + return fmt.Errorf("%s is not a valid TCP/IP port number", v) + } + } + return nil +} + +func validateFromTo(s string, isFirstRule bool, ruleCount int) (err error) { + if s == "" { + return ErrEmptyFromTo + } + + if s == "ANY" || s == "HOME_NET" || s == "!HOME_NET" { + return nil + } + + if isReference(s) { + if isFirstRule { + return ErrReferenceOnFirstRule + } + + return validateReference(s, int64(ruleCount)) + } + + // covers r.To == "IP", r.To == "IP1, IP2, !IP3", r.To == CIDR-netaddr, r.To == "CIDR1, CIDR2, !CIDR3" + sSlice := str.CsvToSlice(s) + for _, v := range sSlice { + if !strings.Contains(v, "/") { + v = v + "/32" + } + isInverse := strings.HasPrefix(v, "!") + if isInverse { + v = str.TrimLeftChar(v) + } + + if _, _, err := net.ParseCIDR(v); err != nil { + return fmt.Errorf("%s is not a valid IPv4 address", v) + } + } + + return nil +} + +func isValidRuleType(t string) bool { + for _, rt := range validRuleType { + if rt == t { + return true + } + } + + return false +} + +func isReference(str string) bool { + return strings.HasPrefix(str, ":") +} + +func validateReference(ref string, ruleCount int64) error { + if !refRegexp.MatchString(ref) { + return ErrInvalidReference + } + + if n, _ := str.RefToDigit(ref); n >= ruleCount { + return ErrInvalidReference + } + + return nil +} diff --git a/internal/pkg/dsiem/siem/validate_test.go b/internal/pkg/dsiem/siem/validate_test.go new file mode 100644 index 00000000..4ff9c24e --- /dev/null +++ b/internal/pkg/dsiem/siem/validate_test.go @@ -0,0 +1,255 @@ +package siem + +import ( + "strings" + "testing" + + "github.com/defenxor/dsiem/internal/pkg/dsiem/rule" + log "github.com/defenxor/dsiem/internal/pkg/shared/logger" +) + +var sampleDirective = &Directive{ + ID: 1337, + Name: "test", + Kingdom: "test", + Category: "test", +} + +func TestValidate(t *testing.T) { + if !log.TestMode { + t.Logf("Enabling log test mode") + log.EnableTestingMode() + } + + t.Run("reference on first rule", func(t *testing.T) { + sample := *sampleDirective + sample.Rules = []rule.DirectiveRule{ + { + Name: "test-rule-1", + Stage: 1, + From: ":1", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + { + Name: "test-rule-2", + Stage: 2, + From: ":1", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + } + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrReferenceOnFirstRule.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrReferenceOnFirstRule.Error(), err.Error()) + } + + sample = *sampleDirective + sample.Rules = []rule.DirectiveRule{ + { + Name: "test-rule-1", + Stage: 1, + From: "ANY", + PortFrom: ":1", + To: "ANY", + PortTo: "ANY", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + { + Name: "test-rule-2", + Stage: 2, + From: ":1", + PortFrom: ":1", + To: ":1", + PortTo: ":1", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + } + + err = ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrReferenceOnFirstRule.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrReferenceOnFirstRule.Error(), err.Error()) + } + }) + + t.Run("negative reference number", func(t *testing.T) { + sample := *sampleDirective + sample.Rules = []rule.DirectiveRule{ + { + Name: "test-rule-1", + Stage: 1, + From: "ANY", + To: "ANY", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + { + Name: "test-rule-2", + Stage: 2, + From: ":-1", + To: ":-1", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + } + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrInvalidReference.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrInvalidReference.Error(), err.Error()) + } + }) + + t.Run("non-number reference", func(t *testing.T) { + sample := *sampleDirective + sample.Rules = []rule.DirectiveRule{ + { + Name: "test-rule-1", + Stage: 1, + From: "ANY", + To: "ANY", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + { + Name: "test-rule-2", + Stage: 2, + From: ":foo", + To: ":bar", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + } + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrInvalidReference.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrInvalidReference.Error(), err.Error()) + } + }) + + t.Run("reference to non-exist rule", func(t *testing.T) { + sample := *sampleDirective + sample.Rules = []rule.DirectiveRule{ + { + Name: "test-rule-1", + Stage: 1, + From: "ANY", + To: "ANY", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + { + Name: "test-rule-2", + Stage: 2, + From: ":3", + To: ":4", + Type: "PluginRule", + PluginID: 1, + PluginSID: []int{1}, + }, + } + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrInvalidReference.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrInvalidReference.Error(), err.Error()) + } + }) + + t.Run("directive with no name", func(t *testing.T) { + sample := *sampleDirective + sample.Name = "" + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrNoDirectiveName.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrNoDirectiveName.Error(), err.Error()) + } + }) + + t.Run("directive with no kingdom", func(t *testing.T) { + sample := *sampleDirective + sample.Kingdom = "" + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrNoDirectiveKingdom.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrNoDirectiveKingdom.Error(), err.Error()) + } + }) + + t.Run("directive with no category", func(t *testing.T) { + sample := *sampleDirective + sample.Category = "" + + err := ValidateDirective(&sample, &Directives{Dirs: []Directive{}}) + if err == nil { + t.Error("expected error") + } + + if !strings.Contains(err.Error(), ErrNoDirectiveCategory.Error()) { + t.Errorf("expected error to contain '%s' but got '%s'", ErrNoDirectiveCategory.Error(), err.Error()) + } + }) +} + +func TestIsReference(t *testing.T) { + tc := []struct { + str string + expected bool + }{ + {":1", true}, + {":2", true}, + {":3", true}, + {":10", true}, + {":99", true}, + {":01", false}, + {":a", false}, + {":x", false}, + {":-1", false}, + {":999", false}, + } + + for _, c := range tc { + result := validateReference(c.str, 999) == nil + if result != c.expected { + t.Errorf("expected reference check of '%s' to be %t but got %t", c.str, c.expected, result) + } + } +}