From 51e05016bef47f8d3796b4fdbcf6ba19f9d37d21 Mon Sep 17 00:00:00 2001 From: Quentin JEROME Date: Thu, 7 Oct 2021 23:13:08 +0200 Subject: [PATCH] Fixed #86 + database path config in Manager configuration --- .github/coverage/coverage.txt | 227 ++++++++++++++++++++++++++++++++++ api/adminapi_test.go | 132 +++++++++----------- api/api_client_test.go | 2 +- api/forwarder_test.go | 104 ++++++++++++---- api/manager.go | 4 +- coverage.sh | 14 +++ logger/search.go | 2 +- tools/manager/whids-man.go | 2 +- 8 files changed, 381 insertions(+), 106 deletions(-) create mode 100644 .github/coverage/coverage.txt create mode 100755 coverage.sh diff --git a/.github/coverage/coverage.txt b/.github/coverage/coverage.txt new file mode 100644 index 0000000..92630bc --- /dev/null +++ b/.github/coverage/coverage.txt @@ -0,0 +1,227 @@ +github.com/0xrawsec/whids/api/api_client.go:40: ManagerIP 60.0% +github.com/0xrawsec/whids/api/api_client.go:52: DialContext 0.0% +github.com/0xrawsec/whids/api/api_client.go:69: DialTLSContext 82.4% +github.com/0xrawsec/whids/api/api_client.go:103: Transport 100.0% +github.com/0xrawsec/whids/api/api_client.go:140: init 60.0% +github.com/0xrawsec/whids/api/api_client.go:150: NewManagerClient 63.6% +github.com/0xrawsec/whids/api/api_client.go:184: Prepare 100.0% +github.com/0xrawsec/whids/api/api_client.go:199: PrepareGzip 90.0% +github.com/0xrawsec/whids/api/api_client.go:219: IsServerAuthEnforced 100.0% +github.com/0xrawsec/whids/api/api_client.go:224: IsServerUp 75.0% +github.com/0xrawsec/whids/api/api_client.go:244: IsServerAuthenticated 80.0% +github.com/0xrawsec/whids/api/api_client.go:275: buildURI 100.0% +github.com/0xrawsec/whids/api/api_client.go:281: GetRulesSha256 68.8% +github.com/0xrawsec/whids/api/api_client.go:309: GetContainer 75.0% +github.com/0xrawsec/whids/api/api_client.go:338: GetContainersList 75.0% +github.com/0xrawsec/whids/api/api_client.go:367: GetContainerSha256 68.8% +github.com/0xrawsec/whids/api/api_client.go:396: GetRules 68.8% +github.com/0xrawsec/whids/api/api_client.go:423: IsFileAboveUploadLimit 0.0% +github.com/0xrawsec/whids/api/api_client.go:434: PostDump 0.0% +github.com/0xrawsec/whids/api/api_client.go:470: PostLogs 68.8% +github.com/0xrawsec/whids/api/api_client.go:502: PostCommand 70.6% +github.com/0xrawsec/whids/api/api_client.go:536: FetchCommand 64.7% +github.com/0xrawsec/whids/api/api_client.go:572: Close 100.0% +github.com/0xrawsec/whids/api/command.go:48: NewCommand 100.0% +github.com/0xrawsec/whids/api/command.go:59: SetCommandLine 87.5% +github.com/0xrawsec/whids/api/command.go:76: AddDropFile 83.3% +github.com/0xrawsec/whids/api/command.go:93: AddDropFileFromPath 0.0% +github.com/0xrawsec/whids/api/command.go:98: AddFetchFile 100.0% +github.com/0xrawsec/whids/api/command.go:102: FromExecCmd 0.0% +github.com/0xrawsec/whids/api/command.go:119: BuildCmd 50.0% +github.com/0xrawsec/whids/api/command.go:128: Unrunnable 0.0% +github.com/0xrawsec/whids/api/command.go:134: Run 76.9% +github.com/0xrawsec/whids/api/command.go:217: String 0.0% +github.com/0xrawsec/whids/api/command.go:223: Strip 100.0% +github.com/0xrawsec/whids/api/command.go:232: Complete 92.3% +github.com/0xrawsec/whids/api/endpoint.go:27: NewEndpoint 100.0% +github.com/0xrawsec/whids/api/endpoint.go:32: Copy 100.0% +github.com/0xrawsec/whids/api/endpoint.go:38: UpdateLastConnection 100.0% +github.com/0xrawsec/whids/api/endpoint.go:53: NewEndpoints 100.0% +github.com/0xrawsec/whids/api/endpoint.go:61: Add 100.0% +github.com/0xrawsec/whids/api/endpoint.go:69: DelByUUID 0.0% +github.com/0xrawsec/whids/api/endpoint.go:90: HasByUUID 0.0% +github.com/0xrawsec/whids/api/endpoint.go:98: GetByUUID 80.0% +github.com/0xrawsec/whids/api/endpoint.go:108: GetMutByUUID 100.0% +github.com/0xrawsec/whids/api/endpoint.go:118: Len 100.0% +github.com/0xrawsec/whids/api/endpoint.go:125: Endpoints 100.0% +github.com/0xrawsec/whids/api/endpoint.go:136: MutEndpoints 100.0% +github.com/0xrawsec/whids/api/forwarder.go:61: NewForwarder 72.7% +github.com/0xrawsec/whids/api/forwarder.go:100: LogfilePath 100.0% +github.com/0xrawsec/whids/api/forwarder.go:108: ArchiveLogs 0.0% +github.com/0xrawsec/whids/api/forwarder.go:125: PipeEvent 100.0% +github.com/0xrawsec/whids/api/forwarder.go:134: Save 84.6% +github.com/0xrawsec/whids/api/forwarder.go:161: HasQueuedEvents 100.0% +github.com/0xrawsec/whids/api/forwarder.go:171: CleanOlderQueued 94.4% +github.com/0xrawsec/whids/api/forwarder.go:201: DiskSpaceQueue 100.0% +github.com/0xrawsec/whids/api/forwarder.go:214: listLogfiles 100.0% +github.com/0xrawsec/whids/api/forwarder.go:227: ProcessQueue 79.4% +github.com/0xrawsec/whids/api/forwarder.go:297: Reset 100.0% +github.com/0xrawsec/whids/api/forwarder.go:304: Collect 72.7% +github.com/0xrawsec/whids/api/forwarder.go:332: Run 100.0% +github.com/0xrawsec/whids/api/forwarder.go:365: Close 87.5% +github.com/0xrawsec/whids/api/log_streamer.go:18: Queue 75.0% +github.com/0xrawsec/whids/api/log_streamer.go:26: Stream 100.0% +github.com/0xrawsec/whids/api/log_streamer.go:40: Close 0.0% +github.com/0xrawsec/whids/api/log_streamer.go:49: NewEventStreamer 100.0% +github.com/0xrawsec/whids/api/log_streamer.go:55: NewStream 100.0% +github.com/0xrawsec/whids/api/log_streamer.go:63: newId 100.0% +github.com/0xrawsec/whids/api/log_streamer.go:73: Queue 83.3% +github.com/0xrawsec/whids/api/manager.go:65: init 75.0% +github.com/0xrawsec/whids/api/manager.go:78: IPFromRequest 0.0% +github.com/0xrawsec/whids/api/manager.go:91: gunzipMiddleware 62.5% +github.com/0xrawsec/whids/api/manager.go:114: Empty 100.0% +github.com/0xrawsec/whids/api/manager.go:119: Verify 50.0% +github.com/0xrawsec/whids/api/manager.go:132: UUIDGen 100.0% +github.com/0xrawsec/whids/api/manager.go:141: KeyGen 100.0% +github.com/0xrawsec/whids/api/manager.go:197: LoadManagerConfig 0.0% +github.com/0xrawsec/whids/api/manager.go:209: SetPath 100.0% +github.com/0xrawsec/whids/api/manager.go:214: Save 0.0% +github.com/0xrawsec/whids/api/manager.go:251: NewManager 70.5% +github.com/0xrawsec/whids/api/manager.go:336: initializeDB 66.7% +github.com/0xrawsec/whids/api/manager.go:355: LoadGeneEngine 85.7% +github.com/0xrawsec/whids/api/manager.go:371: LoadContainers 82.4% +github.com/0xrawsec/whids/api/manager.go:395: updateRules 100.0% +github.com/0xrawsec/whids/api/manager.go:407: updateMispContainer 0.0% +github.com/0xrawsec/whids/api/manager.go:426: AddEndpoint 100.0% +github.com/0xrawsec/whids/api/manager.go:431: UpdateReducer 100.0% +github.com/0xrawsec/whids/api/manager.go:447: Wait 100.0% +github.com/0xrawsec/whids/api/manager.go:452: IsDone 0.0% +github.com/0xrawsec/whids/api/manager.go:457: Shutdown 86.7% +github.com/0xrawsec/whids/api/manager.go:481: Run 66.7% +github.com/0xrawsec/whids/api/manager_admin_api.go:32: admApiParseDuration 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:45: admApiParseTime 66.7% +github.com/0xrawsec/whids/api/manager_admin_api.go:52: muxGetVar 75.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:60: format 100.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:65: readPostAsJSON 80.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:91: NewAdminAPIResponse 100.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:96: NewAdminAPIRespError 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:101: NewAdminAPIRespErrorString 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:106: UnmarshalData 75.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:115: ToJSON 50.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:125: admErr 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:129: admJSONResp 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:133: admMsgStr 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:144: adminAuthorizationMiddleware 66.7% +github.com/0xrawsec/whids/api/manager_admin_api.go:156: admLogHTTPMiddleware 100.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:164: adminRespHeaderMiddleware 100.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:174: admAPIUsers 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:206: admAPIUser 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:256: admAPIEndpoints 83.3% +github.com/0xrawsec/whids/api/manager_admin_api.go:301: admAPIEndpoint 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:380: ToCommand 77.8% +github.com/0xrawsec/whids/api/manager_admin_api.go:402: admAPIEndpointCommand 70.8% +github.com/0xrawsec/whids/api/manager_admin_api.go:447: admAPIEndpointCommandField 52.9% +github.com/0xrawsec/whids/api/manager_admin_api.go:484: admAPIEndpointLogs 57.9% +github.com/0xrawsec/whids/api/manager_admin_api.go:625: admAPIEndpointReport 73.7% +github.com/0xrawsec/whids/api/manager_admin_api.go:665: admAPIEndpointReportArchive 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:740: admAPIEndpointsReports 100.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:763: listEndpointDumps 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:830: admAPIArtifacts 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:861: admAPIEndpointArtifacts 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:892: admAPIEndpointArtifact 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:963: admAPIStats 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:971: admAPIRules 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:1132: admAPIRulesReload 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:1145: admAPIRulesSave 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:1188: wsHandleControlMessage 100.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:1198: admAPIStreamEvents 71.4% +github.com/0xrawsec/whids/api/manager_admin_api.go:1221: admAPIStreamDetections 0.0% +github.com/0xrawsec/whids/api/manager_admin_api.go:1246: runAdminAPI 86.8% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:27: endpointFromRequest 0.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:35: mutEndpointFromRequest 75.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:45: endpointAuthorizationMiddleware 76.2% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:85: isVerboseURL 100.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:94: endptLogHTTPMiddleware 0.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:102: endptQuietLogHTTPMiddleware 100.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:112: runEndpointAPI 78.6% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:178: ServerKey 100.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:183: Rules 100.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:190: RulesSha256 100.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:197: UploadDump 0.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:228: Container 72.7% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:248: ContainerList 80.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:265: ContainerSha256 85.7% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:279: Collect 84.6% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:352: AddCommand 75.0% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:361: GetCommand 66.7% +github.com/0xrawsec/whids/api/manager_endpoint_api.go:371: Command 80.8% +github.com/0xrawsec/whids/api/upload.go:31: NewUploadShrinker 0.0% +github.com/0xrawsec/whids/api/upload.go:65: Size 0.0% +github.com/0xrawsec/whids/api/upload.go:70: Next 0.0% +github.com/0xrawsec/whids/api/upload.go:91: Done 0.0% +github.com/0xrawsec/whids/api/upload.go:96: Err 0.0% +github.com/0xrawsec/whids/api/upload.go:101: Close 0.0% +github.com/0xrawsec/whids/api/upload.go:118: Validate 0.0% +github.com/0xrawsec/whids/api/upload.go:132: Implode 0.0% +github.com/0xrawsec/whids/api/upload.go:137: Dump 0.0% +github.com/0xrawsec/whids/api/upload.go:155: write 0.0% +github.com/0xrawsec/whids/api/users.go:27: NewUsers 100.0% +github.com/0xrawsec/whids/api/users.go:35: Add 87.5% +github.com/0xrawsec/whids/api/users.go:48: Len 0.0% +github.com/0xrawsec/whids/api/users.go:55: List 0.0% +github.com/0xrawsec/whids/api/users.go:65: GetByIdentifier 100.0% +github.com/0xrawsec/whids/api/users.go:72: GetByKey 100.0% +github.com/0xrawsec/whids/api/users.go:79: GetByUUID 0.0% +github.com/0xrawsec/whids/api/users.go:86: exist 40.0% +github.com/0xrawsec/whids/api/users.go:102: Exist 0.0% +github.com/0xrawsec/whids/api/users.go:108: Delete 0.0% +github.com/0xrawsec/whids/logger/events.go:19: NewRawEvent 100.0% +github.com/0xrawsec/whids/logger/events.go:26: DecodeRawEvent 90.9% +github.com/0xrawsec/whids/logger/events.go:43: Less 100.0% +github.com/0xrawsec/whids/logger/events.go:47: Encode 100.0% +github.com/0xrawsec/whids/logger/events.go:55: Event 0.0% +github.com/0xrawsec/whids/logger/index.go:23: OpenIndexFile 80.0% +github.com/0xrawsec/whids/logger/index.go:34: LogfilePath 100.0% +github.com/0xrawsec/whids/logger/index.go:40: Next 81.8% +github.com/0xrawsec/whids/logger/index.go:60: Close 100.0% +github.com/0xrawsec/whids/logger/index.go:75: IndexEntryFromCSV 63.6% +github.com/0xrawsec/whids/logger/index.go:100: UpdateTime 87.5% +github.com/0xrawsec/whids/logger/index.go:117: Less 100.0% +github.com/0xrawsec/whids/logger/index.go:122: In 0.0% +github.com/0xrawsec/whids/logger/index.go:127: Overlaps 100.0% +github.com/0xrawsec/whids/logger/index.go:132: Contains 0.0% +github.com/0xrawsec/whids/logger/index.go:137: Before 0.0% +github.com/0xrawsec/whids/logger/index.go:142: ToCSV 100.0% +github.com/0xrawsec/whids/logger/logfile.go:19: RenameIndexedLogfile 60.0% +github.com/0xrawsec/whids/logger/logfile.go:30: RemoveIndexedLogfile 0.0% +github.com/0xrawsec/whids/logger/logfile.go:41: ArchiveFilename 100.0% +github.com/0xrawsec/whids/logger/logfile.go:59: OpenIndexedLogfile 85.7% +github.com/0xrawsec/whids/logger/logfile.go:74: IndexFileFromPath 100.0% +github.com/0xrawsec/whids/logger/logfile.go:78: resetIndexEntry 75.0% +github.com/0xrawsec/whids/logger/logfile.go:87: IndexFile 100.0% +github.com/0xrawsec/whids/logger/logfile.go:91: size 62.5% +github.com/0xrawsec/whids/logger/logfile.go:113: Size 0.0% +github.com/0xrawsec/whids/logger/logfile.go:121: ReadRawEvents 73.9% +github.com/0xrawsec/whids/logger/logfile.go:165: WriteRawEventWithTimestamp 90.9% +github.com/0xrawsec/whids/logger/logfile.go:191: WriteRawEvent 100.0% +github.com/0xrawsec/whids/logger/logfile.go:195: flush 73.3% +github.com/0xrawsec/whids/logger/logfile.go:225: Flush 0.0% +github.com/0xrawsec/whids/logger/logfile.go:233: Close 75.0% +github.com/0xrawsec/whids/logger/loggers.go:33: stdTime 100.0% +github.com/0xrawsec/whids/logger/loggers.go:37: timestampToDir 100.0% +github.com/0xrawsec/whids/logger/loggers.go:42: fmtTime 100.0% +github.com/0xrawsec/whids/logger/loggers.go:46: parseTime 100.0% +github.com/0xrawsec/whids/logger/loggers.go:64: NewEventLogger 100.0% +github.com/0xrawsec/whids/logger/loggers.go:74: openLogfile 86.4% +github.com/0xrawsec/whids/logger/loggers.go:117: InitTransaction 100.0% +github.com/0xrawsec/whids/logger/loggers.go:129: WriteEvent 77.8% +github.com/0xrawsec/whids/logger/loggers.go:149: CountFiles 100.0% +github.com/0xrawsec/whids/logger/loggers.go:153: endTransaction 100.0% +github.com/0xrawsec/whids/logger/loggers.go:160: CommitTransaction 100.0% +github.com/0xrawsec/whids/logger/loggers.go:166: close 80.0% +github.com/0xrawsec/whids/logger/loggers.go:177: Close 100.0% +github.com/0xrawsec/whids/logger/search.go:21: reverseIndex 100.0% +github.com/0xrawsec/whids/logger/search.go:34: newChunk 100.0% +github.com/0xrawsec/whids/logger/search.go:38: add 80.0% +github.com/0xrawsec/whids/logger/search.go:48: overlaps 100.0% +github.com/0xrawsec/whids/logger/search.go:53: contains 100.0% +github.com/0xrawsec/whids/logger/search.go:66: NewEventSearcher 100.0% +github.com/0xrawsec/whids/logger/search.go:73: keys 0.0% +github.com/0xrawsec/whids/logger/search.go:87: buildIndex 84.4% +github.com/0xrawsec/whids/logger/search.go:146: getFile 64.3% +github.com/0xrawsec/whids/logger/search.go:170: readRawEvents 75.0% +github.com/0xrawsec/whids/logger/search.go:187: Events 89.7% +github.com/0xrawsec/whids/logger/search.go:254: Err 100.0% +github.com/0xrawsec/whids/logger/search.go:258: close 80.0% +github.com/0xrawsec/whids/logger/search.go:269: Close 100.0% +total: (statements) 56.0% diff --git a/api/adminapi_test.go b/api/adminapi_test.go index 2b30b2f..9247f7e 100644 --- a/api/adminapi_test.go +++ b/api/adminapi_test.go @@ -14,6 +14,8 @@ import ( "time" "github.com/0xrawsec/golang-evtx/evtx" + "github.com/0xrawsec/whids/event" + "github.com/0xrawsec/whids/utils" "github.com/gorilla/websocket" ) @@ -171,13 +173,14 @@ func TestAdminAPIPostCommand(t *testing.T) { m.Shutdown() m.Wait() }() - euuid := getEndpointUUID() + euuid := c.config.UUID ca := CommandAPI{ CommandLine: "/bin/ls", FetchFiles: []string{"/etc/fstab"}, } r := post(format("%s/%s/command", AdmAPIEndpointsPath, euuid), JSON(ca)) failOnAdminAPIError(t, r) + time.Sleep(2 * time.Second) if cmd, err := c.FetchCommand(); err != nil { t.Errorf("Failed to Fetch command: %s", err) t.FailNow() @@ -212,7 +215,7 @@ func TestAdminAPIGetCommandField(t *testing.T) { m.Shutdown() m.Wait() }() - euuid := getEndpointUUID() + euuid := c.config.UUID ca := CommandAPI{ CommandLine: "/bin/ls", FetchFiles: []string{"/etc/fstab"}, @@ -280,9 +283,11 @@ func TestAdminAPIGetNewEndpoint(t *testing.T) { func TestAdminAPIGetEndpointReport(t *testing.T) { events := []string{ - `{"Event":{"EventData":{"CreationUtcTime":"2018-02-26 16:28:13.169","Image":"C:\\Program Files\\cagent\\cagent.exe","ProcessGuid":"{49F1AF32-11B0-5A90-0000-0010594E0100}","ProcessId":"1216","TargetFilename":"C:\\commander.exe","UtcTime":"2018-02-26 16:28:13.169"},"GeneInfo":{"Criticality":10,"Signature":["ExecutableFileCreated","NewExeCreatedInRoot"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"11","EventRecordID":"1274413","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"11","TimeCreated":{"SystemTime":"2018-02-26T16:28:13.185436300Z"},"Version":"2"}}}`, - `{"Event":{"EventData":{"CommandLine":"\"powershell\" -command -","Company":"Microsoft Corporation","CurrentDirectory":"C:\\Windows\\system32\\","Description":"Windows PowerShell","FileVersion":"6.1.7600.16385 (win7_rtm.090713-1255)","Hashes":"SHA1=5330FEDAD485E0E4C23B2ABE1075A1F984FDE9FC,MD5=852D67A27E454BD389FA7F02A8CBE23F,SHA256=A8FDBA9DF15E41B6F5C69C79F66A26A9D48E174F9E7018A371600B866867DAB8,IMPHASH=F2C0E8A5BD10DBC167455484050CD683","Image":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","IntegrityLevel":"System","LogonGuid":"{49F1AF32-11AE-5A90-0000-0020E7030000}","LogonId":"0x3e7","ParentCommandLine":"C:\\commander.exe -f","ParentImage":"C:\\commander.exe","ParentProcessGuid":"{49F1AF32-359D-5A94-0000-0010A9530C00}","ParentProcessId":"3068","ProcessGuid":"{49F1AF32-35A0-5A94-0000-0010FE5E0C00}","ProcessId":"1244","Product":"Microsoft® Windows® Operating System","TerminalSessionId":"0","User":"NT AUTHORITY\\SYSTEM","UtcTime":"2018-02-26 16:28:16.514"},"GeneInfo":{"Criticality":10,"Signature":["HeurSpawnShell","PowershellStdin"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"1","EventRecordID":"1274784","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"1","TimeCreated":{"SystemTime":"2018-02-26T16:28:16.530122800Z"},"Version":"5"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"GeneInfo":{"Criticality":10,"Signature":["HeurMaliciousAccess","MaliciousLsassAccess","SuspWriteAccess","SuspiciousLsassAccess"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"2018-02-26T16:43:26.447894800Z"},"Version":"3"}}}`, + `{"Event":{"EventData":{"CommandLine":"\"C:\\Program Files\\Mozilla Firefox\\firefox.exe\"","CurrentDirectory":"C:\\Program Files\\Mozilla Firefox\\","Image":"C:\\Program Files\\Mozilla Firefox\\firefox.exe","ImageHashes":"SHA1=6923508844E6FE0C1DEDD684FE299EBC26D778F3,MD5=988976B1058A1DAE198C93A5688142FD,SHA256=28BE8E0485DBA68F6A4B37F6A68D7AE542B0DA00925A69EA12A4E7AA3B477EC6,IMPHASH=AECE7B7E776840D7A7255A31B309B7E4","ImageSignature":"Mozilla Corporation","ImageSignatureStatus":"Valid","ImageSigned":"true","IntegrityLevel":"Medium","ProcessGuid":"{515cd0d1-c09c-615c-6886-000000008b00}","ProcessId":"9472","ProcessThreatScore":"60","QueryName":"analytics-collector-28944298.us-east-1.elb.amazonaws.com","QueryResults":"-","QueryStatus":"9501","RuleName":"-","Services":"N/A","User":"DESKTOP-LJRVE06\\Generic","UtcTime":"2021-10-04 03:47:27.711"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"DESKTOP-LJRVE06","EventID":22,"Execution":{"ProcessID":3188,"ThreadID":1536},"Keywords":{"Value":9223372036854776000,"Name":""},"Level":{"Value":4,"Name":"Information"},"Opcode":{"Value":0,"Name":"Info"},"Task":{"Value":0,"Name":""},"Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"TimeCreated":{"SystemTime":"2021-10-04T03:47:28.7994921Z"}},"EdrData":{"Endpoint":{"UUID":"03e31275-2277-d8e0-bb5f-480fac7ee4ef","IP":"192.168.56.110","Hostname":"DESKTOP-LJRVE06","Group":"HR"},"Event":{"Hash":"107115af9a7ae294b66499d9f24b4da40840f8dc","Detection":true,"ReceiptTime":"2021-10-06T07:00:47.488763072Z"}},"Detection":{"Signature":["HeurSysmonLongDomain"],"Criticality":6,"Actions":["brief","filedump","regdump"]}}}`, + `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+9c524|C:\\Windows\\System32\\wow64.dll+17014|C:\\Windows\\System32\\wow64.dll+16c85|C:\\Windows\\System32\\wow64.dll+1723b|C:\\Windows\\System32\\wow64.dll+1e5b|C:\\Windows\\System32\\wow64.dll+301d|C:\\Windows\\System32\\wow64.dll+67e3|C:\\Windows\\System32\\wow64cpu.dll+1783|C:\\Windows\\System32\\wow64cpu.dll+1199|C:\\Windows\\System32\\wow64.dll+baea|C:\\Windows\\System32\\wow64.dll+b9a7|C:\\Windows\\SYSTEM32\\ntdll.dll+d3fb3|C:\\Windows\\SYSTEM32\\ntdll.dll+c1dbd|C:\\Windows\\SYSTEM32\\ntdll.dll+717f3|C:\\Windows\\SYSTEM32\\ntdll.dll+7179e|C:\\Windows\\SYSTEM32\\ntdll.dll+71ffc(wow64)|C:\\Windows\\System32\\KERNELBASE.dll+110926(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+f614(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+f89d(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12ef1(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12f58(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12e7b(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12fc9(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+aa418(wow64)","GrantedAccess":"0x1010","RuleName":"-","SourceHashes":"SHA1=12950D906FF703F3A1E0BD973FCA2B433E5AB207,MD5=9A66A3DE2589F7108426AF37AB7F6B41,SHA256=A913415626433D5D0F07D3EC4084A67FF6F5138C3C3F64E36DD0C1AE4C423C65,IMPHASH=7DF1816239C5BC855600D41210406C5B","SourceImage":"C:\\Program Files (x86)\\Google\\Update\\GoogleUpdate.exe","SourceIntegrityLevel":"System","SourceProcessGUID":"{515cd0d1-421a-615d-e087-000000008b00}","SourceProcessId":"6176","SourceProcessThreatScore":"54","SourceServices":"N/A","SourceThreadId":"5788","SourceUser":"NT AUTHORITY\\SYSTEM","TargetHashes":"?","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetIntegrityLevel":"?","TargetParentProcessGuid":"?","TargetProcessGUID":"{515cd0d1-6dae-6154-0c00-000000008b00}","TargetProcessId":"708","TargetProcessThreatScore":"-1","TargetServices":"KeyIso,SamSs,VaultSvc","TargetUser":"?","UtcTime":"2021-10-06 06:28:43.309"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"DESKTOP-LJRVE06","EventID":10,"Execution":{"ProcessID":3188,"ThreadID":3104},"Keywords":{"Value":9223372036854776000,"Name":""},"Level":{"Value":4,"Name":"Information"},"Opcode":{"Value":0,"Name":"Info"},"Task":{"Value":0,"Name":""},"Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"TimeCreated":{"SystemTime":"2021-10-04T03:15:27.1523337Z"}},"EdrData":{"Endpoint":{"UUID":"03e31275-2277-d8e0-bb5f-480fac7ee4ef","IP":"192.168.56.110","Hostname":"DESKTOP-LJRVE06","Group":"HR"},"Event":{"Hash":"af6ee1bef517b5f2d45205f3fb0cf3b48b8d3851","Detection":true,"ReceiptTime":"2021-10-06T06:28:44.894685897Z"}},"Detection":{"Signature":["SuspiciousLsassAccess"],"Criticality":8,"Actions":["report","filedump","regdump","memdump"]}}}`, + `{"Event":{"EventData":{"CommandLine":"\"C:\\Program Files (x86)\\Google\\Update\\Install\\{B29ED602-C455-4B82-80D2-A5992C371348}\\CR_C52DD.tmp\\setup.exe\" --install-archive=\"C:\\Program Files (x86)\\Google\\Update\\Install\\{B29ED602-C455-4B82-80D2-A5992C371348}\\CR_C52DD.tmp\\CHROME_PATCH.PACKED.7Z\" --previous-version=\"94.0.4606.61\" --verbose-logging --do-not-launch-chrome --channel=stable --system-level","Count":"104","CountByExt":"9","CreationUtcTime":"2021-10-06 06:28:31.108","CurrentDirectory":"C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\","Extension":".dll","FrequencyEps":"6","Image":"C:\\Program Files (x86)\\Google\\Update\\Install\\{B29ED602-C455-4B82-80D2-A5992C371348}\\CR_C52DD.tmp\\setup.exe","ImageHashes":"SHA1=0019051003B762EBA424E00BA0D34023608D48D6,MD5=46EB8A20A6B5B16C0BC24B907E0AA684,SHA256=C5360313BD1E95409174C03B71AC83FA13FBFFD3D13412A71D38FB451783FC0E,IMPHASH=44B4DFB0DCCA5DE0AA33EAEC613BAC84","ImageSignature":"Google LLC","ImageSignatureStatus":"Valid","ImageSigned":"true","IntegrityLevel":"System","ProcessGuid":"{515cd0d1-41ff-615d-d587-000000008b00}","ProcessId":"400","ProcessThreatScore":"91","RuleName":"-","Services":"N/A","TargetFilename":"C:\\Program Files\\Google\\Chrome\\Temp\\source400_1020262374\\Chrome-bin\\94.0.4606.71\\vk_swiftshader.dll","User":"NT AUTHORITY\\SYSTEM","UtcTime":"2021-10-06 06:28:31.109"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"DESKTOP-LJRVE06","EventID":11,"Execution":{"ProcessID":3188,"ThreadID":3104},"Keywords":{"Value":9223372036854776000,"Name":""},"Level":{"Value":4,"Name":"Information"},"Opcode":{"Value":0,"Name":"Info"},"Task":{"Value":0,"Name":""},"Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"TimeCreated":{"SystemTime":"2021-10-04T03:15:14.9259991Z"}},"EdrData":{"Endpoint":{"UUID":"03e31275-2277-d8e0-bb5f-480fac7ee4ef","IP":"192.168.56.110","Hostname":"DESKTOP-LJRVE06","Group":"HR"},"Event":{"Hash":"7f21f22e69db69f712798574f04baf28c8d44106","Detection":true,"ReceiptTime":"2021-10-06T06:28:32.368300493Z"}},"Detection":{"Signature":["ExecutableFileCreated"],"Criticality":7,"Actions":["brief","filedump","regdump"]}}}`, + `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+9c524|C:\\Windows\\System32\\wow64.dll+17014|C:\\Windows\\System32\\wow64.dll+16c85|C:\\Windows\\System32\\wow64.dll+1723b|C:\\Windows\\System32\\wow64.dll+1e5b|C:\\Windows\\System32\\wow64.dll+301d|C:\\Windows\\System32\\wow64.dll+67e3|C:\\Windows\\System32\\wow64cpu.dll+1783|C:\\Windows\\System32\\wow64cpu.dll+1199|C:\\Windows\\System32\\wow64.dll+baea|C:\\Windows\\System32\\wow64.dll+b9a7|C:\\Windows\\SYSTEM32\\ntdll.dll+7190b|C:\\Windows\\SYSTEM32\\ntdll.dll+717f3|C:\\Windows\\SYSTEM32\\ntdll.dll+7179e|C:\\Windows\\SYSTEM32\\ntdll.dll+71ffc(wow64)|C:\\Windows\\System32\\KERNELBASE.dll+110926(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+f614(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+f89d(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12ef1(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12f58(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12e7b(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+12aa8(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+1cf31(wow64)|C:\\Program Files (x86)\\Google\\Update\\1.3.36.112\\goopdate.dll+1d691(wow64)","GrantedAccess":"0x1010","RuleName":"-","SourceHashes":"SHA1=12950D906FF703F3A1E0BD973FCA2B433E5AB207,MD5=9A66A3DE2589F7108426AF37AB7F6B41,SHA256=A913415626433D5D0F07D3EC4084A67FF6F5138C3C3F64E36DD0C1AE4C423C65,IMPHASH=7DF1816239C5BC855600D41210406C5B","SourceImage":"C:\\Program Files (x86)\\Google\\Update\\GoogleUpdate.exe","SourceIntegrityLevel":"System","SourceProcessGUID":"{515cd0d1-41e9-615d-a787-000000008b00}","SourceProcessId":"5368","SourceProcessThreatScore":"50","SourceServices":"gupdate","SourceThreadId":"8284","SourceUser":"NT AUTHORITY\\SYSTEM","TargetHashes":"?","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetIntegrityLevel":"?","TargetParentProcessGuid":"?","TargetProcessGUID":"{515cd0d1-6dae-6154-0c00-000000008b00}","TargetProcessId":"708","TargetProcessThreatScore":"-1","TargetServices":"KeyIso,SamSs,VaultSvc","TargetUser":"?","UtcTime":"2021-10-06 06:28:42.590"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"DESKTOP-LJRVE06","EventID":10,"Execution":{"ProcessID":3188,"ThreadID":3104},"Keywords":{"Value":9223372036854776000,"Name":""},"Level":{"Value":4,"Name":"Information"},"Opcode":{"Value":0,"Name":"Info"},"Task":{"Value":0,"Name":""},"Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"TimeCreated":{"SystemTime":"2021-10-04T03:15:26.4082567Z"}},"EdrData":{"Endpoint":{"UUID":"03e31275-2277-d8e0-bb5f-480fac7ee4ef","IP":"192.168.56.110","Hostname":"DESKTOP-LJRVE06","Group":"HR"},"Event":{"Hash":"72241c0e9816fca5d44787752e87715db3ada5f4","Detection":true,"ReceiptTime":"2021-10-06T06:28:43.620696097Z"}},"Detection":{"Signature":["SuspiciousLsassAccess"],"Criticality":8,"Actions":["report","filedump","regdump","memdump"]}}}`, + `{"Event":{"EventData":{"CommandLine":"\"C:\\Program Files\\Mozilla Firefox\\firefox.exe\"","CurrentDirectory":"C:\\Program Files\\Mozilla Firefox\\","Image":"C:\\Program Files\\Mozilla Firefox\\firefox.exe","ImageHashes":"SHA1=6923508844E6FE0C1DEDD684FE299EBC26D778F3,MD5=988976B1058A1DAE198C93A5688142FD,SHA256=28BE8E0485DBA68F6A4B37F6A68D7AE542B0DA00925A69EA12A4E7AA3B477EC6,IMPHASH=AECE7B7E776840D7A7255A31B309B7E4","ImageSignature":"Mozilla Corporation","ImageSignatureStatus":"Valid","ImageSigned":"true","IntegrityLevel":"Medium","ProcessGuid":"{515cd0d1-c09c-615c-6886-000000008b00}","ProcessId":"9472","ProcessThreatScore":"30","QueryName":"analytics-collector-28944298.us-east-1.elb.amazonaws.com","QueryResults":"54.209.192.22;23.21.66.55;54.84.193.129;34.230.149.116;","QueryStatus":"0","RuleName":"-","Services":"N/A","User":"DESKTOP-LJRVE06\\Generic","UtcTime":"2021-10-04 03:46:23.070"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"DESKTOP-LJRVE06","EventID":22,"Execution":{"ProcessID":3188,"ThreadID":1536},"Keywords":{"Value":9223372036854776000,"Name":""},"Level":{"Value":4,"Name":"Information"},"Opcode":{"Value":0,"Name":"Info"},"Task":{"Value":0,"Name":""},"Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"TimeCreated":{"SystemTime":"2021-10-04T03:46:23.342471Z"}},"EdrData":{"Endpoint":{"UUID":"03e31275-2277-d8e0-bb5f-480fac7ee4ef","IP":"192.168.56.110","Hostname":"DESKTOP-LJRVE06","Group":"HR"},"Event":{"Hash":"5b74a882fba6a5a762d6e9cabfa1d3a9883ba203","Detection":true,"ReceiptTime":"2021-10-06T06:59:46.487128874Z"}},"Detection":{"Signature":["HeurSysmonLongDomain"],"Criticality":6,"Actions":["brief","filedump","regdump"]}}}`, } m, mc := prepareTest() @@ -294,8 +299,7 @@ func TestAdminAPIGetEndpointReport(t *testing.T) { m.Shutdown() m.Wait() }() - euuid := getEndpointUUID() - + euuid := mc.config.UUID // creating a new endpoint r := put(AdmAPIEndpointsPath) failOnAdminAPIError(t, r) @@ -330,12 +334,7 @@ func TestAdminAPIGetEndpointLogs(t *testing.T) { // cleanup previous data clean(&mconf, &fconf) - events := []string{ - `{"Event":{"EventData":{"CreationUtcTime":"2018-02-26 16:28:13.169","Image":"C:\\Program Files\\cagent\\cagent.exe","ProcessGuid":"{49F1AF32-11B0-5A90-0000-0010594E0100}","ProcessId":"1216","TargetFilename":"C:\\commander.exe","UtcTime":"2018-02-26 16:28:13.169"},"GeneInfo":{"Criticality":10,"Signature":["ExecutableFileCreated","NewExeCreatedInRoot"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"11","EventRecordID":"1274413","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"11","TimeCreated":{"SystemTime":"` + time.Now().Add(-time.Hour).Format(time.RFC3339Nano) + `"},"Version":"2"}}}`, - `{"Event":{"EventData":{"CommandLine":"\"powershell\" -command -","Company":"Microsoft Corporation","CurrentDirectory":"C:\\Windows\\system32\\","Description":"Windows PowerShell","FileVersion":"6.1.7600.16385 (win7_rtm.090713-1255)","Hashes":"SHA1=5330FEDAD485E0E4C23B2ABE1075A1F984FDE9FC,MD5=852D67A27E454BD389FA7F02A8CBE23F,SHA256=A8FDBA9DF15E41B6F5C69C79F66A26A9D48E174F9E7018A371600B866867DAB8,IMPHASH=F2C0E8A5BD10DBC167455484050CD683","Image":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","IntegrityLevel":"System","LogonGuid":"{49F1AF32-11AE-5A90-0000-0020E7030000}","LogonId":"0x3e7","ParentCommandLine":"C:\\commander.exe -f","ParentImage":"C:\\commander.exe","ParentProcessGuid":"{49F1AF32-359D-5A94-0000-0010A9530C00}","ParentProcessId":"3068","ProcessGuid":"{49F1AF32-35A0-5A94-0000-0010FE5E0C00}","ProcessId":"1244","Product":"Microsoft® Windows® Operating System","TerminalSessionId":"0","User":"NT AUTHORITY\\SYSTEM","UtcTime":"2018-02-26 16:28:16.514"},"GeneInfo":{"Criticality":10,"Signature":["HeurSpawnShell","PowershellStdin"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"1","EventRecordID":"1274784","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"1","TimeCreated":{"SystemTime":"` + time.Now().Add(-4*time.Minute).Format(time.RFC3339Nano) + `"},"Version":"5"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"GeneInfo":{"Criticality":10,"Signature":["HeurMaliciousAccess","MaliciousLsassAccess","SuspWriteAccess","SuspiciousLsassAccess"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - } - + n := 1000 m, mc := prepareTest() mconfBak := mconf defer func() { @@ -345,14 +344,23 @@ func TestAdminAPIGetEndpointLogs(t *testing.T) { m.Shutdown() m.Wait() }() - euuid := getEndpointUUID() + euuid := mc.config.UUID // creating a new endpoint r := put(AdmAPIEndpointsPath) failOnAdminAPIError(t, r) - for _, e := range events { - r, err := mc.PrepareGzip("POST", EptAPIPostLogsPath, bytes.NewBufferString(e)) + npivot := 0 + for e := range emitEvents(n, false) { + switch rand.Int() % 3 { + case 0: + e.Event.System.TimeCreated.SystemTime = e.Event.System.TimeCreated.SystemTime.Add(-time.Hour) + case 1: + e.Event.System.TimeCreated.SystemTime = e.Event.System.TimeCreated.SystemTime.Add(time.Hour) + default: + npivot++ + } + r, err := mc.PrepareGzip("POST", EptAPIPostLogsPath, bytes.NewBuffer(utils.Json(e))) if err != nil { t.Logf("Failed to prepare request: %s", err) t.FailNow() @@ -362,25 +370,16 @@ func TestAdminAPIGetEndpointLogs(t *testing.T) { time.Sleep(1 * time.Second) - // test retrieving all the logs - r = get(AdmAPIEndpointsPath + "/" + euuid + "/logs") - failOnAdminAPIError(t, r) - data := make([]evtx.GoEvtxMap, 0) - r.UnmarshalData(&data) - if len(data) != len(events) { - t.Errorf("Wrong number of events %d instead of %d", len(data), len(events)) - t.FailNow() - } - // test pivoting v := url.Values{} v.Set("pivot", time.Now().Format(time.RFC3339)) + v.Set("delta", "1m") r = get(AdmAPIEndpointsPath + "/" + euuid + "/logs?" + v.Encode()) failOnAdminAPIError(t, r) - data = make([]evtx.GoEvtxMap, 0) + data := make([]event.EdrEvent, 0) r.UnmarshalData(&data) - if len(data) != 2 { - t.Errorf("Wrong number of events %d instead of %d", len(data), 2) + if len(data) != npivot { + t.Errorf("Wrong number of events %d instead of %d", len(data), npivot) t.FailNow() } @@ -390,9 +389,9 @@ func TestAdminAPIGetEndpointLogs(t *testing.T) { v.Set("delta", "3h") r = get(AdmAPIEndpointsPath + "/" + euuid + "/logs?" + v.Encode()) failOnAdminAPIError(t, r) - data = make([]evtx.GoEvtxMap, 0) + data = make([]event.EdrEvent, 0) r.UnmarshalData(&data) - if len(data) != len(events) { + if len(data) != n { t.Errorf("Wrong number of events %d instead of %d", len(data), len(events)) t.FailNow() } @@ -400,12 +399,12 @@ func TestAdminAPIGetEndpointLogs(t *testing.T) { // test with start and stop v = url.Values{} v.Set("start", time.Now().Add(-3*time.Hour).Format(time.RFC3339)) - v.Set("stop", time.Now().Format(time.RFC3339)) + v.Set("stop", time.Now().Add(3*time.Hour).Format(time.RFC3339)) r = get(AdmAPIEndpointsPath + "/" + euuid + "/logs?" + v.Encode()) failOnAdminAPIError(t, r) - data = make([]evtx.GoEvtxMap, 0) + data = make([]event.EdrEvent, 0) r.UnmarshalData(&data) - if len(data) != len(events) { + if len(data) != n { t.Errorf("Wrong number of events %d instead of %d", len(data), len(events)) t.FailNow() } @@ -416,22 +415,6 @@ func TestAdminAPIGetEndpointAlerts(t *testing.T) { // cleanup previous data clean(&mconf, &fconf) - alerts := []string{ - `{"Event":{"EventData":{"CreationUtcTime":"2018-02-26 16:28:13.169","Image":"C:\\Program Files\\cagent\\cagent.exe","ProcessGuid":"{49F1AF32-11B0-5A90-0000-0010594E0100}","ProcessId":"1216","TargetFilename":"C:\\commander.exe","UtcTime":"2018-02-26 16:28:13.169"},"GeneInfo":{"Criticality":10,"Signature":["ExecutableFileCreated","NewExeCreatedInRoot"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"11","EventRecordID":"1274413","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"11","TimeCreated":{"SystemTime":"` + time.Now().Add(-time.Hour).Format(time.RFC3339Nano) + `"},"Version":"2"}}}`, - `{"Event":{"EventData":{"CommandLine":"\"powershell\" -command -","Company":"Microsoft Corporation","CurrentDirectory":"C:\\Windows\\system32\\","Description":"Windows PowerShell","FileVersion":"6.1.7600.16385 (win7_rtm.090713-1255)","Hashes":"SHA1=5330FEDAD485E0E4C23B2ABE1075A1F984FDE9FC,MD5=852D67A27E454BD389FA7F02A8CBE23F,SHA256=A8FDBA9DF15E41B6F5C69C79F66A26A9D48E174F9E7018A371600B866867DAB8,IMPHASH=F2C0E8A5BD10DBC167455484050CD683","Image":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","IntegrityLevel":"System","LogonGuid":"{49F1AF32-11AE-5A90-0000-0020E7030000}","LogonId":"0x3e7","ParentCommandLine":"C:\\commander.exe -f","ParentImage":"C:\\commander.exe","ParentProcessGuid":"{49F1AF32-359D-5A94-0000-0010A9530C00}","ParentProcessId":"3068","ProcessGuid":"{49F1AF32-35A0-5A94-0000-0010FE5E0C00}","ProcessId":"1244","Product":"Microsoft® Windows® Operating System","TerminalSessionId":"0","User":"NT AUTHORITY\\SYSTEM","UtcTime":"2018-02-26 16:28:16.514"},"GeneInfo":{"Criticality":10,"Signature":["HeurSpawnShell","PowershellStdin"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"1","EventRecordID":"1274784","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"1","TimeCreated":{"SystemTime":"` + time.Now().Add(-4*time.Minute).Format(time.RFC3339Nano) + `"},"Version":"5"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"GeneInfo":{"Criticality":10,"Signature":["HeurMaliciousAccess","MaliciousLsassAccess","SuspWriteAccess","SuspiciousLsassAccess"]},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - } - - events := []string{ - // all following should not be in alerts - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - `{"Event":{"EventData":{"CallTrace":"C:\\Windows\\SYSTEM32\\ntdll.dll+4d61a|C:\\Windows\\system32\\KERNELBASE.dll+19577|UNKNOWN(000000001ABD2A68)","GrantedAccess":"0x143a","SourceImage":"C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe","SourceProcessGUID":"{49F1AF32-3922-5A94-0000-0010E3581900}","SourceProcessId":"1916","SourceThreadId":"2068","TargetImage":"C:\\Windows\\system32\\lsass.exe","TargetProcessGUID":"{49F1AF32-11AD-5A90-0000-00102F6F0000}","TargetProcessId":"472","UtcTime":"2018-02-26 16:43:26.380"},"System":{"Channel":"Microsoft-Windows-Sysmon/Operational","Computer":"CALDERA01.caldera.loc","Correlation":{},"EventID":"10","EventRecordID":"1293693","Execution":{"ProcessID":"1408","ThreadID":"1652"},"Keywords":"0x8000000000000000","Level":"4","Opcode":"0","Provider":{"Guid":"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}","Name":"Microsoft-Windows-Sysmon"},"Security":{"UserID":"S-1-5-18"},"Task":"10","TimeCreated":{"SystemTime":"` + time.Now().Format(time.RFC3339Nano) + `"},"Version":"3"}}}`, - } - m, mc := prepareTest() mconfBak := mconf defer func() { @@ -441,17 +424,26 @@ func TestAdminAPIGetEndpointAlerts(t *testing.T) { m.Shutdown() m.Wait() }() - euuid := getEndpointUUID() + euuid := mc.config.UUID // creating a new endpoint r := put(AdmAPIEndpointsPath) failOnAdminAPIError(t, r) - tmp := make([]string, 0) - tmp = append(tmp, alerts...) - tmp = append(tmp, events...) - for _, e := range tmp { - r, err := mc.PrepareGzip("POST", EptAPIPostLogsPath, bytes.NewBufferString(e)) + npivot := 0 + n, ndet := 1000, 100 + for e := range emitMixedEvents(n, ndet) { + if e.IsDetection() { + switch rand.Int() % 3 { + case 0: + e.Event.System.TimeCreated.SystemTime = e.Event.System.TimeCreated.SystemTime.Add(-time.Hour) + case 1: + e.Event.System.TimeCreated.SystemTime = e.Event.System.TimeCreated.SystemTime.Add(time.Hour) + default: + npivot++ + } + } + r, err := mc.PrepareGzip("POST", EptAPIPostLogsPath, bytes.NewBuffer(utils.Json(e))) if err != nil { t.Logf("Failed to prepare request: %s", err) t.FailNow() @@ -461,25 +453,15 @@ func TestAdminAPIGetEndpointAlerts(t *testing.T) { time.Sleep(1 * time.Second) - // test retrieving all the logs - r = get(AdmAPIEndpointsPath + "/" + euuid + AdmAPIDetectionPart) - failOnAdminAPIError(t, r) - data := make([]evtx.GoEvtxMap, 0) - r.UnmarshalData(&data) - if len(data) != len(alerts) { - t.Errorf("Wrong number of events %d instead of %d", len(data), len(events)) - t.FailNow() - } - // test pivoting v := url.Values{} v.Set("pivot", time.Now().Format(time.RFC3339)) - r = get(AdmAPIEndpointsPath + "/" + euuid + "/alerts?" + v.Encode()) + r = get(AdmAPIEndpointsPath + "/" + euuid + AdmAPIDetectionPart + "?" + v.Encode()) failOnAdminAPIError(t, r) - data = make([]evtx.GoEvtxMap, 0) + data := make([]evtx.GoEvtxMap, 0) r.UnmarshalData(&data) - if len(data) != 2 { - t.Errorf("Wrong number of events %d instead of %d", len(data), 2) + if len(data) != npivot { + t.Errorf("Wrong number of events %d instead of %d", len(data), npivot) t.FailNow() } @@ -487,11 +469,11 @@ func TestAdminAPIGetEndpointAlerts(t *testing.T) { v = url.Values{} v.Set("pivot", time.Now().Format(time.RFC3339)) v.Set("delta", "3h") - r = get(AdmAPIEndpointsPath + "/" + euuid + "/alerts?" + v.Encode()) + r = get(AdmAPIEndpointsPath + "/" + euuid + AdmAPIDetectionPart + "?" + v.Encode()) failOnAdminAPIError(t, r) data = make([]evtx.GoEvtxMap, 0) r.UnmarshalData(&data) - if len(data) != len(alerts) { + if len(data) != ndet { t.Errorf("Wrong number of events %d instead of %d", len(data), len(events)) t.FailNow() } @@ -499,12 +481,12 @@ func TestAdminAPIGetEndpointAlerts(t *testing.T) { // test with start and stop v = url.Values{} v.Set("start", time.Now().Add(-3*time.Hour).Format(time.RFC3339)) - v.Set("stop", time.Now().Format(time.RFC3339)) - r = get(AdmAPIEndpointsPath + "/" + euuid + "/alerts?" + v.Encode()) + v.Set("stop", time.Now().Add(3*time.Hour).Format(time.RFC3339)) + r = get(AdmAPIEndpointsPath + "/" + euuid + AdmAPIDetectionPart + "?" + v.Encode()) failOnAdminAPIError(t, r) data = make([]evtx.GoEvtxMap, 0) r.UnmarshalData(&data) - if len(data) != len(alerts) { + if len(data) != ndet { t.Errorf("Wrong number of events %d instead of %d", len(data), len(events)) t.FailNow() } diff --git a/api/api_client_test.go b/api/api_client_test.go index 854fd89..98f6882 100644 --- a/api/api_client_test.go +++ b/api/api_client_test.go @@ -14,7 +14,7 @@ import ( ) var ( - cconf = ClientConfig{ +cconf = ClientConfig{ Proto: "https", Host: "localhost", Port: 8000, diff --git a/api/forwarder_test.go b/api/forwarder_test.go index d1b8dfd..59bcadf 100644 --- a/api/forwarder_test.go +++ b/api/forwarder_test.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "io/ioutil" + "math" "math/rand" "os" "path/filepath" @@ -19,6 +20,7 @@ import ( "github.com/0xrawsec/golang-utils/scanner" "github.com/0xrawsec/golang-utils/sync/semaphore" "github.com/0xrawsec/whids/event" + "github.com/0xrawsec/whids/logger" "github.com/0xrawsec/whids/utils" ) @@ -43,8 +45,8 @@ var ( Logging: ManagerLogConfig{ Root: "./data/logs", LogBasename: "alerts", - //EnEnptLogs: true, }, + Database: "./data/database", RulesDir: "./data", DumpDir: "./data/uploads/", ContainersDir: "./data/containers", @@ -70,26 +72,58 @@ func init() { } } -func emitEvents(count int) (ce chan *event.EdrEvent) { +func emitEvents(count int, detection bool) (ce chan *event.EdrEvent) { ce = make(chan *event.EdrEvent) go func() { defer close(ce) - for i := 0; i < count; i++ { + for count > 0 { i := rand.Int() % len(events) e := events[i] + if detection && !e.IsDetection() { + continue + } + if !detection && e.IsDetection() { + continue + } e.Event.System.TimeCreated.SystemTime = time.Now() ce <- &e + count-- } }() return } -func readerFromEvents(count int) io.Reader { - tmp := make([]string, 0, count) - for event := range emitEvents(count) { - tmp = append(tmp, string(utils.Json(event))) +func emitMixedEvents(ecount, dcount int) (ce chan *event.EdrEvent) { + ce = make(chan *event.EdrEvent) + go func() { + defer close(ce) + for ecount > 0 || dcount > 0 { + i := rand.Int() % len(events) + e := events[i] + if dcount == 0 && e.IsDetection() { + continue + } + if ecount == 0 && !e.IsDetection() { + continue + } + e.Event.System.TimeCreated.SystemTime = time.Now() + ce <- &e + if e.IsDetection() { + dcount-- + } else { + ecount-- + } + } + }() + return +} + +func countEvents(s *logger.EventSearcher) (n int) { + for range s.Events(time.Now().Add(-time.Hour), time.Now().Add(time.Hour), "", math.MaxInt, 0) { + n++ } - return bytes.NewBufferString(strings.Join(tmp, "\n")) + return + } func countLinesInGzFile(filepath string) int { @@ -114,6 +148,14 @@ func countLinesInGzFile(filepath string) int { return line } +func readerFromEvents(count int) io.Reader { + tmp := make([]string, 0, count) + for event := range emitEvents(count, false) { + tmp = append(tmp, string(utils.Json(event))) + } + return bytes.NewBufferString(strings.Join(tmp, "\n")) +} + func clean(mc *ManagerConfig, fc *ForwarderConfig) { os.RemoveAll(mc.Logging.Root) os.RemoveAll(fc.Logging.Dir) @@ -128,8 +170,8 @@ func TestForwarderBasic(t *testing.T) { //defer clean(&mconf, &fconf) nevents := 1000 - testfile := "Testlog.gz" key := KeyGen(DefaultKeySize) + testfile := "Testlog.gz" mconf.Logging.LogBasename = testfile r, err := NewManager(&mconf) @@ -146,9 +188,10 @@ func TestForwarderBasic(t *testing.T) { t.FailNow() } f.Run() + defer f.Close() cnt := 0 - for e := range emitEvents(nevents) { + for e := range emitEvents(nevents, false) { if cnt == 500 { time.Sleep(2 * time.Second) } @@ -158,7 +201,7 @@ func TestForwarderBasic(t *testing.T) { // shuts down the receiver before counting lines r.Shutdown() - if n := countLinesInGzFile(logfileFromConfig(mconf)); n != nevents { + if n := countEvents(r.eventSearcher); n != nevents { t.Errorf("Some events were lost on the way: %d logged by server instead of %d sent", n, nevents) } log.Infof("Shutting down") @@ -193,9 +236,10 @@ func TestCollectorAuthFailure(t *testing.T) { t.FailNow() } f.Run() + defer f.Close() cnt := 0 - for e := range emitEvents(nevents) { + for e := range emitEvents(nevents, false) { if cnt == 500 { time.Sleep(2 * time.Second) } @@ -205,7 +249,7 @@ func TestCollectorAuthFailure(t *testing.T) { // shuts down the receiver before counting lines r.Shutdown() - if n := countLinesInGzFile(logfileFromConfig(mconf)); n != 0 { + if n := countEvents(r.eventSearcher); n != 0 { t.Errorf("Some events were logged while it should not") } } @@ -237,9 +281,10 @@ func TestCollectorAuthSuccess(t *testing.T) { t.FailNow() } f.Run() + defer f.Close() cnt := 0 - for e := range emitEvents(nevents) { + for e := range emitEvents(nevents, false) { if cnt == 500 { time.Sleep(2 * time.Second) } @@ -249,12 +294,17 @@ func TestCollectorAuthSuccess(t *testing.T) { // shuts down the receiver before counting lines r.Shutdown() - if n := countLinesInGzFile(logfileFromConfig(mconf)); n != nevents { + if n := countEvents(r.eventSearcher); n != nevents { t.Errorf("Some events were lost on the way: %d logged by server instead of %d sent", n, nevents) } } func TestForwarderParallel(t *testing.T) { + + if testing.Short() { + t.Skip() + } + clean(&mconf, &fconf) defer clean(&mconf, &fconf) @@ -286,11 +336,11 @@ func TestForwarderParallel(t *testing.T) { t.FailNow() } c.Run() - for e := range emitEvents(nevents) { + defer c.Close() + for e := range emitEvents(nevents, false) { c.PipeEvent(e) } time.Sleep(2 * time.Second) - c.Close() }() } wg.Wait() @@ -298,7 +348,7 @@ func TestForwarderParallel(t *testing.T) { // shuts down the receiver before counting lines r.Shutdown() - if n := countLinesInGzFile(logfileFromConfig(mconf)); n != nclients*nevents { + if n := countEvents(r.eventSearcher); n != nclients*nevents { t.Errorf("Some events were lost on the way: %d logged by server instead of %d sent", n, nclients*nevents) } } @@ -310,7 +360,7 @@ func TestForwarderQueueBasic(t *testing.T) { nevents := 1000 testfile := "TestCollectorQueue.log" - outfile := fmt.Sprintf("%s.1", testfile) + //outfile := fmt.Sprintf("%s.1", testfile) // Initialize the receiver key := KeyGen(DefaultKeySize) @@ -335,7 +385,7 @@ func TestForwarderQueueBasic(t *testing.T) { defer f.Close() // Sending events - for e := range emitEvents(nevents / 2) { + for e := range emitEvents(nevents/2, false) { f.PipeEvent(e) } @@ -351,7 +401,7 @@ func TestForwarderQueueBasic(t *testing.T) { r.Shutdown() // Sending another wave of events - for e := range emitEvents(nevents / 2) { + for e := range emitEvents(nevents/2, false) { f.PipeEvent(e) } @@ -367,7 +417,7 @@ func TestForwarderQueueBasic(t *testing.T) { // shuts down the receiver before counting lines r.Shutdown() - if n := countLinesInGzFile(filepath.Join(mconf.Logging.Root, outfile)); n != nevents { + if n := countEvents(r.eventSearcher); n != nevents { t.Errorf("Some events were lost on the way: %d logged by server instead of %d sent", n, nevents) } } @@ -387,6 +437,7 @@ func TestForwarderCleanup(t *testing.T) { } // Running the forwarder f.Run() + defer f.Close() // create bogus files inside queue directory numberOfFiles := DiskSpaceThreshold / DefaultLogfileSize @@ -409,18 +460,17 @@ func TestForwarderCleanup(t *testing.T) { } // send enough events to trigger cleanup - for i := 0; i < additionalFiles+3; i++ { - for e := range emitEvents(int(f.EventTresh)) { + for i := 0; i < additionalFiles; i++ { + for e := range emitEvents(int(f.EventTresh), false) { f.PipeEvent(e) } time.Sleep(2 * time.Second) } files, _ := ioutil.ReadDir(fconf.Logging.Dir) - if len(files) != numberOfFiles { - t.Errorf("Unexpected number of files remaining in the directory") + if len(files)-1 != numberOfFiles { + t.Errorf("Expecting %d remaining in directory but got %d", numberOfFiles, len(files)-1) t.FailNow() } - defer f.Close() } diff --git a/api/manager.go b/api/manager.go index 6f2c096..90d2ccf 100644 --- a/api/manager.go +++ b/api/manager.go @@ -181,6 +181,7 @@ type MispConfig struct { // ManagerConfig defines manager's configuration structure type ManagerConfig struct { // TOML strings need to be first otherwise issue parsing back config + Database string `toml:"db" comment:"Path to store database"` RulesDir string `toml:"rules-dir" comment:"Gene rule directory\n See: https://github.com/0xrawsec/gene-rules"` DumpDir string `toml:"dump-dir" comment:"Directory where to dump artifacts collected on hosts"` ContainersDir string `toml:"containers-dir" comment:"Gene rules' containers directory\n (c.f. Gene documentation https://github.com/0xrawsec/gene)"` @@ -261,7 +262,8 @@ func NewManager(c *ManagerConfig) (*Manager, error) { m.detectionSearcher = logger.NewEventSearcher(detectionDir) // database initialization - m.db = sod.Open("database") + m.db = sod.Open(c.Database) + // Create a new streamer m.eventStreamer = NewEventStreamer() diff --git a/coverage.sh b/coverage.sh new file mode 100755 index 0000000..6d6a2f9 --- /dev/null +++ b/coverage.sh @@ -0,0 +1,14 @@ +#!/bin/bash +set -e + +pkgs=("./logger" "./api") + +tmp=$(mktemp -d) +coverprofile="${tmp}/coverage.out" +coverage_dir=".github/coverage" +out="${coverage_dir}/coverage.txt" + +mkdir -p "${coverage_dir}" + +GOOS=linux go test -short -failfast -v -coverprofile="${coverprofile}" ${pkgs[*]} +go tool cover -func "${coverprofile}" | tee "${out}" \ No newline at end of file diff --git a/logger/search.go b/logger/search.go index 1f48425..1a1c2ff 100644 --- a/logger/search.go +++ b/logger/search.go @@ -93,7 +93,7 @@ func (s *EventSearcher) buildIndex(start, stop time.Time, key string) (index *da index = datastructs.NewSortedSlice() marked := datastructs.NewSet() - if key != "" { + if key == "" { if keys, err = s.keys(); err != nil { return } diff --git a/tools/manager/whids-man.go b/tools/manager/whids-man.go index 47372bc..2e9f56c 100644 --- a/tools/manager/whids-man.go +++ b/tools/manager/whids-man.go @@ -57,11 +57,11 @@ var ( Logging: api.ManagerLogConfig{ Root: "./data/logs", LogBasename: "forwarded", - //EnEnptLogs: true, }, ContainersDir: "./data/containers", RulesDir: "./data/rules", DumpDir: "./data/dumps", + Database: "./data/database", } )