diff --git a/packages/zeek/_dev/build/docs/README.md b/packages/zeek/_dev/build/docs/README.md index bbd9c2499cb..de94572fb18 100644 --- a/packages/zeek/_dev/build/docs/README.md +++ b/packages/zeek/_dev/build/docs/README.md @@ -110,6 +110,24 @@ contains kerberos data. {{fields "kerberos"}} +### known_certs + +The `known_certs` dataset captures information about SSL/TLS certificates seen on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-certs-log) for more details. + +{{fields "known_certs"}} + +### known_hosts + +The `known_hosts` dataset simply records a timestamp and an IP address when Zeek observes a new system on the local network.. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-hosts-log) for more details. + +{{fields "known_hosts"}} + +### known_services + +The `known_services` dataset records a timestamp, IP, port number, protocol, and service (if available) when Zeek observes a system offering a new service on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-services-log) for more details. + +{{fields "known_services"}} + ### modbus The `modbus` dataset collects the Zeek modbus.log file, which contains @@ -236,6 +254,12 @@ SOCKS proxy requests. {{fields "socks"}} +### software + +The `software` dataset collects details on applications operated by the hosts it sees on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#software-log) for more details. + +{{fields "software"}} + ### ssh The `ssh` dataset collects the Zeek ssh.log file, which contains SSH diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/capture_loss.log b/packages/zeek/_dev/deploy/docker/sample_logs/capture_loss.log index ea7af28778c..8472bc463fe 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/capture_loss.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/capture_loss.log @@ -2,4 +2,4 @@ {"ts":1617062640.941952,"ts_delta":900.0005369186401,"peer":"zeek","gaps":58475,"acks":65665,"percent_lost":89.05048351481003} {"ts":1617063540.942231,"ts_delta":900.0002789497376,"peer":"zeek","gaps":54754,"acks":61818,"percent_lost":88.5729075673752} {"ts":1617064440.942597,"ts_delta":900.0003659725189,"peer":"zeek","gaps":51022,"acks":57974,"percent_lost":88.00841756649533} -{"ts":1617065340.942651,"ts_delta":900.0000541210175,"peer":"zeek","gaps":55105,"acks":62497,"percent_lost":88.17223226714883} \ No newline at end of file +{"ts":1617065340.942651,"ts_delta":900.0000541210175,"peer":"zeek","gaps":55105,"acks":62497,"percent_lost":88.17223226714883} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/dhcp.log b/packages/zeek/_dev/deploy/docker/sample_logs/dhcp.log index 84861a99c7e..4a0150f0689 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/dhcp.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/dhcp.log @@ -1,2 +1,2 @@ {"ts":1476605498.771847,"uids":["CmWOt6VWaNGqXYcH6","CLObLo4YHn0u23Tp8a"],"client_addr":"192.168.199.132","server_addr":"192.168.199.254","mac":"00:0c:29:03:df:ad","host_name":"DESKTOP-2AEFM7G","client_fqdn":"DESKTOP-2AEFM7G","domain":"localdomain","requested_addr":"192.168.199.132","assigned_addr":"192.168.199.132","lease_time":1800.0,"msg_types":["REQUEST","ACK"],"duration":0.000161} -{"ts":1617088722.072416,"uids":["Ck0tsG4wsJxI3lIEZ"],"client_addr":"10.156.0.2","server_addr":"169.254.169.254","mac":"42:01:0a:9c:00:02","domain":"c.elastic-sa.internal","assigned_addr":"10.156.0.2","lease_time":86400.0,"msg_types":["ACK"],"duration":0.0} \ No newline at end of file +{"ts":1617088722.072416,"uids":["Ck0tsG4wsJxI3lIEZ"],"client_addr":"10.156.0.2","server_addr":"169.254.169.254","mac":"42:01:0a:9c:00:02","domain":"c.elastic-sa.internal","assigned_addr":"10.156.0.2","lease_time":86400.0,"msg_types":["ACK"],"duration":0.0} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/files.log b/packages/zeek/_dev/deploy/docker/sample_logs/files.log index 950362180ca..5148950f05f 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/files.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/files.log @@ -5,4 +5,4 @@ {"ts":1617069773.678327,"fuid":"FYszs61e8hIUWMWgL5","tx_hosts":["89.160.20.156"],"rx_hosts":["10.156.0.2"],"conn_uids":["CaB3fq3yLrKCbYLqr4"],"source":"SSL","depth":0,"analyzers":["X509","SHA256","SHA1","MD5"],"mime_type":"application/x-x509-user-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":893,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"5abbc8a9137f6924861596848b7d7794","sha1":"99c1daf07c8d69a8a065492dcaae43c43ff13497","sha256":"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74"} {"ts":1617069783.678588,"fuid":"FdGWZq2wRIvCfjvdI5","tx_hosts":["89.160.20.156"],"rx_hosts":["10.156.0.2"],"conn_uids":["C0vhl91PPOI7LbrPZ8"],"source":"SSL","depth":0,"analyzers":["X509","SHA256","SHA1","MD5"],"mime_type":"application/x-x509-user-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":893,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"5abbc8a9137f6924861596848b7d7794","sha1":"99c1daf07c8d69a8a065492dcaae43c43ff13497","sha256":"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74"} {"ts":1617069792.519193,"fuid":"FSMkdM3YUSoEVpLZN4","tx_hosts":["169.254.169.254"],"rx_hosts":["10.156.0.2"],"conn_uids":["CgbPEj2jf5Ca7Lw0x2"],"source":"HTTP","depth":0,"analyzers":["SHA1","MD5"],"mime_type":"text/html","duration":0.00005316734313964844,"local_orig":false,"is_orig":false,"seen_bytes":1609,"total_bytes":1609,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"1ab1d3a926a99ccfc25acccc5b4289b4","sha1":"1895628784b47ad8da112c699a1b21f5b49c2b80"} -{"ts":1617069793.669729,"fuid":"F1msmE2xRFsdvL2iI","tx_hosts":["89.160.20.156"],"rx_hosts":["10.156.0.2"],"conn_uids":["C0vua63rzjtLaiefyj"],"source":"SSL","depth":0,"analyzers":["X509","SHA256","SHA1","MD5"],"mime_type":"application/x-x509-user-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":893,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"5abbc8a9137f6924861596848b7d7794","sha1":"99c1daf07c8d69a8a065492dcaae43c43ff13497","sha256":"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74"} \ No newline at end of file +{"ts":1617069793.669729,"fuid":"F1msmE2xRFsdvL2iI","tx_hosts":["89.160.20.156"],"rx_hosts":["10.156.0.2"],"conn_uids":["C0vua63rzjtLaiefyj"],"source":"SSL","depth":0,"analyzers":["X509","SHA256","SHA1","MD5"],"mime_type":"application/x-x509-user-cert","duration":0.0,"local_orig":false,"is_orig":false,"seen_bytes":893,"missing_bytes":0,"overflow_bytes":0,"timedout":false,"md5":"5abbc8a9137f6924861596848b7d7794","sha1":"99c1daf07c8d69a8a065492dcaae43c43ff13497","sha256":"1de4074b4e38377f4367303f4a19c986a506180f22a6e53a68cc7679ea6d9c74"} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/known_certs.log b/packages/zeek/_dev/deploy/docker/sample_logs/known_certs.log new file mode 100644 index 00000000000..30d04a47388 --- /dev/null +++ b/packages/zeek/_dev/deploy/docker/sample_logs/known_certs.log @@ -0,0 +1 @@ +{"ts":"2020-12-31T15:15:53.690221Z","host":"192.168.4.1","port_num":443,"subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","issuer_subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","serial":"98D0AD47D748CDD6"} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/known_hosts.log b/packages/zeek/_dev/deploy/docker/sample_logs/known_hosts.log new file mode 100644 index 00000000000..d69fbbc646e --- /dev/null +++ b/packages/zeek/_dev/deploy/docker/sample_logs/known_hosts.log @@ -0,0 +1,5 @@ +{"ts":"2021-01-03T01:19:26.260073Z","host":"192.168.4.25"} +{"ts":"2021-01-03T01:19:27.353353Z","host":"192.168.4.29"} +{"ts":"2021-01-03T01:19:32.488179Z","host":"192.168.4.43"} +{"ts":"2021-01-03T01:19:58.792683Z","host":"192.168.4.142"} +{"ts":"2021-01-03T12:17:22.496004Z","host":"192.168.4.115"} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/known_services.log b/packages/zeek/_dev/deploy/docker/sample_logs/known_services.log new file mode 100644 index 00000000000..c46411c420f --- /dev/null +++ b/packages/zeek/_dev/deploy/docker/sample_logs/known_services.log @@ -0,0 +1 @@ +{"ts":"2021-01-03T01:19:36.242774Z","host":"192.168.4.1","port_num":53,"port_proto":"udp","service":["DNS"]} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/notice.log b/packages/zeek/_dev/deploy/docker/sample_logs/notice.log index 16b4052346c..2ad091bf9bd 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/notice.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/notice.log @@ -1,4 +1,4 @@ {"ts":1320435875.879278,"note":"SSH::Password_Guessing","msg":"172.16.238.1 appears to be guessing SSH passwords (seen in 30 connections).","sub":"Sampled servers: 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136, 172.16.238.136","src":"172.16.238.1","peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false} {"ts":1551393388.426472,"note":"Scan::Port_Scan","msg":"89.160.20.156 scanned at least 15 unique ports of host 89.160.20.156 in 0m0s","sub":"remote","src":"89.160.20.156","dst":"89.160.20.156","peer_descr":"bro","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0,"dropped":false} {"ts":1617097740.958466,"note":"CaptureLoss::Too_Much_Loss","msg":"The capture loss script detected an estimated loss rate above 88.306%","actions":["Notice::ACTION_LOG"],"suppress_for":3600.0} -{"ts":1617097929.601155,"uid":"CmvrSS1wIiuOGYCbfi","id.orig_h":"10.156.0.2","id.orig_p":48818,"id.resp_h":"89.160.20.156","id.resp_p":443,"fuid":"F39b0Bdfm3FW1rNS5","proto":"tcp","note":"SSL::Invalid_Server_Cert","msg":"SSL certificate validation failed with (self signed certificate)","sub":"CN=*.badssl.com,O=BadSSL,L=San Francisco,ST=California,C=US","src":"10.156.0.2","dst":"89.160.20.156","p":443,"actions":["Notice::ACTION_LOG"],"suppress_for":3600.0} \ No newline at end of file +{"ts":1617097929.601155,"uid":"CmvrSS1wIiuOGYCbfi","id.orig_h":"10.156.0.2","id.orig_p":48818,"id.resp_h":"89.160.20.156","id.resp_p":443,"fuid":"F39b0Bdfm3FW1rNS5","proto":"tcp","note":"SSL::Invalid_Server_Cert","msg":"SSL certificate validation failed with (self signed certificate)","sub":"CN=*.badssl.com,O=BadSSL,L=San Francisco,ST=California,C=US","src":"10.156.0.2","dst":"89.160.20.156","p":443,"actions":["Notice::ACTION_LOG"],"suppress_for":3600.0} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/sip.log b/packages/zeek/_dev/deploy/docker/sample_logs/sip.log index 60e4d651f42..e58163306df 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/sip.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/sip.log @@ -2,4 +2,4 @@ {"ts":1105725482.965944,"uid":"ComJz236lSOcuOmix3","id.orig_h":"89.160.20.156","id.orig_p":5061,"id.resp_h":"89.160.20.156","id.resp_p":5060,"trans_depth":0,"method":"INVITE","uri":"sip:francisco@bestel.com:55060","request_from":"","request_to":"\u0022francisco@bestel.com\u0022 ","response_from":"","response_to":"\u0022francisco@bestel.com\u0022 ;tag=298852044","call_id":"12013223@89.160.20.156","seq":"1 INVITE","request_path":["SIP/2.0/UDP 89.160.20.156","SIP/2.0/UDP 89.160.20.156:55061"],"response_path":["SIP/2.0/UDP 89.160.20.156","SIP/2.0/UDP 89.160.20.156:55061","SIP/2.0/UDP 89.160.20.156","SIP/2.0/UDP 89.160.20.156:55061"],"status_code":180,"status_msg":"Ringing","request_body_len":229,"response_body_len":0} {"ts":1105725487.022577,"uid":"CJZDWgixtwqXctWEg","id.orig_h":"89.160.20.156","id.orig_p":5061,"id.resp_h":"89.160.20.156","id.resp_p":5060,"trans_depth":0,"method":"REGISTER","uri":"sip:Verso.com","request_from":"Ivan ","request_to":"Ivan ","response_from":"\u0022Ivan\u0022 ","response_to":"\u0022Ivan\u0022 ","call_id":"46E1C3CB36304F84A020CF6DD3F96461@Verso.com","seq":"37764 REGISTER","request_path":["SIP/2.0/UDP 89.160.20.156:5061;rport"],"response_path":["SIP/2.0/UDP 89.160.20.156:5061;received=89.160.20.156;rport=5061"],"user_agent":"Verso Softphone release 1104w","status_code":200,"status_msg":"OK","request_body_len":0,"response_body_len":0} {"ts":1617119416.928735,"uid":"CR6XQH1Lf2mF9YG7H2","id.orig_h":"89.160.20.156","id.orig_p":5083,"id.resp_h":"10.156.0.2","id.resp_p":5060,"trans_depth":0,"method":"OPTIONS","uri":"sip:100@89.160.20.156","request_from":"\"sipvicious\"","request_to":"\"sipvicious\"","call_id":"767538559354206383610151","seq":"1 OPTIONS","request_path":["SIP/2.0/UDP 89.160.20.156:5083"],"response_path":[],"user_agent":"friendly-scanner","request_body_len":0} -{"ts":1617119923.416653,"uid":"Cf9QMt4ear7ZkX74ti","id.orig_h":"89.160.20.156","id.orig_p":5170,"id.resp_h":"10.156.0.2","id.resp_p":5060,"trans_depth":0,"method":"OPTIONS","uri":"sip:100@89.160.20.156","request_from":"\"sipvicious\"","request_to":"\"sipvicious\"","call_id":"35848812076538877174452","seq":"1 OPTIONS","request_path":["SIP/2.0/UDP 127.0.0.1:5170"],"response_path":[],"user_agent":"friendly-scanner","request_body_len":0} \ No newline at end of file +{"ts":1617119923.416653,"uid":"Cf9QMt4ear7ZkX74ti","id.orig_h":"89.160.20.156","id.orig_p":5170,"id.resp_h":"10.156.0.2","id.resp_p":5060,"trans_depth":0,"method":"OPTIONS","uri":"sip:100@89.160.20.156","request_from":"\"sipvicious\"","request_to":"\"sipvicious\"","call_id":"35848812076538877174452","seq":"1 OPTIONS","request_path":["SIP/2.0/UDP 127.0.0.1:5170"],"response_path":[],"user_agent":"friendly-scanner","request_body_len":0} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/snmp.log b/packages/zeek/_dev/deploy/docker/sample_logs/snmp.log index 4d278a60ad1..efd48762a36 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/snmp.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/snmp.log @@ -1,2 +1,2 @@ {"ts":1543877948.916584,"uid":"CnKW1B4w9fpRa6Nkf2","id.orig_h":"192.168.1.2","id.orig_p":59696,"id.resp_h":"192.168.1.1","id.resp_p":161,"duration":7.849924,"version":"2c","community":"public","get_requests":0,"get_bulk_requests":0,"get_responses":8,"set_requests":0,"up_since":1543631204.766508} -{"ts":1617080496.400704,"uid":"CxtWIB4ECPW89F8mSi","id.orig_h":"89.160.20.156","id.orig_p":37533,"id.resp_h":"10.156.0.2","id.resp_p":161,"duration":0.0,"version":"2c","community":"public","get_requests":4,"get_bulk_requests":0,"get_responses":0,"set_requests":0} \ No newline at end of file +{"ts":1617080496.400704,"uid":"CxtWIB4ECPW89F8mSi","id.orig_h":"89.160.20.156","id.orig_p":37533,"id.resp_h":"10.156.0.2","id.resp_p":161,"duration":0.0,"version":"2c","community":"public","get_requests":4,"get_bulk_requests":0,"get_responses":0,"set_requests":0} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/software.log b/packages/zeek/_dev/deploy/docker/sample_logs/software.log new file mode 100644 index 00000000000..5cb34fbddd5 --- /dev/null +++ b/packages/zeek/_dev/deploy/docker/sample_logs/software.log @@ -0,0 +1 @@ +{"ts":"2021-01-03T00:16:22.694616Z","host":"192.168.4.25","software_type":"HTTP::BROWSER","name":"Windows-Update-Agent","version.major":10,"version.minor":0,"version.minor2":10011,"version.minor3":16384,"version.addl":"Client","unparsed_version":"Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0"} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/ssh.log b/packages/zeek/_dev/deploy/docker/sample_logs/ssh.log index 85ccd975ac4..35bce20dd64 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/ssh.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/ssh.log @@ -1,4 +1,4 @@ {"ts":1562527532.904291,"uid":"CajWfz1b3qnnWT0BU9","id.orig_h":"192.168.1.2","id.orig_p":48380,"id.resp_h":"192.168.1.1","id.resp_p":22,"version":2,"auth_success":false,"auth_attempts":2,"client":"SSH-2.0-OpenSSH_7.9p1 Ubuntu-10","server":"SSH-2.0-OpenSSH_6.6.1p1 Debian-4~bpo70+1","cipher_alg":"chacha20-poly1305@openssh.com","mac_alg":"umac-64-etm@openssh.com","compression_alg":"none","kex_alg":"curve25519-sha256@libssh.org","host_key_alg":"ecdsa-sha2-nistp256","host_key":"86:71:ac:9c:35:1c:28:29:05:81:48:ec:66:67:de:bd"} {"ts":1617123417.413634,"uid":"COXxsJ3dlSh6ECRYQj","id.orig_h":"89.160.20.156","id.orig_p":38204,"id.resp_h":"10.156.0.2","id.resp_p":22,"auth_attempts":0,"direction":"INBOUND","client":"SSH-2.0-libssh-0.6.3"} {"ts":1617123445.61524,"uid":"CZPdXz1jfKSWzIDAeb","id.orig_h":"89.160.20.156","id.orig_p":44164,"id.resp_h":"10.156.0.2","id.resp_p":22,"auth_attempts":0,"direction":"INBOUND","client":"SSH-2.0-libssh-0.6.3"} -{"ts":1617123450.957272,"uid":"Cha1rs3OamonAZ4Nz6","id.orig_h":"89.160.20.156","id.orig_p":33953,"id.resp_h":"10.156.0.2","id.resp_p":22,"auth_attempts":0,"direction":"INBOUND","client":"SSH-2.0-ZGrab ZGrab SSH Survey"} \ No newline at end of file +{"ts":1617123450.957272,"uid":"Cha1rs3OamonAZ4Nz6","id.orig_h":"89.160.20.156","id.orig_p":33953,"id.resp_h":"10.156.0.2","id.resp_p":22,"auth_attempts":0,"direction":"INBOUND","client":"SSH-2.0-ZGrab ZGrab SSH Survey"} diff --git a/packages/zeek/_dev/deploy/docker/sample_logs/ssl.log b/packages/zeek/_dev/deploy/docker/sample_logs/ssl.log index a8a0385eb0a..12fa6598f20 100644 --- a/packages/zeek/_dev/deploy/docker/sample_logs/ssl.log +++ b/packages/zeek/_dev/deploy/docker/sample_logs/ssl.log @@ -6,4 +6,4 @@ {"ts":1617091253.726384,"uid":"C4jH9IqWGZwc1PPUh","id.orig_h":"89.160.20.156","id.orig_p":53368,"id.resp_h":"10.156.0.2","id.resp_p":443,"server_name":"tickets.swiftcrypto.com","resumed":false,"established":false} {"ts":1617091253.91861,"uid":"CXVMSq6Dainy4WFN9","id.orig_h":"89.160.20.156","id.orig_p":53382,"id.resp_h":"10.156.0.2","id.resp_p":443,"server_name":"rundeck.swiftcrypto.com","resumed":false,"established":false} {"ts":1617091254.325291,"uid":"CsgtQe4AikDZBsIM6k","id.orig_h":"10.156.0.2","id.orig_p":55120,"id.resp_h":"89.160.20.156","id.resp_p":443,"version":"TLSv12","cipher":"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256","curve":"secp256r1","resumed":false,"established":false,"cert_chain_fuids":["FeyRIk4nUtwwcUcnRf"],"client_cert_chain_fuids":[],"validation_status":"self signed certificate"} -{"ts":1617091255.065602,"uid":"CPGhJS3UPpcnR96NQc","id.orig_h":"89.160.20.156","id.orig_p":53095,"id.resp_h":"10.156.0.2","id.resp_p":443,"server_name":"splunk-api.swiftcrypto.com","resumed":false,"established":false} \ No newline at end of file +{"ts":1617091255.065602,"uid":"CPGhJS3UPpcnR96NQc","id.orig_h":"89.160.20.156","id.orig_p":53095,"id.resp_h":"10.156.0.2","id.resp_p":443,"server_name":"splunk-api.swiftcrypto.com","resumed":false,"established":false} diff --git a/packages/zeek/changelog.yml b/packages/zeek/changelog.yml index fde5da00436..b58caf339b4 100644 --- a/packages/zeek/changelog.yml +++ b/packages/zeek/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.0" + changes: + - description: Add new data sets for known_hosts, known_certs, known_services, & software logs files. + type: enhancement + link: https://github.com/elastic/integrations/pull/3340 - version: "2.1.0" changes: - description: Add JA3/JA3S parsing & fix certificate data parsing; hash, not valid before/after timestamps diff --git a/packages/zeek/data_stream/capture_loss/manifest.yml b/packages/zeek/data_stream/capture_loss/manifest.yml index 5f66ef81d25..db5171babd1 100644 --- a/packages/zeek/data_stream/capture_loss/manifest.yml +++ b/packages/zeek/data_stream/capture_loss/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-capture-loss - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/connection/manifest.yml b/packages/zeek/data_stream/connection/manifest.yml index 05f23885b62..088dc55b555 100644 --- a/packages/zeek/data_stream/connection/manifest.yml +++ b/packages/zeek/data_stream/connection/manifest.yml @@ -21,6 +21,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-connection - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/dce_rpc/manifest.yml b/packages/zeek/data_stream/dce_rpc/manifest.yml index 557f87bc917..5db353cc78c 100644 --- a/packages/zeek/data_stream/dce_rpc/manifest.yml +++ b/packages/zeek/data_stream/dce_rpc/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-dce-rpc - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/dhcp/manifest.yml b/packages/zeek/data_stream/dhcp/manifest.yml index f99de0f1556..78f6d098b1e 100644 --- a/packages/zeek/data_stream/dhcp/manifest.yml +++ b/packages/zeek/data_stream/dhcp/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-dhcp template_path: log.yml.hbs title: Zeek dhcp.log diff --git a/packages/zeek/data_stream/dns/manifest.yml b/packages/zeek/data_stream/dns/manifest.yml index cfb0e18a570..d655e5a773a 100644 --- a/packages/zeek/data_stream/dns/manifest.yml +++ b/packages/zeek/data_stream/dns/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-dns - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/ftp/manifest.yml b/packages/zeek/data_stream/ftp/manifest.yml index 3b1974cbb3a..d3f67f3067c 100644 --- a/packages/zeek/data_stream/ftp/manifest.yml +++ b/packages/zeek/data_stream/ftp/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-ftp - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/http/manifest.yml b/packages/zeek/data_stream/http/manifest.yml index 6fae84056ff..5e5e0e36fa1 100644 --- a/packages/zeek/data_stream/http/manifest.yml +++ b/packages/zeek/data_stream/http/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-http - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/intel/manifest.yml b/packages/zeek/data_stream/intel/manifest.yml index 723e99f395f..de479c71e76 100644 --- a/packages/zeek/data_stream/intel/manifest.yml +++ b/packages/zeek/data_stream/intel/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-intel - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/irc/manifest.yml b/packages/zeek/data_stream/irc/manifest.yml index 42e693551ec..dd8a3894675 100644 --- a/packages/zeek/data_stream/irc/manifest.yml +++ b/packages/zeek/data_stream/irc/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-irc - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/kerberos/manifest.yml b/packages/zeek/data_stream/kerberos/manifest.yml index 8de52255d55..20b96a58013 100644 --- a/packages/zeek/data_stream/kerberos/manifest.yml +++ b/packages/zeek/data_stream/kerberos/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-kerberos - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-common-config.yml b/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..946e0d498f7 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,4 @@ +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log b/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log new file mode 100644 index 00000000000..2966451c2f2 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log @@ -0,0 +1 @@ +{"ts":"2020-12-31T15:15:53.690221Z","host":"192.168.4.1","port_num":443,"subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","issuer_subject":"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US","serial":"98D0AD47D748CDD6"} \ No newline at end of file diff --git a/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log-expected.json b/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log-expected.json new file mode 100644 index 00000000000..f4acc25c3d7 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/_dev/test/pipeline/test-known_certs.log-expected.json @@ -0,0 +1,57 @@ +{ + "expected": [ + { + "@timestamp": "2020-12-31T15:15:53.690Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "file" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2020-12-31T15:15:53.690221Z\",\"host\":\"192.168.4.1\",\"port_num\":443,\"subject\":\"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US\",\"issuer_subject\":\"L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US\",\"serial\":\"98D0AD47D748CDD6\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.1" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.1" + ] + }, + "server": { + "ip": "192.168.4.1", + "port": 443 + }, + "tags": [ + "preserve_original_event" + ], + "tls": { + "server": { + "issuer": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US", + "subject": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US", + "x509": { + "issuer": { + "common_name": "UBNT Router UI", + "distinguished_name": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US" + }, + "serial_number": "98D0AD47D748CDD6", + "subject": { + "common_name": "UBNT Router UI", + "distinguished_name": "L=San Jose,ST=CA,O=Ubiquiti Networks,CN=UBNT Router UI,C=US" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/packages/zeek/data_stream/known_certs/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/known_certs/_dev/test/system/test-logs-config.yml new file mode 100644 index 00000000000..5cfff500299 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/_dev/test/system/test-logs-config.yml @@ -0,0 +1,6 @@ +vars: + base_paths: + - "{{SERVICE_LOGS_DIR}}" +input: logfile +data_stream: + vars: ~ diff --git a/packages/zeek/data_stream/known_certs/agent/stream/log.yml.hbs b/packages/zeek/data_stream/known_certs/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..9dd9f724a5d --- /dev/null +++ b/packages/zeek/data_stream/known_certs/agent/stream/log.yml.hbs @@ -0,0 +1,21 @@ +paths: +{{#each base_paths}} + {{#each ../filenames}} + - {{../this}}/{{this}} + {{/each}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..1a08b943668 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,129 @@ +--- +description: Pipeline for normalizing Zeek known_certs.log +processors: + - rename: + field: message + target_field: event.original + - json: + field: event.original + target_field: json + - drop: + description: Drop if no timestamp (invalid json) + if: 'ctx?.json?.ts == null' + +# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down + - set: + field: event.created + copy_from: "@timestamp" + - set: + field: ecs.version + value: '8.2.0' + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - network + - file + - set: + field: event.type + value: + - info + - date: + field: json.ts + formats: + - UNIX + - ISO8601 + - rename: + field: json.host + target_field: host.ip + ignore_missing: true + - set: + field: network.type + value: ipv4 + if: ctx.host?.ip.contains('.') + - set: + field: network.type + value: ipv6 + if: ctx.host?.ip.contains(':') + - append: + field: related.ip + value: "{{host.ip}}" + if: ctx?.host?.ip != null + allow_duplicates: false + - geoip: + field: host.ip + target_field: host.geo + ignore_missing: true + - set: + field: server + copy_from: host + ignore_empty_value: true + - rename: + field: json.port_num + target_field: server.port + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + - rename: + field: json.subject + target_field: tls.server.x509.subject.distinguished_name + ignore_missing: true + - rename: + field: json.issuer_subject + target_field: tls.server.x509.issuer.distinguished_name + ignore_missing: true + - rename: + field: json.serial + target_field: tls.server.x509.serial_number + ignore_missing: true + - grok: + field: tls.server.x509.subject.distinguished_name + ignore_missing: true + patterns: + - 'CN=%{CN:tls.server.x509.subject.common_name}' + pattern_definitions: + CN: '[^,]+' + - grok: + field: tls.server.x509.issuer.distinguished_name + ignore_missing: true + patterns: + - 'CN=%{CN:tls.server.x509.issuer.common_name}' + pattern_definitions: + CN: '[^,]+' + - set: + field: tls.server.issuer + copy_from: tls.server.x509.issuer.distinguished_name + ignore_empty_value: true + - set: + field: tls.server.subject + copy_from: tls.server.x509.subject.distinguished_name + ignore_empty_value: true + - remove: + field: + - json + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/zeek/data_stream/known_certs/fields/agent.yml b/packages/zeek/data_stream/known_certs/fields/agent.yml new file mode 100644 index 00000000000..79a7a39864b --- /dev/null +++ b/packages/zeek/data_stream/known_certs/fields/agent.yml @@ -0,0 +1,180 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + level: core + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/zeek/data_stream/known_certs/fields/base-fields.yml b/packages/zeek/data_stream/known_certs/fields/base-fields.yml new file mode 100644 index 00000000000..2867f9687f2 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: zeek +- name: event.dataset + type: constant_keyword + description: Event dataset + value: zeek.known_certs +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/zeek/data_stream/known_certs/fields/beats.yml b/packages/zeek/data_stream/known_certs/fields/beats.yml new file mode 100644 index 00000000000..470f5fae484 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/fields/beats.yml @@ -0,0 +1,23 @@ +- description: Unique container id. + ignore_above: 1024 + name: container.id + type: keyword +- description: Type of Filebeat input. + name: input.type + type: keyword +- description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + name: log.file.path + type: keyword +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset + type: long +- description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + name: tags + type: keyword diff --git a/packages/zeek/data_stream/known_certs/fields/ecs.yml b/packages/zeek/data_stream/known_certs/fields/ecs.yml new file mode 100644 index 00000000000..9abb66fa435 --- /dev/null +++ b/packages/zeek/data_stream/known_certs/fields/ecs.yml @@ -0,0 +1,68 @@ +- name: ecs.version + external: ecs +- name: event.category + external: ecs +- name: event.created + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: network.type + external: ecs +- name: related.ip + external: ecs +- name: host.geo.city_name + external: ecs +- name: host.geo.continent_name + external: ecs +- name: host.geo.country_iso_code + external: ecs +- name: host.geo.country_name + external: ecs +- name: host.geo.location + external: ecs +- name: host.geo.name + external: ecs +- name: host.geo.region_iso_code + external: ecs +- name: host.geo.region_name + external: ecs +- name: host.ip + external: ecs +- name: server.geo.city_name + external: ecs +- name: server.geo.continent_name + external: ecs +- name: server.geo.country_iso_code + external: ecs +- name: server.geo.country_name + external: ecs +- name: server.geo.location + external: ecs +- name: server.geo.name + external: ecs +- name: server.geo.region_iso_code + external: ecs +- name: server.geo.region_name + external: ecs +- name: server.ip + external: ecs +- name: server.port + external: ecs +- name: tls.server.x509.issuer.distinguished_name + external: ecs +- name: tls.server.x509.serial_number + external: ecs +- name: tls.server.x509.subject.distinguished_name + external: ecs +- name: tls.server.x509.issuer.common_name + external: ecs +- name: tls.server.x509.subject.common_name + external: ecs +- name: tls.server.issuer + external: ecs +- name: tls.server.subject + external: ecs diff --git a/packages/zeek/data_stream/known_certs/manifest.yml b/packages/zeek/data_stream/known_certs/manifest.yml new file mode 100644 index 00000000000..19a7c17b2db --- /dev/null +++ b/packages/zeek/data_stream/known_certs/manifest.yml @@ -0,0 +1,41 @@ +type: logs +title: Zeek Known Certs logs +streams: + - input: logfile + template_path: log.yml.hbs + title: Zeek known_certs.log + description: Collect Zeek Known Certs logs + vars: + - name: filenames + type: text + title: Filename of Known Certs log + multi: true + required: true + show_user: true + default: + - known_certs.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - zeek-known_certs + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-common-config.yml b/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..946e0d498f7 --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,4 @@ +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log b/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log new file mode 100644 index 00000000000..1dce8949c6d --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log @@ -0,0 +1,5 @@ +{"ts":"2021-01-03T01:19:26.260073Z","host":"192.168.4.25"} +{"ts":"2021-01-03T01:19:27.353353Z","host":"192.168.4.29"} +{"ts":"2021-01-03T01:19:32.488179Z","host":"192.168.4.43"} +{"ts":"2021-01-03T01:19:58.792683Z","host":"192.168.4.142"} +{"ts":"2021-01-03T12:17:22.496004Z","host":"192.168.4.115"} \ No newline at end of file diff --git a/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log-expected.json b/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log-expected.json new file mode 100644 index 00000000000..c4fae98020f --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/_dev/test/pipeline/test-known_hosts.log-expected.json @@ -0,0 +1,164 @@ +{ + "expected": [ + { + "@timestamp": "2021-01-03T01:19:26.260Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "host" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T01:19:26.260073Z\",\"host\":\"192.168.4.25\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.25" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.25" + ] + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-01-03T01:19:27.353Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "host" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T01:19:27.353353Z\",\"host\":\"192.168.4.29\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.29" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.29" + ] + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-01-03T01:19:32.488Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "host" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T01:19:32.488179Z\",\"host\":\"192.168.4.43\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.43" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.43" + ] + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-01-03T01:19:58.792Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "host" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T01:19:58.792683Z\",\"host\":\"192.168.4.142\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.142" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.142" + ] + }, + "tags": [ + "preserve_original_event" + ] + }, + { + "@timestamp": "2021-01-03T12:17:22.496Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "host" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T12:17:22.496004Z\",\"host\":\"192.168.4.115\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.115" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.115" + ] + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/zeek/data_stream/known_hosts/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/known_hosts/_dev/test/system/test-logs-config.yml new file mode 100644 index 00000000000..5cfff500299 --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/_dev/test/system/test-logs-config.yml @@ -0,0 +1,6 @@ +vars: + base_paths: + - "{{SERVICE_LOGS_DIR}}" +input: logfile +data_stream: + vars: ~ diff --git a/packages/zeek/data_stream/known_hosts/agent/stream/log.yml.hbs b/packages/zeek/data_stream/known_hosts/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..9dd9f724a5d --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/agent/stream/log.yml.hbs @@ -0,0 +1,21 @@ +paths: +{{#each base_paths}} + {{#each ../filenames}} + - {{../this}}/{{this}} + {{/each}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..74babbe4c71 --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,71 @@ +--- +description: Pipeline for normalizing Zeek known_hosts.log +processors: + - rename: + field: message + target_field: event.original + - json: + field: event.original + target_field: json + - drop: + description: Drop if no timestamp (invalid json) + if: 'ctx?.json?.ts == null' + +# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down + - set: + field: event.created + copy_from: "@timestamp" + - set: + field: ecs.version + value: '8.2.0' + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - network + - host + - set: + field: event.type + value: + - info + - date: + field: json.ts + formats: + - UNIX + - ISO8601 + - rename: + field: json.host + target_field: host.ip + ignore_missing: true + - set: + field: network.type + value: ipv4 + if: ctx.host?.ip.contains('.') + - set: + field: network.type + value: ipv6 + if: ctx.host?.ip.contains(':') + - append: + field: related.ip + value: "{{host.ip}}" + if: ctx?.host?.ip != null + allow_duplicates: false + - geoip: + field: host.ip + target_field: host.geo + ignore_missing: true + - remove: + field: + - json + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/zeek/data_stream/known_hosts/fields/agent.yml b/packages/zeek/data_stream/known_hosts/fields/agent.yml new file mode 100644 index 00000000000..79a7a39864b --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/fields/agent.yml @@ -0,0 +1,180 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + level: core + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/zeek/data_stream/known_hosts/fields/base-fields.yml b/packages/zeek/data_stream/known_hosts/fields/base-fields.yml new file mode 100644 index 00000000000..3f252f24b8a --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: zeek +- name: event.dataset + type: constant_keyword + description: Event dataset + value: zeek.known_hosts +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/zeek/data_stream/known_hosts/fields/beats.yml b/packages/zeek/data_stream/known_hosts/fields/beats.yml new file mode 100644 index 00000000000..470f5fae484 --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/fields/beats.yml @@ -0,0 +1,23 @@ +- description: Unique container id. + ignore_above: 1024 + name: container.id + type: keyword +- description: Type of Filebeat input. + name: input.type + type: keyword +- description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + name: log.file.path + type: keyword +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset + type: long +- description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + name: tags + type: keyword diff --git a/packages/zeek/data_stream/known_hosts/fields/ecs.yml b/packages/zeek/data_stream/known_hosts/fields/ecs.yml new file mode 100644 index 00000000000..588dc3de030 --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/fields/ecs.yml @@ -0,0 +1,34 @@ +- name: ecs.version + external: ecs +- name: event.category + external: ecs +- name: event.created + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: network.type + external: ecs +- name: related.ip + external: ecs +- name: host.geo.city_name + external: ecs +- name: host.geo.continent_name + external: ecs +- name: host.geo.country_iso_code + external: ecs +- name: host.geo.country_name + external: ecs +- name: host.geo.location + external: ecs +- name: host.geo.name + external: ecs +- name: host.geo.region_iso_code + external: ecs +- name: host.geo.region_name + external: ecs +- name: host.ip + external: ecs diff --git a/packages/zeek/data_stream/known_hosts/manifest.yml b/packages/zeek/data_stream/known_hosts/manifest.yml new file mode 100644 index 00000000000..af25548e320 --- /dev/null +++ b/packages/zeek/data_stream/known_hosts/manifest.yml @@ -0,0 +1,41 @@ +type: logs +title: Zeek Known Hosts logs +streams: + - input: logfile + template_path: log.yml.hbs + title: Zeek known_hosts.log + description: Collect Zeek Known Hosts logs + vars: + - name: filenames + type: text + title: Filename of Known Hosts log + multi: true + required: true + show_user: true + default: + - known_hosts.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - zeek-known_hosts + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-common-config.yml b/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..946e0d498f7 --- /dev/null +++ b/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,4 @@ +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log b/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log new file mode 100644 index 00000000000..cfa59481be7 --- /dev/null +++ b/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log @@ -0,0 +1 @@ +{"ts":"2021-01-03T01:19:36.242774Z","host":"192.168.4.1","port_num":53,"port_proto":"udp","service":["DNS"]} \ No newline at end of file diff --git a/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log-expected.json b/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log-expected.json new file mode 100644 index 00000000000..97efbd8caaf --- /dev/null +++ b/packages/zeek/data_stream/known_services/_dev/test/pipeline/test-known_services.log-expected.json @@ -0,0 +1,43 @@ +{ + "expected": [ + { + "@timestamp": "2021-01-03T01:19:36.242Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T01:19:36.242774Z\",\"host\":\"192.168.4.1\",\"port_num\":53,\"port_proto\":\"udp\",\"service\":[\"DNS\"]}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.1" + }, + "network": { + "application": [ + "DNS" + ], + "transport": "udp", + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.1" + ] + }, + "server": { + "ip": "192.168.4.1", + "port": 53 + }, + "tags": [ + "preserve_original_event" + ] + } + ] +} \ No newline at end of file diff --git a/packages/zeek/data_stream/known_services/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/known_services/_dev/test/system/test-logs-config.yml new file mode 100644 index 00000000000..5cfff500299 --- /dev/null +++ b/packages/zeek/data_stream/known_services/_dev/test/system/test-logs-config.yml @@ -0,0 +1,6 @@ +vars: + base_paths: + - "{{SERVICE_LOGS_DIR}}" +input: logfile +data_stream: + vars: ~ diff --git a/packages/zeek/data_stream/known_services/agent/stream/log.yml.hbs b/packages/zeek/data_stream/known_services/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..9dd9f724a5d --- /dev/null +++ b/packages/zeek/data_stream/known_services/agent/stream/log.yml.hbs @@ -0,0 +1,21 @@ +paths: +{{#each base_paths}} + {{#each ../filenames}} + - {{../this}}/{{this}} + {{/each}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..c824b8a42c6 --- /dev/null +++ b/packages/zeek/data_stream/known_services/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,102 @@ +--- +description: Pipeline for normalizing Zeek known_services.log +processors: + - rename: + field: message + target_field: event.original + - json: + field: event.original + target_field: json + - drop: + description: Drop if no timestamp (invalid json) + if: 'ctx?.json?.ts == null' + +# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down + - set: + field: event.created + copy_from: "@timestamp" + - set: + field: ecs.version + value: '8.2.0' + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - network + - set: + field: event.type + value: + - info + - date: + field: json.ts + formats: + - UNIX + - ISO8601 + - rename: + field: json.host + target_field: host.ip + ignore_missing: true + - set: + field: network.type + value: ipv4 + if: ctx.host?.ip.contains('.') + - set: + field: network.type + value: ipv6 + if: ctx.host?.ip.contains(':') + - append: + field: related.ip + value: "{{host.ip}}" + if: ctx?.host?.ip != null + allow_duplicates: false + - geoip: + field: host.ip + target_field: host.geo + ignore_missing: true + - set: + field: server + copy_from: host + ignore_empty_value: true + - rename: + field: json.port_num + target_field: server.port + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: server.ip + target_field: server.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: server.as.asn + target_field: server.as.number + ignore_missing: true + - rename: + field: server.as.organization_name + target_field: server.as.organization.name + ignore_missing: true + - rename: + field: json.port_proto + target_field: network.transport + ignore_missing: true + - rename: + field: json.service + target_field: network.application + ignore_missing: true + - remove: + field: + - json + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/zeek/data_stream/known_services/fields/agent.yml b/packages/zeek/data_stream/known_services/fields/agent.yml new file mode 100644 index 00000000000..79a7a39864b --- /dev/null +++ b/packages/zeek/data_stream/known_services/fields/agent.yml @@ -0,0 +1,180 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + level: core + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/zeek/data_stream/known_services/fields/base-fields.yml b/packages/zeek/data_stream/known_services/fields/base-fields.yml new file mode 100644 index 00000000000..ecbd3a015c6 --- /dev/null +++ b/packages/zeek/data_stream/known_services/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: zeek +- name: event.dataset + type: constant_keyword + description: Event dataset + value: zeek.known_services +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/zeek/data_stream/known_services/fields/beats.yml b/packages/zeek/data_stream/known_services/fields/beats.yml new file mode 100644 index 00000000000..470f5fae484 --- /dev/null +++ b/packages/zeek/data_stream/known_services/fields/beats.yml @@ -0,0 +1,23 @@ +- description: Unique container id. + ignore_above: 1024 + name: container.id + type: keyword +- description: Type of Filebeat input. + name: input.type + type: keyword +- description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + name: log.file.path + type: keyword +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset + type: long +- description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + name: tags + type: keyword diff --git a/packages/zeek/data_stream/known_services/fields/ecs.yml b/packages/zeek/data_stream/known_services/fields/ecs.yml new file mode 100644 index 00000000000..65b327b55e9 --- /dev/null +++ b/packages/zeek/data_stream/known_services/fields/ecs.yml @@ -0,0 +1,58 @@ +- name: ecs.version + external: ecs +- name: event.category + external: ecs +- name: event.created + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: network.type + external: ecs +- name: network.application + external: ecs +- name: network.transport + external: ecs +- name: related.ip + external: ecs +- name: host.geo.city_name + external: ecs +- name: host.geo.continent_name + external: ecs +- name: host.geo.country_iso_code + external: ecs +- name: host.geo.country_name + external: ecs +- name: host.geo.location + external: ecs +- name: host.geo.name + external: ecs +- name: host.geo.region_iso_code + external: ecs +- name: host.geo.region_name + external: ecs +- name: host.ip + external: ecs +- name: server.geo.city_name + external: ecs +- name: server.geo.continent_name + external: ecs +- name: server.geo.country_iso_code + external: ecs +- name: server.geo.country_name + external: ecs +- name: server.geo.location + external: ecs +- name: server.geo.name + external: ecs +- name: server.geo.region_iso_code + external: ecs +- name: server.geo.region_name + external: ecs +- name: server.ip + external: ecs +- name: server.port + external: ecs diff --git a/packages/zeek/data_stream/known_services/manifest.yml b/packages/zeek/data_stream/known_services/manifest.yml new file mode 100644 index 00000000000..4b5ebb150d2 --- /dev/null +++ b/packages/zeek/data_stream/known_services/manifest.yml @@ -0,0 +1,41 @@ +type: logs +title: Zeek Known Services logs +streams: + - input: logfile + template_path: log.yml.hbs + title: Zeek known_services.log + description: Collect Zeek Known Services logs + vars: + - name: filenames + type: text + title: Filename of Known Services log + multi: true + required: true + show_user: true + default: + - known_services.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - zeek-known_services + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/zeek/data_stream/modbus/manifest.yml b/packages/zeek/data_stream/modbus/manifest.yml index eb770cf866d..ab505158c15 100644 --- a/packages/zeek/data_stream/modbus/manifest.yml +++ b/packages/zeek/data_stream/modbus/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-modbus - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/mysql/manifest.yml b/packages/zeek/data_stream/mysql/manifest.yml index 9acee92a449..1d8e9acee69 100644 --- a/packages/zeek/data_stream/mysql/manifest.yml +++ b/packages/zeek/data_stream/mysql/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-mysql - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/notice/manifest.yml b/packages/zeek/data_stream/notice/manifest.yml index 230a456c4b1..59afd2ab57e 100644 --- a/packages/zeek/data_stream/notice/manifest.yml +++ b/packages/zeek/data_stream/notice/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-notice - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/ntlm/manifest.yml b/packages/zeek/data_stream/ntlm/manifest.yml index 38c3afa8a3d..d56238fb0c9 100644 --- a/packages/zeek/data_stream/ntlm/manifest.yml +++ b/packages/zeek/data_stream/ntlm/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-ntlm - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/ntp/manifest.yml b/packages/zeek/data_stream/ntp/manifest.yml index 59b3e3bae58..f450b993be3 100644 --- a/packages/zeek/data_stream/ntp/manifest.yml +++ b/packages/zeek/data_stream/ntp/manifest.yml @@ -21,6 +21,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-ntp - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/ocsp/manifest.yml b/packages/zeek/data_stream/ocsp/manifest.yml index 1066b168a89..6cca1eabd2e 100644 --- a/packages/zeek/data_stream/ocsp/manifest.yml +++ b/packages/zeek/data_stream/ocsp/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-ocsp - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/pe/manifest.yml b/packages/zeek/data_stream/pe/manifest.yml index 7387997eba4..529d8abb401 100644 --- a/packages/zeek/data_stream/pe/manifest.yml +++ b/packages/zeek/data_stream/pe/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-pe - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/radius/manifest.yml b/packages/zeek/data_stream/radius/manifest.yml index 5ca5cd766b8..b703d8bfce3 100644 --- a/packages/zeek/data_stream/radius/manifest.yml +++ b/packages/zeek/data_stream/radius/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-radius - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/rdp/manifest.yml b/packages/zeek/data_stream/rdp/manifest.yml index ba3f66a0e50..02303490edd 100644 --- a/packages/zeek/data_stream/rdp/manifest.yml +++ b/packages/zeek/data_stream/rdp/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-rdp - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/rfb/manifest.yml b/packages/zeek/data_stream/rfb/manifest.yml index dc620d9e21a..b5513bb69d9 100644 --- a/packages/zeek/data_stream/rfb/manifest.yml +++ b/packages/zeek/data_stream/rfb/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-rfb - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/signature/manifest.yml b/packages/zeek/data_stream/signature/manifest.yml index 355c10e5a05..bd9ca42ed43 100644 --- a/packages/zeek/data_stream/signature/manifest.yml +++ b/packages/zeek/data_stream/signature/manifest.yml @@ -21,6 +21,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-signature - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/sip/manifest.yml b/packages/zeek/data_stream/sip/manifest.yml index 010396ae006..d922c5d29ce 100644 --- a/packages/zeek/data_stream/sip/manifest.yml +++ b/packages/zeek/data_stream/sip/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-sip - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/smb_cmd/manifest.yml b/packages/zeek/data_stream/smb_cmd/manifest.yml index d8387b5cc86..835b2e365ed 100644 --- a/packages/zeek/data_stream/smb_cmd/manifest.yml +++ b/packages/zeek/data_stream/smb_cmd/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-smb-cmd - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/smb_files/manifest.yml b/packages/zeek/data_stream/smb_files/manifest.yml index dcc309d2b6c..a8906ac4d09 100644 --- a/packages/zeek/data_stream/smb_files/manifest.yml +++ b/packages/zeek/data_stream/smb_files/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-smb-files - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/smb_mapping/manifest.yml b/packages/zeek/data_stream/smb_mapping/manifest.yml index 65d967d3c5f..8f43d7dba22 100644 --- a/packages/zeek/data_stream/smb_mapping/manifest.yml +++ b/packages/zeek/data_stream/smb_mapping/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: true default: + - forwarded - zeek.smb_mapping template_path: log.yml.hbs title: Zeek smb_mapping.log diff --git a/packages/zeek/data_stream/smtp/manifest.yml b/packages/zeek/data_stream/smtp/manifest.yml index f6c149387ec..d9f7afcd411 100644 --- a/packages/zeek/data_stream/smtp/manifest.yml +++ b/packages/zeek/data_stream/smtp/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-smtp - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/snmp/manifest.yml b/packages/zeek/data_stream/snmp/manifest.yml index ee4659f751b..8fadc2cfd4b 100644 --- a/packages/zeek/data_stream/snmp/manifest.yml +++ b/packages/zeek/data_stream/snmp/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-snmp - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/socks/manifest.yml b/packages/zeek/data_stream/socks/manifest.yml index b9e1f9af306..5e001fd7a38 100644 --- a/packages/zeek/data_stream/socks/manifest.yml +++ b/packages/zeek/data_stream/socks/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-socks - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log b/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log new file mode 100644 index 00000000000..c6318fa6936 --- /dev/null +++ b/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log @@ -0,0 +1 @@ +{"ts":"2021-01-03T00:16:22.694616Z","host":"192.168.4.25","software_type":"HTTP::BROWSER","name":"Windows-Update-Agent","version.major":10,"version.minor":0,"version.minor2":10011,"version.minor3":16384,"version.addl":"Client","unparsed_version":"Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0"} \ No newline at end of file diff --git a/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log-config.yml b/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log-config.yml new file mode 100644 index 00000000000..946e0d498f7 --- /dev/null +++ b/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log-config.yml @@ -0,0 +1,4 @@ +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + tags: + - preserve_original_event diff --git a/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log-expected.json b/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log-expected.json new file mode 100644 index 00000000000..773da936f8c --- /dev/null +++ b/packages/zeek/data_stream/software/_dev/test/pipeline/test-software.log-expected.json @@ -0,0 +1,50 @@ +{ + "expected": [ + { + "@timestamp": "2021-01-03T00:16:22.694Z", + "ecs": { + "version": "8.2.0" + }, + "event": { + "category": [ + "network", + "file" + ], + "created": "2020-04-28T11:07:58.223Z", + "kind": "event", + "original": "{\"ts\":\"2021-01-03T00:16:22.694616Z\",\"host\":\"192.168.4.25\",\"software_type\":\"HTTP::BROWSER\",\"name\":\"Windows-Update-Agent\",\"version.major\":10,\"version.minor\":0,\"version.minor2\":10011,\"version.minor3\":16384,\"version.addl\":\"Client\",\"unparsed_version\":\"Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0\"}", + "type": [ + "info" + ] + }, + "host": { + "ip": "192.168.4.25" + }, + "network": { + "type": "ipv4" + }, + "related": { + "ip": [ + "192.168.4.25" + ] + }, + "tags": [ + "preserve_original_event" + ], + "zeek": { + "software": { + "name": "Windows-Update-Agent", + "type": "HTTP::BROWSER", + "version": { + "additional": "Client", + "full": "Windows-Update-Agent/10.0.10011.16384 Client-Protocol/2.0", + "major": 10, + "minor": 0, + "minor2": 10011, + "minor3": 16384 + } + } + } + } + ] +} \ No newline at end of file diff --git a/packages/zeek/data_stream/software/_dev/test/system/test-logs-config.yml b/packages/zeek/data_stream/software/_dev/test/system/test-logs-config.yml new file mode 100644 index 00000000000..5cfff500299 --- /dev/null +++ b/packages/zeek/data_stream/software/_dev/test/system/test-logs-config.yml @@ -0,0 +1,6 @@ +vars: + base_paths: + - "{{SERVICE_LOGS_DIR}}" +input: logfile +data_stream: + vars: ~ diff --git a/packages/zeek/data_stream/software/agent/stream/log.yml.hbs b/packages/zeek/data_stream/software/agent/stream/log.yml.hbs new file mode 100644 index 00000000000..9dd9f724a5d --- /dev/null +++ b/packages/zeek/data_stream/software/agent/stream/log.yml.hbs @@ -0,0 +1,21 @@ +paths: +{{#each base_paths}} + {{#each ../filenames}} + - {{../this}}/{{this}} + {{/each}} +{{/each}} +exclude_files: [".gz$"] +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml b/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..48db0aff86b --- /dev/null +++ b/packages/zeek/data_stream/software/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,102 @@ +--- +description: Pipeline for normalizing Zeek software.log +processors: + - rename: + field: message + target_field: event.original + - json: + field: event.original + target_field: _temp_ + - drop: + description: Drop if no timestamp (invalid json) + if: 'ctx?._temp_?.ts == null' + - rename: + field: _temp_ + target_field: zeek.software + ignore_failure: true + +# Sets event.created from the @timestamp field generated by filebeat before being overwritten further down + - set: + field: event.created + copy_from: "@timestamp" + - set: + field: ecs.version + value: '8.2.0' + - set: + field: event.kind + value: event + - set: + field: event.category + value: + - network + - file + - set: + field: event.type + value: + - info + - rename: + field: zeek.software.host + target_field: host.ip + ignore_missing: true + - date: + field: zeek.software.ts + formats: + - UNIX + - ISO8601 + - set: + field: network.type + value: ipv4 + if: ctx.host?.ip.contains('.') + - set: + field: network.type + value: ipv6 + if: ctx.host?.ip.contains(':') + - append: + field: related.ip + value: "{{host.ip}}" + if: ctx?.host?.ip != null + allow_duplicates: false + - geoip: + field: host.ip + target_field: host.geo + ignore_missing: true + - rename: + field: zeek.software.software_type + target_field: zeek.software.type + ignore_missing: true + - rename: + field: zeek.software.unparsed_version + target_field: zeek.software.version.full + ignore_missing: true + - dot_expander: + field: version.major + path: zeek.software + - dot_expander: + field: version.minor + path: zeek.software + - dot_expander: + field: version.minor2 + path: zeek.software + - dot_expander: + field: version.minor3 + path: zeek.software + - dot_expander: + field: version.addl + path: zeek.software + - rename: + field: zeek.software.version.addl + target_field: zeek.software.version.additional + ignore_missing: true + - remove: + field: + - zeek.software.ts + ignore_missing: true + - remove: + field: event.original + if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true +on_failure: + - set: + field: error.message + value: "{{ _ingest.on_failure_message }}" diff --git a/packages/zeek/data_stream/software/fields/agent.yml b/packages/zeek/data_stream/software/fields/agent.yml new file mode 100644 index 00000000000..79a7a39864b --- /dev/null +++ b/packages/zeek/data_stream/software/fields/agent.yml @@ -0,0 +1,180 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: "Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on." + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: "The cloud account or organization id used to identify different entities in a multi-tenant environment.\nExamples: AWS account id, Google Cloud ORG Id, or other unique identifier." + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: "Container fields are used for meta information about the specific container that is the source of information.\nThese fields help correlate data based containers from any runtime." + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: "A host is defined as a general computing instance.\nECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes." + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: "Name of the domain of which the host is a member.\nFor example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider." + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: "Hostname of the host.\nIt normally contains what the `hostname` command returns on the host machine." + - name: id + level: core + type: keyword + ignore_above: 1024 + description: "Unique host id.\nAs hostname is not always unique, use values that are meaningful in your environment.\nExample: The current usage of `beat.name`." + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: "Name of the host.\nIt can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use." + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: "Type of host.\nFor Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment." + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/zeek/data_stream/software/fields/base-fields.yml b/packages/zeek/data_stream/software/fields/base-fields.yml new file mode 100644 index 00000000000..642369cceaa --- /dev/null +++ b/packages/zeek/data_stream/software/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: zeek +- name: event.dataset + type: constant_keyword + description: Event dataset + value: zeek.software +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/zeek/data_stream/software/fields/beats.yml b/packages/zeek/data_stream/software/fields/beats.yml new file mode 100644 index 00000000000..470f5fae484 --- /dev/null +++ b/packages/zeek/data_stream/software/fields/beats.yml @@ -0,0 +1,23 @@ +- description: Unique container id. + ignore_above: 1024 + name: container.id + type: keyword +- description: Type of Filebeat input. + name: input.type + type: keyword +- description: Full path to the log file this event came from. + example: /var/log/fun-times.log + ignore_above: 1024 + name: log.file.path + type: keyword +- description: Flags for the log file. + name: log.flags + type: keyword +- description: Offset of the entry in the log file. + name: log.offset + type: long +- description: List of keywords used to tag each event. + example: '["production", "env2"]' + ignore_above: 1024 + name: tags + type: keyword diff --git a/packages/zeek/data_stream/software/fields/ecs.yml b/packages/zeek/data_stream/software/fields/ecs.yml new file mode 100644 index 00000000000..588dc3de030 --- /dev/null +++ b/packages/zeek/data_stream/software/fields/ecs.yml @@ -0,0 +1,34 @@ +- name: ecs.version + external: ecs +- name: event.category + external: ecs +- name: event.created + external: ecs +- name: event.ingested + external: ecs +- name: event.kind + external: ecs +- name: event.type + external: ecs +- name: network.type + external: ecs +- name: related.ip + external: ecs +- name: host.geo.city_name + external: ecs +- name: host.geo.continent_name + external: ecs +- name: host.geo.country_iso_code + external: ecs +- name: host.geo.country_name + external: ecs +- name: host.geo.location + external: ecs +- name: host.geo.name + external: ecs +- name: host.geo.region_iso_code + external: ecs +- name: host.geo.region_name + external: ecs +- name: host.ip + external: ecs diff --git a/packages/zeek/data_stream/software/fields/fields.yml b/packages/zeek/data_stream/software/fields/fields.yml new file mode 100644 index 00000000000..b9011e9f043 --- /dev/null +++ b/packages/zeek/data_stream/software/fields/fields.yml @@ -0,0 +1,46 @@ +- name: zeek.software + type: group + default_field: false + description: > + Fields exported by the Zeek Software log. + + fields: + - name: name + type: keyword + description: > + Name of the software (e.g. Apache). + + - name: type + type: keyword + description: > + The type of software detected + + - name: version.full + type: keyword + description: > + Full unparsed version of the software. + + - name: version.major + type: long + description: > + Major version of software. + + - name: version.minor + type: long + description: > + minor version of software. + + - name: version.minor2 + type: long + description: > + 2nd minor version of software. + + - name: version.minor3 + type: long + description: > + 3rd minor version of software. + + - name: version.additional + type: keyword + description: >- + Additional version information diff --git a/packages/zeek/data_stream/software/manifest.yml b/packages/zeek/data_stream/software/manifest.yml new file mode 100644 index 00000000000..d12de67c7c9 --- /dev/null +++ b/packages/zeek/data_stream/software/manifest.yml @@ -0,0 +1,41 @@ +type: logs +title: Zeek software logs +streams: + - input: logfile + template_path: log.yml.hbs + title: Zeek software.log + description: Collect Zeek software logs + vars: + - name: filenames + type: text + title: Filename of software log + multi: true + required: true + show_user: true + default: + - software.log + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - zeek-software + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: >- + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/zeek/data_stream/ssh/manifest.yml b/packages/zeek/data_stream/ssh/manifest.yml index f01683502d8..be5e501a70e 100644 --- a/packages/zeek/data_stream/ssh/manifest.yml +++ b/packages/zeek/data_stream/ssh/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-ssh - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/ssl/manifest.yml b/packages/zeek/data_stream/ssl/manifest.yml index c9b7afdf922..93ba40223c9 100644 --- a/packages/zeek/data_stream/ssl/manifest.yml +++ b/packages/zeek/data_stream/ssl/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-ssl - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/stats/manifest.yml b/packages/zeek/data_stream/stats/manifest.yml index 215920bc6da..665b9b9781f 100644 --- a/packages/zeek/data_stream/stats/manifest.yml +++ b/packages/zeek/data_stream/stats/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-stats - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/syslog/manifest.yml b/packages/zeek/data_stream/syslog/manifest.yml index f3f6db4679c..ac982fd6b3f 100644 --- a/packages/zeek/data_stream/syslog/manifest.yml +++ b/packages/zeek/data_stream/syslog/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-syslog - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/traceroute/manifest.yml b/packages/zeek/data_stream/traceroute/manifest.yml index d4452526704..15b6db76853 100644 --- a/packages/zeek/data_stream/traceroute/manifest.yml +++ b/packages/zeek/data_stream/traceroute/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-traceroute - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/tunnel/manifest.yml b/packages/zeek/data_stream/tunnel/manifest.yml index 575db110bac..01956ef6809 100644 --- a/packages/zeek/data_stream/tunnel/manifest.yml +++ b/packages/zeek/data_stream/tunnel/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-tunnel - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/weird/manifest.yml b/packages/zeek/data_stream/weird/manifest.yml index d8ec7ea27f5..d2619e9ebe8 100644 --- a/packages/zeek/data_stream/weird/manifest.yml +++ b/packages/zeek/data_stream/weird/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-weird - name: preserve_original_event required: true diff --git a/packages/zeek/data_stream/x509/manifest.yml b/packages/zeek/data_stream/x509/manifest.yml index 98bc3c67d32..ae5b23ca7ee 100644 --- a/packages/zeek/data_stream/x509/manifest.yml +++ b/packages/zeek/data_stream/x509/manifest.yml @@ -18,6 +18,7 @@ streams: required: true show_user: false default: + - forwarded - zeek-x509 - name: preserve_original_event required: true diff --git a/packages/zeek/docs/README.md b/packages/zeek/docs/README.md index 2eb68159225..eba27c20f1b 100644 --- a/packages/zeek/docs/README.md +++ b/packages/zeek/docs/README.md @@ -1420,6 +1420,236 @@ contains kerberos data. | zeek.session_id | A unique identifier of the session | keyword | +### known_certs + +The `known_certs` dataset captures information about SSL/TLS certificates seen on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-certs-log) for more details. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.geo.city_name | City name. | keyword | +| host.geo.continent_name | Name of the continent. | keyword | +| host.geo.country_iso_code | Country ISO code. | keyword | +| host.geo.country_name | Country name. | keyword | +| host.geo.location | Longitude and latitude. | geo_point | +| host.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | +| host.geo.region_iso_code | Region ISO code. | keyword | +| host.geo.region_name | Region name. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| server.geo.city_name | City name. | keyword | +| server.geo.continent_name | Name of the continent. | keyword | +| server.geo.country_iso_code | Country ISO code. | keyword | +| server.geo.country_name | Country name. | keyword | +| server.geo.location | Longitude and latitude. | geo_point | +| server.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | +| server.geo.region_iso_code | Region ISO code. | keyword | +| server.geo.region_name | Region name. | keyword | +| server.ip | IP address of the server (IPv4 or IPv6). | ip | +| server.port | Port of the server. | long | +| tags | List of keywords used to tag each event. | keyword | +| tls.server.issuer | Subject of the issuer of the x.509 certificate presented by the server. | keyword | +| tls.server.subject | Subject of the x.509 certificate presented by the server. | keyword | +| tls.server.x509.issuer.common_name | List of common name (CN) of issuing certificate authority. | keyword | +| tls.server.x509.issuer.distinguished_name | Distinguished name (DN) of issuing certificate authority. | keyword | +| tls.server.x509.serial_number | Unique serial number issued by the certificate authority. For consistency, if this value is alphanumeric, it should be formatted without colons and uppercase characters. | keyword | +| tls.server.x509.subject.common_name | List of common names (CN) of subject. | keyword | +| tls.server.x509.subject.distinguished_name | Distinguished name (DN) of the certificate subject entity. | keyword | + + +### known_hosts + +The `known_hosts` dataset simply records a timestamp and an IP address when Zeek observes a new system on the local network.. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-hosts-log) for more details. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.geo.city_name | City name. | keyword | +| host.geo.continent_name | Name of the continent. | keyword | +| host.geo.country_iso_code | Country ISO code. | keyword | +| host.geo.country_name | Country name. | keyword | +| host.geo.location | Longitude and latitude. | geo_point | +| host.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | +| host.geo.region_iso_code | Region ISO code. | keyword | +| host.geo.region_name | Region name. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| tags | List of keywords used to tag each event. | keyword | + + +### known_services + +The `known_services` dataset records a timestamp, IP, port number, protocol, and service (if available) when Zeek observes a system offering a new service on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#known-services-log) for more details. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.geo.city_name | City name. | keyword | +| host.geo.continent_name | Name of the continent. | keyword | +| host.geo.country_iso_code | Country ISO code. | keyword | +| host.geo.country_name | Country name. | keyword | +| host.geo.location | Longitude and latitude. | geo_point | +| host.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | +| host.geo.region_iso_code | Region ISO code. | keyword | +| host.geo.region_name | Region name. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| network.application | When a specific application or service is identified from network connection details (source/dest IPs, ports, certificates, or wire format), this field captures the application's or service's name. For example, the original event identifies the network connection being from a specific web service in a `https` network connection, like `facebook` or `twitter`. The field value must be normalized to lowercase for querying. | keyword | +| network.transport | Same as network.iana_number, but instead using the Keyword name of the transport layer (udp, tcp, ipv6-icmp, etc.) The field value must be normalized to lowercase for querying. | keyword | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| server.geo.city_name | City name. | keyword | +| server.geo.continent_name | Name of the continent. | keyword | +| server.geo.country_iso_code | Country ISO code. | keyword | +| server.geo.country_name | Country name. | keyword | +| server.geo.location | Longitude and latitude. | geo_point | +| server.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | +| server.geo.region_iso_code | Region ISO code. | keyword | +| server.geo.region_name | Region name. | keyword | +| server.ip | IP address of the server (IPv4 or IPv6). | ip | +| server.port | Port of the server. | long | +| tags | List of keywords used to tag each event. | keyword | + + ### modbus The `modbus` dataset collects the Zeek modbus.log file, which contains @@ -3312,6 +3542,81 @@ SOCKS proxy requests. | zeek.socks.version | Protocol version of SOCKS. | integer | +### software + +The `software` dataset collects details on applications operated by the hosts it sees on the local network. See the [documentation](https://docs.zeek.org/en/master/logs/known-and-software.html#software-log) for more details. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | event.created contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from @timestamp in that @timestamp typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, @timestamp should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.geo.city_name | City name. | keyword | +| host.geo.continent_name | Name of the continent. | keyword | +| host.geo.country_iso_code | Country ISO code. | keyword | +| host.geo.country_name | Country name. | keyword | +| host.geo.location | Longitude and latitude. | geo_point | +| host.geo.name | User-defined description of a location, at the level of granularity they care about. Could be the name of their data centers, the floor number, if this describes a local physical entity, city names. Not typically used in automated geolocation. | keyword | +| host.geo.region_iso_code | Region ISO code. | keyword | +| host.geo.region_name | Region name. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Full path to the log file this event came from. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| network.type | In the OSI Model this would be the Network Layer. ipv4, ipv6, ipsec, pim, etc The field value must be normalized to lowercase for querying. | keyword | +| related.ip | All of the IPs seen on your event. | ip | +| tags | List of keywords used to tag each event. | keyword | +| zeek.software.name | Name of the software (e.g. Apache). | keyword | +| zeek.software.type | The type of software detected | keyword | +| zeek.software.version.additional | Additional version information | keyword | +| zeek.software.version.full | Full unparsed version of the software. | keyword | +| zeek.software.version.major | Major version of software. | long | +| zeek.software.version.minor | minor version of software. | long | +| zeek.software.version.minor2 | 2nd minor version of software. | long | +| zeek.software.version.minor3 | 3rd minor version of software. | long | + + ### ssh The `ssh` dataset collects the Zeek ssh.log file, which contains SSH diff --git a/packages/zeek/manifest.yml b/packages/zeek/manifest.yml index 4f66bdcf7f1..f0e8a759886 100644 --- a/packages/zeek/manifest.yml +++ b/packages/zeek/manifest.yml @@ -1,6 +1,6 @@ name: zeek title: Zeek Logs -version: 2.1.0 +version: 2.2.0 release: ga description: Collect and parse logs from Zeek network security with Elastic Agent. type: integration