diff --git a/doc/protocols.rst b/doc/protocols.rst index b9c360fc3ad..b987111e460 100644 --- a/doc/protocols.rst +++ b/doc/protocols.rst @@ -856,3 +856,12 @@ References: `Oracle site `_ and `Documentation `_ + + +.. _Proto 418: + +`NDPI_PROTOCOL_HLS` +===================== +HTTP Live Streaming (HLS) is an adaptive bitrate streaming communications protocol developed by Apple Inc. It allows for the delivery of media content over the internet by breaking the stream into small segments and adjusting the quality of the stream in real time based on the viewer's network conditions. + +References: `RFC `_ diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h index d21af9e4fab..8b8208204a9 100644 --- a/src/include/ndpi_protocol_ids.h +++ b/src/include/ndpi_protocol_ids.h @@ -446,6 +446,7 @@ typedef enum { NDPI_PROTOCOL_ZUG = 415, NDPI_PROTOCOL_JRMI = 416, NDPI_PROTOCOL_RIPE_ATLAS = 417, + NDPI_PROTOCOL_HLS = 418, #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h" diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index a07037b92e3..e9911e903fb 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -2279,9 +2279,13 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp ndpi_build_default_ports(ports_a, 1099, 0, 0, 0, 0) /* TCP */, ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_RIPE_ATLAS, - "RipeAtlas", NDPI_PROTOCOL_CATEGORY_NETWORK, - ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, - ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + "RipeAtlas", NDPI_PROTOCOL_CATEGORY_NETWORK, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); + ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_HLS, + "HLS", NDPI_PROTOCOL_CATEGORY_MEDIA, + ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */, + ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */); #ifdef CUSTOM_NDPI_PROTOCOLS #include "../../../nDPI-custom/custom_ndpi_main.c" diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c index c57ed8441ba..bf365c46fc5 100644 --- a/src/lib/protocols/http.c +++ b/src/lib/protocols/http.c @@ -489,6 +489,15 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, master_protocol, NDPI_CONFIDENCE_DPI); } + /* HTTP Live Streaming */ + if (packet->content_line.len > 28 && + (strncmp((const char *)packet->content_line.ptr, "application/vnd.apple.mpegurl", 29) == 0 || + strncmp((const char *)packet->content_line.ptr, "application/x-mpegURL", 21) == 0 || + strncmp((const char *)packet->content_line.ptr, "application/x-mpegurl", 21) == 0)) { + NDPI_LOG_DBG2(ndpi_struct, "Found HLS\n"); + ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HLS, master_protocol, NDPI_CONFIDENCE_DPI); + } + if((flow->http.method == NDPI_HTTP_METHOD_RPC_CONNECT) || (flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA) || (flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA)) { diff --git a/tests/cfgs/default/pcap/hls.pcapng b/tests/cfgs/default/pcap/hls.pcapng new file mode 100644 index 00000000000..713e4305035 Binary files /dev/null and b/tests/cfgs/default/pcap/hls.pcapng differ diff --git a/tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out b/tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out index 389ada5e114..5f3c773cd31 100644 --- a/tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out +++ b/tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out @@ -25,6 +25,6 @@ CustomProtocolC 3 222 1 Acceptable 8 592 3 - 1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.424/TLS.CustomProtocolA][IP: 424/CustomProtocolA][Encrypted][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.425/TLS.CustomProtocolA][IP: 425/CustomProtocolA][Encrypted][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 800/CustomProtocolC][IP: 800/CustomProtocolC][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 425/CustomProtocolB][IP: 425/CustomProtocolB][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 426/CustomProtocolB][IP: 426/CustomProtocolB][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/cfgs/default/result/hls.pcapng.out b/tests/cfgs/default/result/hls.pcapng.out new file mode 100644 index 00000000000..5766d4ddff9 --- /dev/null +++ b/tests/cfgs/default/result/hls.pcapng.out @@ -0,0 +1,26 @@ +DPI Packets (TCP): 6 (6.00 pkts/flow) +Confidence DPI : 1 (flows) +Num dissector calls: 15 (15.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +Automa host: 1/0 (search/found) +Automa domain: 1/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 1/0 (search/found) +Automa common alpns: 0/0 (search/found) +Patricia risk mask: 2/0 (search/found) +Patricia risk mask IPv6: 0/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia risk IPv6: 0/0 (search/found) +Patricia protocols: 2/0 (search/found) +Patricia protocols IPv6: 0/0 (search/found) + +HLS 13 1019 1 + +Fun 13 1019 1 + + 1 TCP 10.215.173.1:41644 <-> 192.168.88.231:8080 [proto: 7.418/HTTP.HLS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Web/5][7 pkts/448 bytes <-> 6 pkts/571 bytes][Goodput ratio: 33/56][5.34 sec][Hostname/SNI: 192.168.88.231][bytes ratio: -0.121 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 1060/0 5298/1 2119/0][Pkt Len c2s/s2c min/avg/max/stddev: 40/40 64/95 188/246 51/80][URL: 192.168.88.231:8080/output.m3u8][StatusCode: 200][Content-Type: application/vnd.apple.mpegurl][Server: SimpleHTTP/0.6 Python/3.11.4][User-Agent: VLC/3.0.18 LibVLC/3.0.18][Risk: ** Known Proto on Non Std Port **** HTTP/TLS/QUIC Numeric Hostname/SNI **][Risk Score: 60][Risk Info: Found host 192.168.88.231 / Expected on port 80][PLAIN TEXT (GET /output.m)][Plen Bins: 0,0,0,33,33,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] diff --git a/tests/cfgs/default/result/synscan.pcap.out b/tests/cfgs/default/result/synscan.pcap.out index bf392fa5e1d..51991810034 100644 --- a/tests/cfgs/default/result/synscan.pcap.out +++ b/tests/cfgs/default/result/synscan.pcap.out @@ -144,7 +144,7 @@ Unrated 1848 107192 1844 48 TCP 172.16.0.8:36050 -> 64.13.134.52:2605 [proto: 13/BGP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 49 TCP 172.16.0.8:36050 -> 64.13.134.52:3000 [proto: 26/ntop][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 50 TCP 172.16.0.8:36050 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 51 TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 418/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 51 TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 419/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 52 TCP 172.16.0.8:36050 -> 64.13.134.52:3300 [proto: 381/Ceph][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: DataTransfer/4][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 53 TCP 172.16.0.8:36050 -> 64.13.134.52:3306 [proto: 20/MySQL][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 54 TCP 172.16.0.8:36050 -> 64.13.134.52:3389 [proto: 88/RDP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Desktop/File Sharing **** Unidirectional Traffic **** Probing attempt **][Risk Score: 70][Risk Info: No server to client traffic / TCP connection with unidirectional traffic / Found RDP][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] @@ -217,7 +217,7 @@ Unrated 1848 107192 1844 121 TCP 172.16.0.8:36051 -> 64.13.134.52:2605 [proto: 13/BGP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 122 TCP 172.16.0.8:36051 -> 64.13.134.52:3000 [proto: 26/ntop][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 123 TCP 172.16.0.8:36051 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] - 124 TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 418/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 124 TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 419/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 125 TCP 172.16.0.8:36051 -> 64.13.134.52:3300 [proto: 381/Ceph][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: DataTransfer/4][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 126 TCP 172.16.0.8:36051 -> 64.13.134.52:3306 [proto: 20/MySQL][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] 127 TCP 172.16.0.8:36051 -> 64.13.134.52:3389 [proto: 88/RDP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Desktop/File Sharing **** Unidirectional Traffic **** Probing attempt **][Risk Score: 70][Risk Info: No server to client traffic / TCP connection with unidirectional traffic / Found RDP][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]