Skip to content

Commit

Permalink
Add HLS support
Browse files Browse the repository at this point in the history
  • Loading branch information
@0xA50C1A1
0xA50C1A1 committed Jul 16, 2024
1 parent 996cddb commit 74ea9b5
Show file tree
Hide file tree
Showing 8 changed files with 56 additions and 7 deletions.
9 changes: 9 additions & 0 deletions doc/protocols.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -856,3 +856,12 @@ References: `Oracle site <https://docs.oracle.com/en/java/javase/21/docs/specs/
The RIPE Atlas probe protocol is used for the world's largest active Internet measurement network.

References: `Main Site <https://atlas.ripe.net/>`_ and `Documentation <https://ripe-atlas-tools.readthedocs.io/en/latest/index.html>`_


.. _Proto 418:

`NDPI_PROTOCOL_HLS`
=====================
HTTP Live Streaming (HLS) is an adaptive bitrate streaming communications protocol developed by Apple Inc. It allows for the delivery of media content over the internet by breaking the stream into small segments and adjusting the quality of the stream in real time based on the viewer's network conditions.

References: `RFC <https://datatracker.ietf.org/doc/html/rfc8216>`_
1 change: 1 addition & 0 deletions src/include/ndpi_protocol_ids.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,7 @@ typedef enum {
NDPI_PROTOCOL_ZUG = 415,
NDPI_PROTOCOL_JRMI = 416,
NDPI_PROTOCOL_RIPE_ATLAS = 417,
NDPI_PROTOCOL_HLS = 418,

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"
Expand Down
10 changes: 7 additions & 3 deletions src/lib/ndpi_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2279,9 +2279,13 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
ndpi_build_default_ports(ports_a, 1099, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_RIPE_ATLAS,
"RipeAtlas", NDPI_PROTOCOL_CATEGORY_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
"RipeAtlas", NDPI_PROTOCOL_CATEGORY_NETWORK,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_HLS,
"HLS", NDPI_PROTOCOL_CATEGORY_MEDIA,
ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_main.c"
Expand Down
9 changes: 9 additions & 0 deletions src/lib/protocols/http.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,15 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, master_protocol, NDPI_CONFIDENCE_DPI);
}

/* HTTP Live Streaming */
if (packet->content_line.len > 28 &&
(strncmp((const char *)packet->content_line.ptr, "application/vnd.apple.mpegurl", 29) == 0 ||
strncmp((const char *)packet->content_line.ptr, "application/x-mpegURL", 21) == 0 ||
strncmp((const char *)packet->content_line.ptr, "application/x-mpegurl", 21) == 0)) {
NDPI_LOG_DBG2(ndpi_struct, "Found HLS\n");
ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HLS, master_protocol, NDPI_CONFIDENCE_DPI);
}

if((flow->http.method == NDPI_HTTP_METHOD_RPC_CONNECT) ||
(flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA) ||
(flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA)) {
Expand Down
Binary file added tests/cfgs/default/pcap/hls.pcapng
View file
Binary file not shown.
4 changes: 2 additions & 2 deletions tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@ CustomProtocolC 3 222 1

Acceptable 8 592 3

1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.424/TLS.CustomProtocolA][IP: 424/CustomProtocolA][Encrypted][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.425/TLS.CustomProtocolA][IP: 425/CustomProtocolA][Encrypted][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:58288 -> 3.3.3.3:446 [proto: 800/CustomProtocolC][IP: 800/CustomProtocolC][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.04 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 425/CustomProtocolB][IP: 425/CustomProtocolB][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
3 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 426/CustomProtocolB][IP: 426/CustomProtocolB][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
26 changes: 26 additions & 0 deletions tests/cfgs/default/result/hls.pcapng.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
DPI Packets (TCP): 6 (6.00 pkts/flow)
Confidence DPI : 1 (flows)
Num dissector calls: 15 (15.00 diss/flow)
LRU cache ookla: 0/0/0 (insert/search/found)
LRU cache bittorrent: 0/0/0 (insert/search/found)
LRU cache stun: 0/0/0 (insert/search/found)
LRU cache tls_cert: 0/0/0 (insert/search/found)
LRU cache mining: 0/0/0 (insert/search/found)
LRU cache msteams: 0/0/0 (insert/search/found)
Automa host: 1/0 (search/found)
Automa domain: 1/0 (search/found)
Automa tls cert: 0/0 (search/found)
Automa risk mask: 1/0 (search/found)
Automa common alpns: 0/0 (search/found)
Patricia risk mask: 2/0 (search/found)
Patricia risk mask IPv6: 0/0 (search/found)
Patricia risk: 0/0 (search/found)
Patricia risk IPv6: 0/0 (search/found)
Patricia protocols: 2/0 (search/found)
Patricia protocols IPv6: 0/0 (search/found)

HLS 13 1019 1

Fun 13 1019 1

1 TCP 10.215.173.1:41644 <-> 192.168.88.231:8080 [proto: 7.418/HTTP.HLS][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Web/5][7 pkts/448 bytes <-> 6 pkts/571 bytes][Goodput ratio: 33/56][5.34 sec][Hostname/SNI: 192.168.88.231][bytes ratio: -0.121 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 1060/0 5298/1 2119/0][Pkt Len c2s/s2c min/avg/max/stddev: 40/40 64/95 188/246 51/80][URL: 192.168.88.231:8080/output.m3u8][StatusCode: 200][Content-Type: application/vnd.apple.mpegurl][Server: SimpleHTTP/0.6 Python/3.11.4][User-Agent: VLC/3.0.18 LibVLC/3.0.18][Risk: ** Known Proto on Non Std Port **** HTTP/TLS/QUIC Numeric Hostname/SNI **][Risk Score: 60][Risk Info: Found host 192.168.88.231 / Expected on port 80][PLAIN TEXT (GET /output.m)][Plen Bins: 0,0,0,33,33,0,33,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 changes: 2 additions & 2 deletions tests/cfgs/default/result/synscan.pcap.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ Unrated 1848 107192 1844
48 TCP 172.16.0.8:36050 -> 64.13.134.52:2605 [proto: 13/BGP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
49 TCP 172.16.0.8:36050 -> 64.13.134.52:3000 [proto: 26/ntop][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
50 TCP 172.16.0.8:36050 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
51 TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 418/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
51 TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 419/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
52 TCP 172.16.0.8:36050 -> 64.13.134.52:3300 [proto: 381/Ceph][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: DataTransfer/4][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
53 TCP 172.16.0.8:36050 -> 64.13.134.52:3306 [proto: 20/MySQL][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
54 TCP 172.16.0.8:36050 -> 64.13.134.52:3389 [proto: 88/RDP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Desktop/File Sharing **** Unidirectional Traffic **** Probing attempt **][Risk Score: 70][Risk Info: No server to client traffic / TCP connection with unidirectional traffic / Found RDP][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Expand Down Expand Up @@ -217,7 +217,7 @@ Unrated 1848 107192 1844
121 TCP 172.16.0.8:36051 -> 64.13.134.52:2605 [proto: 13/BGP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
122 TCP 172.16.0.8:36051 -> 64.13.134.52:3000 [proto: 26/ntop][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
123 TCP 172.16.0.8:36051 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
124 TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 418/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
124 TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 419/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by custom rule][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
125 TCP 172.16.0.8:36051 -> 64.13.134.52:3300 [proto: 381/Ceph][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: DataTransfer/4][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
126 TCP 172.16.0.8:36051 -> 64.13.134.52:3306 [proto: 20/MySQL][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Probing attempt **][Risk Score: 60][Risk Info: No server to client traffic / TCP connection with unidirectional traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
127 TCP 172.16.0.8:36051 -> 64.13.134.52:3389 [proto: 88/RDP][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 1][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Desktop/File Sharing **** Unidirectional Traffic **** Probing attempt **][Risk Score: 70][Risk Info: No server to client traffic / TCP connection with unidirectional traffic / Found RDP][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Expand Down

0 comments on commit 74ea9b5

Please sign in to comment.