Add HLS support

ntop · Jul 16, 2024 · 9d286ab · 9d286ab
1 parent 996cddb
commit 9d286ab
Show file tree

Hide file tree

Showing 82 changed files with 184 additions and 134 deletions.
diff --git a/doc/protocols.rst b/doc/protocols.rst
@@ -856,3 +856,12 @@ References:  `Oracle site <https://docs.oracle.com/en/java/javase/21/docs/specs/
 The RIPE Atlas probe protocol is used for the world's largest active Internet measurement network.
 
 References: `Main Site <https://atlas.ripe.net/>`_ and `Documentation <https://ripe-atlas-tools.readthedocs.io/en/latest/index.html>`_
+
+
+.. _Proto 418:
+
+`NDPI_PROTOCOL_HLS`
+=====================
+HTTP Live Streaming (HLS) is an adaptive bitrate streaming communications protocol developed by Apple Inc. It allows for the delivery of media content over the internet by breaking the stream into small segments and adjusting the quality of the stream in real time based on the viewer's network conditions.
+
+References: `RFC <https://datatracker.ietf.org/doc/html/rfc8216>`_
diff --git a/src/include/ndpi_protocol_ids.h b/src/include/ndpi_protocol_ids.h
@@ -446,6 +446,7 @@ typedef enum {
   NDPI_PROTOCOL_ZUG                   = 415,
   NDPI_PROTOCOL_JRMI                  = 416, 
   NDPI_PROTOCOL_RIPE_ATLAS            = 417,
+  NDPI_PROTOCOL_HLS                   = 418,
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"

diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c
@@ -1029,6 +1029,7 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			      NDPI_PROTOCOL_APACHE_THRIFT,
 			      NDPI_PROTOCOL_JSON_RPC,
 			      NDPI_PROTOCOL_HL7,
+			      NDPI_PROTOCOL_HLS,
 			      NDPI_PROTOCOL_MATCHED_BY_CONTENT,
 			      NDPI_PROTOCOL_NO_MORE_SUBPROTOCOLS); /* NDPI_PROTOCOL_HTTP can have (content-matched) subprotocols */
   ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_MDNS,
@@ -2279,9 +2280,13 @@ static void ndpi_init_protocol_defaults(struct ndpi_detection_module_struct *ndp
 			  ndpi_build_default_ports(ports_a, 1099, 0, 0, 0, 0) /* TCP */,
 			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
   ndpi_set_proto_defaults(ndpi_str, 0 /* encrypted */, 0 /* nw proto */, NDPI_PROTOCOL_ACCEPTABLE, NDPI_PROTOCOL_RIPE_ATLAS,
-              "RipeAtlas", NDPI_PROTOCOL_CATEGORY_NETWORK,
-              ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
-              ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+			  "RipeAtlas", NDPI_PROTOCOL_CATEGORY_NETWORK,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
+  ndpi_set_proto_defaults(ndpi_str, 1 /* cleartext */, 0 /* nw proto */, NDPI_PROTOCOL_FUN, NDPI_PROTOCOL_HLS,
+			  "HLS", NDPI_PROTOCOL_CATEGORY_MEDIA,
+			  ndpi_build_default_ports(ports_a, 0, 0, 0, 0, 0) /* TCP */,
+			  ndpi_build_default_ports(ports_b, 0, 0, 0, 0, 0) /* UDP */);
 
 #ifdef CUSTOM_NDPI_PROTOCOLS
 #include "../../../nDPI-custom/custom_ndpi_main.c"

diff --git a/src/lib/protocols/http.c b/src/lib/protocols/http.c
@@ -489,6 +489,15 @@ static void ndpi_http_parse_subprotocol(struct ndpi_detection_module_struct *ndp
     ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_OCSP, master_protocol, NDPI_CONFIDENCE_DPI);
   }
 
+  /* HTTP Live Streaming */
+  if (packet->content_line.len > 28 &&
+      (strncmp((const char *)packet->content_line.ptr, "application/vnd.apple.mpegurl", 29) == 0 ||
+      strncmp((const char *)packet->content_line.ptr, "application/x-mpegURL", 21) == 0 ||
+      strncmp((const char *)packet->content_line.ptr, "application/x-mpegurl", 21) == 0)) {
+    NDPI_LOG_DBG2(ndpi_struct, "Found HLS\n");
+    ndpi_set_detected_protocol(ndpi_struct, flow, NDPI_PROTOCOL_HLS, master_protocol, NDPI_CONFIDENCE_DPI);
+  }
+
   if((flow->http.method == NDPI_HTTP_METHOD_RPC_CONNECT) ||
      (flow->http.method == NDPI_HTTP_METHOD_RPC_IN_DATA) ||
      (flow->http.method == NDPI_HTTP_METHOD_RPC_OUT_DATA)) {

diff --git a/tests/cfgs/caches_cfg/result/ookla.pcap.out b/tests/cfgs/caches_cfg/result/ookla.pcap.out
@@ -3,7 +3,7 @@ Guessed flow protos:	1
 DPI Packets (TCP):	40	(6.67 pkts/flow)
 Confidence Match by port    : 1 (flows)
 Confidence DPI              : 5 (flows)
-Num dissector calls: 571 (95.17 diss/flow)
+Num dissector calls: 572 (95.33 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/3/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

diff --git a/tests/cfgs/caches_global/result/ookla.pcap.out b/tests/cfgs/caches_global/result/ookla.pcap.out
@@ -4,7 +4,7 @@ DPI Packets (TCP):	40	(6.67 pkts/flow)
 Confidence DPI (partial cache): 1 (flows)
 Confidence DPI              : 4 (flows)
 Confidence DPI (aggressive) : 1 (flows)
-Num dissector calls: 571 (95.17 diss/flow)
+Num dissector calls: 572 (95.33 diss/flow)
 LRU cache ookla:      4/2/2 (insert/search/found)
 LRU cache bittorrent: 0/3/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

diff --git a/tests/cfgs/default/pcap/hls.pcapng b/tests/cfgs/default/pcap/hls.pcapng
diff --git a/tests/cfgs/default/result/1kxun.pcap.out b/tests/cfgs/default/result/1kxun.pcap.out
@@ -5,7 +5,7 @@ DPI Packets (UDP):	120	(1.21 pkts/flow)
 Confidence Unknown          : 14 (flows)
 Confidence Match by port    : 6 (flows)
 Confidence DPI              : 177 (flows)
-Num dissector calls: 4968 (25.22 diss/flow)
+Num dissector calls: 5055 (25.66 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/60/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

diff --git a/tests/cfgs/default/result/6in4tunnel.pcap.out b/tests/cfgs/default/result/6in4tunnel.pcap.out
@@ -2,7 +2,7 @@ DPI Packets (TCP):	29	(5.80 pkts/flow)
 DPI Packets (UDP):	4	(2.00 pkts/flow)
 DPI Packets (other):	3	(1.00 pkts/flow)
 Confidence DPI              : 10 (flows)
-Num dissector calls: 28 (2.80 diss/flow)
+Num dissector calls: 29 (2.90 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

diff --git a/tests/cfgs/default/result/EAQ.pcap.out b/tests/cfgs/default/result/EAQ.pcap.out
@@ -1,7 +1,7 @@
 DPI Packets (TCP):	12	(6.00 pkts/flow)
 DPI Packets (UDP):	116	(4.00 pkts/flow)
 Confidence DPI              : 31 (flows)
-Num dissector calls: 5068 (163.48 diss/flow)
+Num dissector calls: 5070 (163.55 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

diff --git a/tests/cfgs/default/result/KakaoTalk_chat.pcap.out b/tests/cfgs/default/result/KakaoTalk_chat.pcap.out
@@ -5,7 +5,7 @@ DPI Packets (UDP):	36	(2.00 pkts/flow)
 DPI Packets (other):	1	(1.00 pkts/flow)
 Confidence Match by port    : 5 (flows)
 Confidence DPI              : 33 (flows)
-Num dissector calls: 551 (14.50 diss/flow)
+Num dissector calls: 553 (14.55 diss/flow)
 LRU cache ookla:      0/1/0 (insert/search/found)
 LRU cache bittorrent: 0/15/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)

diff --git a/tests/cfgs/default/result/WebattackRCE.pcap.out b/tests/cfgs/default/result/WebattackRCE.pcap.out
@@ -1,6 +1,6 @@
 DPI Packets (TCP):	797	(1.00 pkts/flow)
 Confidence DPI              : 797 (flows)
-Num dissector calls: 11955 (15.00 diss/flow)
+Num dissector calls: 12752 (16.00 diss/flow)
 LRU cache ookla:      0/0/0 (insert/search/found)
 LRU cache bittorrent: 0/0/0 (insert/search/found)
 LRU cache stun:       0/0/0 (insert/search/found)