Skip to content

Commit

Permalink
config: follow-up (#2268)
Browse files Browse the repository at this point in the history 
Some changes in the parameters names.
Add a fuzzer to fuzz the configuration file format.
Add the infrastructure to configuratin callbacks.
Add an helper to map LRU cache indexes to names.
  • Loading branch information
@IvanNardi
IvanNardi authored Jan 20, 2024
1 parent 8651ce9 commit 42d23cf
Show file tree
Hide file tree
Showing 62 changed files with 264 additions and 118 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@
/fuzz/fuzz_filecfg_malicious_sha1
/fuzz/fuzz_filecfg_malicious_ja3
/fuzz/fuzz_filecfg_risk_domains
/fuzz/fuzz_filecfg_config
/fuzz/fuzz_readerutils_workflow
/fuzz/fuzz_readerutils_parseprotolist
/fuzz/fuzz_ndpi_reader_alloc_fail_seed_corpus.zip
Expand Down Expand Up @@ -117,6 +118,7 @@
/fuzz/fuzz_filecfg_malicious_sha1_seed_corpus.zip
/fuzz/fuzz_filecfg_malicious_ja3_seed_corpus.zip
/fuzz/fuzz_filecfg_risk_domains_seed_corpus.zip
/fuzz/fuzz_filecfg_config_seed_corpus.zip
/fuzz/fuzz_dga_seed_corpus.zip
/fuzz/fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip
/fuzz/fuzz_readerutils_workflow_seed_corpus.zip
Expand Down
34 changes: 17 additions & 17 deletions doc/configuration_parameters.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,12 @@ TODO
| Protocol | Parameter | Default value | Min value | Max value | Description | Notes |
| ------ | ------ | ------ | ------ | ------ | ------ | ------ |
| NULL | "packets_limit_per_flow" | 32 | 0 | 255 | The upper limit on the number of packets per flow that will be subject to DPI, after which classification will be considered complete (0 = no limit) |
| NULL | "flow.direction_detection.enable" | 1 | NULL | NULL | Enable/disable internal detection of packet direction (client to server or server to client) |
| NULL | "flow.track_payload.enable" | 0 | NULL | NULL | Enable/disable tracking/export of flow payload (i.e. L5/7 data) |
| NULL | "tcp_ack_payload_heuristic.enable" | 0 | NULL | NULL | In some networks, there are some anomalous TCP flows where the smallest ACK packets have some kind of zero padding. It looks like the IP and TCP headers in those frames wrongly consider the 0x00 Ethernet padding bytes as part of the TCP payload. While this kind of packets is perfectly valid per-se, in some conditions they might be treated by the TCP reassembler logic as (partial) overlaps, deceiving the classification engine. This parameter enable/disable an heuristic to detect these packets and to ignore them, allowing correct detection/classification. See #1946 for other details |
| NULL | "fully_encrypted_heuristic.enable" | 1 | NULL | NULL | Enable/disable an heuristic to detect fully encrypted sessions, i.e. flows where every bytes of the payload is encrypted in an attempt to “look like nothing”. This heuristic only analyzes the first packet of the flow. See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf |
| NULL | "flow.direction_detection" | enable | NULL | NULL | Enable/disable internal detection of packet direction (client to server or server to client) |
| NULL | "flow.track_payload" | disable | NULL | NULL | Enable/disable tracking/export of flow payload (i.e. L5/7 data): if enabled, the library exports the first 1024 bytes of payload for each flow |
| NULL | "tcp_ack_payload_heuristic" | disable | NULL | NULL | In some networks, there are some anomalous TCP flows where the smallest ACK packets have some kind of zero padding. It looks like the IP and TCP headers in those frames wrongly consider the 0x00 Ethernet padding bytes as part of the TCP payload. While this kind of packets is perfectly valid per-se, in some conditions they might be treated by the TCP reassembler logic as (partial) overlaps, deceiving the classification engine. This parameter enable/disable an heuristic to detect these packets and to ignore them, allowing correct detection/classification. See #1946 for other details |
| NULL | "fully_encrypted_heuristic" | enable | NULL | NULL | Enable/disable an heuristic to detect fully encrypted sessions, i.e. flows where every bytes of the payload is encrypted in an attempt to “look like nothing”. This heuristic only analyzes the first packet of the flow. See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf |
| NULL | "libgcrypt.init" | 1 | NULL | NULL | Enable/disable initialization of libgcrypt. When using the external libgcrypt (instead of the internal crypto code) the libgcrypt runtime must be initialized. If, for whatever reasons, the application alread does it, nDPI must be told to skip it. Note that, by default, nDPI uses the crypto code and not libgcrypt: in that case this parameter is ignored |
| NULL | "guess_on_giveup" | 0x03 | 0x00 | 0x03 | Tell the library to guess flow classification, if any DPI algorithms/logics fail. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable guessing by port; 0x02 = enable guessing by ip |
| NULL | "dpi.guess_on_giveup" | 0x03 | 0x00 | 0x03 | Tell the library to guess flow classification, if any DPI algorithms/logics fail. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable guessing by port; 0x02 = enable guessing by ip |
| NULL | "flow_risk_lists.load" | 1 | NULL | NULL | Enable/disable loading of every IP addresses lists used to check any flow risks |
| NULL | "flow_risk.anonymous_subscriber.list.icloudprivaterelay.load" | 1 | NULL | NULL | Enable/disable loading of internal iCouldPrivateRealy IP address list used to check `NDPI_ANONYMOUS_SUBSCRIBER` flow risk |
| NULL | "flow_risk.anonymous_subscriber.list.protonvpn.load" | 1 | NULL | NULL | Enable/disable loading of internal IP address list of ProtonVPN exit nodes used to check `NDPI_ANONYMOUS_SUBSCRIBER` flow risk |
Expand All @@ -21,16 +21,16 @@ TODO
| NULL | "lru.$CACHE_NAME.size" | See description | 0 | 16777215 | Set the size (in number of elements) of the specified LRU cache (0 = the cache is disabled). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom. The default value is "32768" for the bittorrent cache, "512" for the zoom cache and "1024" for all the other caches |
| NULL | "lru.$CACHE_NAME.ttl" | See description | 0 | 16777215 | Set the TTL (in seconds) for the elements of the specified LRU cache (0 = the elements never explicitly expire). The keyword "$CACHE_NAME" is a placeholder for the cache name and the possible values are: ookla, bittorrent, zoom, stun, tls_cert, mining, msteams, stun_zoom. The default value is "120" for the ookla cache, "60" for the msteams and stun_zoom caches and "0" for all the other caches |
| "tls" | "certificate_expiration_threshold" | 30 | 0 | 365 | The threshold (in days) used to trigger the `NDPI_TLS_CERTIFICATE_ABOUT_TO_EXPIRE` flow risk |
| "tls" | "application_blocks_tracking.enable" | 0 | NULL | NULL | Enable/disable processing of TLS Application Blocks (post handshake) to extract statistical information about the flow |
| "tls" | "metadata.sha1_fingerprint.enable" | 1 | NULL | NULL | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked |
| "smtp" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext SMTP flows (because of opportunistic TLS, via STARTTLS msg) |
| "imap" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext IMAP flows (because of opportunistic TLS, via STARTTLS msg) |
| "pop" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext POP flows (because of opportunistic TLS, via STARTTLS msg) |
| "ftp" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets in cleartext FTP flows (because of opportunistic TLS, via AUTH TLS msg) |
| "stun" | "tls_dissection.enable" | 1 | NULL | NULL | Enable/disable dissection of TLS packets multiplexed into STUN flows |
| "dns" | "subclassification.enable" | 1 | NULL | NULL | Enable/disable sub-classification of DNS flows (via query/response domain name). If disabled, some flow risks are not checked |
| "dns" | "process_response.enable" | 1 | NULL | NULL | Enable/disable processing of DNS responses. By default, DNS flows are fully classified after the first request/response pair (or after the first response, if the request is missing). If this parameter is disabled, the flows are fully classified after the first packet, i.e. usually after the first request; in that case, some flow risks are not checked and some metadata are not exported |
| "http" | "process_response.enable" | 1 | NULL | NULL | Enable/disable processing of HTTP responses. By default, HTTP flows are usually fully classified after the first request/response pair. If this parameter is disabled, the flows are fully classified after the first request (or after the first response, if the request is missing); in that case, some flow risks are not checked and some metadata are not exported |
| "ookla" | "aggressiveness", | 0x01 | 0x00 | 0x01 | Detection aggressiveness for Ookla. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable heuristic for detection over TLS (via Ookla LRU cache) |
| $PROTO_NAME | "log.enable" | 0 | NULL | NULL | Enable/disable logging/debug for specific protocol. Use "any" as protocol name if you want to easily enable/disable logging/debug for all protocols |
| "tls" | "application_blocks_tracking" | disable | NULL | NULL | Enable/disable processing of TLS Application Blocks (post handshake) to extract statistical information about the flow |
| "tls" | "metadata.sha1_fingerprint" | enable | NULL | NULL | Enable/disable computation and export of SHA1 fingerprint for TLS flows. Note that if it is disable, the flow risk `NDPI_MALICIOUS_SHA1_CERTIFICATE` is not checked |
| "smtp" | "tls_dissection" | enable | NULL | NULL | Enable/disable dissection of TLS packets in cleartext SMTP flows (because of opportunistic TLS, via STARTTLS msg) |
| "imap" | "tls_dissection" | enable | NULL | NULL | Enable/disable dissection of TLS packets in cleartext IMAP flows (because of opportunistic TLS, via STARTTLS msg) |
| "pop" | "tls_dissection" | enable | NULL | NULL | Enable/disable dissection of TLS packets in cleartext POP flows (because of opportunistic TLS, via STARTTLS msg) |
| "ftp" | "tls_dissection" | enable | NULL | NULL | Enable/disable dissection of TLS packets in cleartext FTP flows (because of opportunistic TLS, via AUTH TLS msg) |
| "stun" | "tls_dissection" | enable | NULL | NULL | Enable/disable dissection of TLS packets multiplexed into STUN flows |
| "dns" | "subclassification" | enable | NULL | NULL | Enable/disable sub-classification of DNS flows (via query/response domain name). If disabled, some flow risks are not checked |
| "dns" | "process_response" | enable | NULL | NULL | Enable/disable processing of DNS responses. By default, DNS flows are fully classified after the first request/response pair (or after the first response, if the request is missing). If this parameter is disabled, the flows are fully classified after the first packet, i.e. usually after the first request; in that case, some flow risks are not checked and some metadata are not exported |
| "http" | "process_response" | enable | NULL | NULL | Enable/disable processing of HTTP responses. By default, HTTP flows are usually fully classified after the first request/response pair. If this parameter is disabled, the flows are fully classified after the first request (or after the first response, if the request is missing); in that case, some flow risks are not checked and some metadata are not exported |
| "ookla" | "dpi.aggressiveness", | 0x01 | 0x00 | 0x01 | Detection aggressiveness for Ookla. The value is a bitmask. Values: 0x0 = disabled; 0x01 = enable heuristic for detection over TLS (via Ookla LRU cache) |
| $PROTO_NAME | "log" | disable | NULL | NULL | Enable/disable logging/debug for specific protocol. Use "any" as protocol name if you want to easily enable/disable logging/debug for all protocols |
| $PROTO_NAME | "ip_list.load" | 1 | NULL | NULL | Enable/disable loading of internal list of IP addresses (used for (sub)classification) specific to that protocol. Use "any" as protocol name if you want to easily enable/disable all lists. This knob is valid only for the following protocols: Alibaba, Amazon AWS, Apple, Avast, Bloomberg, Cachefly, Cloudflare, Discord, Disney+, Dropbox, Edgecast, EpicGames, Ethereum, Facebook, Github, Google, Google Cloud, GoTo, Hotspot Shield, Hulu, Line, Microsoft 365, Microsoft Azure, Microsoft One Drive, Microsoft Outlook, Mullvad, Netflix, Nvidia, OpenDNS, ProtonVPN, RiotGames, Roblox, Skype/Teams, Starcraft, Steam, Teamviewer, Telegram, Tencent, Threema, TOR, Twitch, Twitter, UbuntuONE, VK, Yandex, Yandex Cloud, Webex, Whatsapp, Zoom |
4 changes: 2 additions & 2 deletions example/config.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@
#See doc/configuration_parameters.md for a complete list and description of all the accepted knobs

packets_limit_per_flow,32
ookla,aggressiveness,0x1
tls,metadata.sha1_fingerprint.enable,1
ookla,dpi.aggressiveness,0x1
tls,metadata.sha1_fingerprint,1

lru.bittorrent.ttl,0

Expand Down
16 changes: 8 additions & 8 deletions example/ndpiReader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ static void ndpiCheckIPMatch(char *testChar) {
for(i = 0; i < num_cfgs; i++) {
rc = ndpi_set_config(ndpi_str,
cfgs[i].proto, cfgs[i].param, cfgs[i].value);
if (rc < NDPI_CFG_OK)
if (rc != NDPI_CFG_OK)
fprintf(stderr, "Error setting config [%s][%s][%s]: %d\n",
cfgs[i].proto, cfgs[i].param, cfgs[i].value, rc);
}
Expand Down Expand Up @@ -557,7 +557,7 @@ static void help(u_int long_help) {
" | 0 - List known protocols\n"
" | 1 - List known categories\n"
" | 2 - List known risks\n"
" -d | Disable protocol guess (by ip and by port) and use only DPI. It is a shortcut to --cfg=,NULL,guess_on_giveup,0\n"
" -d | Disable protocol guess (by ip and by port) and use only DPI. It is a shortcut to --cfg=dpi.guess_on_giveup,0\n"
" -e <len> | Min human readeable string match len. Default %u\n"
" -q | Quiet mode\n"
" -F | Enable flow stats\n"
Expand Down Expand Up @@ -1021,7 +1021,7 @@ static void parseOptions(int argc, char **argv) {

case 'd':
enable_protocol_guess = 0;
if(reader_add_cfg(NULL, "guess_on_giveup", "0", 1) == 1) {
if(reader_add_cfg(NULL, "dpi.guess_on_giveup", "0", 1) == 1) {
printf("Invalid parameter [%s] [num:%d/%d]\n", optarg, num_cfgs, MAX_NUM_CFGS);
exit(1);
}
Expand Down Expand Up @@ -1134,7 +1134,7 @@ static void parseOptions(int argc, char **argv) {
}
if(log_level > NDPI_LOG_DEBUG_EXTRA) {
log_level = NDPI_LOG_DEBUG_EXTRA;
if(reader_add_cfg("all", "log.enable", "1", 1) == 1) {
if(reader_add_cfg("all", "log", "enable", 1) == 1) {
printf("Invalid cfg [num:%d/%d]\n", num_cfgs, MAX_NUM_CFGS);
exit(1);
}
Expand All @@ -1155,7 +1155,7 @@ static void parseOptions(int argc, char **argv) {
int inverted_logic;

/* Reset any previous call to this knob */
if(reader_add_cfg("all", "log.enable", "0", 1) == 1) {
if(reader_add_cfg("all", "log", "disable", 1) == 1) {
printf("Invalid cfg [num:%d/%d]\n", num_cfgs, MAX_NUM_CFGS);
exit(1);
}
Expand All @@ -1166,7 +1166,7 @@ static void parseOptions(int argc, char **argv) {
inverted_logic = 1;
n++;
}
if(reader_add_cfg(n, "log.enable", inverted_logic ? "0" : "1", 1) == 1) {
if(reader_add_cfg(n, "log", inverted_logic ? "disable" : "enable", 1) == 1) {
printf("Invalid parameter [%s] [num:%d/%d]\n", n, num_cfgs, MAX_NUM_CFGS);
exit(1);
}
Expand Down Expand Up @@ -2829,7 +2829,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
if(_protoFilePath != NULL)
ndpi_load_protocols_file(ndpi_thread_info[thread_id].workflow->ndpi_struct, _protoFilePath);

ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic.enable", "1");
ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, NULL, "tcp_ack_payload_heuristic", "enable");

for(i = 0; i < num_cfgs; i++) {
rc = ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct,
Expand All @@ -2840,7 +2840,7 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) {
}

if(enable_doh_dot_detection)
ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, "tls", "application_blocks_tracking.enable", "1");
ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, "tls", "application_blocks_tracking", "enable");

ret = ndpi_finalize_initialization(ndpi_thread_info[thread_id].workflow->ndpi_struct);
if(ret != 0) {
Expand Down
Loading

0 comments on commit 42d23cf

Please sign in to comment.