Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added scripts to auto generate hostname/SNI *.inc files. #1984

Merged
merged 1 commit into from
May 20, 2023
Merged

Added scripts to auto generate hostname/SNI *.inc files. #1984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/include/ndpi_protocol_ids.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,7 @@ typedef enum {
NDPI_PROTOCOL_HOTS = 336, /* Heroes of the Storm */
NDPI_PROTOCOL_FACEBOOK_REEL_STORY = 337,
NDPI_PROTOCOL_SRTP = 338,
NDPI_PROTOCOL_GAMBLING = 339,

#ifdef CUSTOM_NDPI_PROTOCOLS
#include "../../../nDPI-custom/custom_ndpi_protocol_ids.h"
Expand Down
452 changes: 452 additions & 0 deletions src/lib/inc_generated/ndpi_gambling_match.c.inc
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 4 additions & 0 deletions src/lib/ndpi_main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
#include "inc_generated/ndpi_ms_onedrive_match.c.inc"
#include "inc_generated/ndpi_ms_outlook_match.c.inc"
#include "inc_generated/ndpi_ms_skype_teams_match.c.inc"
#include "inc_generated/ndpi_gambling_match.c.inc"
#include "inc_generated/ndpi_google_match.c.inc"
#include "inc_generated/ndpi_google_cloud_match.c.inc"
#include "inc_generated/ndpi_crawlers_match.c.inc"
Expand Down Expand Up @@ -928,6 +929,9 @@ static void init_string_based_protocols(struct ndpi_detection_module_struct *ndp
for(i = 0; host_match[i].string_to_match != NULL; i++)
ndpi_init_protocol_match(ndpi_str, &host_match[i]);

for(i = 0; ndpi_protocol_gambling_hostname_list[i].string_to_match != NULL; i++)
ndpi_init_protocol_match(ndpi_str, &ndpi_protocol_gambling_hostname_list[i]);

/* ************************ */

for(i = 0; tls_certificate_match[i].string_to_match != NULL; i++) {
Expand Down
4 changes: 2 additions & 2 deletions tests/cfgs/default/result/custom_rules_same-ip_multiple_ports.pcapng.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,5 +23,5 @@ Patricia protocols: 2/2 (search/found)
CustomProtocolA 3 222 1
CustomProtocolB 2 148 1

1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.345/TLS.CustomProtocolA][IP: 345/CustomProtocolA][Encrypted][Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 346/CustomProtocolB][IP: 346/CustomProtocolB][ClearText][Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.245:56866 -> 3.3.3.3:443 [proto: 91.346/TLS.CustomProtocolA][IP: 346/CustomProtocolA][Encrypted][Confidence: Unknown][DPI packets: 1][cat: Web/5][3 pkts/222 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][3.05 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
2 TCP 192.168.1.245:59682 -> 3.3.3.3:444 [proto: 347/CustomProtocolB][IP: 347/CustomProtocolB][ClearText][Confidence: Unknown][DPI packets: 1][2 pkts/148 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][1.02 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 changes: 2 additions & 2 deletions tests/cfgs/default/result/synscan.pcap.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ iSCSI 2 116 2
44 TCP 172.16.0.8:36050 -> 64.13.134.52:2605 [proto: 13/BGP][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
45 TCP 172.16.0.8:36050 -> 64.13.134.52:3000 [proto: 26/ntop][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
46 TCP 172.16.0.8:36050 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
47 TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 339/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
47 TCP 172.16.0.8:36050 -> 64.13.134.52:3260 [proto: 340/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
48 TCP 172.16.0.8:36050 -> 64.13.134.52:3306 [proto: 20/MySQL][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
49 TCP 172.16.0.8:36050 -> 64.13.134.52:3389 [proto: 88/RDP][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Desktop/File Sharing **** Unidirectional Traffic **][Risk Score: 20][Risk Info: No server to client traffic / Found RDP][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
50 TCP 172.16.0.8:36050 -> 64.13.134.52:4343 [proto: 170/Whois-DAS][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Expand Down Expand Up @@ -188,7 +188,7 @@ iSCSI 2 116 2
108 TCP 172.16.0.8:36051 -> 64.13.134.52:2605 [proto: 13/BGP][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
109 TCP 172.16.0.8:36051 -> 64.13.134.52:3000 [proto: 26/ntop][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
110 TCP 172.16.0.8:36051 -> 64.13.134.52:3128 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Web/5][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
111 TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 339/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
111 TCP 172.16.0.8:36051 -> 64.13.134.52:3260 [proto: 340/iSCSI][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
112 TCP 172.16.0.8:36051 -> 64.13.134.52:3306 [proto: 20/MySQL][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Database/11][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
113 TCP 172.16.0.8:36051 -> 64.13.134.52:3389 [proto: 88/RDP][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: RemoteAccess/12][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Desktop/File Sharing **** Unidirectional Traffic **][Risk Score: 20][Risk Info: No server to client traffic / Found RDP][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
114 TCP 172.16.0.8:36051 -> 64.13.134.52:4343 [proto: 170/Whois-DAS][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 1][cat: Network/14][1 pkts/58 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **][Risk Score: 10][Risk Info: No server to client traffic][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Expand Down
17 changes: 17 additions & 0 deletions utils/gambling_sites_download.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/usr/bin/env sh

set -e

cd "$(dirname "${0}")" || exit 1

DEST=../src/lib/inc_generated/ndpi_gambling_match.c.inc
LIST=/tmp/gambling.list

printf '(1) %s\n' "Scraping Illegal Gambling Sites (Belgium)"
DOMAINS="$(curl -s 'https://www.gamingcommission.be/en/gaming-commission/illegal-games-of-chance/list-of-illegal-gambling-sites' | sed -n 's/^<td[^>]\+>\(.\+\.[a-zA-Z0-9]\+\)\(\|\/.*[^<]*\)<\/td>/\1/gp' || exit 1)"

echo "${DOMAINS}" >${LIST}
./hostname2list.py "${LIST}" "Gambling" NDPI_PROTOCOL_GAMBLING NDPI_PROTOCOL_CATEGORY_WEB NDPI_PROTOCOL_UNSAFE >${DEST}
rm -f "${LIST}"

exit 0
59 changes: 59 additions & 0 deletions utils/hostname2list.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
#!/usr/bin/env python3

import sys
import socket, struct

# This scripts is used to create "hostname/sni -> protocols" lists.
if len(sys.argv) < 6:
print("Usage: {} <file> <name> <protocol> <category> <breed>".format(sys.argv[0]))
sys.exit (1)

name = sys.argv[2]
proto = sys.argv[3]
category = sys.argv[4]
breed = sys.argv[5]


print("""/*
*
* This file is generated automatically and part of nDPI
*
* nDPI is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* nDPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with nDPI. If not, see <http://www.gnu.org/licenses/>.
*
*/

/* ****************************************************** */

""")

print("static ndpi_protocol_match "+proto.lower()+"_hostname_list[] = {")

with open(sys.argv[1]) as fp:
for cnt, line in enumerate(fp):
line = line.rstrip()

if(line != ""):
x = line.split("/")

if(len(x) == 2):
host = x[0]
else:
host = line

if(host != ""):
print(' { ' + f'"{host}", "{name}", {proto}, {category}, {breed}, NDPI_PROTOCOL_DEFAULT_LEVEL' + ' },')

print(" /* End */")
print(" { NULL, NULL, NDPI_PROTOCOL_UNKNOWN, NDPI_PROTOCOL_CATEGORY_UNSPECIFIED, NDPI_PROTOCOL_SAFE, NDPI_PROTOCOL_DEFAULT_LEVEL }")
print("};")
3 changes: 3 additions & 0 deletions utils/update_every_lists.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,8 @@ RETVAL=$(( RETVAL + $? ))
./asn_update.sh
RETVAL=$(( RETVAL + $? ))

./gambling_sites_download.sh
RETVAL=$(( RETVAL + $? ))

test ${RETVAL} -ne 0 && printf '%s: %s\n' "${0}" "${RETVAL} script(s) failed"
exit ${RETVAL}