Skip to content

Commit

Permalink
ndpiReader: improve printing of payload statistics
Browse files Browse the repository at this point in the history 
Add a basic unit test

Fix an endianess issue
  • Loading branch information
@IvanNardi
IvanNardi committed May 29, 2023
1 parent 7ce14da commit 67bf1f4
Show file tree
Hide file tree
Showing 7 changed files with 316 additions and 36 deletions.
12 changes: 6 additions & 6 deletions example/ndpiReader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -252,7 +252,7 @@ static int dpdk_port_id = 0, dpdk_run_capture = 1;

void test_lib(); /* Forward */

extern void ndpi_report_payload_stats(int print);
extern void ndpi_report_payload_stats(FILE *out);
extern int parse_proto_name_list(char *str, NDPI_PROTOCOL_BITMASK *bitmask, int inverted_logic);

/* ********************************** */
Expand Down Expand Up @@ -420,10 +420,10 @@ flowGetBDMeanandVariance(struct ndpi_flow_info* flow) {
if(csv_fp) {
fprintf(csv_fp, ",%.3f,%.3f,%.3f,%.3f", mean, variance, entropy, entropy * num_bytes);
} else {
fprintf(out, "[byte_dist_mean: %f", mean);
fprintf(out, "][byte_dist_std: %f]", variance);
fprintf(out, "[entropy: %f]", entropy);
fprintf(out, "[total_entropy: %f]", entropy * num_bytes);
fprintf(out, "[byte_dist_mean: %.3f", mean);
fprintf(out, "][byte_dist_std: %.3f]", variance);
fprintf(out, "[entropy: %.3f]", entropy);
fprintf(out, "[total_entropy: %.3f]", entropy * num_bytes);
}
} else {
if(csv_fp)
Expand Down Expand Up @@ -2747,7 +2747,7 @@ static void printFlowsStats() {
FILE *out = results_file ? results_file : stdout;

if(enable_payload_analyzer)
ndpi_report_payload_stats(1);
ndpi_report_payload_stats(out);

for(thread_id = 0; thread_id < num_threads; thread_id++)
total_flows += ndpi_thread_info[thread_id].workflow->num_allocated_flows;
Expand Down
55 changes: 28 additions & 27 deletions example/reader_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ void ndpi_analyze_payload(struct ndpi_flow_info *flow,
struct packet_id_stats *p;

#ifdef DEBUG_PAYLOAD
u_int16_t i;
for(i=0; i<payload_len; i++)
printf("%c", isprint(payload[i]) ? payload[i] : '.');
printf("\n");
Expand Down Expand Up @@ -216,68 +217,68 @@ static int payload_stats_sort_asc(void *_a, void *_b) {

/* ***************************************************** */

void print_payload_stat(struct payload_stats *p) {
static void print_payload_stat(struct payload_stats *p, FILE *out) {
u_int i;
struct flow_id_stats *s, *tmp;
struct packet_id_stats *s1, *tmp1;

printf("\t[");
fprintf(out, "\t[");

for(i=0; i<p->pattern_len; i++) {
printf("%c", isprint(p->pattern[i]) ? p->pattern[i] : '.');
fprintf(out, "%c", isprint(p->pattern[i]) ? p->pattern[i] : '.');
}

printf("]");
for(; i<16; i++) printf(" ");
printf("[");
fprintf(out, "]");
for(; i<16; i++) fprintf(out, " ");
fprintf(out, "[");

for(i=0; i<p->pattern_len; i++) {
printf("%s%02X", (i > 0) ? " " : "", isprint(p->pattern[i]) ? p->pattern[i] : '.');
fprintf(out, "%s%02X", (i > 0) ? " " : "", isprint(p->pattern[i]) ? p->pattern[i] : '.');
}

printf("]");
fprintf(out, "]");

for(; i<16; i++) printf(" ");
for(i=p->pattern_len; i<max_pattern_len; i++) printf(" ");
for(; i<16; i++) fprintf(out, " ");
for(i=p->pattern_len; i<max_pattern_len; i++) fprintf(out, " ");

printf("[len: %u][num_occurrencies: %u][flowId: ",
p->pattern_len, p->num_occurrencies);
fprintf(out, "[len: %u][num_occurrencies: %u][flowId: ",
p->pattern_len, p->num_occurrencies);

i = 0;
HASH_ITER(hh, p->flows, s, tmp) {
printf("%s%u", (i > 0) ? " " : "", s->flow_id);
fprintf(out, "%s%u", (i > 0) ? " " : "", s->flow_id);
i++;
}

printf("][packetIds: ");
fprintf(out, "][packetIds: ");

/* ******************************** */

i = 0;
HASH_ITER(hh, p->packets, s1, tmp1) {
printf("%s%u", (i > 0) ? " " : "", s1->packet_id);
fprintf(out, "%s%u", (i > 0) ? " " : "", s1->packet_id);
i++;
}

printf("]\n");
fprintf(out, "]\n");


}

/* ***************************************************** */

void ndpi_report_payload_stats(int print) {
void ndpi_report_payload_stats(FILE *out) {
struct payload_stats *p, *tmp;
u_int num = 0;

if(print)
printf("\n\nPayload Analysis\n");
if(out)
fprintf(out, "\n\nPayload Analysis\n");

HASH_SORT(pstats, payload_stats_sort_asc);

HASH_ITER(hh, pstats, p, tmp) {
if(print && num <= max_num_reported_top_payloads)
print_payload_stat(p);
if(out && num <= max_num_reported_top_payloads)
print_payload_stat(p, out);

ndpi_free(p->pattern);

Expand Down Expand Up @@ -711,20 +712,20 @@ ndpi_flow_update_byte_dist_mean_var(ndpi_flow_info_t *flow, const void *x,

/* ***************************************************** */

float ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256],
double ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256],
unsigned int num_bytes)
{
int i;
float sum = 0.0;
double sum = 0.0;

for(i=0; i<256; i++) {
float tmp = (float) byte_count[i] / (float) num_bytes;
double tmp = (double) byte_count[i] / (double) num_bytes;

if(tmp > FLT_EPSILON) {
sum -= tmp * logf(tmp);
}
}
return(sum / logf(2.0));
return(sum / log(2.0));
}

/* ***************************************************** */
Expand Down Expand Up @@ -1584,15 +1585,15 @@ static struct ndpi_proto packet_processing(struct ndpi_workflow * workflow,
flow->entropy->score = ndpi_classify(flow->entropy->src2dst_pkt_len, flow->entropy->src2dst_pkt_time,
flow->entropy->dst2src_pkt_len, flow->entropy->dst2src_pkt_time,
flow->entropy->src2dst_start, flow->entropy->dst2src_start,
max_num_packets_per_flow, flow->src_port, flow->dst_port,
max_num_packets_per_flow, ntohs(flow->src_port), ntohs(flow->dst_port),
flow->src2dst_packets, flow->dst2src_packets,
flow->entropy->src2dst_opackets, flow->entropy->dst2src_opackets,
flow->entropy->src2dst_l4_bytes, flow->entropy->dst2src_l4_bytes, 1,
flow->entropy->src2dst_byte_count, flow->entropy->dst2src_byte_count);
else
flow->entropy->score = ndpi_classify(flow->entropy->src2dst_pkt_len, flow->entropy->src2dst_pkt_time,
NULL, NULL, flow->entropy->src2dst_start, flow->entropy->src2dst_start,
max_num_packets_per_flow, flow->src_port, flow->dst_port,
max_num_packets_per_flow, ntohs(flow->src_port), ntohs(flow->dst_port),
flow->src2dst_packets, 0,
flow->entropy->src2dst_opackets, 0,
flow->entropy->src2dst_l4_bytes, 0, 1,
Expand Down
2 changes: 1 addition & 1 deletion example/reader_util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
void ndpi_flow_info_free_data(struct ndpi_flow_info *flow);
void ndpi_flow_info_freer(void *node);
const char* print_cipher_id(u_int32_t cipher);
float ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256], unsigned int num_bytes);
double ndpi_flow_get_byte_count_entropy(const uint32_t byte_count[256], unsigned int num_bytes);

extern int nDPI_LogLevel;

Expand Down
4 changes: 2 additions & 2 deletions fuzz/fuzz_ndpi_reader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ int malloc_size_stats = 0;
int max_malloc_bins = 14;
struct ndpi_bin malloc_bins; /* unused */

extern void ndpi_report_payload_stats(int print);
extern void ndpi_report_payload_stats(FILE *out);

#ifdef CRYPT_FORCE_NO_AESNI
extern int force_no_aesni;
Expand Down Expand Up @@ -152,7 +152,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) {
ndpi_free(workflow->ndpi_flows_root);
/* Free payload analyzer data, without printing */
if(enable_payload_analyzer)
ndpi_report_payload_stats(0);
ndpi_report_payload_stats(NULL);

return 0;
}
1 change: 1 addition & 0 deletions tests/cfgs/enable_payload_stat/config.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
-P 4:8:10:16:25
1 change: 1 addition & 0 deletions tests/cfgs/enable_payload_stat/pcap/1kxun.pcap
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
../../default/pcap/1kxun.pcap
Loading

0 comments on commit 67bf1f4

Please sign in to comment.