Skip to content

Commit

Permalink
First batch of changes for flexible csv format output mode - still co…
Browse files Browse the repository at this point in the history 
…mpatible
  • Loading branch information
@phaag
phaag committed Jun 9, 2024
1 parent e7df924 commit 0b20860
Show file tree
Hide file tree
Showing 10 changed files with 109 additions and 86 deletions.
6 changes: 3 additions & 3 deletions src/libnffile/conf/nfconf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -135,19 +135,19 @@ int ConfOpen(char *filename, char *section) {
return 1;
} // ConfOpen

// recursive iterate fmt entries from config file
// recursive iterate fmt or csv entries from config file
// return
// 0 if end of list
// i for entry
// -1 for error
int ConfGetFMTentry(char **key, char **value) {
int ConfGetFormatEntry(char *format, char **key, char **value) {
static toml_table_t *fmtConf = NULL;
static int i = 0;

if (!nfconfFile.valid) return 0;

if (!fmtConf) {
fmtConf = toml_table_in(nfconfFile.sectionConf, "fmt");
fmtConf = toml_table_in(nfconfFile.sectionConf, format);
if (!fmtConf) {
*key = NULL;
*value = NULL;
Expand Down
2 changes: 1 addition & 1 deletion src/libnffile/conf/nfconf.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@

int ConfOpen(char *filename, char *section);

int ConfGetFMTentry(char **key, char **value);
int ConfGetFormatEntry(char *format, char **key, char **value);

int ConfGetExporter(char **ident, char **ip, char **flowdir);

Expand Down
3 changes: 3 additions & 0 deletions src/libnffile/conf/nfdump.conf.dist
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ fmt.geolong = "%ts %td %pr %sc %gsap -> %dc %gdap %flg %pkt %byt %fl"
# Nokia NAT
fmt.nokia = "%ts %te %flid %pr %sap -> %dap %isid %osid %nats"

# default csv format
csv.line = "%ts,%td,%pr,%sa,%sp,%da,%dp,%pkt,%byt,%fl"

# OpenBSD pf logs
fmt.pflog = "%ts %pfact %pfrea %pfdir on %pfifn %pfrule %pr %sap -> %dap %pkt %byt"

Expand Down
6 changes: 5 additions & 1 deletion src/nfdump/nfdump.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1266,7 +1266,11 @@ int main(int argc, char **argv) {

if (!outputParams->quiet) {
switch (outputParams->mode) {
case MODE_PLAIN:
case MODE_NULL:
break;
case MODE_RAW:
break;
case MODE_FMT:
PrintSummary(&sum_stat, outputParams);
if (t_last_flow == 0) {
printf("Time window: <unknown>\n");
Expand Down
6 changes: 3 additions & 3 deletions src/nfdump/nflowcache.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1728,7 +1728,7 @@ int SetBidirAggregation(void) {
// print -s record/xx statistics with as many print orders as required
void PrintFlowStat(RecordPrinter_t print_record, outputParams_t *outputParams) {
dbg_printf("Enter %s\n", __func__);
if (outputParams->mode != MODE_PLAIN) {
if (outputParams->mode != MODE_FMT) {
printf("Output format not implemented\n");
return;
}
Expand All @@ -1755,7 +1755,7 @@ void PrintFlowStat(RecordPrinter_t print_record, outputParams_t *outputParams) {
blocksort(SortList, maxindex);

if (!outputParams->quiet) {
if (outputParams->mode == MODE_PLAIN) {
if (outputParams->mode == MODE_FMT) {
if (outputParams->topN != 0)
printf("Top %i flows ordered by %s:\n", outputParams->topN, order_mode[order_index].string);
else
Expand All @@ -1773,7 +1773,7 @@ void PrintFlowStat(RecordPrinter_t print_record, outputParams_t *outputParams) {
// print Flow cache
void PrintFlowTable(RecordPrinter_t print_record, outputParams_t *outputParams, int GuessDir) {
dbg_printf("Enter %s\n", __func__);
if (outputParams->mode != MODE_PLAIN) {
if (outputParams->mode != MODE_FMT) {
printf("Output format not implemented\n");
return;
}
Expand Down
7 changes: 5 additions & 2 deletions src/nfdump/nfstat.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1365,7 +1365,7 @@ void PrintElementStat(stat_record_t *sum_stat, outputParams_t *outputParams, Rec
SortElement_t *topN_element_list = StatTopN(outputParams->topN, &numflows, hash_num, order_index, direction);

// this output formatting is pretty ugly - and needs to be cleaned up - improved
if (outputParams->mode == MODE_PLAIN && !outputParams->quiet) {
if (outputParams->mode == MODE_FMT && !outputParams->quiet) {
if (outputParams->topN != 0) {
printf("Top %i %s ordered by %s:\n", outputParams->topN, StatParameters[stat].HeaderInfo, orderByTable[order_index].string);
} else {
Expand Down Expand Up @@ -1421,7 +1421,10 @@ void PrintElementStat(stat_record_t *sum_stat, outputParams_t *outputParams, Rec
int index = startIndex;
while (index != endIndex) {
switch (outputParams->mode) {
case MODE_PLAIN:
case MODE_NULL:
case MODE_RAW:
break;
case MODE_FMT:
PrintStatLine(sum_stat, outputParams, &topN_element_list[index], type, StatRequest[hash_num].order_proto,
orderByTable[order_index].inout);
break;
Expand Down
131 changes: 73 additions & 58 deletions src/output/output.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,8 @@
#endif
#define DefaultGeoMode "gline"

static void AddFormat(char *format, char *name, char *fmtString);

static void null_record(FILE *stream, recordHandle_t *record, int tag);

static void null_prolog(void);
Expand All @@ -81,23 +83,35 @@ static void null_epilog(void);

// Assign print functions for all output options -o
// Terminated with a NULL record
printmap_t printmap[MAXFORMATS] = {{"raw", raw_record, raw_prolog, raw_epilog, NULL, "Raw format - multi line"},
{"line", fmt_record, fmt_prolog, fmt_epilog, FORMAT_line, "predefined"},
{"gline", fmt_record, fmt_prolog, fmt_epilog, FORMAT_gline, "predefined"},
{"long", fmt_record, fmt_prolog, fmt_epilog, FORMAT_long, "predefined"},
{"glong", fmt_record, fmt_prolog, fmt_epilog, FORMAT_glong, "predefined"},
{"extended", fmt_record, fmt_prolog, fmt_epilog, FORMAT_extended, "predefined"},
{"biline", fmt_record, fmt_prolog, fmt_epilog, FORMAT_biline, "predefined"},
{"bilong", fmt_record, fmt_prolog, fmt_epilog, FORMAT_bilong, "predefined"},
{"nsel", fmt_record, fmt_prolog, fmt_epilog, FORMAT_nsel, "predefined"},
{"nat", fmt_record, fmt_prolog, fmt_epilog, FORMAT_nat, "predefined"},
{"json", flow_record_to_json_human, json_prolog, json_epilog, NULL, "json output"},
{"json-log", flow_record_to_json_log, null_prolog, null_epilog, NULL, "json output for logging"},
{"csv", csv_record, csv_prolog, csv_epilog, NULL, "csv predefined"},
{"null", null_record, null_prolog, null_epilog, NULL, "do not print any output"},
printmap_t printmap[MAXFORMATS] = {{"raw", MODE_RAW, NULL, "Raw format - multi line"},
{"line", MODE_FMT, FORMAT_line, "predefined"},
{"gline", MODE_FMT, FORMAT_gline, "predefined"},
{"long", MODE_FMT, FORMAT_long, "predefined"},
{"glong", MODE_FMT, FORMAT_glong, "predefined"},
{"extended", MODE_FMT, FORMAT_extended, "predefined"},
{"biline", MODE_FMT, FORMAT_biline, "predefined"},
{"bilong", MODE_FMT, FORMAT_bilong, "predefined"},
{"nsel", MODE_FMT, FORMAT_nsel, "predefined"},
{"nat", MODE_FMT, FORMAT_nat, "predefined"},
{"json", MODE_JSON, NULL, "json output"},
{"json-log", MODE_JSON, NULL, "json output for logging"},
{"csv", MODE_CSV, NULL, "csv predefined"},
{"null", MODE_NULL, NULL, "do not print any output"},

// This is always the last line
{NULL, NULL, NULL, NULL, "", NULL}};
{NULL, MODE_NULL, "", NULL}};

// table with appropriate printer function for given format
static struct printerFunc_s {
RecordPrinter_t func_record; // prints the record
PrologPrinter_t func_prolog; // prints the output prolog
PrologPrinter_t func_epilog; // prints the output epilog
} printFuncMap[] = {[MODE_NULL] = {null_record, null_prolog, null_epilog},
[MODE_FMT] = {fmt_record, fmt_prolog, fmt_epilog},
[MODE_RAW] = {raw_record, raw_prolog, raw_epilog},
[MODE_CSV] = {csv_record, csv_prolog, csv_epilog},
[MODE_JSON] = {flow_record_to_json_human, json_prolog, json_epilog},
[MODE_JSON_LOG] = {flow_record_to_json_log, json_prolog, json_epilog}};

static PrologPrinter_t print_prolog; // prints the output prolog
static PrologPrinter_t print_epilog; // prints the output epilog
Expand All @@ -116,15 +130,14 @@ static void null_epilog(void) {
// empty epilog
} // End of null_epilog

void AddFormat(char *name, char *fmtString) {
static void AddFormat(char *format, char *name, char *fmtString) {
int csvMode = strcmp(format, "csv") == 0;
int i = 0;
while (printmap[i].printmode) {
if (strncasecmp(name, printmap[i].printmode, MAXMODELEN) == 0) {
// default format exists - overwrite
printmap[i].Format = fmtString;
printmap[i].func_record = fmt_record;
printmap[i].func_prolog = fmt_prolog;
printmap[i].func_epilog = fmt_epilog;
printmap[i].outputMode = csvMode ? MODE_CSV : MODE_FMT;
dbg_printf("Overwrite format: %s\n", name);
free(name);
return;
Expand All @@ -136,12 +149,10 @@ void AddFormat(char *name, char *fmtString) {
printmap[i].printmode = name;
printmap[i].Format = fmtString;
printmap[i].help = "user defined";
printmap[i].func_record = fmt_record;
printmap[i].func_prolog = fmt_prolog;
printmap[i].func_epilog = fmt_epilog;
printmap[i].outputMode = csvMode ? MODE_CSV : MODE_FMT;
i++;
printmap[i].printmode = NULL;
dbg_printf("Insert format: %s\n", name);
dbg_printf("Insert format: %s - %s\n", csvMode ? "csv" : "fmt", name);
} else {
LogError("Number of print format slots exhaustet: %d", MAXFORMATS);
}
Expand All @@ -151,16 +162,19 @@ static void UpdateFormatList(void) {
char *key = NULL;
char *value = NULL;

int ret;
do {
ret = ConfGetFMTentry(&key, &value);
if (ret > 0) {
dbg_printf("key: %s, value %s\n", key, value);
AddFormat(key, value);
} else {
break;
}
} while (1);
char *formats[2] = {"fmt", "csv"};

for (int i = 0; i < 2; i++) {
do {
int ret = ConfGetFormatEntry(formats[i], &key, &value);
if (ret > 0) {
dbg_printf("format: %s, key: %s, value %s\n", formats[i], key, value);
AddFormat(formats[i], key, value);
} else {
break;
}
} while (1);
}

} // End of UpdateFormatList

Expand All @@ -172,15 +186,27 @@ RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams

if (print_format == NULL) print_format = outputParams->hasGeoDB ? DefaultGeoMode : DefaultMode;

if (strncasecmp(print_format, "fmt:", 4) == 0 || print_format[0] == '%') {
int fmtFormat = strncasecmp(print_format, "fmt:", 4) == 0;
int csvFormat = strncasecmp(print_format, "csv:", 4) == 0;
if (fmtFormat || csvFormat || print_format[0] == '%') {
// special user defined output format
char *format = &print_format[4]; // for 'fmt:%xxx'
if (print_format[0] == '%') format = print_format; // for '%xxx' - forgot to add fmt:
char *format = &print_format[4]; // for 'fmt:%xxx' or 'csv:%xxx'
if (print_format[0] == '%') {
fmtFormat = 1;
format = print_format; // for '%xxx' - forgot to add fmt: assume fmt
}

if (strlen(format)) {
if (!ParseOutputFormat(format, outputParams->printPlain, printmap)) exit(EXIT_FAILURE);
print_record = fmt_record;
print_prolog = fmt_prolog;
print_epilog = fmt_epilog;
if (!ParseOutputFormat(csvFormat, format, outputParams->printPlain, printmap)) exit(EXIT_FAILURE);
if (csvFormat) {
print_record = csv_record;
print_prolog = csv_prolog;
print_epilog = csv_epilog;
} else {
print_record = fmt_record;
print_prolog = fmt_prolog;
print_epilog = fmt_epilog;
}
} else {
LogError("Missing format description for user defined output format!\n");
exit(EXIT_FAILURE);
Expand All @@ -201,27 +227,16 @@ RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams
i = 0;
while (printmap[i].printmode) {
if (strncasecmp(print_format, printmap[i].printmode, MAXMODELEN) == 0) {
outputParams->mode = printmap[i].outputMode;
if (printmap[i].Format) {
if (!ParseOutputFormat(printmap[i].Format, outputParams->printPlain, printmap)) exit(EXIT_FAILURE);
// predefined custom format
print_record = printmap[i].func_record;
print_prolog = printmap[i].func_prolog;
print_epilog = printmap[i].func_epilog;
} else {
if (strncasecmp(print_format, "csv", MAXMODELEN) == 0) {
outputParams->mode = MODE_CSV;
} else if (strncasecmp(print_format, "json", MAXMODELEN) == 0) {
outputParams->mode = MODE_JSON;
} else if (strncasecmp(print_format, "json-log", MAXMODELEN) == 0) {
outputParams->mode = MODE_JSON_LOG;
} else {
outputParams->mode = MODE_PLAIN;
}
// predefined static format
print_record = printmap[i].func_record;
print_prolog = printmap[i].func_prolog;
print_epilog = printmap[i].func_epilog;
if (!ParseOutputFormat(outputParams->mode == MODE_CSV, printmap[i].Format, outputParams->printPlain, printmap))
exit(EXIT_FAILURE);
}
// else - predefined static format
print_record = printFuncMap[outputParams->mode].func_record;
print_prolog = printFuncMap[outputParams->mode].func_prolog;
print_epilog = printFuncMap[outputParams->mode].func_epilog;
break;
}
i++;
Expand Down
16 changes: 6 additions & 10 deletions src/output/output.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,29 +40,25 @@ typedef void (*RecordPrinter_t)(FILE *, recordHandle_t *, int);
typedef void (*PrologPrinter_t)(void);
typedef void (*EpilogPrinter_t)(void);

enum { MODE_PLAIN = 0, MODE_JSON, MODE_CSV, MODE_JSON_LOG };
typedef enum { MODE_NULL = 0, MODE_RAW, MODE_FMT, MODE_CSV, MODE_JSON, MODE_JSON_LOG } outputMode_t;

typedef struct outputParams_s {
bool printPlain;
bool doTag;
bool quiet;
bool hasGeoDB;
bool hasTorDB;
int mode;
outputMode_t mode;
int topN;
} outputParams_t;

typedef struct printmap_s {
char *printmode; // name of the output format
RecordPrinter_t func_record; // prints the record
PrologPrinter_t func_prolog; // prints the output prolog
PrologPrinter_t func_epilog; // prints the output epilog
char *Format; // output format definition
char *help; // help text
char *printmode; // name of the output format
outputMode_t outputMode; // type of output mode
char *Format; // output format definition
char *help; // help text
} printmap_t;

void AddFormat(char *name, char *fmtString);

RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams);

void PrintProlog(outputParams_t *outputParams);
Expand Down
14 changes: 8 additions & 6 deletions src/output/output_fmt.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ static int max_format_index = 0;

static int do_tag = 0;
static int long_v6 = 0;
static int modeCSV = 0;
static int printPlain = 0;
static double duration = 0;

Expand Down Expand Up @@ -354,7 +355,7 @@ static void String_natString(FILE *stream, recordHandle_t *recordHandle);
static struct format_token_list_s {
char *token; // token
int is_address; // is an IP address
char *header; // header line description
char *fmtHeader; // header line description
string_function_t string_function; // function generation output string
} format_token_list[] = {
// v3 header info
Expand Down Expand Up @@ -659,6 +660,7 @@ void fmt_record(FILE *stream, recordHandle_t *recordHandle, int tag) {

void fmt_prolog(void) {
// header
modeCSV = 0;
printf("%s\n", header_string);
} // End of fmt_prolog

Expand Down Expand Up @@ -774,7 +776,7 @@ static char *RecursiveReplace(char *format, printmap_t *printmap) {

} // End of RecursiveReplace

int ParseOutputFormat(char *format, int plain_numbers, printmap_t *printmap) {
int ParseOutputFormat(int csvFormat, char *format, int plain_numbers, printmap_t *printmap) {
char *c, *s, *h;
int i, remaining;

Expand Down Expand Up @@ -806,9 +808,9 @@ int ParseOutputFormat(char *format, int plain_numbers, printmap_t *printmap) {
if (strncmp(format_token_list[i].token, c, len) == 0) { // token found
AddToken(i, NULL);
if (long_v6 && format_token_list[i].is_address)
snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%23s%s", "", format_token_list[i].header);
snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%23s%s", "", format_token_list[i].fmtHeader);
else
snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%s", format_token_list[i].header);
snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%s", format_token_list[i].fmtHeader);
h += strlen(h);
c[len] = p;
c += len;
Expand Down Expand Up @@ -1302,9 +1304,9 @@ static void String_SrcAddr(FILE *stream, recordHandle_t *recordHandle) {
tmp_str[IP_STRING_LEN - 1] = 0;

if (long_v6)
fprintf(stream, "%s%39s", tag_string, tmp_str);
fprintf(stream, "%s%*s", tag_string, 39, tmp_str);

This comment has been minimized.

Sign in to view

Copy link
@gabrielmocan

gabrielmocan Jul 18, 2024

@phaag why only this string is formatted in this way now? This change broke a couple things here 😔

This comment has been minimized.

Sign in to view

Copy link
@phaag

phaag Jul 19, 2024

Author Owner

@gabrielmocan
the syntax:
fprintf(stream, "%s%39s", tag_string, tmp_str) and
fprintf(stream, "%s%*s", tag_string, 39, tmp_str)
results in the same code but is simply a different syntax. So I don't see the point, why this should break anything.
The csv format in general did change, but not that specific string. If you see a problem somewhere, please tell me and open an issue.
else
fprintf(stream, "%s%16s", tag_string, tmp_str);
fprintf(stream, "%s%*s", tag_string, 16, tmp_str);

} // End of String_SrcAddr

Expand Down
Loading

0 comments on commit 0b20860

Please sign in to comment.