From 0b2086014bfa29a89f98515bd0bb5f26ac291023 Mon Sep 17 00:00:00 2001 From: Peter Haag Date: Sun, 9 Jun 2024 16:39:51 +0200 Subject: [PATCH] First batch of changes for flexible csv format output mode - still compatible --- src/libnffile/conf/nfconf.c | 6 +- src/libnffile/conf/nfconf.h | 2 +- src/libnffile/conf/nfdump.conf.dist | 3 + src/nfdump/nfdump.c | 6 +- src/nfdump/nflowcache.c | 6 +- src/nfdump/nfstat.c | 7 +- src/output/output.c | 131 ++++++++++++++++------------ src/output/output.h | 16 ++-- src/output/output_fmt.c | 14 +-- src/output/output_fmt.h | 4 +- 10 files changed, 109 insertions(+), 86 deletions(-) diff --git a/src/libnffile/conf/nfconf.c b/src/libnffile/conf/nfconf.c index c6e8fa15..03b6231f 100644 --- a/src/libnffile/conf/nfconf.c +++ b/src/libnffile/conf/nfconf.c @@ -135,19 +135,19 @@ int ConfOpen(char *filename, char *section) { return 1; } // ConfOpen -// recursive iterate fmt entries from config file +// recursive iterate fmt or csv entries from config file // return // 0 if end of list // i for entry // -1 for error -int ConfGetFMTentry(char **key, char **value) { +int ConfGetFormatEntry(char *format, char **key, char **value) { static toml_table_t *fmtConf = NULL; static int i = 0; if (!nfconfFile.valid) return 0; if (!fmtConf) { - fmtConf = toml_table_in(nfconfFile.sectionConf, "fmt"); + fmtConf = toml_table_in(nfconfFile.sectionConf, format); if (!fmtConf) { *key = NULL; *value = NULL; diff --git a/src/libnffile/conf/nfconf.h b/src/libnffile/conf/nfconf.h index ceeed716..4d8cc1c3 100644 --- a/src/libnffile/conf/nfconf.h +++ b/src/libnffile/conf/nfconf.h @@ -38,7 +38,7 @@ int ConfOpen(char *filename, char *section); -int ConfGetFMTentry(char **key, char **value); +int ConfGetFormatEntry(char *format, char **key, char **value); int ConfGetExporter(char **ident, char **ip, char **flowdir); diff --git a/src/libnffile/conf/nfdump.conf.dist b/src/libnffile/conf/nfdump.conf.dist index de34f551..fab2aa2e 100644 --- a/src/libnffile/conf/nfdump.conf.dist +++ b/src/libnffile/conf/nfdump.conf.dist @@ -26,6 +26,9 @@ fmt.geolong = "%ts %td %pr %sc %gsap -> %dc %gdap %flg %pkt %byt %fl" # Nokia NAT fmt.nokia = "%ts %te %flid %pr %sap -> %dap %isid %osid %nats" +# default csv format +csv.line = "%ts,%td,%pr,%sa,%sp,%da,%dp,%pkt,%byt,%fl" + # OpenBSD pf logs fmt.pflog = "%ts %pfact %pfrea %pfdir on %pfifn %pfrule %pr %sap -> %dap %pkt %byt" diff --git a/src/nfdump/nfdump.c b/src/nfdump/nfdump.c index f04f0f59..18c4f620 100644 --- a/src/nfdump/nfdump.c +++ b/src/nfdump/nfdump.c @@ -1266,7 +1266,11 @@ int main(int argc, char **argv) { if (!outputParams->quiet) { switch (outputParams->mode) { - case MODE_PLAIN: + case MODE_NULL: + break; + case MODE_RAW: + break; + case MODE_FMT: PrintSummary(&sum_stat, outputParams); if (t_last_flow == 0) { printf("Time window: \n"); diff --git a/src/nfdump/nflowcache.c b/src/nfdump/nflowcache.c index cbd981bc..9f14c2fd 100755 --- a/src/nfdump/nflowcache.c +++ b/src/nfdump/nflowcache.c @@ -1728,7 +1728,7 @@ int SetBidirAggregation(void) { // print -s record/xx statistics with as many print orders as required void PrintFlowStat(RecordPrinter_t print_record, outputParams_t *outputParams) { dbg_printf("Enter %s\n", __func__); - if (outputParams->mode != MODE_PLAIN) { + if (outputParams->mode != MODE_FMT) { printf("Output format not implemented\n"); return; } @@ -1755,7 +1755,7 @@ void PrintFlowStat(RecordPrinter_t print_record, outputParams_t *outputParams) { blocksort(SortList, maxindex); if (!outputParams->quiet) { - if (outputParams->mode == MODE_PLAIN) { + if (outputParams->mode == MODE_FMT) { if (outputParams->topN != 0) printf("Top %i flows ordered by %s:\n", outputParams->topN, order_mode[order_index].string); else @@ -1773,7 +1773,7 @@ void PrintFlowStat(RecordPrinter_t print_record, outputParams_t *outputParams) { // print Flow cache void PrintFlowTable(RecordPrinter_t print_record, outputParams_t *outputParams, int GuessDir) { dbg_printf("Enter %s\n", __func__); - if (outputParams->mode != MODE_PLAIN) { + if (outputParams->mode != MODE_FMT) { printf("Output format not implemented\n"); return; } diff --git a/src/nfdump/nfstat.c b/src/nfdump/nfstat.c index 23d3299c..d8b34cc3 100644 --- a/src/nfdump/nfstat.c +++ b/src/nfdump/nfstat.c @@ -1365,7 +1365,7 @@ void PrintElementStat(stat_record_t *sum_stat, outputParams_t *outputParams, Rec SortElement_t *topN_element_list = StatTopN(outputParams->topN, &numflows, hash_num, order_index, direction); // this output formatting is pretty ugly - and needs to be cleaned up - improved - if (outputParams->mode == MODE_PLAIN && !outputParams->quiet) { + if (outputParams->mode == MODE_FMT && !outputParams->quiet) { if (outputParams->topN != 0) { printf("Top %i %s ordered by %s:\n", outputParams->topN, StatParameters[stat].HeaderInfo, orderByTable[order_index].string); } else { @@ -1421,7 +1421,10 @@ void PrintElementStat(stat_record_t *sum_stat, outputParams_t *outputParams, Rec int index = startIndex; while (index != endIndex) { switch (outputParams->mode) { - case MODE_PLAIN: + case MODE_NULL: + case MODE_RAW: + break; + case MODE_FMT: PrintStatLine(sum_stat, outputParams, &topN_element_list[index], type, StatRequest[hash_num].order_proto, orderByTable[order_index].inout); break; diff --git a/src/output/output.c b/src/output/output.c index 3ad46b03..e14e4dcf 100644 --- a/src/output/output.c +++ b/src/output/output.c @@ -73,6 +73,8 @@ #endif #define DefaultGeoMode "gline" +static void AddFormat(char *format, char *name, char *fmtString); + static void null_record(FILE *stream, recordHandle_t *record, int tag); static void null_prolog(void); @@ -81,23 +83,35 @@ static void null_epilog(void); // Assign print functions for all output options -o // Terminated with a NULL record -printmap_t printmap[MAXFORMATS] = {{"raw", raw_record, raw_prolog, raw_epilog, NULL, "Raw format - multi line"}, - {"line", fmt_record, fmt_prolog, fmt_epilog, FORMAT_line, "predefined"}, - {"gline", fmt_record, fmt_prolog, fmt_epilog, FORMAT_gline, "predefined"}, - {"long", fmt_record, fmt_prolog, fmt_epilog, FORMAT_long, "predefined"}, - {"glong", fmt_record, fmt_prolog, fmt_epilog, FORMAT_glong, "predefined"}, - {"extended", fmt_record, fmt_prolog, fmt_epilog, FORMAT_extended, "predefined"}, - {"biline", fmt_record, fmt_prolog, fmt_epilog, FORMAT_biline, "predefined"}, - {"bilong", fmt_record, fmt_prolog, fmt_epilog, FORMAT_bilong, "predefined"}, - {"nsel", fmt_record, fmt_prolog, fmt_epilog, FORMAT_nsel, "predefined"}, - {"nat", fmt_record, fmt_prolog, fmt_epilog, FORMAT_nat, "predefined"}, - {"json", flow_record_to_json_human, json_prolog, json_epilog, NULL, "json output"}, - {"json-log", flow_record_to_json_log, null_prolog, null_epilog, NULL, "json output for logging"}, - {"csv", csv_record, csv_prolog, csv_epilog, NULL, "csv predefined"}, - {"null", null_record, null_prolog, null_epilog, NULL, "do not print any output"}, +printmap_t printmap[MAXFORMATS] = {{"raw", MODE_RAW, NULL, "Raw format - multi line"}, + {"line", MODE_FMT, FORMAT_line, "predefined"}, + {"gline", MODE_FMT, FORMAT_gline, "predefined"}, + {"long", MODE_FMT, FORMAT_long, "predefined"}, + {"glong", MODE_FMT, FORMAT_glong, "predefined"}, + {"extended", MODE_FMT, FORMAT_extended, "predefined"}, + {"biline", MODE_FMT, FORMAT_biline, "predefined"}, + {"bilong", MODE_FMT, FORMAT_bilong, "predefined"}, + {"nsel", MODE_FMT, FORMAT_nsel, "predefined"}, + {"nat", MODE_FMT, FORMAT_nat, "predefined"}, + {"json", MODE_JSON, NULL, "json output"}, + {"json-log", MODE_JSON, NULL, "json output for logging"}, + {"csv", MODE_CSV, NULL, "csv predefined"}, + {"null", MODE_NULL, NULL, "do not print any output"}, // This is always the last line - {NULL, NULL, NULL, NULL, "", NULL}}; + {NULL, MODE_NULL, "", NULL}}; + +// table with appropriate printer function for given format +static struct printerFunc_s { + RecordPrinter_t func_record; // prints the record + PrologPrinter_t func_prolog; // prints the output prolog + PrologPrinter_t func_epilog; // prints the output epilog +} printFuncMap[] = {[MODE_NULL] = {null_record, null_prolog, null_epilog}, + [MODE_FMT] = {fmt_record, fmt_prolog, fmt_epilog}, + [MODE_RAW] = {raw_record, raw_prolog, raw_epilog}, + [MODE_CSV] = {csv_record, csv_prolog, csv_epilog}, + [MODE_JSON] = {flow_record_to_json_human, json_prolog, json_epilog}, + [MODE_JSON_LOG] = {flow_record_to_json_log, json_prolog, json_epilog}}; static PrologPrinter_t print_prolog; // prints the output prolog static PrologPrinter_t print_epilog; // prints the output epilog @@ -116,15 +130,14 @@ static void null_epilog(void) { // empty epilog } // End of null_epilog -void AddFormat(char *name, char *fmtString) { +static void AddFormat(char *format, char *name, char *fmtString) { + int csvMode = strcmp(format, "csv") == 0; int i = 0; while (printmap[i].printmode) { if (strncasecmp(name, printmap[i].printmode, MAXMODELEN) == 0) { // default format exists - overwrite printmap[i].Format = fmtString; - printmap[i].func_record = fmt_record; - printmap[i].func_prolog = fmt_prolog; - printmap[i].func_epilog = fmt_epilog; + printmap[i].outputMode = csvMode ? MODE_CSV : MODE_FMT; dbg_printf("Overwrite format: %s\n", name); free(name); return; @@ -136,12 +149,10 @@ void AddFormat(char *name, char *fmtString) { printmap[i].printmode = name; printmap[i].Format = fmtString; printmap[i].help = "user defined"; - printmap[i].func_record = fmt_record; - printmap[i].func_prolog = fmt_prolog; - printmap[i].func_epilog = fmt_epilog; + printmap[i].outputMode = csvMode ? MODE_CSV : MODE_FMT; i++; printmap[i].printmode = NULL; - dbg_printf("Insert format: %s\n", name); + dbg_printf("Insert format: %s - %s\n", csvMode ? "csv" : "fmt", name); } else { LogError("Number of print format slots exhaustet: %d", MAXFORMATS); } @@ -151,16 +162,19 @@ static void UpdateFormatList(void) { char *key = NULL; char *value = NULL; - int ret; - do { - ret = ConfGetFMTentry(&key, &value); - if (ret > 0) { - dbg_printf("key: %s, value %s\n", key, value); - AddFormat(key, value); - } else { - break; - } - } while (1); + char *formats[2] = {"fmt", "csv"}; + + for (int i = 0; i < 2; i++) { + do { + int ret = ConfGetFormatEntry(formats[i], &key, &value); + if (ret > 0) { + dbg_printf("format: %s, key: %s, value %s\n", formats[i], key, value); + AddFormat(formats[i], key, value); + } else { + break; + } + } while (1); + } } // End of UpdateFormatList @@ -172,15 +186,27 @@ RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams if (print_format == NULL) print_format = outputParams->hasGeoDB ? DefaultGeoMode : DefaultMode; - if (strncasecmp(print_format, "fmt:", 4) == 0 || print_format[0] == '%') { + int fmtFormat = strncasecmp(print_format, "fmt:", 4) == 0; + int csvFormat = strncasecmp(print_format, "csv:", 4) == 0; + if (fmtFormat || csvFormat || print_format[0] == '%') { // special user defined output format - char *format = &print_format[4]; // for 'fmt:%xxx' - if (print_format[0] == '%') format = print_format; // for '%xxx' - forgot to add fmt: + char *format = &print_format[4]; // for 'fmt:%xxx' or 'csv:%xxx' + if (print_format[0] == '%') { + fmtFormat = 1; + format = print_format; // for '%xxx' - forgot to add fmt: assume fmt + } + if (strlen(format)) { - if (!ParseOutputFormat(format, outputParams->printPlain, printmap)) exit(EXIT_FAILURE); - print_record = fmt_record; - print_prolog = fmt_prolog; - print_epilog = fmt_epilog; + if (!ParseOutputFormat(csvFormat, format, outputParams->printPlain, printmap)) exit(EXIT_FAILURE); + if (csvFormat) { + print_record = csv_record; + print_prolog = csv_prolog; + print_epilog = csv_epilog; + } else { + print_record = fmt_record; + print_prolog = fmt_prolog; + print_epilog = fmt_epilog; + } } else { LogError("Missing format description for user defined output format!\n"); exit(EXIT_FAILURE); @@ -201,27 +227,16 @@ RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams i = 0; while (printmap[i].printmode) { if (strncasecmp(print_format, printmap[i].printmode, MAXMODELEN) == 0) { + outputParams->mode = printmap[i].outputMode; if (printmap[i].Format) { - if (!ParseOutputFormat(printmap[i].Format, outputParams->printPlain, printmap)) exit(EXIT_FAILURE); // predefined custom format - print_record = printmap[i].func_record; - print_prolog = printmap[i].func_prolog; - print_epilog = printmap[i].func_epilog; - } else { - if (strncasecmp(print_format, "csv", MAXMODELEN) == 0) { - outputParams->mode = MODE_CSV; - } else if (strncasecmp(print_format, "json", MAXMODELEN) == 0) { - outputParams->mode = MODE_JSON; - } else if (strncasecmp(print_format, "json-log", MAXMODELEN) == 0) { - outputParams->mode = MODE_JSON_LOG; - } else { - outputParams->mode = MODE_PLAIN; - } - // predefined static format - print_record = printmap[i].func_record; - print_prolog = printmap[i].func_prolog; - print_epilog = printmap[i].func_epilog; + if (!ParseOutputFormat(outputParams->mode == MODE_CSV, printmap[i].Format, outputParams->printPlain, printmap)) + exit(EXIT_FAILURE); } + // else - predefined static format + print_record = printFuncMap[outputParams->mode].func_record; + print_prolog = printFuncMap[outputParams->mode].func_prolog; + print_epilog = printFuncMap[outputParams->mode].func_epilog; break; } i++; diff --git a/src/output/output.h b/src/output/output.h index fafa01f6..f865916e 100644 --- a/src/output/output.h +++ b/src/output/output.h @@ -40,7 +40,7 @@ typedef void (*RecordPrinter_t)(FILE *, recordHandle_t *, int); typedef void (*PrologPrinter_t)(void); typedef void (*EpilogPrinter_t)(void); -enum { MODE_PLAIN = 0, MODE_JSON, MODE_CSV, MODE_JSON_LOG }; +typedef enum { MODE_NULL = 0, MODE_RAW, MODE_FMT, MODE_CSV, MODE_JSON, MODE_JSON_LOG } outputMode_t; typedef struct outputParams_s { bool printPlain; @@ -48,21 +48,17 @@ typedef struct outputParams_s { bool quiet; bool hasGeoDB; bool hasTorDB; - int mode; + outputMode_t mode; int topN; } outputParams_t; typedef struct printmap_s { - char *printmode; // name of the output format - RecordPrinter_t func_record; // prints the record - PrologPrinter_t func_prolog; // prints the output prolog - PrologPrinter_t func_epilog; // prints the output epilog - char *Format; // output format definition - char *help; // help text + char *printmode; // name of the output format + outputMode_t outputMode; // type of output mode + char *Format; // output format definition + char *help; // help text } printmap_t; -void AddFormat(char *name, char *fmtString); - RecordPrinter_t SetupOutputMode(char *print_format, outputParams_t *outputParams); void PrintProlog(outputParams_t *outputParams); diff --git a/src/output/output_fmt.c b/src/output/output_fmt.c index a9b3a260..72d36893 100755 --- a/src/output/output_fmt.c +++ b/src/output/output_fmt.c @@ -75,6 +75,7 @@ static int max_format_index = 0; static int do_tag = 0; static int long_v6 = 0; +static int modeCSV = 0; static int printPlain = 0; static double duration = 0; @@ -354,7 +355,7 @@ static void String_natString(FILE *stream, recordHandle_t *recordHandle); static struct format_token_list_s { char *token; // token int is_address; // is an IP address - char *header; // header line description + char *fmtHeader; // header line description string_function_t string_function; // function generation output string } format_token_list[] = { // v3 header info @@ -659,6 +660,7 @@ void fmt_record(FILE *stream, recordHandle_t *recordHandle, int tag) { void fmt_prolog(void) { // header + modeCSV = 0; printf("%s\n", header_string); } // End of fmt_prolog @@ -774,7 +776,7 @@ static char *RecursiveReplace(char *format, printmap_t *printmap) { } // End of RecursiveReplace -int ParseOutputFormat(char *format, int plain_numbers, printmap_t *printmap) { +int ParseOutputFormat(int csvFormat, char *format, int plain_numbers, printmap_t *printmap) { char *c, *s, *h; int i, remaining; @@ -806,9 +808,9 @@ int ParseOutputFormat(char *format, int plain_numbers, printmap_t *printmap) { if (strncmp(format_token_list[i].token, c, len) == 0) { // token found AddToken(i, NULL); if (long_v6 && format_token_list[i].is_address) - snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%23s%s", "", format_token_list[i].header); + snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%23s%s", "", format_token_list[i].fmtHeader); else - snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%s", format_token_list[i].header); + snprintf(h, STRINGSIZE - 1 - strlen(header_string), "%s", format_token_list[i].fmtHeader); h += strlen(h); c[len] = p; c += len; @@ -1302,9 +1304,9 @@ static void String_SrcAddr(FILE *stream, recordHandle_t *recordHandle) { tmp_str[IP_STRING_LEN - 1] = 0; if (long_v6) - fprintf(stream, "%s%39s", tag_string, tmp_str); + fprintf(stream, "%s%*s", tag_string, 39, tmp_str); else - fprintf(stream, "%s%16s", tag_string, tmp_str); + fprintf(stream, "%s%*s", tag_string, 16, tmp_str); } // End of String_SrcAddr diff --git a/src/output/output_fmt.h b/src/output/output_fmt.h index 2678f26e..685b85aa 100644 --- a/src/output/output_fmt.h +++ b/src/output/output_fmt.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009-2022, Peter Haag + * Copyright (c) 2009-2024, Peter Haag * Copyright (c) 2004-2008, SWITCH - Teleinformatikdienste fuer Lehre und Forschung * All rights reserved. * @@ -53,7 +53,7 @@ void fmt_epilog(void); void CondenseV6(char *s); -int ParseOutputFormat(char *format, int printPlain, printmap_t *printmap); +int ParseOutputFormat(int csvFormat, char *format, int printPlain, printmap_t *printmap); void fmt_record(FILE *stream, recordHandle_t *recordHandle, int tag);