diff --git a/lib/apphook.c b/lib/apphook.c index 222ff79314f..b676ead0cdf 100644 --- a/lib/apphook.c +++ b/lib/apphook.c @@ -33,7 +33,7 @@ #include "logsource.h" #include "logwriter.h" #include "afinter.h" -#include "template/templates.h" +#include "template/globals.h" #include "hostname.h" #include "mainloop-call.h" #include "service-management.h" diff --git a/lib/cfg-grammar-internal.h b/lib/cfg-grammar-internal.h index 966332e0d05..4d39598242f 100644 --- a/lib/cfg-grammar-internal.h +++ b/lib/cfg-grammar-internal.h @@ -77,7 +77,6 @@ extern LogSchedulerOptions *last_scheduler_options; extern LogParser *last_parser; extern FilterExprNode *last_filter_expr; extern LogTemplateOptions *last_template_options; -extern LogTemplate *last_template; extern ValuePairs *last_value_pairs; extern ValuePairsTransformSet *last_vp_transset; extern LogMatcherOptions *last_matcher_options; diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index 11a6677df9d..a09e925521d 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -143,9 +143,10 @@ %token LL_CONTEXT_SERVER_PROTO 18 %token LL_CONTEXT_OPTIONS 19 %token LL_CONTEXT_CONFIG 20 +%token LL_CONTEXT_TEMPLATE_REF 21 /* this is a placeholder for unit tests, must be the latest & largest */ -%token LL_CONTEXT_MAX 21 +%token LL_CONTEXT_MAX 22 /* statements */ @@ -337,6 +338,7 @@ %token LL_TOKEN 10434 %token LL_BLOCK 10435 %token LL_PLUGIN 10436 +%token LL_TEMPLATE_REF 10437 %destructor { free($$); } @@ -388,6 +390,8 @@ %type template_content %type template_content_list +%type template_name_or_content +%type template_name_or_content_tail %type filter_content @@ -833,9 +837,9 @@ template_stmt } | template_fn { - user_template_function_register(configuration, last_template->name, last_template); - log_template_unref(last_template); - last_template = NULL; + LogTemplate *template = $1; + user_template_function_register(configuration, template->name, template); + log_template_unref(template); } ; @@ -847,15 +851,15 @@ template_def template_block : KW_TEMPLATE string { - $$ = last_template = log_template_new(configuration, $2); + $$ = log_template_new(configuration, $2); } - '{' template_items '}' { $$ = $3; free($2); } + '{' { $$ = $3; } template_items '}' { $$ = $3; free($2); } ; template_simple : KW_TEMPLATE string { - $$ = last_template = log_template_new(configuration, $2); + $$ = log_template_new(configuration, $2); } template_content_inner { $$ = $3; free($2); } ; @@ -863,34 +867,40 @@ template_simple template_fn : KW_TEMPLATE_FUNCTION string { - $$ = last_template = log_template_new(configuration, $2); + $$ = log_template_new(configuration, $2); } template_content_inner { $$ = $3; free($2); } ; template_items - : template_item semicolons template_items + : { $$ = $0; } template_item semicolons template_items | ; +template_item + : KW_TEMPLATE '(' { $$ = $0; } template_content_inner ')' + | KW_TEMPLATE_ESCAPE '(' yesno ')' { log_template_set_escape($0, $3); } + ; + /* START_RULES */ +/* $0 must be the for the LogTemplate to be populated */ template_content_inner : string { GError *error = NULL; - CHECK_ERROR_GERROR(log_template_compile(last_template, $1, &error), @1, error, "Error compiling template"); + CHECK_ERROR_GERROR(log_template_compile($0, $1, &error), @1, error, "Error compiling template"); free($1); } | LL_IDENTIFIER '(' string_or_number ')' { GError *error = NULL; - CHECK_ERROR_GERROR(log_template_compile(last_template, $3, &error), @3, error, "Error compiling template"); + CHECK_ERROR_GERROR(log_template_compile($0, $3, &error), @3, error, "Error compiling template"); free($3); - CHECK_ERROR_GERROR(log_template_set_type_hint(last_template, $1, &error), @1, error, "Error setting the template type-hint \"%s\"", $1); + CHECK_ERROR_GERROR(log_template_set_type_hint($0, $1, &error), @1, error, "Error setting the template type-hint \"%s\"", $1); free($1); } | LL_NUMBER @@ -898,18 +908,20 @@ template_content_inner gchar decimal[32]; g_snprintf(decimal, sizeof(decimal), "%" G_GINT64_FORMAT, $1); - log_template_compile_literal_string(last_template, decimal); - log_template_set_type_hint(last_template, "int64", NULL); + log_template_compile_literal_string($0, decimal); + log_template_set_type_hint($0, "int64", NULL); } | LL_FLOAT { - log_template_compile_literal_string(last_template, lexer->token_text->str); - log_template_set_type_hint(last_template, "float", NULL); + log_template_compile_literal_string($0, lexer->token_text->str); + log_template_set_type_hint($0, "float", NULL); } ; template_content - : { $$ = last_template = log_template_new(configuration, NULL); } template_content_inner { $$ = $1; } + : { + $$ = log_template_new(configuration, NULL); + } template_content_inner { $$ = $1; } ; template_content_list @@ -917,12 +929,17 @@ template_content_list | { $$ = NULL; } ; +template_name_or_content + : _template_ref_context_push template_name_or_content_tail { $$ = $2; }; + ; + +template_name_or_content_tail + : LL_TEMPLATE_REF { $$ = cfg_tree_lookup_template(&configuration->tree, $1); free($1); } + | template_content { $$ = $1; }; + ; + /* END_RULES */ -template_item - : KW_TEMPLATE '(' template_content_inner ')' - | KW_TEMPLATE_ESCAPE '(' yesno ')' { log_template_set_escape(last_template, $3); } - ; block_stmt @@ -1207,16 +1224,8 @@ facility_string ; parser_opt - : KW_TEMPLATE '(' string ')' { - LogTemplate *template; - GError *error = NULL; - - template = cfg_tree_check_inline_template(&configuration->tree, $3, &error); - CHECK_ERROR_GERROR(template != NULL, @3, error, "Error compiling template"); - log_parser_set_template(last_parser, template); - free($3); - } - | KW_INTERNAL '(' yesno ')' { log_pipe_set_internal(&last_parser->super, $3); } + : KW_TEMPLATE '(' template_name_or_content ')' { log_parser_set_template(last_parser, $3); } + | KW_INTERNAL '(' yesno ')' { log_pipe_set_internal(&last_parser->super, $3); } ; driver_option @@ -1422,14 +1431,7 @@ dest_writer_option | KW_FLUSH_LINES '(' nonnegative_integer ')' { last_writer_options->flush_lines = $3; } | KW_FLUSH_TIMEOUT '(' positive_integer ')' { } | KW_SUPPRESS '(' nonnegative_integer ')' { last_writer_options->suppress = $3; } - | KW_TEMPLATE '(' string ')' { - GError *error = NULL; - - last_writer_options->template = cfg_tree_check_inline_template(&configuration->tree, $3, &error); - CHECK_ERROR_GERROR(last_writer_options->template != NULL, @3, error, "Error compiling template"); - free($3); - } - | KW_TEMPLATE_ESCAPE '(' yesno ')' { log_writer_options_set_template_escape(last_writer_options, $3); } + | KW_TEMPLATE '(' template_name_or_content ')' { last_writer_options->template = $3; } | KW_PAD_SIZE '(' nonnegative_integer ')' { last_writer_options->padding = $3; } | KW_TRUNCATE_SIZE '(' nonnegative_integer ')' { last_writer_options->truncate_size = $3; } | KW_MARK_FREQ '(' nonnegative_integer ')' { last_writer_options->mark_freq = $3; } @@ -1470,6 +1472,7 @@ template_option | KW_TIME_ZONE '(' string ')' { last_template_options->time_zone[LTZ_SEND] = g_strdup($3); free($3); } | KW_SEND_TIME_ZONE '(' string ')' { last_template_options->time_zone[LTZ_SEND] = g_strdup($3); free($3); } | KW_LOCAL_TIME_ZONE '(' string ')' { last_template_options->time_zone[LTZ_LOCAL] = g_strdup($3); free($3); } + | KW_TEMPLATE_ESCAPE '(' yesno ')' { last_template_options->escape = $3; } | KW_ON_ERROR '(' string ')' { gint on_error; @@ -1628,6 +1631,7 @@ _block_content_context_push: { cfg_lexer_push_context(lexer, LL_CONTEXT_BLOCK_CO _block_content_context_pop: { cfg_lexer_pop_context(lexer); }; _block_arg_context_push: { cfg_lexer_push_context(lexer, LL_CONTEXT_BLOCK_ARG, NULL, "block argument"); }; _block_arg_context_pop: { cfg_lexer_pop_context(lexer); }; +_template_ref_context_push: { cfg_lexer_push_context(lexer, LL_CONTEXT_TEMPLATE_REF, NULL, "template reference"); }; _inner_dest_context_push: { cfg_lexer_push_context(lexer, LL_CONTEXT_INNER_DEST, NULL, "within destination"); }; _inner_dest_context_pop: { cfg_lexer_pop_context(lexer); }; _inner_src_context_push: { cfg_lexer_push_context(lexer, LL_CONTEXT_INNER_SRC, NULL, "within source"); }; diff --git a/lib/cfg-lexer.c b/lib/cfg-lexer.c index c50cdb15823..659f5c05711 100644 --- a/lib/cfg-lexer.c +++ b/lib/cfg-lexer.c @@ -1115,6 +1115,22 @@ cfg_lexer_lex(CfgLexer *self, CFG_STYPE *yylval, CFG_LTYPE *yylloc) tok = cfg_lexer_lex_next_token(self, yylval, yylloc); cfg_lexer_append_preprocessed_output(self, self->token_pretext->str); + + if (cfg_lexer_get_context_type(self) == LL_CONTEXT_TEMPLATE_REF) + { + cfg_lexer_pop_context(self); + + if ((tok == LL_IDENTIFIER || tok == LL_STRING)) + { + + LogTemplate *template = cfg_tree_lookup_template(&configuration->tree, yylval->cptr); + if (template != NULL) + { + tok = LL_TEMPLATE_REF; + log_template_unref(template); + } + } + } } preprocess_result = cfg_lexer_preprocess(self, tok, yylval, yylloc); @@ -1232,6 +1248,7 @@ static const gchar *lexer_contexts[] = [LL_CONTEXT_SERVER_PROTO] = "server-proto", [LL_CONTEXT_OPTIONS] = "options", [LL_CONTEXT_CONFIG] = "config", + [LL_CONTEXT_TEMPLATE_REF] = "template-ref", }; gint diff --git a/lib/logwriter.c b/lib/logwriter.c index ff4659e078b..ee0b542ca26 100644 --- a/lib/logwriter.c +++ b/lib/logwriter.c @@ -1937,19 +1937,6 @@ log_writer_options_defaults(LogWriterOptions *options) host_resolve_options_defaults(&options->host_resolve_options); } -void -log_writer_options_set_template_escape(LogWriterOptions *options, gboolean enable) -{ - if (options->template && options->template->def_inline) - { - log_template_set_escape(options->template, enable); - } - else - { - msg_error("Macro escaping can only be specified for inline templates"); - } -} - void log_writer_options_set_mark_mode(LogWriterOptions *options, const gchar *mark_mode) { diff --git a/lib/template/CMakeLists.txt b/lib/template/CMakeLists.txt index 31b636ccaf4..6e644b0f539 100644 --- a/lib/template/CMakeLists.txt +++ b/lib/template/CMakeLists.txt @@ -2,6 +2,7 @@ set(TEMPLATE_HEADERS template/templates.h template/macros.h template/function.h + template/globals.h template/eval.h template/simple-function.h template/repr.h @@ -15,6 +16,7 @@ set(TEMPLATE_SOURCES template/templates.c template/macros.c template/eval.c + template/globals.c template/simple-function.c template/repr.c template/compiler.c diff --git a/lib/template/Makefile.am b/lib/template/Makefile.am index 4319cce6e33..af6795e2e7c 100644 --- a/lib/template/Makefile.am +++ b/lib/template/Makefile.am @@ -7,6 +7,7 @@ templateinclude_HEADERS = \ lib/template/templates.h \ lib/template/macros.h \ lib/template/function.h \ + lib/template/globals.h \ lib/template/eval.h \ lib/template/simple-function.h \ lib/template/repr.h \ @@ -18,6 +19,7 @@ templateinclude_HEADERS = \ template_sources = \ lib/template/templates.c \ lib/template/macros.c \ + lib/template/globals.c \ lib/template/eval.c \ lib/template/simple-function.c \ lib/template/repr.c \ diff --git a/lib/template/common-template-typedefs.h b/lib/template/common-template-typedefs.h index 55a9a58de50..12f99ab7929 100644 --- a/lib/template/common-template-typedefs.h +++ b/lib/template/common-template-typedefs.h @@ -25,7 +25,35 @@ #ifndef COMMON_TYPEDEFS_H_INCLUDED #define COMMON_TYPEDEFS_H_INCLUDED +#include "timeutils/zoneinfo.h" + +#define LTZ_LOCAL 0 +#define LTZ_SEND 1 +#define LTZ_MAX 2 + typedef struct _LogTemplateOptions LogTemplateOptions; typedef struct _LogTemplate LogTemplate; +/* template expansion options that can be influenced by the user and + * is static throughout the runtime for a given configuration. There + * are call-site specific options too, those are specified as + * arguments to log_template_format() */ +struct _LogTemplateOptions +{ + gboolean initialized; + /* timestamp format as specified by ts_format() */ + gint ts_format; + /* number of digits in the fraction of a second part, specified using frac_digits() */ + gint frac_digits; + gboolean use_fqdn; + gboolean escape; + + /* timezone for LTZ_LOCAL/LTZ_SEND settings */ + gchar *time_zone[LTZ_MAX]; + TimeZoneInfo *time_zone_info[LTZ_MAX]; + + /* Template error handling settings */ + gint on_error; +}; + #endif diff --git a/lib/template/escaping.c b/lib/template/escaping.c index 9755ed174fa..f81124cdb42 100644 --- a/lib/template/escaping.c +++ b/lib/template/escaping.c @@ -28,32 +28,24 @@ #include void -result_append(GString *result, const gchar *sstr, gssize len, gboolean escape) +log_template_default_escape_method(GString *result, const gchar *sstr, gsize len) { - gint i; + gsize i; const guchar *ustr = (const guchar *) sstr; - if (len < 0) - len = strlen(sstr); - - if (escape) + for (i = 0; i < len; i++) { - for (i = 0; i < len; i++) + if (ustr[i] == '\'' || ustr[i] == '"' || ustr[i] == '\\') + { + g_string_append_c(result, '\\'); + g_string_append_c(result, ustr[i]); + } + else if (ustr[i] < ' ') { - if (ustr[i] == '\'' || ustr[i] == '"' || ustr[i] == '\\') - { - g_string_append_c(result, '\\'); - g_string_append_c(result, ustr[i]); - } - else if (ustr[i] < ' ') - { - g_string_append_c(result, '\\'); - format_uint32_padded(result, 3, '0', 8, ustr[i]); - } - else - g_string_append_c(result, ustr[i]); + g_string_append_c(result, '\\'); + format_uint32_padded(result, 3, '0', 8, ustr[i]); } + else + g_string_append_c(result, ustr[i]); } - else - g_string_append_len(result, sstr, len); } diff --git a/lib/template/escaping.h b/lib/template/escaping.h index 039141f84b5..0ccc5db99c5 100644 --- a/lib/template/escaping.h +++ b/lib/template/escaping.h @@ -27,6 +27,8 @@ #include "syslog-ng.h" -void result_append(GString *result, const gchar *sstr, gssize len, gboolean escape); +typedef void (*LogTemplateEscapeFunction)(GString *target, const gchar *value, gsize value_len); + +void log_template_default_escape_method(GString *result, const gchar *sstr, gsize len); #endif diff --git a/lib/template/eval.c b/lib/template/eval.c index e226a572d1b..f3fe2a9ede4 100644 --- a/lib/template/eval.c +++ b/lib/template/eval.c @@ -29,6 +29,7 @@ #include "cfg.h" #include "scratch-buffers.h" #include "templates.h" +#include "globals.h" static LogMessageValueType _propagate_type(LogMessageValueType acc_type, LogMessageValueType elem_type) @@ -54,6 +55,73 @@ _should_render(const gchar *value, LogMessageValueType value_type, LogMessageVal return !!value[0]; } +static void +log_template_append_elem_value(LogTemplate *self, LogTemplateElem *e, LogTemplateEvalOptions *options, + LogMessage *msg, LogMessageValueType *type, GString *result) +{ + const gchar *value = NULL; + gssize value_len = -1; + LogMessageValueType value_type = LM_VT_NONE; + + value = log_msg_get_value_with_type(msg, e->value_handle, &value_len, &value_type); + if (value && _should_render(value, value_type, self->type_hint)) + { + g_string_append_len(result, value, value_len); + } + else if (e->default_value) + { + g_string_append_len(result, e->default_value, -1); + value_type = LM_VT_STRING; + } + else if (value_type == LM_VT_BYTES || value_type == LM_VT_PROTOBUF) + { + value_type = LM_VT_NULL; + } + *type = _propagate_type(*type, value_type); +} + +static void +log_template_append_elem_macro(LogTemplate *self, LogTemplateElem *e, LogTemplateEvalOptions *options, + LogMessage *msg, LogMessageValueType *type, GString *result) +{ + gint len = result->len; + LogMessageValueType value_type = LM_VT_NONE; + + if (e->macro) + { + log_macro_expand(e->macro, options, msg, result, &value_type); + if (len == result->len && e->default_value) + g_string_append(result, e->default_value); + *type = _propagate_type(*type, value_type); + } +} + +static void +log_template_append_elem_func(LogTemplate *self, LogTemplateElem *e, LogTemplateEvalOptions *options, + LogMessage **messages, gint num_messages, gint msg_ndx, + LogMessageValueType *type, GString *result) +{ + LogTemplateInvokeArgs args = + { + e->msg_ref ? &messages[msg_ndx] : messages, + e->msg_ref ? 1 : num_messages, + options, + }; + LogMessageValueType value_type = LM_VT_NONE; + + + /* if a function call is called with an msg_ref, we only + * pass that given logmsg to argument resolution, otherwise + * we pass the whole set so the arguments can individually + * specify which message they want to resolve from + */ + if (e->func.ops->eval) + e->func.ops->eval(e->func.ops, e->func.state, &args); + e->func.ops->call(e->func.ops, e->func.state, &args, result, &value_type); + + *type = _propagate_type(*type, value_type); +} + void log_template_append_format_value_and_type_with_context(LogTemplate *self, LogMessage **messages, gint num_messages, LogTemplateEvalOptions *options, @@ -62,12 +130,21 @@ log_template_append_format_value_and_type_with_context(LogTemplate *self, LogMes LogTemplateElem *e; LogMessageValueType t = LM_VT_NONE; gboolean first_elem = TRUE; + GString *target_buffer = result; if (!options->opts) - options->opts = &self->cfg->template_options; + { + /* try the configuration first */ + + if (self->cfg) + options->opts = &self->cfg->template_options; + else + options->opts = log_template_get_global_template_options(); + } - if (self->escape) - t = LM_VT_STRING; + gboolean escape = (self->escape || (self->top_level && options->opts->escape)); + if (escape) + target_buffer = scratch_buffers_alloc(); for (GList *p = self->compiled_template; p; p = g_list_next(p), first_elem = FALSE) { @@ -108,71 +185,33 @@ log_template_append_format_value_and_type_with_context(LogTemplate *self, LogMes if (e->msg_ref == 0) msg_ndx--; + if (escape) + g_string_truncate(target_buffer, 0); + switch (e->type) { case LTE_VALUE: - { - const gchar *value = NULL; - gssize value_len = -1; - LogMessageValueType value_type = LM_VT_NONE; - - value = log_msg_get_value_with_type(messages[msg_ndx], e->value_handle, &value_len, &value_type); - if (value && _should_render(value, value_type, self->type_hint)) - { - result_append(result, value, value_len, self->escape); - } - else if (e->default_value) - { - result_append(result, e->default_value, -1, self->escape); - value_type = LM_VT_STRING; - } - else if (value_type == LM_VT_BYTES || value_type == LM_VT_PROTOBUF) - { - value_type = LM_VT_NULL; - } - t = _propagate_type(t, value_type); + log_template_append_elem_value(self, e, options, messages[msg_ndx], &t, target_buffer); break; - } case LTE_MACRO: - { - gint len = result->len; - LogMessageValueType value_type = LM_VT_NONE; - - if (e->macro) - { - log_macro_expand(e->macro, self->escape, options, messages[msg_ndx], result, &value_type); - if (len == result->len && e->default_value) - g_string_append(result, e->default_value); - t = _propagate_type(t, value_type); - } + log_template_append_elem_macro(self, e, options, messages[msg_ndx], &t, target_buffer); break; - } case LTE_FUNC: - { - LogTemplateInvokeArgs args = - { - e->msg_ref ? &messages[msg_ndx] : messages, - e->msg_ref ? 1 : num_messages, - options, - }; - LogMessageValueType value_type = LM_VT_NONE; - - - /* if a function call is called with an msg_ref, we only - * pass that given logmsg to argument resolution, otherwise - * we pass the whole set so the arguments can individually - * specify which message they want to resolve from - */ - if (e->func.ops->eval) - e->func.ops->eval(e->func.ops, e->func.state, &args); - e->func.ops->call(e->func.ops, e->func.state, &args, result, &value_type); - t = _propagate_type(t, value_type); + log_template_append_elem_func(self, e, options, messages, num_messages, msg_ndx, &t, target_buffer); break; - } default: g_assert_not_reached(); break; } + + if (escape) + { + if (options->escape) + options->escape(result, target_buffer->str, target_buffer->len); + else + log_template_default_escape_method(result, target_buffer->str, target_buffer->len); + t = LM_VT_STRING; + } } if (type) { diff --git a/lib/template/eval.h b/lib/template/eval.h index 952ce76bd36..b041f9a3eb1 100644 --- a/lib/template/eval.h +++ b/lib/template/eval.h @@ -27,34 +27,9 @@ #include "syslog-ng.h" #include "common-template-typedefs.h" -#include "timeutils/zoneinfo.h" +#include "escaping.h" #include "logmsg/logmsg.h" -#define LTZ_LOCAL 0 -#define LTZ_SEND 1 -#define LTZ_MAX 2 - -/* template expansion options that can be influenced by the user and - * is static throughout the runtime for a given configuration. There - * are call-site specific options too, those are specified as - * arguments to log_template_format() */ -struct _LogTemplateOptions -{ - gboolean initialized; - /* timestamp format as specified by ts_format() */ - gint ts_format; - /* number of digits in the fraction of a second part, specified using frac_digits() */ - gint frac_digits; - gboolean use_fqdn; - - /* timezone for LTZ_LOCAL/LTZ_SEND settings */ - gchar *time_zone[LTZ_MAX]; - TimeZoneInfo *time_zone_info[LTZ_MAX]; - - /* Template error handling settings */ - gint on_error; -}; - typedef struct _LogTemplateEvalOptions { /* options for recursive template evaluation, inherited from the parent */ @@ -63,6 +38,7 @@ typedef struct _LogTemplateEvalOptions gint seq_num; const gchar *context_id; LogMessageValueType context_id_type; + LogTemplateEscapeFunction escape; } LogTemplateEvalOptions; #define DEFAULT_TEMPLATE_EVAL_OPTIONS ((LogTemplateEvalOptions){NULL, LTZ_LOCAL, 0, NULL, LM_VT_STRING}) diff --git a/lib/template/globals.c b/lib/template/globals.c new file mode 100644 index 00000000000..d69ecfdff2d --- /dev/null +++ b/lib/template/globals.c @@ -0,0 +1,23 @@ +#include "templates.h" +#include "macros.h" + +static LogTemplateOptions global_template_options; + +LogTemplateOptions * +log_template_get_global_template_options(void) +{ + return &global_template_options; +} + +void +log_template_global_init(void) +{ + log_template_options_global_defaults(&global_template_options); + log_macros_global_init(); +} + +void +log_template_global_deinit(void) +{ + log_macros_global_deinit(); +} diff --git a/lib/template/globals.h b/lib/template/globals.h new file mode 100644 index 00000000000..baef8d0e9a4 --- /dev/null +++ b/lib/template/globals.h @@ -0,0 +1,11 @@ +#ifndef TEMPLATES_GLOBALS_INCLUDED +#define TEMPLATES_GLOBALS_INCLUDED + +#include "common-template-typedefs.h" + +LogTemplateOptions *log_template_get_global_template_options(void); + +void log_template_global_init(void); +void log_template_global_deinit(void); + +#endif diff --git a/lib/template/macros.c b/lib/template/macros.c index 7a110ed96de..c27b1887d54 100644 --- a/lib/template/macros.c +++ b/lib/template/macros.c @@ -23,6 +23,7 @@ */ #include "template/macros.h" +#include "template/globals.h" #include "template/escaping.h" #include "timeutils/cache.h" #include "timeutils/names.h" @@ -229,16 +230,15 @@ LogMacroDef macros[] = static GTimeVal app_uptime; static GHashTable *macro_hash; -static LogTemplateOptions template_options_for_macro_expand; static void -_result_append_value(GString *result, const LogMessage *lm, NVHandle handle, gboolean escape) +_result_append_value(GString *result, const LogMessage *lm, NVHandle handle) { const gchar *str; gssize len = 0; str = log_msg_get_value(lm, handle, &len); - result_append(result, str, len, escape); + g_string_append_len(result, str, len); } static gboolean @@ -270,7 +270,7 @@ _is_message_dest_an_ip_address(const LogMessage *msg) } static void -log_macro_expand_date_time(gint id, gboolean escape, +log_macro_expand_date_time(gint id, LogTemplateEvalOptions *options, const LogMessage *msg, GString *result, LogMessageValueType *type) { @@ -423,7 +423,7 @@ log_macro_expand_date_time(gint id, gboolean escape, } gboolean -log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, const LogMessage *msg, +log_macro_expand(gint id, LogTemplateEvalOptions *options, const LogMessage *msg, GString *result, LogMessageValueType *type) { LogMessageValueType t = LM_VT_STRING; @@ -524,28 +524,17 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons length = p2 ? p2 - p1 : host_len - (p1 - host); - result_append(result, p1, length, escape); + g_string_append_len(result, p1, length); } else { - _result_append_value(result, msg, LM_V_HOST, escape); + _result_append_value(result, msg, LM_V_HOST); } break; } case M_SDATA: { - if (escape) - { - GString *sdstr = g_string_sized_new(0); - - log_msg_append_format_sdata(msg, sdstr, options->seq_num); - result_append(result, sdstr->str, sdstr->len, TRUE); - g_string_free(sdstr, TRUE); - } - else - { - log_msg_append_format_sdata(msg, result, options->seq_num); - } + log_msg_append_format_sdata(msg, result, options->seq_num); break; } case M_MSGHDR: @@ -555,29 +544,29 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons p = log_msg_get_value(msg, LM_V_LEGACY_MSGHDR, &len); if (len > 0) - result_append(result, p, len, escape); + g_string_append_len(result, p, len); else { /* message, complete with program name and pid */ len = result->len; - _result_append_value(result, msg, LM_V_PROGRAM, escape); + _result_append_value(result, msg, LM_V_PROGRAM); if (len != result->len) { const gchar *pid = log_msg_get_value(msg, LM_V_PID, &len); if (len > 0) { - result_append(result, "[", 1, FALSE); - result_append(result, pid, len, escape); - result_append(result, "]", 1, FALSE); + g_string_append_c(result, '['); + g_string_append_len(result, pid, len); + g_string_append_c(result, ']'); } - result_append(result, ": ", 2, FALSE); + g_string_append_len(result, ": ", 2); } } break; } case M_MESSAGE: { - _result_append_value(result, msg, LM_V_MESSAGE, escape); + _result_append_value(result, msg, LM_V_MESSAGE); break; } case M_SOURCE_IP: @@ -594,7 +583,7 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons { ip = "127.0.0.1"; } - result_append(result, ip, strlen(ip), escape); + g_string_append(result, ip); break; } case M_DEST_IP: @@ -611,7 +600,7 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons { ip = "127.0.0.1"; } - result_append(result, ip, strlen(ip), escape); + g_string_append(result, ip); break; } case M_DEST_PORT: @@ -654,7 +643,7 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons { if (options->context_id) { - result_append(result, options->context_id, strlen(options->context_id), escape); + g_string_append(result, options->context_id); t = options->context_id_type; } break; @@ -696,7 +685,7 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons ? get_local_hostname_fqdn() : get_local_hostname_short(); - result_append(result, hname, -1, escape); + g_string_append(result, hname); break; } case M_SYSUPTIME: @@ -710,13 +699,13 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons default: { - log_macro_expand_date_time(id, escape, options, msg, result, &t); + log_macro_expand_date_time(id, options, msg, result, &t); break; } } if (type) - *type = escape ? LM_VT_STRING : t; + *type = t; return TRUE; } @@ -724,8 +713,8 @@ log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, cons gboolean log_macro_expand_simple(gint id, const LogMessage *msg, GString *result, LogMessageValueType *type) { - LogTemplateEvalOptions options = {&template_options_for_macro_expand, LTZ_LOCAL, 0, NULL, LM_VT_STRING}; - return log_macro_expand(id, FALSE, &options, msg, result, type); + LogTemplateEvalOptions options = {log_template_get_global_template_options(), LTZ_LOCAL, 0, NULL, LM_VT_STRING}; + return log_macro_expand(id, &options, msg, result, type); } guint @@ -749,7 +738,6 @@ log_macros_global_init(void) /* init the uptime (SYSUPTIME macro) */ g_get_current_time(&app_uptime); - log_template_options_global_defaults(&template_options_for_macro_expand); macro_hash = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); for (i = 0; macros[i].name; i++) diff --git a/lib/template/macros.h b/lib/template/macros.h index 20db018c8e8..ac6da5e3a3c 100644 --- a/lib/template/macros.h +++ b/lib/template/macros.h @@ -119,7 +119,7 @@ extern LogMacroDef macros[]; /* low level macro functions */ guint log_macro_lookup(const gchar *macro, gint len); -gboolean log_macro_expand(gint id, gboolean escape, LogTemplateEvalOptions *options, +gboolean log_macro_expand(gint id, LogTemplateEvalOptions *options, const LogMessage *msg, GString *result, LogMessageValueType *type); gboolean log_macro_expand_simple(gint id, const LogMessage *msg, diff --git a/lib/template/simple-function.c b/lib/template/simple-function.c index 50b7dd22e69..ab2e0b68ba5 100644 --- a/lib/template/simple-function.c +++ b/lib/template/simple-function.c @@ -52,8 +52,7 @@ tf_simple_func_prepare(LogTemplateFunction *self, gpointer s, LogTemplate *paren * one. */ for (i = 0; i < argc - 1; i++) { - state->argv_templates[i] = log_template_new(parent->cfg, NULL); - log_template_set_escape(state->argv_templates[i], parent->escape); + state->argv_templates[i] = log_template_new_embedded(parent->cfg); if (!log_template_compile(state->argv_templates[i], argv[i + 1], error)) { state->argc = i + 1; diff --git a/lib/template/templates.c b/lib/template/templates.c index fd2465c645a..560bab0b41d 100644 --- a/lib/template/templates.c +++ b/lib/template/templates.c @@ -367,6 +367,15 @@ log_template_new(GlobalConfig *cfg, const gchar *name) else self->type_hint = LM_VT_NONE; self->explicit_type_hint = LM_VT_NONE; + self->top_level = TRUE; + return self; +} + +LogTemplate * +log_template_new_embedded(GlobalConfig *cfg) +{ + LogTemplate *self = log_template_new(cfg, NULL); + self->top_level = FALSE; return self; } @@ -478,18 +487,6 @@ log_template_error_quark(void) return g_quark_from_static_string("log-template-error-quark"); } -void -log_template_global_init(void) -{ - log_macros_global_init(); -} - -void -log_template_global_deinit(void) -{ - log_macros_global_deinit(); -} - gboolean log_template_on_error_parse(const gchar *strictness, gint *out) { diff --git a/lib/template/templates.h b/lib/template/templates.h index 5b44c7e4672..e12399a2b0f 100644 --- a/lib/template/templates.h +++ b/lib/template/templates.h @@ -59,8 +59,7 @@ struct _LogTemplate gchar *template_str; GList *compiled_template; GlobalConfig *cfg; - guint escape:1, def_inline:1, trivial:1, literal:1; - + guint top_level:1, escape:1, def_inline:1, trivial:1, literal:1; /* This value stores the type-hint the user _explicitly_ specified. If * this is an automatic cast to string (in compat mode), this would be @@ -92,6 +91,7 @@ const gchar *log_template_get_trivial_value_and_type(LogTemplate *self, LogMessa void log_template_set_name(LogTemplate *self, const gchar *name); LogTemplate *log_template_new(GlobalConfig *cfg, const gchar *name); +LogTemplate *log_template_new_embedded(GlobalConfig *cfg); LogTemplate *log_template_ref(LogTemplate *s); void log_template_unref(LogTemplate *s); @@ -101,9 +101,6 @@ void log_template_options_destroy(LogTemplateOptions *options); void log_template_options_defaults(LogTemplateOptions *options); void log_template_options_global_defaults(LogTemplateOptions *options); -void log_template_global_init(void); -void log_template_global_deinit(void); - gboolean log_template_on_error_parse(const gchar *on_error, gint *out); void log_template_options_set_on_error(LogTemplateOptions *options, gint on_error); diff --git a/lib/template/tests/test_macro.c b/lib/template/tests/test_macro.c index 968282a15de..c13e23c78b5 100644 --- a/lib/template/tests/test_macro.c +++ b/lib/template/tests/test_macro.c @@ -110,7 +110,7 @@ Test(macro, test_context_id_type_is_returned) LogTemplateEvalOptions options = {NULL, LTZ_SEND, 5555, "5678", LM_VT_INTEGER}; - gboolean result = log_macro_expand(M_CONTEXT_ID, FALSE, &options, msg, resolved, &type); + gboolean result = log_macro_expand(M_CONTEXT_ID, &options, msg, resolved, &type); cr_assert(result); cr_assert_str_eq(resolved->str, "5678"); cr_assert_eq(type, LM_VT_INTEGER); diff --git a/lib/template/tests/test_template.c b/lib/template/tests/test_template.c index 2dae271447e..05a27269eed 100644 --- a/lib/template/tests/test_template.c +++ b/lib/template/tests/test_template.c @@ -414,6 +414,11 @@ Test(template, test_multi_thread) Test(template, test_escaping) { + assert_template_format_with_escaping("$(echo ${APP.QVALUE})", TRUE, "\\\"value\\\""); + assert_template_format_with_escaping("$(echo ${APP.QVALUE}) ${APP.QVALUE}", TRUE, "\\\"value\\\" \\\"value\\\""); + assert_template_format_with_escaping("$(echo $(echo $(echo ${APP.QVALUE})))", TRUE, "\\\"value\\\""); + assert_template_format_with_escaping("$(echo $(echo $(length ${APP.QVALUE})))", TRUE, "7"); + assert_template_format_with_escaping("${APP.QVALUE}", FALSE, "\"value\""); assert_template_format_with_escaping("${APP.QVALUE}", TRUE, "\\\"value\\\""); assert_template_format_with_escaping("$(if (\"${APP.VALUE}\" eq \"value\") \"${APP.QVALUE}\" \"${APP.QVALUE}\")", diff --git a/lib/template/tests/test_template_compile.c b/lib/template/tests/test_template_compile.c index 393f8a29d68..fa4d5c387d3 100644 --- a/lib/template/tests/test_template_compile.c +++ b/lib/template/tests/test_template_compile.c @@ -28,6 +28,7 @@ #include "template/templates.c" #include "template/simple-function.h" +#include "template/globals.h" #include "logmsg/logmsg.h" #include "apphook.h" #include "cfg.h" diff --git a/lib/value-pairs/value-pairs.c b/lib/value-pairs/value-pairs.c index 8056690df97..a7f44e2c809 100644 --- a/lib/value-pairs/value-pairs.c +++ b/lib/value-pairs/value-pairs.c @@ -387,7 +387,7 @@ vp_merge_builtins(ValuePairs *vp, VPResults *results, LogMessage *msg, LogTempla switch (spec->type) { case VPT_MACRO: - log_macro_expand(spec->id, FALSE, options, msg, sb, &type); + log_macro_expand(spec->id, options, msg, sb, &type); break; case VPT_NVPAIR: { diff --git a/modules/diskq/dqtool.c b/modules/diskq/dqtool.c index adf7545236c..fdd6326a5df 100644 --- a/modules/diskq/dqtool.c +++ b/modules/diskq/dqtool.c @@ -23,7 +23,7 @@ #include "syslog-ng.h" #include "logqueue.h" -#include "template/templates.h" +#include "template/globals.h" #include "logmsg/logmsg.h" #include "messages.h" #include "logpipe.h" diff --git a/modules/http/http-grammar.ym b/modules/http/http-grammar.ym index 48ee75b83db..4407907858a 100644 --- a/modules/http/http-grammar.ym +++ b/modules/http/http-grammar.ym @@ -144,7 +144,12 @@ response_action_item ; http_option - : KW_URL '(' string_list ')' { http_dd_set_urls(last_driver, $3); g_list_free_full($3, free); } + : KW_URL '(' string_list ')' + { + GError *error = NULL; + CHECK_ERROR_GERROR(http_dd_set_urls(last_driver, $3, &error), @3, error, "Error setting url"); + g_list_free_full($3, free); + } | KW_USER '(' string ')' { http_dd_set_user(last_driver, $3); free($3); } | KW_PASSWORD '(' string ')' { http_dd_set_password(last_driver, $3); free($3); } | KW_USER_AGENT '(' string ')' { http_dd_set_user_agent(last_driver, $3); free($3); } diff --git a/modules/http/http-loadbalancer.c b/modules/http/http-loadbalancer.c index b7448e63b85..adc06d134da 100644 --- a/modules/http/http-loadbalancer.c +++ b/modules/http/http-loadbalancer.c @@ -23,24 +23,71 @@ #include "http-loadbalancer.h" #include "messages.h" +#include "str-utils.h" #include /* HTTPLoadBalancerTarget */ -void -http_lb_target_init(HTTPLoadBalancerTarget *self, const gchar *url, gint index_) +gboolean +http_lb_target_init(HTTPLoadBalancerTarget *self, const gchar *url, gint index_, GError **error) { memset(self, 0, sizeof(*self)); - self->url = g_strdup(url); + LogTemplate *url_template = log_template_new(configuration, NULL); + log_template_set_escape(url_template, TRUE); + if (!log_template_compile(url_template, url, error)) + { + log_template_unref(url_template); + return FALSE; + } + + log_template_unref(self->url_template); + self->url_template = url_template; self->state = HTTP_TARGET_OPERATIONAL; self->index = index_; + g_snprintf(self->formatted_index, sizeof(self->formatted_index), "%d", index_); + + return TRUE; } void http_lb_target_deinit(HTTPLoadBalancerTarget *self) { - g_free(self->url); + log_template_unref(self->url_template); +} + +gboolean +http_lb_target_is_url_templated(HTTPLoadBalancerTarget *self) +{ + return !log_template_is_literal_string(self->url_template); +} + +const gchar * +http_lb_target_get_literal_url(HTTPLoadBalancerTarget *self) +{ + return log_template_get_literal_value(self->url_template, NULL); +} + +static void +_escape_urlencoded(GString *result, const gchar *value, gsize value_len) +{ + APPEND_ZERO(value, value, value_len); + g_string_append_uri_escaped(result, value, NULL, FALSE); +} + +void +http_lb_target_format_templated_url(HTTPLoadBalancerTarget *self, LogMessage *msg, + const LogTemplateOptions *template_options, GString *result) +{ + LogTemplateEvalOptions template_eval_options = { + .opts = template_options, + .tz = LTZ_LOCAL, + .seq_num = -1, + .context_id = self->formatted_index, + .context_id_type = LM_VT_INTEGER, + .escape = _escape_urlencoded, + }; + log_template_format(self->url_template, msg, &template_eval_options, result); } /* HTTPLoadBalancerClient */ @@ -86,7 +133,7 @@ _recalculate_clients_per_target_goals(HTTPLoadBalancer *self) remainder--; } msg_trace("Setting maximum number of workers for HTTP destination", - evt_tag_str("url", target->url), + evt_tag_str("url", target->url_template->template_str), evt_tag_int("max_clients", target->max_clients)); } } @@ -202,13 +249,13 @@ http_load_balancer_choose_target(HTTPLoadBalancer *self, HTTPLoadBalancerClient return lbc->target; } -void -http_load_balancer_add_target(HTTPLoadBalancer *self, const gchar *url) +gboolean +http_load_balancer_add_target(HTTPLoadBalancer *self, const gchar *url, GError **error) { gint n = self->num_targets++; self->targets = g_renew(HTTPLoadBalancerTarget, self->targets, self->num_targets); - http_lb_target_init(&self->targets[n], url, n); + return http_lb_target_init(&self->targets[n], url, n, error); } void @@ -235,7 +282,7 @@ http_load_balancer_set_target_failed(HTTPLoadBalancer *self, HTTPLoadBalancerTar if (target->state != HTTP_TARGET_FAILED) { msg_debug("Load balancer target failed, removing from rotation", - evt_tag_str("url", target->url)); + evt_tag_str("url", target->url_template->template_str)); self->num_failed_targets++; target->state = HTTP_TARGET_FAILED; _recalculate_clients_per_target_goals(self); @@ -251,7 +298,7 @@ http_load_balancer_set_target_successful(HTTPLoadBalancer *self, HTTPLoadBalance if (target->state != HTTP_TARGET_OPERATIONAL) { msg_debug("Load balancer target recovered, adding back to rotation", - evt_tag_str("url", target->url)); + evt_tag_str("url", target->url_template->template_str)); self->num_failed_targets--; target->state = HTTP_TARGET_OPERATIONAL; _recalculate_clients_per_target_goals(self); @@ -259,6 +306,18 @@ http_load_balancer_set_target_successful(HTTPLoadBalancer *self, HTTPLoadBalance g_mutex_unlock(&self->lock); } +gboolean +http_load_balancer_is_url_templated(HTTPLoadBalancer *self) +{ + for (gint i = 0; i < self->num_targets; i++) + { + if (http_lb_target_is_url_templated(&self->targets[i])) + return TRUE; + } + + return FALSE; +} + void http_load_balancer_set_recovery_timeout(HTTPLoadBalancer *self, gint recovery_timeout) { diff --git a/modules/http/http-loadbalancer.h b/modules/http/http-loadbalancer.h index 0e915083f2e..7e2a4498c1a 100644 --- a/modules/http/http-loadbalancer.h +++ b/modules/http/http-loadbalancer.h @@ -24,7 +24,7 @@ #ifndef HTTP_LOADBALANCER_H_INCLUDED #define HTTP_LOADBALANCER_H_INCLUDED 1 -#include "syslog-ng.h" +#include "template/templates.h" typedef enum { @@ -46,15 +46,21 @@ typedef struct _HTTPLoadBalancer HTTPLoadBalancer; struct _HTTPLoadBalancerTarget { /* read-only data, no need to lock */ - gchar *url; + LogTemplate *url_template; gint index; /* read-write data, locking must be in effect */ HTTPLoadBalancerTargetState state; gint number_of_clients; gint max_clients; time_t last_failure_time; + gchar formatted_index[16]; }; +gboolean http_lb_target_is_url_templated(HTTPLoadBalancerTarget *self); +const gchar *http_lb_target_get_literal_url(HTTPLoadBalancerTarget *self); +void http_lb_target_format_templated_url(HTTPLoadBalancerTarget *self, LogMessage *msg, + const LogTemplateOptions *template_options, GString *result); + struct _HTTPLoadBalancerClient { HTTPLoadBalancerTarget *target; @@ -75,11 +81,12 @@ struct _HTTPLoadBalancer }; HTTPLoadBalancerTarget *http_load_balancer_choose_target(HTTPLoadBalancer *self, HTTPLoadBalancerClient *lbc); -void http_load_balancer_add_target(HTTPLoadBalancer *self, const gchar *url); +gboolean http_load_balancer_add_target(HTTPLoadBalancer *self, const gchar *url, GError **error); void http_load_balancer_drop_all_targets(HTTPLoadBalancer *self); void http_load_balancer_track_client(HTTPLoadBalancer *self, HTTPLoadBalancerClient *lbc); void http_load_balancer_set_target_failed(HTTPLoadBalancer *self, HTTPLoadBalancerTarget *target); void http_load_balancer_set_target_successful(HTTPLoadBalancer *self, HTTPLoadBalancerTarget *target); +gboolean http_load_balancer_is_url_templated(HTTPLoadBalancer *self); void http_load_balancer_set_recovery_timeout(HTTPLoadBalancer *self, gint recovery_timeout); HTTPLoadBalancer *http_load_balancer_new(void); diff --git a/modules/http/http-worker.c b/modules/http/http-worker.c index d139083eafb..be37f136184 100644 --- a/modules/http/http-worker.c +++ b/modules/http/http-worker.c @@ -442,7 +442,7 @@ _finish_request_body(HTTPDestinationWorker *self) } static void -_debug_response_info(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target, glong http_code) +_debug_response_info(HTTPDestinationWorker *self, const gchar *url, glong http_code) { HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; @@ -452,7 +452,7 @@ _debug_response_info(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target curl_easy_getinfo(self->curl, CURLINFO_TOTAL_TIME, &total_time); curl_easy_getinfo(self->curl, CURLINFO_REDIRECT_COUNT, &redirect_count); msg_debug("curl: HTTP response received", - evt_tag_str("url", target->url), + evt_tag_str("url", url), evt_tag_int("status_code", http_code), evt_tag_int("body_size", self->request_body->len), evt_tag_int("batch_size", self->super.batch_size), @@ -517,14 +517,14 @@ _custom_map_http_result(HTTPDestinationWorker *self, const gchar *url, HttpRespo } static gboolean -_curl_perform_request(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target) +_curl_perform_request(HTTPDestinationWorker *self, const gchar *url) { HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; msg_trace("Sending HTTP request", - evt_tag_str("url", target->url)); + evt_tag_str("url", url)); - curl_easy_setopt(self->curl, CURLOPT_URL, target->url); + curl_easy_setopt(self->curl, CURLOPT_URL, url); if (owner->message_compression != CURL_COMPRESSION_UNCOMPRESSED) { if (compressor_compress(self->compressor, self->request_body_compressed, self->request_body)) @@ -547,7 +547,7 @@ _curl_perform_request(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *targe if (ret != CURLE_OK) { msg_error("curl: error sending HTTP request", - evt_tag_str("url", target->url), + evt_tag_str("url", url), evt_tag_str("error", curl_easy_strerror(ret)), evt_tag_int("worker_index", self->super.worker_index), evt_tag_str("driver", owner->super.super.super.id), @@ -559,7 +559,7 @@ _curl_perform_request(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *targe } static gboolean -_curl_get_status_code(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target, glong *http_code) +_curl_get_status_code(HTTPDestinationWorker *self, const gchar *url, glong *http_code) { HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; CURLcode ret = curl_easy_getinfo(self->curl, CURLINFO_RESPONSE_CODE, http_code); @@ -567,7 +567,7 @@ _curl_get_status_code(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *targe if (ret != CURLE_OK) { msg_error("curl: error querying response code", - evt_tag_str("url", target->url), + evt_tag_str("url", url), evt_tag_str("error", curl_easy_strerror(ret)), evt_tag_int("worker_index", self->super.worker_index), evt_tag_str("driver", owner->super.super.super.id), @@ -603,20 +603,20 @@ _map_http_status_code(HTTPDestinationWorker *self, const gchar *url, glong http_ } static LogThreadedResult -_flush_on_target(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target) +_flush_on_target(HTTPDestinationWorker *self, const gchar *url) { HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; - if (!_curl_perform_request(self, target)) + if (!_curl_perform_request(self, url)) return LTR_NOT_CONNECTED; glong http_code = 0; - if (!_curl_get_status_code(self, target, &http_code)) + if (!_curl_get_status_code(self, url, &http_code)) return LTR_NOT_CONNECTED; if (debug_flag) - _debug_response_info(self, target, http_code); + _debug_response_info(self, url, http_code); HttpResponseReceivedSignalData signal_data = { @@ -633,7 +633,7 @@ _flush_on_target(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target) return LTR_RETRY; } - return _map_http_status_code(self, target->url, http_code); + return _map_http_status_code(self, url, http_code); } static gboolean @@ -671,6 +671,17 @@ _format_request_headers_catch_error(GError **error) return !unhandled; } +static const gchar * +_get_url(HTTPDestinationWorker *self, HTTPLoadBalancerTarget *target) +{ + if (!http_lb_target_is_url_templated(target)) + return http_lb_target_get_literal_url(target); + + HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; + http_lb_target_format_templated_url(target, self->msg_for_templated_url, &owner->template_options, self->url_buffer); + return self->url_buffer->str; +} + /* we flush the accumulated data if * 1) we reach batch_size, * 2) the message queue becomes empty @@ -700,10 +711,11 @@ _flush(LogThreadedDestWorker *s, LogThreadedFlushMode mode) } target = http_load_balancer_choose_target(owner->load_balancer, &self->lbc); + const gchar *url = _get_url(self, target); while (--retry_attempts >= 0) { - retval = _flush_on_target(self, target); + retval = _flush_on_target(self, url); if (retval == LTR_SUCCESS) { gsize msg_length = self->request_body->len; @@ -719,26 +731,31 @@ _flush(LogThreadedDestWorker *s, LogThreadedFlushMode mode) if (alt_target == target) { msg_debug("Target server down, but no alternative server available. Falling back to retrying after time-reopen()", - evt_tag_str("url", target->url), + evt_tag_str("url", url), evt_tag_int("worker_index", self->super.worker_index), evt_tag_str("driver", owner->super.super.super.id), log_pipe_location_tag(&owner->super.super.super.super)); break; } + const gchar *alt_url = _get_url(self, alt_target); msg_debug("Target server down, trying an alternative server", - evt_tag_str("url", target->url), - evt_tag_str("alternative_url", alt_target->url), + evt_tag_str("url", url), + evt_tag_str("alternative_url", alt_url), evt_tag_int("worker_index", self->super.worker_index), evt_tag_str("driver", owner->super.super.super.id), log_pipe_location_tag(&owner->super.super.super.super)); target = alt_target; + url = alt_url; } _reinit_request_headers(self); _reinit_request_body(self); + log_msg_unref(self->msg_for_templated_url); + self->msg_for_templated_url = NULL; + return retval; } @@ -761,6 +778,9 @@ _insert_batched(LogThreadedDestWorker *s, LogMessage *msg) gsize diff_msg_len = self->request_body->len - orig_msg_len; log_threaded_dest_driver_insert_msg_length_stats(self->super.owner, diff_msg_len); + if (!self->msg_for_templated_url) + self->msg_for_templated_url = log_msg_ref(msg); + if (_should_initiate_flush(self)) { return log_threaded_dest_worker_flush(&self->super, LTF_FLUSH_NORMAL); @@ -780,6 +800,8 @@ _insert_single(LogThreadedDestWorker *s, LogMessage *msg) _add_msg_specific_headers(self, msg); + self->msg_for_templated_url = log_msg_ref(msg); + return log_threaded_dest_worker_flush(&self->super, LTF_FLUSH_NORMAL); } @@ -789,6 +811,11 @@ _init(LogThreadedDestWorker *s) HTTPDestinationWorker *self = (HTTPDestinationWorker *) s; HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; + if (http_load_balancer_is_url_templated(owner->load_balancer)) + { + self->url_buffer = g_string_new(NULL); + } + self->request_body = g_string_sized_new(32768); if (owner->message_compression != CURL_COMPRESSION_UNCOMPRESSED) { @@ -828,6 +855,9 @@ _deinit(LogThreadedDestWorker *s) HTTPDestinationWorker *self = (HTTPDestinationWorker *) s; HTTPDestinationDriver *owner = (HTTPDestinationDriver *) self->super.owner; + if (self->url_buffer) + g_string_free(self->url_buffer, TRUE); + g_string_free(self->request_body, TRUE); if (self->request_body_compressed) g_string_free(self->request_body_compressed, TRUE); diff --git a/modules/http/http-worker.h b/modules/http/http-worker.h index 02977a4d2ef..95b022c6635 100644 --- a/modules/http/http-worker.h +++ b/modules/http/http-worker.h @@ -39,6 +39,8 @@ typedef struct _HTTPDestinationWorker GString *request_body_compressed; Compressor *compressor; List *request_headers; + GString *url_buffer; + LogMessage *msg_for_templated_url; } HTTPDestinationWorker; LogThreadedResult default_map_http_status_to_worker_status(HTTPDestinationWorker *self, const gchar *url, diff --git a/modules/http/http.c b/modules/http/http.c index 2765f3280ec..dcff4a54812 100644 --- a/modules/http/http.c +++ b/modules/http/http.c @@ -33,8 +33,8 @@ http_dd_insert_response_handler(LogDriver *d, HttpResponseHandler *response_hand http_response_handlers_insert(self->response_handlers, response_handler); } -void -http_dd_set_urls(LogDriver *d, GList *url_strings) +gboolean +http_dd_set_urls(LogDriver *d, GList *url_strings, GError **error) { HTTPDestinationDriver *self = (HTTPDestinationDriver *) d; @@ -46,10 +46,16 @@ http_dd_set_urls(LogDriver *d, GList *url_strings) for (gint url = 0; urls[url]; url++) { - http_load_balancer_add_target(self->load_balancer, urls[url]); + if (!http_load_balancer_add_target(self->load_balancer, urls[url], error)) + { + g_strfreev(urls); + return FALSE; + } } g_strfreev(urls); } + + return TRUE; } void @@ -400,7 +406,10 @@ http_dd_init(LogPipe *s) GlobalConfig *cfg = log_pipe_get_config(s); if (self->load_balancer->num_targets == 0) - http_load_balancer_add_target(self->load_balancer, HTTP_DEFAULT_URL); + { + GError *error = NULL; + g_assert(http_load_balancer_add_target(self->load_balancer, HTTP_DEFAULT_URL, &error)); + } if (self->load_balancer->num_targets > 1 && s->persist_name == NULL) { @@ -408,7 +417,7 @@ http_dd_init(LogPipe *s) "It is recommended that you set persist-name() in this case as syslog-ng will be " "using the first URL in urls() to register persistent data, such as the disk queue " "name, which might change", - evt_tag_str("url", self->load_balancer->targets[0].url), + evt_tag_str("url", self->load_balancer->targets[0].url_template->template_str), log_pipe_location_tag(&self->super.super.super.super)); } if (self->load_balancer->num_targets > self->super.num_workers) @@ -421,11 +430,22 @@ http_dd_init(LogPipe *s) log_pipe_location_tag(&self->super.super.super.super)); } /* we need to set up url before we call the inherited init method, so our stats key is correct */ - self->url = self->load_balancer->targets[0].url; + self->url = self->load_balancer->targets[0].url_template->template_str; if (!log_threaded_dest_driver_init_method(s)) return FALSE; + if ((self->super.batch_lines || self->batch_bytes) && http_load_balancer_is_url_templated(self->load_balancer) && + self->super.num_workers > 1 && !self->super.worker_partition_key) + { + msg_error("worker-partition-key() must be set if using templates in the url() option " + "while batching is enabled and multiple workers are configured. " + "Make sure to set worker-partition-key() with a template that contains all the templates " + "used in the url() option", + log_pipe_location_tag(&self->super.super.super.super)); + return FALSE; + } + log_template_options_init(&self->template_options, cfg); http_load_balancer_set_recovery_timeout(self->load_balancer, self->super.time_reopen); @@ -483,6 +503,8 @@ http_dd_new(GlobalConfig *cfg) self->super.stats_source = stats_register_type("http"); self->super.worker.construct = http_dw_new; + log_threaded_dest_driver_set_flush_on_worker_key_change(&self->super.super.super, TRUE); + curl_global_init(CURL_GLOBAL_ALL); self->ssl_version = CURL_SSLVERSION_DEFAULT; diff --git a/modules/http/http.h b/modules/http/http.h index 49732317780..ac34c9db5a9 100644 --- a/modules/http/http.h +++ b/modules/http/http.h @@ -39,8 +39,8 @@ typedef struct HTTPLoadBalancer *load_balancer; /* this is the first URL in load-balanced configurations and serves as the - * identifier in persist/stats */ - gchar *url; + * identifier in persist/stats. TODO: Templated URLs should have dynamic counters. */ + const gchar *url; gchar *user; gchar *password; GList *headers; @@ -73,7 +73,7 @@ gboolean http_dd_init(LogPipe *s); gboolean http_dd_deinit(LogPipe *s); LogDriver *http_dd_new(GlobalConfig *cfg); -void http_dd_set_urls(LogDriver *d, GList *urls); +gboolean http_dd_set_urls(LogDriver *d, GList *urls, GError **error); void http_dd_set_user(LogDriver *d, const gchar *user); void http_dd_set_password(LogDriver *d, const gchar *password); void http_dd_set_method(LogDriver *d, const gchar *method); diff --git a/modules/http/tests/test_http-loadbalancer.c b/modules/http/tests/test_http-loadbalancer.c index e6ac384e99c..32772958796 100644 --- a/modules/http/tests/test_http-loadbalancer.c +++ b/modules/http/tests/test_http-loadbalancer.c @@ -41,7 +41,8 @@ _construct_load_balancer(void) { gchar url[256]; g_snprintf(url, sizeof(url), "http://localhost:%d", 8000 + i); - http_load_balancer_add_target(lb, url); + GError *error = NULL; + cr_assert(http_load_balancer_add_target(lb, url, &error)); } return lb; } @@ -96,7 +97,7 @@ Test(http_loadbalancer, choose_target_selects_the_first_operational_target) http_lb_client_init(&lbc, lb); HTTPLoadBalancerTarget *target = http_load_balancer_choose_target(lb, &lbc); - cr_assert(target->url, "http://localhost:8000"); + cr_assert(log_template_get_literal_value(target->url_template, NULL), "http://localhost:8000"); cr_assert(target->state == HTTP_TARGET_OPERATIONAL); http_lb_client_deinit(&lbc); diff --git a/news/feature-4663.md b/news/feature-4663.md new file mode 100644 index 00000000000..f778bcdae09 --- /dev/null +++ b/news/feature-4663.md @@ -0,0 +1,11 @@ +`http()`: Added support for using templates in the `url()` option. + +In syslog-ng a template can only be resolved on a single message, as the same +template might have different resolutions on different messages. A http batch +consists of multiple messages, so it is not trivial to decide which message should +be used for the resolution. + +When batching is enabled and multiple workers are configured it is important to +only batch messages which generate identical URLs. In this scenario one must set +the `worker-partition-key()` option with a template that contains all the templates +used in the `url()` option, otherwise messages will be mixed. diff --git a/tests/light/functional_tests/templates/test_template_stmt.py b/tests/light/functional_tests/templates/test_template_stmt.py new file mode 100644 index 00000000000..b5c2cc311c9 --- /dev/null +++ b/tests/light/functional_tests/templates/test_template_stmt.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +############################################################################# +# Copyright (c) 2019 Balabit +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 as published +# by the Free Software Foundation, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# As an additional exemption you are allowed to compile & link against the +# OpenSSL libraries as published by the OpenSSL project. See the file +# COPYING for details. +# +############################################################################# + + +def test_template_stmt_with_identifier_reference(config, syslog_ng): + template = config.create_template("template with $(format-welf test.*)\n") + config.add_template(template) + + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2") + file_destination = config.create_file_destination(file_name="output.log", template=template.name) + + config.create_logpath(statements=[generator_source, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["template with test.key1=value1 test.key2=value2\n"] + + +def test_template_with_non_string_values(config, syslog_ng): + + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2") + rewrites = [ + config.create_rewrite_set("int(10)", value="values_int_hint"), + config.create_rewrite_set("float(4.5)", value="values_float_hint"), + config.create_rewrite_set("10", value="values_int_literal"), + config.create_rewrite_set("4.5", value="values_float_literal") + ] + file_destination = config.create_file_destination(file_name="output.log", template=config.stringify("$(format-json values_*)\n")) + + config.create_logpath(statements=[generator_source, *rewrites, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["""{"values_int_literal":10,"values_int_hint":10,"values_float_literal":4.5,"values_float_hint":4.5}\n"""] + + +def test_template_stmt_with_indirect_invocation(config, syslog_ng): + template = config.create_template("template with $(format-welf test.*)\n") + config.add_template(template) + + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2 template_fn => {}".format(template.name)) + file_destination = config.create_file_destination(file_name="output.log", template=config.stringify("$(template {} error resolving template)\n".format(template.name))) + + config.create_logpath(statements=[generator_source, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["template with test.key1=value1 test.key2=value2\n"] + + +def test_simple_template_stmt_with_identifier_reference(config, syslog_ng): + template = config.create_template("template with $(format-welf test.*)\n", use_simple_statement=True) + config.add_template(template) + + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2") + file_destination = config.create_file_destination(file_name="output.log", template=template.name) + + config.create_logpath(statements=[generator_source, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["template with test.key1=value1 test.key2=value2\n"] + + +def test_template_stmt_with_string_reference(config, syslog_ng): + template = config.create_template("template with $(format-welf test.*)\n") + config.add_template(template) + + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2") + file_destination = config.create_file_destination(file_name="output.log", template=config.stringify(template.name)) + + config.create_logpath(statements=[generator_source, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["template with test.key1=value1 test.key2=value2\n"] + + +def test_inline_template(config, syslog_ng): + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2") + file_destination = config.create_file_destination(file_name="output.log", template=config.stringify("template with $(format-welf test.*)\n")) + + config.create_logpath(statements=[generator_source, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["template with test.key1=value1 test.key2=value2\n"] + + +def test_template_function(config, syslog_ng): + template = config.create_template_function("template with $(format-welf test.*)\n", name="test_template_fn") + config.add_template(template) + + generator_source = config.create_example_msg_generator_source(num=1, values="test.key1 => value1 test.key2 => value2") + file_destination = config.create_file_destination(file_name="output.log", template=config.stringify("$(test_template_fn)\n")) + + config.create_logpath(statements=[generator_source, file_destination]) + syslog_ng.start(config) + log = file_destination.read_logs(1) + assert log == ["template with test.key1=value1 test.key2=value2\n"] diff --git a/tests/light/src/syslog_ng_config/__init__.py b/tests/light/src/syslog_ng_config/__init__.py index e69de29bb2d..c9f88abe587 100644 --- a/tests/light/src/syslog_ng_config/__init__.py +++ b/tests/light/src/syslog_ng_config/__init__.py @@ -0,0 +1,3 @@ + +def stringify(s): + return '"' + s.replace('\\', "\\\\").replace('"', '\\"').replace('\n', '\\n') + '"' diff --git a/tests/light/src/syslog_ng_config/renderer.py b/tests/light/src/syslog_ng_config/renderer.py index 6475d6a7c44..1fa09ff8aff 100644 --- a/tests/light/src/syslog_ng_config/renderer.py +++ b/tests/light/src/syslog_ng_config/renderer.py @@ -21,7 +21,8 @@ # ############################################################################# from src.syslog_ng_config.statements.filters.filter import Filter - +from src.syslog_ng_config.statements.template.template import Template, TemplateFunction +from src.syslog_ng_config import stringify def render_version(version): return "@version: {}\n".format(version) @@ -61,6 +62,36 @@ def render_options(name, options): return config_snippet +def render_template_statement(template): + if not template.use_simple_statement: + config_snippet = "template {} {{\n".format(template.name) + config_snippet += " template({});\n".format(stringify(template.template)) + if template.template_escape is not None: + config_snippet += " template-escape({});\n".format(template.template_escape) + config_snippet += "};\n" + else: + config_snippet = "template {} {};\n".format(template.name, stringify(template.template)) + return config_snippet + + +def render_template_function(template): + return "template-function {} {};\n".format(template.name, stringify(template.template)); + + +def render_template(template): + if isinstance(template, Template): + return render_template_statement(template) + elif isinstance(template, TemplateFunction): + return render_template_function(template) + + +def render_templates(templates): + config_snippet = "" + for template in templates: + config_snippet += render_template(template) + return config_snippet + + def render_list(name, options): config_snippet = "" @@ -164,6 +195,7 @@ def __render(self, re_create_config=None): version = self.__syslog_ng_config["version"] includes = self.__syslog_ng_config["includes"] global_options = self.__syslog_ng_config["global_options"] + templates = self.__syslog_ng_config["templates"] statement_groups = self.__syslog_ng_config["statement_groups"] logpath_groups = self.__syslog_ng_config["logpath_groups"] @@ -175,6 +207,8 @@ def __render(self, re_create_config=None): config += render_includes(includes) if global_options: config += render_global_options(global_options) + if templates: + config += render_templates(templates) if statement_groups: config += render_statement_groups(statement_groups) if logpath_groups: diff --git a/tests/light/src/syslog_ng_config/statements/rewrite/rewrite.py b/tests/light/src/syslog_ng_config/statements/rewrite/rewrite.py index f4cd46b6013..aa4bf5bef2c 100644 --- a/tests/light/src/syslog_ng_config/statements/rewrite/rewrite.py +++ b/tests/light/src/syslog_ng_config/statements/rewrite/rewrite.py @@ -36,6 +36,11 @@ def __init__(self, tag, **options): super(SetTag, self).__init__("set_tag", [tag], **options) +class Set(Rewrite): + def __init__(self, template, **options): + super(Set, self).__init__("set", [template], **options) + + class SetPri(Rewrite): def __init__(self, pri, **options): super(SetPri, self).__init__("set_pri", [pri], **options) diff --git a/tests/light/src/syslog_ng_config/statements/template/__init__.py b/tests/light/src/syslog_ng_config/statements/template/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/light/src/syslog_ng_config/statements/template/template.py b/tests/light/src/syslog_ng_config/statements/template/template.py new file mode 100644 index 00000000000..c2ef14d47b9 --- /dev/null +++ b/tests/light/src/syslog_ng_config/statements/template/template.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +############################################################################# +# Copyright (c) 2015-2019 Balabit +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 as published +# by the Free Software Foundation, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +# +# As an additional exemption you are allowed to compile & link against the +# OpenSSL libraries as published by the OpenSSL project. See the file +# COPYING for details. +# +############################################################################# + +from src.common.random_id import get_unique_id + + +class Template(object): + def __init__(self, template, name=None, escape=None, use_simple_statement=False): + self.template = template + self.template_escape = escape + self.name = name or "template_{}".format(get_unique_id()) + self.use_simple_statement = escape is not None or use_simple_statement + + +class TemplateFunction(object): + def __init__(self, template, name): + self.template = template + self.name = name diff --git a/tests/light/src/syslog_ng_config/syslog_ng_config.py b/tests/light/src/syslog_ng_config/syslog_ng_config.py index 5c0637e8ac9..137497d353b 100644 --- a/tests/light/src/syslog_ng_config/syslog_ng_config.py +++ b/tests/light/src/syslog_ng_config/syslog_ng_config.py @@ -24,6 +24,7 @@ from src.common.file import File from src.common.operations import cast_to_list +from src.syslog_ng_config import stringify from src.syslog_ng_config.renderer import ConfigRenderer from src.syslog_ng_config.statement_group import StatementGroup from src.syslog_ng_config.statements.destinations.example_destination import ExampleDestination @@ -41,10 +42,12 @@ from src.syslog_ng_config.statements.rewrite.rewrite import CreditCardMask from src.syslog_ng_config.statements.rewrite.rewrite import SetPri from src.syslog_ng_config.statements.rewrite.rewrite import SetTag +from src.syslog_ng_config.statements.rewrite.rewrite import Set from src.syslog_ng_config.statements.sources.example_msg_generator_source import ExampleMsgGeneratorSource from src.syslog_ng_config.statements.sources.file_source import FileSource from src.syslog_ng_config.statements.sources.internal_source import InternalSource from src.syslog_ng_config.statements.sources.network_source import NetworkSource +from src.syslog_ng_config.statements.template.template import Template, TemplateFunction logger = logging.getLogger(__name__) @@ -57,14 +60,13 @@ def __init__(self, version, teardown): "version": version, "includes": [], "global_options": {}, + "templates": [], "statement_groups": [], "logpath_groups": [], } self.teardown = teardown - @staticmethod - def stringify(s): - return '"' + s.replace('\\', "\\\\").replace('"', '\\"').replace('\n', '\\n') + '"' + stringify = staticmethod(stringify) def set_raw_config(self, raw_config): self.__raw_config = raw_config @@ -91,6 +93,9 @@ def add_include(self, include): def update_global_options(self, **options): self.__syslog_ng_config["global_options"].update(options) + def add_template(self, template): + self.__syslog_ng_config["templates"].append(template) + def create_file_source(self, **options): file_source = FileSource(**options) self.teardown.register(file_source.close_file) @@ -105,12 +110,21 @@ def create_internal_source(self, **options): def create_network_source(self, **options): return NetworkSource(**options) + def create_rewrite_set(self, template, **options): + return Set(template, **options) + def create_rewrite_set_tag(self, tag, **options): return SetTag(tag, **options) def create_rewrite_set_pri(self, pri, **options): return SetPri(pri, **options) + def create_template(self, template, **options): + return Template(template, **options) + + def create_template_function(self, template, **options): + return TemplateFunction(template, **options) + def create_filter(self, expr=None, **options): return Filter("", [expr] if expr else [], **options)