From 2b0501dd3f7a5439729ec48987d162cd62c59e13 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 5 Jan 2019 00:55:04 +1000 Subject: [PATCH 1/3] Prototype log filtering system --- src/MicroLogging.jl | 7 ++- src/filters.jl | 126 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 src/filters.jl diff --git a/src/MicroLogging.jl b/src/MicroLogging.jl index 64bd199..16d3236 100644 --- a/src/MicroLogging.jl +++ b/src/MicroLogging.jl @@ -13,7 +13,7 @@ export ## Log creation @debug, @info, @warn, @error, @logmsg, ## Logger installation and control - with_logger, current_logger, global_logger, disable_logging + with_logger, current_logger, global_logger, disable_logging, filterlogs # ----- MicroLogging & stdlib Logging API ----- export @@ -21,6 +21,10 @@ export # config system) configure_logging, ConsoleLogger, + SimpleLogFilter, + LogLevelFilter, + MaxlogFilter, + CatchLogErrors, InteractiveLogger import Base.CoreLogging: @@ -37,6 +41,7 @@ include("StickyMessages.jl") include("ConsoleLogger.jl") include("InteractiveLogger.jl") # deprecated +include("filters.jl") include("config.jl") end diff --git a/src/filters.jl b/src/filters.jl new file mode 100644 index 0000000..a3820df --- /dev/null +++ b/src/filters.jl @@ -0,0 +1,126 @@ + +abstract type SimpleLogFilter end + +# TODO: implement ∘(::SimpleLogFilter, ::SimpleLogFilter) + +# Simple filters don't affect early filtering by default +min_enabled_level(f::SimpleLogFilter) = nothing +shouldlog(f::SimpleLogFilter, args...; kwargs...) = true +catch_exceptions(f::SimpleLogFilter) = nothing + +# Simple filters pass message directly through to sink by default, but they can +# modify the message by overriding this function. +handle_message(f::SimpleLogFilter, sink, args...; kwargs...) = + handle_message(sink, args...; kwargs...) + +struct FilteringLogger{P<:AbstractLogger,F<:SimpleLogFilter} <: AbstractLogger + parent::P + filter::F +end + +shouldlog(f::FilteringLogger, args...) = shouldlog(f.filter, args...) && shouldlog(f.parent, args...) +catch_exceptions(f::FilteringLogger) = something(catch_exceptions(f.filter), catch_exceptions(f.parent)) + +min_enabled_level(f::FilteringLogger) = something(min_enabled_level(f.filter), min_enabled_level(f.parent)) + +function handle_message(f::FilteringLogger, args...; kwargs...) + if !shouldlog(f.filter, args...; kwargs...) + return + end + handle_message(f.filter, f.parent, args...; kwargs...) +end + + +#------------------------------------------------------------------------------- + +struct LogLevelFilter <: SimpleLogFilter + default_min_level::LogLevel + module_limits::Dict{Module,LogLevel} +end + +function LogLevelFilter(min_level=Info, limits::Pair...) + LogLevelFilter(min_level, Dict{Module,LogLevel}(limits...)) +end + +function min_enabled_level(f::LogLevelFilter) + min_level = f.default_min_level + for (_,level) ∈ f.module_limits + if level < min_level + min_level = level + end + end + min_level +end + +shouldlog(f::LogLevelFilter, level, _module, group, id) = + !(level < get(f.module_limits, _module, f.default_min_level)) + +shouldlog(f::LogLevelFilter, args...; kwargs...) = true + + +""" + MaxlogFilter() + +Filter messages from log statements with a `maxlog=N` key value pair, which +occur more than `N` times. +""" +struct MaxlogFilter <: SimpleLogFilter + message_limits::Dict{Any,Int} +end + +shouldlog(f::MaxlogFilter, level, _module, group, id) = get(f.message_limits, id, 1) > 0 + +function shouldlog(f::MaxlogFilter, args...; maxlog=nothing, kwargs...) + if maxlog === nothing || !(maxlog isa Integer) + return true + end + remaining = get!(f.message_limits, id, maxlog) + f.message_limits[id] = remaining - 1 + remaining > 0 +end + +struct CatchLogErrors <: SimpleLogFilter + catch_exceptions::Bool +end + +catch_exceptions(f::CatchLogErrors) = f.catch_exceptions + +# TODO: Sticky filter + +#------------------------------------------------------------------------------- +function filterlogs(func, sf::SimpleLogFilter) + parent = current_logger() + logger = FilteringLogger(parent, sf) + with_logger(func, logger) + # TODO: For sticky filter, make `with_logger` `attach` and `detach` the + # loggers? + # + # replace_logger(logger) +end + +# Prototypes + +#= +struct MaxlogFilter{ParentLogger<:AbstractLogger} <: AbstractLogger + parent::ParentLogger + message_limits::Dict{Any,Int} +end + +MaxlogFilter(parent<:AbstractLogger) = MaxlogFilter(parent, Dict{Any,Int}()) + +shouldlog(f::MaxlogFilter, level, _module, group, id) = + get(f.message_limits, id, 1) > 0 && shouldlog(f.parent) + +min_enabled_level(f::MaxlogFilter) = f.parent.min_level + +function handle_message(f::MaxlogFilter, args...; maxlog=nothing, kwargs...) + if maxlog !== nothing && maxlog isa Integer + remaining = get!(logger.message_limits, id, maxlog) + logger.message_limits[id] = remaining - 1 + remaining > 0 || return + end + + handle_message(f.parent, args...; kwargs...) +end +=# + From 0e7f15963c9e1ee528631dbc3335cd4a2324d393 Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Sat, 5 Jan 2019 01:11:19 +1000 Subject: [PATCH 2/3] Add log filter composition --- src/filters.jl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/filters.jl b/src/filters.jl index a3820df..d19e341 100644 --- a/src/filters.jl +++ b/src/filters.jl @@ -1,8 +1,6 @@ abstract type SimpleLogFilter end -# TODO: implement ∘(::SimpleLogFilter, ::SimpleLogFilter) - # Simple filters don't affect early filtering by default min_enabled_level(f::SimpleLogFilter) = nothing shouldlog(f::SimpleLogFilter, args...; kwargs...) = true @@ -31,6 +29,16 @@ function handle_message(f::FilteringLogger, args...; kwargs...) end +struct ComposedLogFilter{F,G} <: SimpleLogFilter + filter1::F + filter2::G +end + +Base.:∘(f1::SimpleLogFilter, f2::SimpleLogFilter) = ComposedLogFilter(f1, f2) + +FilteringLogger(parent::AbstractLogger, f::ComposedLogFilter) = + FilteringLogger(FilteringLogger(parent, f.filter2), f.filter1) + #------------------------------------------------------------------------------- struct LogLevelFilter <: SimpleLogFilter From 25d1ca62074f1a0413771f9ba7f2ba15ac4415ad Mon Sep 17 00:00:00 2001 From: Chris Foster Date: Wed, 20 Mar 2019 19:45:41 +1000 Subject: [PATCH 3/3] Add some docs and TODOs --- src/MicroLogging.jl | 2 +- src/filters.jl | 65 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/MicroLogging.jl b/src/MicroLogging.jl index 16d3236..ed3e58e 100644 --- a/src/MicroLogging.jl +++ b/src/MicroLogging.jl @@ -21,7 +21,7 @@ export # config system) configure_logging, ConsoleLogger, - SimpleLogFilter, + AbstractLogFilter, LogLevelFilter, MaxlogFilter, CatchLogErrors, diff --git a/src/filters.jl b/src/filters.jl index d19e341..adfb975 100644 --- a/src/filters.jl +++ b/src/filters.jl @@ -1,17 +1,43 @@ +""" + AbstractLogFilter + +An `AbstractLogFilter` defines filtering and mapping rules for log events. +Log filters can be composed together using `∘` and applied to the log stream +generated by a function call using `filterlogs`. + + +Implementing a log filter means overriding some subset of the following +methods. -abstract type SimpleLogFilter end +* `min_enabled_level` +* `shouldlog` +* `catch_exceptions` +* `handle_message` + +TODO: Methods for attach and detach, to handle stateful filters? +TODO: Configurable log record matching seems to be a thing, not just for testing but for applying filter actions. +""" +abstract type AbstractLogFilter end # Simple filters don't affect early filtering by default -min_enabled_level(f::SimpleLogFilter) = nothing -shouldlog(f::SimpleLogFilter, args...; kwargs...) = true -catch_exceptions(f::SimpleLogFilter) = nothing +min_enabled_level(f::AbstractLogFilter) = nothing +shouldlog(f::AbstractLogFilter, args...; kwargs...) = true +catch_exceptions(f::AbstractLogFilter) = nothing # Simple filters pass message directly through to sink by default, but they can # modify the message by overriding this function. -handle_message(f::SimpleLogFilter, sink, args...; kwargs...) = +handle_message(f::AbstractLogFilter, sink, args...; kwargs...) = handle_message(sink, args...; kwargs...) -struct FilteringLogger{P<:AbstractLogger,F<:SimpleLogFilter} <: AbstractLogger + +""" + FilteringLogger(parent, filter) + +A logger which applies `filter` to all incoming messages and then passes them +on to the `parent` logger for further processing. `filter` may both map and +filter messages; see `AbstractLogFilter`. +""" +struct FilteringLogger{P<:AbstractLogger,F<:AbstractLogFilter} <: AbstractLogger parent::P filter::F end @@ -29,19 +55,24 @@ function handle_message(f::FilteringLogger, args...; kwargs...) end -struct ComposedLogFilter{F,G} <: SimpleLogFilter +struct ComposedLogFilter{F,G} <: AbstractLogFilter filter1::F filter2::G end -Base.:∘(f1::SimpleLogFilter, f2::SimpleLogFilter) = ComposedLogFilter(f1, f2) +Base.:∘(f1::AbstractLogFilter, f2::AbstractLogFilter) = ComposedLogFilter(f1, f2) FilteringLogger(parent::AbstractLogger, f::ComposedLogFilter) = FilteringLogger(FilteringLogger(parent, f.filter2), f.filter1) + #------------------------------------------------------------------------------- +# Concrete log filter implementations -struct LogLevelFilter <: SimpleLogFilter +""" +Disables messages below a given level +""" +struct LogLevelFilter <: AbstractLogFilter default_min_level::LogLevel module_limits::Dict{Module,LogLevel} end @@ -66,13 +97,14 @@ shouldlog(f::LogLevelFilter, level, _module, group, id) = shouldlog(f::LogLevelFilter, args...; kwargs...) = true +#------------------------------------------------------------------------------- """ MaxlogFilter() Filter messages from log statements with a `maxlog=N` key value pair, which occur more than `N` times. """ -struct MaxlogFilter <: SimpleLogFilter +struct MaxlogFilter <: AbstractLogFilter message_limits::Dict{Any,Int} end @@ -87,18 +119,23 @@ function shouldlog(f::MaxlogFilter, args...; maxlog=nothing, kwargs...) remaining > 0 end -struct CatchLogErrors <: SimpleLogFilter +""" +Catch log exceptions, or allow them to propagate. +""" +struct CatchLogErrors <: AbstractLogFilter catch_exceptions::Bool end catch_exceptions(f::CatchLogErrors) = f.catch_exceptions # TODO: Sticky filter +# TODO: Downgrade filter +# TODO: Filter to conditionally add keywords #------------------------------------------------------------------------------- -function filterlogs(func, sf::SimpleLogFilter) +function filterlogs(func, f::AbstractLogFilter) parent = current_logger() - logger = FilteringLogger(parent, sf) + logger = FilteringLogger(parent, f) with_logger(func, logger) # TODO: For sticky filter, make `with_logger` `attach` and `detach` the # loggers? @@ -119,7 +156,7 @@ MaxlogFilter(parent<:AbstractLogger) = MaxlogFilter(parent, Dict{Any,Int}()) shouldlog(f::MaxlogFilter, level, _module, group, id) = get(f.message_limits, id, 1) > 0 && shouldlog(f.parent) -min_enabled_level(f::MaxlogFilter) = f.parent.min_level +min_enabled_level(f::MaxlogFilter) = min_enabled_level(f.parent) function handle_message(f::MaxlogFilter, args...; maxlog=nothing, kwargs...) if maxlog !== nothing && maxlog isa Integer