Merge pull request #2685 from ashie/strict-config-value

Make config values more strictly and enable to set nil or default

lib/fluent/command/fluentd.rb

@@ -131,6 +131,10 @@
   opts[:use_v1_config] = !b
 }
 
+op.on('--strict-config-value', "Parse config values strictly", TrueClass) {|b|
+  opts[:strict_config_value] = b
+}
+
 op.on('-v', '--verbose', "increase verbose level (-v: debug, -vv: trace)", TrueClass) {|b|
   if b
     opts[:log_level] = [opts[:log_level] - 1, Fluent::Log::LEVEL_TRACE].max

lib/fluent/config/element.rb

@@ -142,7 +142,7 @@ def to_s(nest = 0)
         indent = "  " * nest
         nindent = "  " * (nest + 1)
         out = ""
-        if @arg.empty?
+        if @arg.nil? || @arg.empty?
           out << "#{indent}<#{@name}>\n"
         else
           out << "#{indent}<#{@name} #{@arg}>\n"
@@ -194,23 +194,36 @@ def param_type(key)
         opts[:type]
       end
 
+      def default_value(key)
+        return nil if @corresponding_proxies.empty?
+
+        param_key = key.to_sym
+        proxy = @corresponding_proxies.detect do |_proxy|
+          _proxy.params.has_key?(param_key)
+        end
+        return nil unless proxy
+        proxy.defaults[param_key]
+      end
+
       def dump_value(k, v, nindent)
-        if secret_param?(k)
-          "#{nindent}#{k} xxxxxx\n"
+        return "#{nindent}#{k} xxxxxx\n" if secret_param?(k)
+        return "#{nindent}#{k} #{v}\n" unless @v1_config
+
+        # for v1 config
+        if v.nil?
+          "#{nindent}#{k} \n"
+        elsif v == :default
+          "#{nindent}#{k} #{default_value(k)}\n"
         else
-          if @v1_config
-            case param_type(k)
-            when :string
-              "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
-            when :enum, :integer, :float, :size, :bool, :time
-              "#{nindent}#{k} #{v}\n"
-            when :hash, :array
-              "#{nindent}#{k} #{v}\n"
-            else
-              # Unknown type
-              "#{nindent}#{k} #{v}\n"
-            end
+          case param_type(k)
+          when :string
+            "#{nindent}#{k} \"#{self.class.unescape_parameter(v)}\"\n"
+          when :enum, :integer, :float, :size, :bool, :time
+            "#{nindent}#{k} #{v}\n"
+          when :hash, :array
+            "#{nindent}#{k} #{v}\n"
           else
+            # Unknown type
             "#{nindent}#{k} #{v}\n"
           end
         end

lib/fluent/config/error.rb

@@ -23,4 +23,10 @@ class ConfigParseError < ConfigError
 
   class ObsoletedParameterError < ConfigError
   end
+
+  class SetNil < Exception
+  end
+
+  class SetDefault < Exception
+  end
 end

lib/fluent/config/literal_parser.rb

@@ -52,6 +52,16 @@ def self.unescape_char(c)
       def initialize(strscan, eval_context)
         super(strscan)
         @eval_context = eval_context
+        unless @eval_context.respond_to?(:use_nil)
+          def @eval_context.use_nil
+            raise SetNil
+          end
+        end
+        unless @eval_context.respond_to?(:use_default)
+          def @eval_context.use_default
+            raise SetDefault
+          end
+        end
       end
 
       def parse_literal(string_boundary_charset = LINE_END)
@@ -81,7 +91,13 @@ def scan_double_quoted_string
         string = []
         while true
           if skip(/\"/)
-            return string.join
+            if string.include?(nil)
+              return nil
+            elsif string.include?(:default)
+              return :default
+            else
+              return string.join
+            end
           elsif check(/[^"]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
             if s = check(/[^\\]#{LINE_END_WITHOUT_SPACING_AND_COMMENT}/)
               string << s
@@ -168,7 +184,13 @@ def eval_embedded_code(code)
 hostname = Socket.gethostname
 worker_id = ENV['SERVERENGINE_WORKER_ID'] || ''
 EOM
-        @eval_context.instance_eval(code)
+        begin
+          @eval_context.instance_eval(code)
+        rescue SetNil => e
+          nil
+        rescue SetDefault => e
+          :default
+        end
       end
 
       def eval_escape_char(c)

lib/fluent/config/section.rb

@@ -106,7 +106,7 @@ def method_missing(name, *args)
     end
 
     module SectionGenerator
-      def self.generate(proxy, conf, logger, plugin_class, stack = [])
+      def self.generate(proxy, conf, logger, plugin_class, stack = [], strict_config_value = false)
         return nil if conf.nil?
 
         section_stack = ""
@@ -122,9 +122,23 @@ def self.generate(proxy, conf, logger, plugin_class, stack = [])
         end
 
         if proxy.argument
-          unless conf.arg.empty?
+          unless conf.arg.nil? || conf.arg.empty?
             key, block, opts = proxy.argument
-            section_params[key] = self.instance_exec(conf.arg, opts, name, &block)
+            opts = opts.merge(strict: true) if strict_config_value
+
+            if conf.arg == :default
+              unless section_params.has_key?(key)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{key}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[key] = self.instance_exec(conf.arg, opts, key, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
           end
           unless section_params.has_key?(proxy.argument.first)
             logger.error "config error in:\n#{conf}" if logger # logger should exist, but somethimes it's nil (e.g, in tests)
@@ -136,13 +150,36 @@ def self.generate(proxy, conf, logger, plugin_class, stack = [])
         proxy.params.each_pair do |name, defval|
           varname = name.to_sym
           block, opts = defval
+          opts = opts.merge(strict: true) if strict_config_value
+
           if conf.has_key?(name.to_s) || opts[:alias] && conf.has_key?(opts[:alias].to_s)
             val = if conf.has_key?(name.to_s)
                     conf[name.to_s]
                   else
                     conf[opts[:alias].to_s]
                   end
-            section_params[varname] = self.instance_exec(val, opts, name, &block)
+
+            if val == :default
+              # default value is already set if it exists
+              unless section_params.has_key?(varname)
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{varname}' doesn't have default value"
+              end
+            else
+              begin
+                section_params[varname] = self.instance_exec(val, opts, name, &block)
+              rescue ConfigError => e
+                logger.error "config error in:\n#{conf}" if logger
+                raise e
+              end
+            end
+
+            if section_params[varname].nil?
+              unless proxy.defaults.has_key?(varname) and proxy.defaults[varname].nil?
+                logger.error "config error in:\n#{conf}" if logger
+                raise ConfigError, "'#{name}' parameter is required but nil is specified"
+              end
+            end
 
             # Source of definitions of deprecated/obsoleted:
             # https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Deprecated_and_obsolete_features
@@ -190,13 +227,13 @@ def self.generate(proxy, conf, logger, plugin_class, stack = [])
             raise ConfigError, "'<#{subproxy.name}>' sections are required" + section_stack
           end
           if subproxy.multi?
-            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name]) }
+            section_params[varname] = elements.map{ |e| generate(subproxy, e, logger, plugin_class, stack + [subproxy.name], strict_config_value) }
           else
             if elements.size > 1
               logger.error "config error in:\n#{conf}" if logger
               raise ConfigError, "'<#{subproxy.name}>' section cannot be written twice or more" + section_stack
             end
-            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name])
+            section_params[varname] = generate(subproxy, elements.first, logger, plugin_class, stack + [subproxy.name], strict_config_value)
           end
         end