Fix to do require for required files straightforward, per files

* Currently, all required files are loaded by `lib/fluent/load.rb` even for tests
  * This file creates the implicit order of files to be loaded
  * There's no explicit dependency definition, and there are many circular dependencies
  * This situation is totally broken
* Without this change, we can't run tests by `bundle exec rake test TEST=test/file.rb`
  * Running whole tests consumes long time, and it's not acceptable

This change is to make straightforward dependency from all files to others.
We can load any file in fluentd source code, make clean dependencies, and
also can run each files itself to test it.

lib/fluent/agent.rb

@@ -13,10 +13,13 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
-  require 'fluent/configurable'
-  require 'fluent/engine'
 
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/plugin'
+require 'fluent/output'
+
+module Fluent
   #
   # Agent is a resource unit who manages emittable plugins
   #

lib/fluent/buffer.rb

@@ -13,6 +13,12 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
+
+require 'monitor'
+require 'fileutils'
+
+require 'fluent/configurable'
+
 module Fluent
   class BufferError < StandardError
   end

lib/fluent/command/bundler_injection.rb

@@ -14,6 +14,9 @@
 #    limitations under the License.
 #
 
+require 'rbconfig'
+require 'rubygems'
+
 if ENV['BUNDLE_BIN_PATH']
   puts 'error: You seem to use `bundle exec` already.'
   exit 1

lib/fluent/command/cat.rb

@@ -66,7 +66,7 @@
   message_key = s
 }
 
-(class<<self;self;end).module_eval do
+(class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg

lib/fluent/command/debug.rb

@@ -34,7 +34,7 @@
   unix = b
 }
 
-(class<<self;self;end).module_eval do
+(class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg
@@ -60,7 +60,8 @@
   uri = "druby://#{host}:#{port}"
 end
 
-require 'fluent/load'
+require 'fluent/log'
+require 'fluent/engine'
 
 $log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
 Fluent::Engine.init

lib/fluent/command/fluentd.rb

@@ -15,6 +15,7 @@
 #
 
 require 'optparse'
+
 require 'fluent/supervisor'
 
 op = OptionParser.new
@@ -119,7 +120,7 @@
   opts[:gem_install_path] = s
 }
 
-(class<<self;self;end).module_eval do
+(class << self; self; end).module_eval do
   define_method(:usage) do |msg|
     puts op.to_s
     puts "error: #{msg}" if msg

lib/fluent/config.rb

@@ -14,10 +14,11 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/error'
-  require 'fluent/config/element'
+require 'fluent/config/error'
+require 'fluent/config/element'
+require 'fluent/configurable'
 
+module Fluent
   module Config
     def self.parse(str, fname, basepath = Dir.pwd, v1_config = false)
       if fname =~ /\.rb$/
@@ -39,8 +40,6 @@ def self.new(name = '')
     end
   end
 
-  require 'fluent/configurable'
-
   module PluginId
     def configure(conf)
       @id = conf['@id'] || conf['id']

lib/fluent/config/basic_parser.rb

@@ -14,12 +14,11 @@
 #    limitations under the License.
 #
 
+require 'stringio'
+require 'fluent/config/error'
+
 module Fluent
   module Config
-
-    require 'stringio'
-    require 'fluent/config/error'
-
     class BasicParser
       def initialize(strscan)
         @ss = strscan

lib/fluent/config/configure_proxy.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/configurable'
+
 module Fluent
   module Config
     class ConfigureProxy

lib/fluent/config/dsl.rb

@@ -14,7 +14,10 @@
 #    limitations under the License.
 #
 
+require 'json'
+
 require 'fluent/config'
+require 'fluent/config/element'
 
 module Fluent
   module Config

lib/fluent/config/element.rb

@@ -14,9 +14,10 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/error'
+require 'fluent/config/error'
+require 'fluent/config/literal_parser'
 
+module Fluent
   module Config
     class Element < Hash
       def initialize(name, arg, attrs, elements, unused = nil)

lib/fluent/config/literal_parser.rb

@@ -14,13 +14,16 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Config
+require 'stringio'
+
+require 'json'
+require 'yajl'
+require 'irb/ruby-lex'  # RubyLex
 
-    require 'yajl'
-    require 'fluent/config/basic_parser'
-    require 'irb/ruby-lex'  # RubyLex
+require 'fluent/config/basic_parser'
 
+module Fluent
+  module Config
     class LiteralParser < BasicParser
       def self.unescape_char(c)
         case c

lib/fluent/config/parser.rb

@@ -14,11 +14,13 @@
 #    limitations under the License.
 #
 
+require 'uri'
+
+require 'fluent/config/error'
+require 'fluent/config/element'
+
 module Fluent
   module Config
-    require 'fluent/config/error'
-    require 'fluent/config/element'
-
     class Parser
       def self.parse(io, fname, basepath = Dir.pwd)
         attrs, elems = Parser.new(basepath, io.each_line, fname).parse!(true)

lib/fluent/config/section.rb

@@ -16,10 +16,10 @@
 
 require 'json'
 
-module Fluent
-  require 'fluent/config/error'
-  require 'fluent/config/v1_parser'
+require 'fluent/config/error'
+require 'fluent/config/v1_parser'
 
+module Fluent
   module Config
     class Section < BasicObject
       def self.name

lib/fluent/config/types.rb

@@ -14,9 +14,12 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'json'
+require 'json'
+
+require 'fluent/config/error'
+require 'fluent/configurable'
 
+module Fluent
   module Config
     def self.size_value(str)
       case str.to_s

lib/fluent/config/v1_parser.rb

@@ -14,14 +14,16 @@
 #    limitations under the License.
 #
 
-module Fluent
-  module Config
+require 'strscan'
+require 'uri'
 
-    require 'strscan'
-    require 'fluent/config/error'
-    require 'fluent/config/literal_parser'
-    require 'fluent/config/element'
+require 'fluent/config/error'
+require 'fluent/config/basic_parser'
+require 'fluent/config/literal_parser'
+require 'fluent/config/element'
 
+module Fluent
+  module Config
     class V1Parser < LiteralParser
       ELEMENT_NAME = /[a-zA-Z0-9_]+/
 

lib/fluent/configurable.rb

@@ -14,13 +14,13 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/config/configure_proxy'
-  require 'fluent/config/section'
-  require 'fluent/config/error'
-  require 'fluent/registry'
-  require 'fluent/plugin'
+require 'fluent/config/configure_proxy'
+require 'fluent/config/section'
+require 'fluent/config/error'
+require 'fluent/registry'
+require 'fluent/plugin'
 
+module Fluent
   module Configurable
     def self.included(mod)
       mod.extend(ClassMethods)

lib/fluent/engine.rb

@@ -14,12 +14,21 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/event_router'
-  require 'fluent/root_agent'
-  require 'fluent/time'
-  require 'fluent/system_config'
+require 'socket'
+require 'rubygems'
+
+require 'msgpack'
+require 'cool.io'
 
+require 'fluent/config'
+require 'fluent/event'
+require 'fluent/event_router'
+require 'fluent/root_agent'
+require 'fluent/time'
+require 'fluent/system_config'
+require 'fluent/plugin'
+
+module Fluent
   class EngineClass
     class DummyMessagePackFactory
       def packer(*args)

lib/fluent/event.rb

@@ -14,6 +14,8 @@
 #    limitations under the License.
 #
 
+require 'fluent/engine'
+
 module Fluent
   class EventStream
     include Enumerable

lib/fluent/event_router.rb

@@ -13,9 +13,11 @@
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
 #
-module Fluent
-  require 'fluent/match'
 
+require 'fluent/match'
+require 'fluent/event'
+
+module Fluent
   #
   # EventRouter is responsible to route events to a collector.
   #

lib/fluent/filter.rb

@@ -14,6 +14,12 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/event'
+require 'fluent/log'
+
 module Fluent
   class Filter
     include Configurable

lib/fluent/formatter.rb

@@ -14,9 +14,11 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/registry'
+require 'fluent/configurable'
+require 'fluent/registry'
+require 'fluent/mixin'
 
+module Fluent
   class Formatter
     include Configurable
 

lib/fluent/input.rb

@@ -14,6 +14,11 @@
 #    limitations under the License.
 #
 
+require 'fluent/config'
+require 'fluent/configurable'
+require 'fluent/engine'
+require 'fluent/log'
+
 module Fluent
   class Input
     include Configurable

lib/fluent/label.rb

@@ -14,9 +14,9 @@
 #    limitations under the License.
 #
 
-module Fluent
-  require 'fluent/agent'
+require 'fluent/agent'
 
+module Fluent
   class Label < Agent
     def initialize(name, opts = {})
       super(opts)

lib/fluent/log.rb

@@ -65,6 +65,9 @@ def initialize(out=STDERR, level=LEVEL_TRACE, opts={})
       # TODO: This variable name is unclear so we should change to better name.
       @threads_exclude_events = []
 
+      # Fluent::Engine requires Fluent::Log, so we must take that object lazily
+      @engine = Fluent.const_get('Engine')
+
       if opts.has_key?(:suppress_repeated_stacktrace)
         @suppress_repeated_stacktrace = opts[:suppress_repeated_stacktrace]
       end
@@ -285,7 +288,7 @@ def event(level, args)
           record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
         }
         record['message'] = message.dup
-        Engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
+        @engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
       end
 
       return time, message