diff --git a/.travis.yml b/.travis.yml index b332bbd..90cfa61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,14 +6,14 @@ env: global: DOCKER_COMPOSE_VERSION: 1.8.1 matrix: - - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/Gemfile - - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/Gemfile - - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/Gemfile - - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/Gemfile - - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile - - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile - - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile - - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile + - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_with_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_with_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_without_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_without_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_with_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_with_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_without_backport_dig.gemfile + - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_without_backport_dig.gemfile before_install: #- sudo apt-get update diff --git a/Appraisals b/Appraisals index cfa0051..9e25252 100644 --- a/Appraisals +++ b/Appraisals @@ -1,4 +1,17 @@ -appraise "fluentd v0.12" do +appraise "fluentd v0.12 with backport_dig" do gem "fluentd", "~> 0.12.0" + gem "backport_dig", "~> 1.0.0" end +appraise "fluentd v0.12 without backport_dig" do + gem "fluentd", "~> 0.12.0" +end + +appraise "fluentd v1.0 with backport_dig" do + gem "fluentd", "~> 1.0.0" + gem "backport_dig", "~> 1.0.0" +end + +appraise "fluentd v1.0 without backport_dig" do + gem "fluentd", "~> 1.0.0" +end diff --git a/data/COPYRIGHT.txt b/data/COPYRIGHT.txt new file mode 100644 index 0000000..8b0907d --- /dev/null +++ b/data/COPYRIGHT.txt @@ -0,0 +1 @@ +Database and Contents Copyright (c) 2017 MaxMind, Inc. diff --git a/data/GeoLite2-City.mmdb b/data/GeoLite2-City.mmdb new file mode 100644 index 0000000..6e7b6df Binary files /dev/null and b/data/GeoLite2-City.mmdb differ diff --git a/data/GeoLiteCity.dat b/data/GeoLiteCity.dat index 07cb172..3bb313b 100644 Binary files a/data/GeoLiteCity.dat and b/data/GeoLiteCity.dat differ diff --git a/data/LICENSE.txt b/data/LICENSE.txt new file mode 100644 index 0000000..8216b8f --- /dev/null +++ b/data/LICENSE.txt @@ -0,0 +1,3 @@ +This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/. + +This database incorporates GeoNames [http://www.geonames.org] geographical data, which is made available under the Creative Commons Attribution 3.0 License. To view a copy of this license, visit http://www.creativecommons.org/licenses/by/3.0/us/. diff --git a/data/README.txt b/data/README.txt new file mode 100644 index 0000000..16e29ad --- /dev/null +++ b/data/README.txt @@ -0,0 +1 @@ +Latitude and longitude are not precise and should not be used to identify a particular street address or household. diff --git a/gemfiles/fluentd_v0.12_with_backport_dig.gemfile b/gemfiles/fluentd_v0.12_with_backport_dig.gemfile new file mode 100644 index 0000000..f27dadb --- /dev/null +++ b/gemfiles/fluentd_v0.12_with_backport_dig.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "fluentd", "~> 0.12.0" +gem "backport_dig", "~> 1.0.0" + +gemspec path: "../" diff --git a/gemfiles/fluentd_v0.12.gemfile b/gemfiles/fluentd_v0.12_without_backport_dig.gemfile similarity index 56% rename from gemfiles/fluentd_v0.12.gemfile rename to gemfiles/fluentd_v0.12_without_backport_dig.gemfile index 596b99e..c050989 100644 --- a/gemfiles/fluentd_v0.12.gemfile +++ b/gemfiles/fluentd_v0.12_without_backport_dig.gemfile @@ -1,8 +1,7 @@ # This file was generated by Appraisal -source "http://rubygems.org" +source "https://rubygems.org" gem "fluentd", "~> 0.12.0" -gemspec :path => "../" - +gemspec path: "../" diff --git a/gemfiles/fluentd_v1.0_with_backport_dig.gemfile b/gemfiles/fluentd_v1.0_with_backport_dig.gemfile new file mode 100644 index 0000000..a893e0a --- /dev/null +++ b/gemfiles/fluentd_v1.0_with_backport_dig.gemfile @@ -0,0 +1,8 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "fluentd", "~> 1.0.0" +gem "backport_dig", "~> 1.0.0" + +gemspec path: "../" diff --git a/gemfiles/fluentd_v1.0_without_backport_dig.gemfile b/gemfiles/fluentd_v1.0_without_backport_dig.gemfile new file mode 100644 index 0000000..00f28fa --- /dev/null +++ b/gemfiles/fluentd_v1.0_without_backport_dig.gemfile @@ -0,0 +1,7 @@ +# This file was generated by Appraisal + +source "https://rubygems.org" + +gem "fluentd", "~> 1.0.0" + +gemspec path: "../" diff --git a/lib/fluent/plugin/geoip.rb b/lib/fluent/plugin/geoip.rb index a72a35c..98ebbca 100644 --- a/lib/fluent/plugin/geoip.rb +++ b/lib/fluent/plugin/geoip.rb @@ -1,6 +1,14 @@ require 'geoip' require 'yajl' -require 'dig_rb' +unless {}.respond_to?(:dig) + begin + # backport_dig is faster than dig_rb so prefer backport_dig. + # And Fluentd v1.0.1 uses backport_dig + require 'backport_dig' + rescue LoadError + require 'dig_rb' + end +end module Fluent class GeoIP diff --git a/test/plugin/test_filter_geoip.rb b/test/plugin/test_filter_geoip.rb index 9a2e7a6..1e70c67 100644 --- a/test/plugin/test_filter_geoip.rb +++ b/test/plugin/test_filter_geoip.rb @@ -177,13 +177,13 @@ def test_filter_with_skip_unknown_address messages = [ {'host' => '203.0.113.1', 'message' => 'invalid ip'}, {'host' => '0', 'message' => 'invalid ip'}, - {'host' => '8.8.8.8', 'message' => 'google public dns'} + {'host' => '66.102.3.80', 'message' => 'google bot'} ] expected = [ {'host' => '203.0.113.1', 'message' => 'invalid ip'}, {'host' => '0', 'message' => 'invalid ip'}, - {'host' => '8.8.8.8', 'message' => 'google public dns', - 'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]} + {'host' => '66.102.3.80', 'message' => 'google bot', + 'geoip_city' => 'Mountain View', 'geopoint' => [-122.0574, 37.419200000000004]} ] filtered = filter(config, messages) assert_equal(expected, filtered) @@ -446,13 +446,13 @@ def test_filter_with_skip_unknown_address messages = [ {'host' => '203.0.113.1', 'message' => 'invalid ip'}, {'host' => '0', 'message' => 'invalid ip'}, - {'host' => '8.8.8.8', 'message' => 'google public dns'} + {'host' => '66.102.3.80', 'message' => 'google bot'} ] expected = [ {'host' => '203.0.113.1', 'message' => 'invalid ip'}, {'host' => '0', 'message' => 'invalid ip'}, - {'host' => '8.8.8.8', 'message' => 'google public dns', - 'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]} + {'host' => '66.102.3.80', 'message' => 'google bot', + 'geoip_city' => 'Mountain View', 'geopoint' => [-122.0574, 37.419200000000004]} ] filtered = filter(config, messages) assert_equal(expected, filtered) @@ -746,13 +746,13 @@ def test_filter_with_skip_unknown_address messages = [ {'host' => '203.0.113.1', 'message' => 'invalid ip'}, {'host' => '0', 'message' => 'invalid ip'}, - {'host' => '8.8.8.8', 'message' => 'google public dns'} + {'host' => '66.102.3.80', 'message' => 'google bot'} ] expected = [ {'host' => '203.0.113.1', 'message' => 'invalid ip'}, {'host' => '0', 'message' => 'invalid ip'}, - {'host' => '8.8.8.8', 'message' => 'google public dns', - 'geoip_city' => 'Mountain View', 'geopoint' => [-122.08380126953125, 37.38600158691406]} + {'host' => '66.102.3.80', 'message' => 'google bot', + 'geoip_city' => 'Mountain View', 'geopoint' => [-122.05740356445312, 37.4192008972168]} ] filtered = filter(config, messages) assert_equal(expected, filtered) diff --git a/test/plugin/test_out_geoip.rb b/test/plugin/test_out_geoip.rb index 5562fab..0fb634d 100644 --- a/test/plugin/test_out_geoip.rb +++ b/test/plugin/test_out_geoip.rb @@ -229,7 +229,7 @@ def test_emit_with_skip_unknown_address # 203.0.113.1 is a test address described in RFC5737 d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'}) d1.emit({'host' => '0', 'message' => 'invalid ip'}) - d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'}) + d1.emit({'host' => '66.102.3.80', 'message' => 'google bot'}) end emits = d1.emits assert_equal 3, emits.length @@ -240,7 +240,7 @@ def test_emit_with_skip_unknown_address assert_equal nil, emits[1][2]['geoip_city'] assert_equal nil, emits[1][2]['geopoint'] assert_equal 'Mountain View', emits[2][2]['geoip_city'] - assert_equal [-122.0838, 37.386], emits[2][2]['geopoint'] + assert_equal [-122.0574, 37.419200000000004], emits[2][2]['geopoint'] end def test_emit_record_directive @@ -523,7 +523,7 @@ def test_emit_with_skip_unknown_address # 203.0.113.1 is a test address described in RFC5737 d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'}) d1.emit({'host' => '0', 'message' => 'invalid ip'}) - d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'}) + d1.emit({'host' => '66.102.3.80', 'message' => 'google bot'}) end emits = d1.emits assert_equal 3, emits.length @@ -534,7 +534,7 @@ def test_emit_with_skip_unknown_address assert_equal nil, emits[1][2]['geoip_city'] assert_equal nil, emits[1][2]['geopoint'] assert_equal 'Mountain View', emits[2][2]['geoip_city'] - assert_equal [-122.0838, 37.386], emits[2][2]['geopoint'] + assert_equal [-122.0574, 37.419200000000004], emits[2][2]['geopoint'] end def test_emit_record_directive @@ -848,7 +848,7 @@ def test_emit_with_skip_unknown_address # 203.0.113.1 is a test address described in RFC5737 d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'}) d1.emit({'host' => '0', 'message' => 'invalid ip'}) - d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'}) + d1.emit({'host' => '66.102.3.80', 'message' => 'google bot'}) end emits = d1.emits assert_equal 3, emits.length @@ -859,7 +859,8 @@ def test_emit_with_skip_unknown_address assert_equal nil, emits[1][2]['geoip_city'] assert_equal nil, emits[1][2]['geopoint'] assert_equal 'Mountain View', emits[2][2]['geoip_city'] - assert_equal [-122.08380126953125, 37.38600158691406], emits[2][2]['geopoint'] + assert_equal [-122.05740356445312, 37.4192008972168], emits[2][2]['geopoint'] + end def test_emit_multiple_key