Merge pull request #46 from y-ken/update-geoip-database

Updated to latest GeoLiteCity Binary on June 2017.

.travis.yml

@@ -6,14 +6,14 @@ env:
   global:
     DOCKER_COMPOSE_VERSION: 1.8.1
   matrix:
-    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/Gemfile
-    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/Gemfile
-    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/Gemfile
-    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/Gemfile
-    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
-    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
-    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
-    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12.gemfile
+    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_with_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_with_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_without_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v1.0_without_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.1 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_with_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.2 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_with_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.3 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_without_backport_dig.gemfile
+    - TARGET_RUBY_VERSION=2.4 BUNDLE_GEMFILE=/app/gemfiles/fluentd_v0.12_without_backport_dig.gemfile
 
 before_install:
   #- sudo apt-get update

Appraisals

@@ -1,4 +1,17 @@
-appraise "fluentd v0.12" do
+appraise "fluentd v0.12 with backport_dig" do
   gem "fluentd", "~> 0.12.0"
+  gem "backport_dig", "~> 1.0.0"
 end
 
+appraise "fluentd v0.12 without backport_dig" do
+  gem "fluentd", "~> 0.12.0"
+end
+
+appraise "fluentd v1.0 with backport_dig" do
+  gem "fluentd", "~> 1.0.0"
+  gem "backport_dig", "~> 1.0.0"
+end
+
+appraise "fluentd v1.0 without backport_dig" do
+  gem "fluentd", "~> 1.0.0"
+end

data/COPYRIGHT.txt

@@ -0,0 +1 @@
+Database and Contents Copyright (c) 2017 MaxMind, Inc.

data/LICENSE.txt

@@ -0,0 +1,3 @@
+This work is licensed under the Creative Commons Attribution-ShareAlike 4.0 International License. To view a copy of this license, visit http://creativecommons.org/licenses/by-sa/4.0/.
+
+This database incorporates GeoNames [http://www.geonames.org] geographical data, which is made available under the Creative Commons Attribution 3.0 License. To view a copy of this license, visit http://www.creativecommons.org/licenses/by/3.0/us/.

data/README.txt

@@ -0,0 +1 @@
+Latitude and longitude are not precise and should not be used to identify a particular street address or household.

gemfiles/fluentd_v0.12_with_backport_dig.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~> 0.12.0"
+gem "backport_dig", "~> 1.0.0"
+
+gemspec path: "../"

gemfiles/fluentd_v0.12.gemfile → gemfiles/fluentd_v0.12_without_backport_dig.gemfile

@@ -1,8 +1,7 @@
 # This file was generated by Appraisal
 
-source "http://rubygems.org"
+source "https://rubygems.org"
 
 gem "fluentd", "~> 0.12.0"
 
-gemspec :path => "../"
-
+gemspec path: "../"

gemfiles/fluentd_v1.0_with_backport_dig.gemfile

@@ -0,0 +1,8 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~> 1.0.0"
+gem "backport_dig", "~> 1.0.0"
+
+gemspec path: "../"

gemfiles/fluentd_v1.0_without_backport_dig.gemfile

@@ -0,0 +1,7 @@
+# This file was generated by Appraisal
+
+source "https://rubygems.org"
+
+gem "fluentd", "~> 1.0.0"
+
+gemspec path: "../"

lib/fluent/plugin/geoip.rb

@@ -1,6 +1,14 @@
 require 'geoip'
 require 'yajl'
-require 'dig_rb'
+unless {}.respond_to?(:dig)
+  begin
+    # backport_dig is faster than dig_rb so prefer backport_dig.
+    # And Fluentd v1.0.1 uses backport_dig
+    require 'backport_dig'
+  rescue LoadError
+    require 'dig_rb'
+  end
+end
 
 module Fluent
   class GeoIP

test/plugin/test_filter_geoip.rb

@@ -177,13 +177,13 @@ def test_filter_with_skip_unknown_address
       messages = [
         {'host' => '203.0.113.1', 'message' => 'invalid ip'},
         {'host' => '0', 'message' => 'invalid ip'},
-        {'host' => '8.8.8.8', 'message' => 'google public dns'}
+        {'host' => '66.102.3.80', 'message' => 'google bot'}
       ]
       expected = [
         {'host' => '203.0.113.1', 'message' => 'invalid ip'},
         {'host' => '0', 'message' => 'invalid ip'},
-        {'host' => '8.8.8.8', 'message' => 'google public dns',
-         'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]}
+        {'host' => '66.102.3.80', 'message' => 'google bot',
+         'geoip_city' => 'Mountain View', 'geopoint' => [-122.0574, 37.419200000000004]}
       ]
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
@@ -446,13 +446,13 @@ def test_filter_with_skip_unknown_address
       messages = [
         {'host' => '203.0.113.1', 'message' => 'invalid ip'},
         {'host' => '0', 'message' => 'invalid ip'},
-        {'host' => '8.8.8.8', 'message' => 'google public dns'}
+        {'host' => '66.102.3.80', 'message' => 'google bot'}
       ]
       expected = [
         {'host' => '203.0.113.1', 'message' => 'invalid ip'},
         {'host' => '0', 'message' => 'invalid ip'},
-        {'host' => '8.8.8.8', 'message' => 'google public dns',
-         'geoip_city' => 'Mountain View', 'geopoint' => [-122.0838, 37.386]}
+        {'host' => '66.102.3.80', 'message' => 'google bot',
+         'geoip_city' => 'Mountain View', 'geopoint' => [-122.0574, 37.419200000000004]}
       ]
       filtered = filter(config, messages)
       assert_equal(expected, filtered)
@@ -746,13 +746,13 @@ def test_filter_with_skip_unknown_address
       messages = [
         {'host' => '203.0.113.1', 'message' => 'invalid ip'},
         {'host' => '0', 'message' => 'invalid ip'},
-        {'host' => '8.8.8.8', 'message' => 'google public dns'}
+        {'host' => '66.102.3.80', 'message' => 'google bot'}
       ]
       expected = [
         {'host' => '203.0.113.1', 'message' => 'invalid ip'},
         {'host' => '0', 'message' => 'invalid ip'},
-        {'host' => '8.8.8.8', 'message' => 'google public dns',
-         'geoip_city' => 'Mountain View', 'geopoint' => [-122.08380126953125, 37.38600158691406]}
+        {'host' => '66.102.3.80', 'message' => 'google bot',
+         'geoip_city' => 'Mountain View', 'geopoint' => [-122.05740356445312, 37.4192008972168]}
       ]
       filtered = filter(config, messages)
       assert_equal(expected, filtered)

test/plugin/test_out_geoip.rb

@@ -229,7 +229,7 @@ def test_emit_with_skip_unknown_address
         # 203.0.113.1 is a test address described in RFC5737
         d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
         d1.emit({'host' => '0', 'message' => 'invalid ip'})
-        d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'})
+        d1.emit({'host' => '66.102.3.80', 'message' => 'google bot'})
       end
       emits = d1.emits
       assert_equal 3, emits.length
@@ -240,7 +240,7 @@ def test_emit_with_skip_unknown_address
       assert_equal nil, emits[1][2]['geoip_city']
       assert_equal nil, emits[1][2]['geopoint']
       assert_equal 'Mountain View', emits[2][2]['geoip_city']
-      assert_equal [-122.0838, 37.386], emits[2][2]['geopoint']
+      assert_equal [-122.0574, 37.419200000000004], emits[2][2]['geopoint']
     end
 
     def test_emit_record_directive
@@ -523,7 +523,7 @@ def test_emit_with_skip_unknown_address
         # 203.0.113.1 is a test address described in RFC5737
         d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
         d1.emit({'host' => '0', 'message' => 'invalid ip'})
-        d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'})
+        d1.emit({'host' => '66.102.3.80', 'message' => 'google bot'})
       end
       emits = d1.emits
       assert_equal 3, emits.length
@@ -534,7 +534,7 @@ def test_emit_with_skip_unknown_address
       assert_equal nil, emits[1][2]['geoip_city']
       assert_equal nil, emits[1][2]['geopoint']
       assert_equal 'Mountain View', emits[2][2]['geoip_city']
-      assert_equal [-122.0838, 37.386], emits[2][2]['geopoint']
+      assert_equal [-122.0574, 37.419200000000004], emits[2][2]['geopoint']
     end
 
     def test_emit_record_directive
@@ -848,7 +848,7 @@ def test_emit_with_skip_unknown_address
         # 203.0.113.1 is a test address described in RFC5737
         d1.emit({'host' => '203.0.113.1', 'message' => 'invalid ip'})
         d1.emit({'host' => '0', 'message' => 'invalid ip'})
-        d1.emit({'host' => '8.8.8.8', 'message' => 'google public dns'})
+        d1.emit({'host' => '66.102.3.80', 'message' => 'google bot'})
       end
       emits = d1.emits
       assert_equal 3, emits.length
@@ -859,7 +859,8 @@ def test_emit_with_skip_unknown_address
       assert_equal nil, emits[1][2]['geoip_city']
       assert_equal nil, emits[1][2]['geopoint']
       assert_equal 'Mountain View', emits[2][2]['geoip_city']
-      assert_equal [-122.08380126953125, 37.38600158691406], emits[2][2]['geopoint']
+      assert_equal [-122.05740356445312, 37.4192008972168], emits[2][2]['geopoint']
+
     end
 
     def test_emit_multiple_key