Add GeoIP db fixtures for filebeat tests

filebeat/module/haproxy/log/test/haproxy.log-expected.json

@@ -13,10 +13,13 @@
         "haproxy.connections.retries": 0, 
         "haproxy.connections.server": 0, 
         "haproxy.frontend_name": "incoming~", 
-        "haproxy.geoip.continent_name": "North America", 
-        "haproxy.geoip.country_iso_code": "US", 
-        "haproxy.geoip.location.lat": 37.751, 
-        "haproxy.geoip.location.lon": -97.822, 
+        "haproxy.geoip.city_name": "Minas Tirith", 
+        "haproxy.geoip.continent_name": "Middle Earth", 
+        "haproxy.geoip.country_iso_code": "GO", 
+        "haproxy.geoip.location.lat": 40.7143, 
+        "haproxy.geoip.location.lon": -74.006, 
+        "haproxy.geoip.region_iso_code": "GO-PE", 
+        "haproxy.geoip.region_name": "Pelennor", 
         "haproxy.http.request.captured_cookie": "-", 
         "haproxy.http.request.captured_headers": [
             "docs.example.internal"

filebeat/module/iis/access/test/test.log-expected.json

@@ -3,13 +3,13 @@
         "@timestamp": "2018-01-01T08:09:10.000Z", 
         "fileset.module": "iis", 
         "fileset.name": "access", 
-        "iis.access.geoip.city_name": "Berlin", 
-        "iis.access.geoip.continent_name": "Europe", 
-        "iis.access.geoip.country_iso_code": "DE", 
+        "iis.access.geoip.city_name": "Hobbiton", 
+        "iis.access.geoip.continent_name": "Middle Earth", 
+        "iis.access.geoip.country_iso_code": "AR", 
         "iis.access.geoip.location.lat": 52.4908, 
         "iis.access.geoip.location.lon": 13.3275, 
-        "iis.access.geoip.region_iso_code": "DE-BE", 
-        "iis.access.geoip.region_name": "Land Berlin", 
+        "iis.access.geoip.region_iso_code": "AR-SH", 
+        "iis.access.geoip.region_name": "The Shire", 
         "iis.access.method": "GET", 
         "iis.access.port": "80", 
         "iis.access.query_string": "q=100", 
@@ -71,13 +71,13 @@
         "iis.access.body_received.bytes": "456", 
         "iis.access.body_sent.bytes": "123", 
         "iis.access.cookie": "-", 
-        "iis.access.geoip.city_name": "Berlin", 
-        "iis.access.geoip.continent_name": "Europe", 
-        "iis.access.geoip.country_iso_code": "DE", 
+        "iis.access.geoip.city_name": "Hobbiton", 
+        "iis.access.geoip.continent_name": "Middle Earth", 
+        "iis.access.geoip.country_iso_code": "AR", 
         "iis.access.geoip.location.lat": 52.4908, 
         "iis.access.geoip.location.lon": 13.3275, 
-        "iis.access.geoip.region_iso_code": "DE-BE", 
-        "iis.access.geoip.region_name": "Land Berlin", 
+        "iis.access.geoip.region_iso_code": "AR-SH", 
+        "iis.access.geoip.region_name": "The Shire", 
         "iis.access.hostname": "example.com", 
         "iis.access.http_version": "1.1", 
         "iis.access.method": "GET", 

filebeat/module/iis/error/test/test.log-expected.json

@@ -21,13 +21,13 @@
         "@timestamp": "2018-01-01T09:10:11.000Z", 
         "fileset.module": "iis", 
         "fileset.name": "error", 
-        "iis.error.geoip.city_name": "Berlin", 
-        "iis.error.geoip.continent_name": "Europe", 
-        "iis.error.geoip.country_iso_code": "DE", 
+        "iis.error.geoip.city_name": "Hobbiton", 
+        "iis.error.geoip.continent_name": "Middle Earth", 
+        "iis.error.geoip.country_iso_code": "AR", 
         "iis.error.geoip.location.lat": 52.4908, 
         "iis.error.geoip.location.lon": 13.3275, 
-        "iis.error.geoip.region_iso_code": "DE-BE", 
-        "iis.error.geoip.region_name": "Land Berlin", 
+        "iis.error.geoip.region_iso_code": "AR-SH", 
+        "iis.error.geoip.region_name": "The Shire", 
         "iis.error.http_version": "1.1", 
         "iis.error.method": "GET", 
         "iis.error.queue_name": "-", 
@@ -46,13 +46,13 @@
         "@timestamp": "2018-01-01T10:11:12.000Z", 
         "fileset.module": "iis", 
         "fileset.name": "error", 
-        "iis.error.geoip.city_name": "Berlin", 
-        "iis.error.geoip.continent_name": "Europe", 
-        "iis.error.geoip.country_iso_code": "DE", 
+        "iis.error.geoip.city_name": "Hobbiton", 
+        "iis.error.geoip.continent_name": "Middle Earth", 
+        "iis.error.geoip.country_iso_code": "AR", 
         "iis.error.geoip.location.lat": 52.4908, 
         "iis.error.geoip.location.lon": 13.3275, 
-        "iis.error.geoip.region_iso_code": "DE-BE", 
-        "iis.error.geoip.region_name": "Land Berlin", 
+        "iis.error.geoip.region_iso_code": "AR-SH", 
+        "iis.error.geoip.region_name": "The Shire", 
         "iis.error.http_version": "2.0", 
         "iis.error.method": "GET", 
         "iis.error.queue_name": "-", 
@@ -71,13 +71,13 @@
         "@timestamp": "2018-01-01T11:12:13.000Z", 
         "fileset.module": "iis", 
         "fileset.name": "error", 
-        "iis.error.geoip.city_name": "Berlin", 
-        "iis.error.geoip.continent_name": "Europe", 
-        "iis.error.geoip.country_iso_code": "DE", 
+        "iis.error.geoip.city_name": "Hobbiton", 
+        "iis.error.geoip.continent_name": "Middle Earth", 
+        "iis.error.geoip.country_iso_code": "AR", 
         "iis.error.geoip.location.lat": 52.4908, 
         "iis.error.geoip.location.lon": 13.3275, 
-        "iis.error.geoip.region_iso_code": "DE-BE", 
-        "iis.error.geoip.region_name": "Land Berlin", 
+        "iis.error.geoip.region_iso_code": "AR-SH", 
+        "iis.error.geoip.region_name": "The Shire", 
         "iis.error.queue_name": "-", 
         "iis.error.reason_phrase": "Timer_MinBytesPerSecond", 
         "iis.error.remote_ip": "85.181.35.98", 

filebeat/module/nginx/access/test/test.log-expected.json

@@ -62,13 +62,13 @@
         "fileset.name": "access", 
         "input.type": "log", 
         "nginx.access.body_sent.bytes": "571", 
-        "nginx.access.geoip.city_name": "Berlin", 
-        "nginx.access.geoip.continent_name": "Europe", 
-        "nginx.access.geoip.country_iso_code": "DE", 
+        "nginx.access.geoip.city_name": "Hobbiton", 
+        "nginx.access.geoip.continent_name": "Middle Earth", 
+        "nginx.access.geoip.country_iso_code": "AR", 
         "nginx.access.geoip.location.lat": 52.4908, 
         "nginx.access.geoip.location.lon": 13.3275, 
-        "nginx.access.geoip.region_iso_code": "DE-BE", 
-        "nginx.access.geoip.region_name": "Land Berlin", 
+        "nginx.access.geoip.region_iso_code": "AR-SH", 
+        "nginx.access.geoip.region_name": "The Shire", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
         "nginx.access.referrer": "-", 
@@ -99,13 +99,13 @@
         "fileset.name": "access", 
         "input.type": "log", 
         "nginx.access.body_sent.bytes": "571", 
-        "nginx.access.geoip.city_name": "Berlin", 
-        "nginx.access.geoip.continent_name": "Europe", 
-        "nginx.access.geoip.country_iso_code": "DE", 
+        "nginx.access.geoip.city_name": "Hobbiton", 
+        "nginx.access.geoip.continent_name": "Middle Earth", 
+        "nginx.access.geoip.country_iso_code": "AR", 
         "nginx.access.geoip.location.lat": 52.4908, 
         "nginx.access.geoip.location.lon": 13.3275, 
-        "nginx.access.geoip.region_iso_code": "DE-BE", 
-        "nginx.access.geoip.region_name": "Land Berlin", 
+        "nginx.access.geoip.region_iso_code": "AR-SH", 
+        "nginx.access.geoip.region_name": "The Shire", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
         "nginx.access.referrer": "-", 
@@ -134,13 +134,13 @@
         "fileset.name": "access", 
         "input.type": "log", 
         "nginx.access.body_sent.bytes": "25507", 
-        "nginx.access.geoip.city_name": "Springfield", 
-        "nginx.access.geoip.continent_name": "North America", 
-        "nginx.access.geoip.country_iso_code": "US", 
-        "nginx.access.geoip.location.lat": 39.772, 
-        "nginx.access.geoip.location.lon": -89.6859, 
-        "nginx.access.geoip.region_iso_code": "US-IL", 
-        "nginx.access.geoip.region_name": "Illinois", 
+        "nginx.access.geoip.city_name": "Minas Tirith", 
+        "nginx.access.geoip.continent_name": "Middle Earth", 
+        "nginx.access.geoip.country_iso_code": "GO", 
+        "nginx.access.geoip.location.lat": 40.7143, 
+        "nginx.access.geoip.location.lon": -74.006, 
+        "nginx.access.geoip.region_iso_code": "GO-PE", 
+        "nginx.access.geoip.region_name": "Pelennor", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
         "nginx.access.referrer": "-", 
@@ -168,10 +168,13 @@
         "fileset.name": "access", 
         "input.type": "log", 
         "nginx.access.body_sent.bytes": "8571", 
-        "nginx.access.geoip.continent_name": "Europe", 
-        "nginx.access.geoip.country_iso_code": "PT", 
-        "nginx.access.geoip.location.lat": 39.5, 
-        "nginx.access.geoip.location.lon": -8.0, 
+        "nginx.access.geoip.city_name": "Minas Tirith", 
+        "nginx.access.geoip.continent_name": "Middle Earth", 
+        "nginx.access.geoip.country_iso_code": "GO", 
+        "nginx.access.geoip.location.lat": 40.7143, 
+        "nginx.access.geoip.location.lon": -74.006, 
+        "nginx.access.geoip.region_iso_code": "GO-PE", 
+        "nginx.access.geoip.region_name": "Pelennor", 
         "nginx.access.http_version": "1.1", 
         "nginx.access.method": "GET", 
         "nginx.access.referrer": "-", 

filebeat/module/system/auth/test/test.log-expected.json

@@ -56,12 +56,13 @@
         "system.auth.hostname": "slave22", 
         "system.auth.pid": "5774", 
         "system.auth.ssh.event": "Failed", 
-        "system.auth.ssh.geoip.continent_name": "Asia", 
-        "system.auth.ssh.geoip.country_iso_code": "CN", 
-        "system.auth.ssh.geoip.location.lat": 23.1167, 
-        "system.auth.ssh.geoip.location.lon": 113.25, 
-        "system.auth.ssh.geoip.region_iso_code": "CN-GD", 
-        "system.auth.ssh.geoip.region_name": "Guangdong", 
+        "system.auth.ssh.geoip.city_name": "Minas Tirith", 
+        "system.auth.ssh.geoip.continent_name": "Middle Earth", 
+        "system.auth.ssh.geoip.country_iso_code": "GO", 
+        "system.auth.ssh.geoip.location.lat": 40.7143, 
+        "system.auth.ssh.geoip.location.lon": -74.006, 
+        "system.auth.ssh.geoip.region_iso_code": "GO-PE", 
+        "system.auth.ssh.geoip.region_name": "Pelennor", 
         "system.auth.ssh.ip": "116.31.116.24", 
         "system.auth.ssh.method": "password", 
         "system.auth.ssh.port": "29160", 

filebeat/module/traefik/access/test/test.log-expected.json

@@ -31,13 +31,13 @@
         "offset": 280, 
         "prospector.type": "log", 
         "traefik.access.body_sent.bytes": "0", 
-        "traefik.access.geoip.city_name": "Berlin", 
-        "traefik.access.geoip.continent_name": "Europe", 
-        "traefik.access.geoip.country_iso_code": "DE", 
+        "traefik.access.geoip.city_name": "Hobbiton", 
+        "traefik.access.geoip.continent_name": "Middle Earth", 
+        "traefik.access.geoip.country_iso_code": "AR", 
         "traefik.access.geoip.location.lat": 52.4908, 
         "traefik.access.geoip.location.lon": 13.3275, 
-        "traefik.access.geoip.region_iso_code": "DE-BE", 
-        "traefik.access.geoip.region_name": "Land Berlin", 
+        "traefik.access.geoip.region_iso_code": "AR-SH", 
+        "traefik.access.geoip.region_name": "The Shire", 
         "traefik.access.http_version": "1.1", 
         "traefik.access.method": "GET", 
         "traefik.access.referrer": "http://example.com/login", 

testing/environments/geoip/Dockerfile

@@ -0,0 +1,9 @@
+FROM ubuntu:18.04
+
+RUN apt-get update && apt-get install -y perl build-essential
+
+# Auto-configure CPAN
+RUN cpan < /dev/null
+
+# Install MaxMind DB writer without running tests
+RUN cpan -T MaxMind::DB::Writer

testing/environments/geoip/Makefile

@@ -0,0 +1,2 @@
+GeoLite2-City.mmdb: createdb.pl
+	docker-compose run --rm maxmind-writer perl /src/createdb.pl /src/$@

testing/environments/geoip/README.md

@@ -0,0 +1,8 @@
+Files to generate GeoIP2 fixtures for testing in order to avoid unexpected
+changes on modules using geoip plugin.
+
+Data is directly provided from the `createdb.pl` script. To update the database
+just run edit the script and run `make`, it builds a docker image and runs the
+script on it.
+
+To read from the DB, `readdb.pl` can be used.

testing/environments/geoip/createdb.pl

@@ -0,0 +1,127 @@
+#!/usr/bin/env perl
+
+use strict;
+use warnings;
+use feature qw( say );
+
+use MaxMind::DB::Writer::Tree;
+
+my $filename = $ARGV[0];
+
+# See https://metacpan.org/pod/MaxMind::DB::Writer::Tree#DATA-TYPES
+my %types = (
+    city                => 'map',
+    continent           => 'map',
+    country             => 'map',
+    registered_country  => 'map',
+    subdivisions        => ['array', 'map'],
+
+    code                => 'utf8_string',
+    iso_code            => 'utf8_string',
+    geoname_id          => 'uint32',
+
+    names               => 'map',
+    en                  => 'utf8_string',
+
+    location            => 'map',
+    latitude            => 'double',
+    longitude           => 'double',
+    time_zone           => 'utf8_string',
+);
+
+my $tree = MaxMind::DB::Writer::Tree->new(
+    database_type => 'GeoIP2-City',
+    description => { en => 'GeoIP test Fixtures' },
+    ip_version => 6,
+
+    # add a callback to validate data going in to the database
+    map_key_type_callback => sub { $types{ $_[0] } },
+
+    # "record_size" is the record size in bits.  Either 24, 28 or 32.
+    record_size => 24,
+);
+
+my $middle_earth = {
+    names => {
+        en => 'Middle Earth',
+    },
+    code  => 'ME',
+};
+
+my $arnor = {
+    names => {
+        en => 'Kingdom of Arnor',
+    },
+    iso_code => 'AR',
+};
+
+my $gondor = {
+    names => {
+        en => 'Kingdom of Gondor',
+    },
+    iso_code => 'GO',
+};
+
+my $shire = {
+    names => {
+        en => 'The Shire',
+    },
+    iso_code => 'SH',
+};
+
+my $pelennor = {
+    names => {
+        en => 'Pelennor',
+    },
+    iso_code => 'PE',
+};
+
+my $hobbiton = {
+    continent => $middle_earth,
+    country => $arnor,
+    subdivisions => [$shire],
+    city => {
+        names => {
+            en => 'Hobbiton',
+        },
+    },
+    location => {
+        latitude => 52.4908,
+        longitude => 13.3275,
+    },
+};
+
+my $minas_tirith = {
+    continent => $middle_earth,
+    country => $gondor,
+    subdivisions => [$pelennor],
+    city => {
+        names => {
+            en => 'Minas Tirith',
+        },
+    },
+    location => {
+        latitude => 40.7143,
+        longitude => -74.0060,
+    },
+};
+
+my %networks = (
+    '85.181.35.0/24' => $hobbiton,
+    '1.2.0.0/16' => $minas_tirith,
+    '199.96.0.0/16' => $minas_tirith,
+    '116.31.0.0/16' => $minas_tirith,
+
+    '2a03:0000:10ff:f00f:0000:0000:0:8000/64' => $minas_tirith,
+);
+
+for my $network ( keys %networks ) {
+    $tree->insert_network( $network, $networks{$network} );
+}
+
+# Write the database to disk.
+open my $fh, '>:raw', $filename;
+$tree->write_tree( $fh );
+close $fh;
+
+say "$filename has now been created";

testing/environments/geoip/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2'
+
+services:
+  maxmind-writer:
+    build:
+      context: .
+    volumes:
+      - .:/src