added LRU cache and ip addr validation

CHANGELOG.md

@@ -10,4 +10,6 @@
   - Updated key check for len
 ## 0.1.5
   - Fixed nil key error
+## 0.1.6
+  - Added LRU cache options and filter for invalid ips
 

CONTRIBUTORS

@@ -2,7 +2,7 @@ The following is a list of people who have contributed ideas, code, bug
 reports, or in general have helped logstash along its way.
 
 Contributors:
-* nsherron90 - [email protected]
+* nicksherron - [email protected]
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

README.md

@@ -19,10 +19,10 @@ $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise
 
 Or you can build it yourself:
 ```
-git clone https://github.com/nsherron90/logstash-filter-greynoise.git
+git clone https://github.com/nicksherron/logstash-filter-greynoise.git
 bundle install
 gem build logstash-filter-greynoise.gemspec
-$LS_HOME/bin/logstash-plugin install logstash-filter-greynoise-0.1.4.gem
+$LS_HOME/bin/logstash-plugin install logstash-filter-greynoise-0.1.6.gem
 ```
 
 ### 2. Filter Configuration
@@ -34,6 +34,8 @@ filter {
     ip => "ip_value"                 # string (required, reference to ip address field)
     key => "your_greynoise_key"      # string (optional, no default)
     target => "greynoise"            # string (optional, default = greynoise)
+    hit_cache_size => 100            # number (optional, default = 0)
+    hit_cache_ttl => 6               # number (optional, default = 60) 
   }
 }
 ```

docs/index.asciidoc

@@ -61,6 +61,22 @@ If this field is an array, only the first value will be used.
 
 Your Greynoise API key. If you don't have a key, either signup for a free enterprise trial or
 simply leave out this field to perform lookups via the Alpha(free) api. NOTE, the enterpise and alpha api do not return the same data.
+
+[id="plugins-{type}s-{plugin}-hit_cache_size"]
+===== `hit_cache_size`
+
+  * Value type is <<number,number>>
+  * Default value is `0`
+
+Number of items to store in LRU cache.
+
+id="plugins-{type}s-{plugin}-hit_cache_ttl"]
+===== `hit_cache_ttl`
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+Time in seconds for LRU cache item eviction.
 //
 //[id="plugins-{type}s-{plugin}-setting_name_3"]
 //===== `setting_name_3`

lib/logstash/filters/greynoise.rb

@@ -1,8 +1,10 @@
 # encoding: utf-8
-require "logstash/filters/base"
+require 'logstash/filters/base'
 require "json"
 require "logstash/namespace"
-require 'faraday'
+require "faraday"
+require "ipaddr"
+require "lru_redux"
 
 
 # This  filter will replace the contents of the default
@@ -14,48 +16,113 @@ class LogStash::Filters::Greynoise < LogStash::Filters::Base
   # Setting the config_name here is required. This is how you
   # configure this filter from your Logstash config.
   #
-   #  filter {
-   #   greynoise {
-   #     ip => "ip"
-   #   }
-   #  }
+  #  filter {
+  #   greynoise {
+  #     ip => "ip"
+  #   }
+  #  }
 
   config_name "greynoise"
 
-  # Replace the message with this value.
-
+  # ip address to use for greynoise query
   config :ip, :validate => :string, :required => true
+
+  # greynoise enterprise api key
   config :key, :validate => :string, :default => ""
+
+  # target top level key of hash response
   config :target, :validate => :string, :default => "greynoise"
 
+  # tag if ip address supplied is invalid
+  config :tag_on_failure, :validate => :string, :default => '_greynoise_filter_invalid_ip'
 
+  # set the size of cache for successful requests
+  config :hit_cache_size, :validate => :number, :default => 0
+
+  # how long to cache successful requests (in seconds)
+  config :hit_cache_ttl, :validate => :number, :default => 60
 
   public
+
   def register
-  end # def register
+    if @hit_cache_size > 0
+      @hit_cache = LruRedux::TTL::ThreadSafeCache.new(@hit_cache_size, @hit_cache_ttl)
+    end
 
-  public
-  def filter(event)
+  end # def register
 
-    # check if api key exists and has len of 25 or more to prevent forbidden response
-    if @key.length >= 25
-      url = "https://enterprise.api.greynoise.io/v2/noise/context/" + event.sprintf(ip)
-      uri = URI.parse(URI.encode(url.strip))
+  private
 
-      response = Faraday.get(uri, nil, 'User-Agent' => 'logstash-filter-greynoise', Key: event.sprintf(key))
-    # if no key then use alpha(free) api
+  def get_free(target_ip)
+    url = "https://api.greynoise.io/v1/query/ip"
+    response = Faraday.post url, {:ip => target_ip}, 'User-Agent' => 'logstash-filter-greynoise'
+    if response.success?
+      JSON.parse(response.body)
     else
-      url = "https://api.greynoise.io/v1/query/ip"
-      response = Faraday.post url, { :ip => event.sprintf(ip) }, 'User-Agent' => 'logstash-filter-greynoise'
+      nil
+    end
+  end
+
+  private
 
+  def get_enterprise(target_ip, api_key)
+    url = "https://enterprise.api.greynoise.io/v2/noise/context/" + target_ip
+    uri = URI.parse(URI.encode(url.strip))
+    response = Faraday.get(uri, nil, 'User-Agent' => 'logstash-filter-greynoise', Key: api_key)
+    if response.success?
+      JSON.parse(response.body)
+    else
+      nil
     end
+  end
 
-    result = JSON.parse(response.body)
+  public
 
-    event.set(@target, result)
-    # filter_matched should go in the last line of our successful code
-    filter_matched(event)
+  def filter(event)
+    valid = nil
+    begin
+      IPAddr.new(event.sprintf(ip))
+    rescue ArgumentError => e
+      valid = e
+    end
 
+    if valid
+      @logger.error("Invalid IP address, skipping", :ip => event.sprintf(ip), :event => event.to_hash)
+      event.tag(@tag_on_failure)
+    else
+      if @hit_cache
+        result = @hit_cache[event.sprintf(ip)]
+        if result
+          event.set(@target, result)
+          filter_matched(event)
+        else
+          # check if api key exists and has len of 25 or more to prevent forbidden response
+          if @key.length >= 25
+            result = get_enterprise(event.sprintf(ip), event.sprintf(key))
+            # if no key then use alpha(free) api
+          else
+            result = get_free(event.sprintf(ip))
+          end
+          unless result.nil?
+            @hit_cache[event.sprintf(ip)] = result
+            event.set(@target, result)
+            # filter_matched should go in the last line of our successful code
+            filter_matched(event)
+          end
+        end
+      else
+        if @key.length >= 25
+          result = get_enterprise(event.sprintf(ip), event.sprintf(key))
+        else
+          result = get_free(event.sprintf(ip))
+        end
+
+        unless result.nil?
+          event.set(@target, result)
+          filter_matched(event)
+        end
+      end
+    end
   end # def filter
-end # class LogStash::Filters::Greynoise
+end # def LogStash::Filters::Greynoise
 

logstash-filter-greynoise.gemspec

@@ -1,10 +1,10 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-filter-greynoise'
-  s.version       = '0.1.5'
+  s.version       = '0.1.6'
   s.licenses      = ['Apache-2.0']
   s.summary = 'This greynoise filter takes contents in the ip field and returns greynoise api data (see https://greynoise.io/ for more info).'
   s.description     = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install logstash-filter-greynoise. This gem is not a stand-alone program'
-  s.homepage = 'https://github.com/nsherron90/logstash-filter-greynoise'
+  s.homepage = 'https://github.com/nicksherron/logstash-filter-greynoise'
   s.authors       = ['nsherron90']
   s.email         = '[email protected]'
   s.require_paths = ['lib']
@@ -18,8 +18,10 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash-core-plugin-api', '~> 2.0'
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_development_dependency 'logstash-devutils'
-  s.add_runtime_dependency  'faraday', '~> 0.9.2'
+  s.add_runtime_dependency  'faraday', '= 0.17.1'
+  s.add_runtime_dependency 'lru_redux', "~> 1.1.0"
+
 
 end