Elasticsearch v2 Compatibility

circle.yml

@@ -32,6 +32,11 @@ dependencies:
     # fresh clone.
     - if [[ ! -e build/work/git-restore-mtime ]]; then curl https://raw.githubusercontent.com/MestreLion/git-tools/0fc841a3e49d041576e5b21d1644c8df2d2ef801/git-restore-mtime > build/work/git-restore-mtime && chmod +x build/work/git-restore-mtime; fi
     - ./build/work/git-restore-mtime -f .
+    - if [[ ! -e build/work/elasticsearch-2.3.3/bin/elasticsearch ]]; then mkdir -p build/work/elasticsearch-2.3.3 && curl -L https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.3.3/elasticsearch-2.3.3.tar.gz | tar --strip-components 1 -C build/work/elasticsearch-2.3.3 -xz; fi
+  post:
+    - rm -rf /tmp/elasticsearch-v2-data
+    - ./build/work/elasticsearch-2.3.3/bin/elasticsearch --path.data=/tmp/elasticsearch-v2-data:
+        background: true
   override:
     # Build all the API Umbrella software dependencies.
     - env PATH=build/work/cmake-3.4.3/bin:$PATH ./configure --enable-test-dependencies

config/default.yml

@@ -152,6 +152,7 @@ elasticsearch:
     - "http://127.0.0.1:14002"
   embedded_server_env:
     heap_size: 512m
+  api_version: 1
   embedded_server_config:
     http:
       port: 14002
@@ -167,11 +168,6 @@ elasticsearch:
       breaker:
         fielddata:
           limit: 60%
-    # Disable Groovy scripting for security: CVE-2015-1427
-    script:
-      groovy:
-        sandbox:
-          enabled: false
 analytics:
   adapter: elasticsearch
   timezone: UTC

config/elasticsearch_templates.json

@@ -8,11 +8,10 @@
           "_all": {
             "enabled": false
           },
-          "_timestamp": {
-            "enabled": true,
-            "path": "updated_at"
-          },
           "properties": {
+            "updated_at": {
+              "type": "date"
+            },
             "country": {
               "type": "string",
               "index": "not_analyzed"
@@ -62,10 +61,6 @@
           "_all": {
             "enabled": false
           },
-          "_timestamp": {
-            "enabled": true,
-            "path": "request_at"
-          },
           "date_detection": false,
           "numeric_detection": false,
           "dynamic_templates": [

docs/server/db-config.md

@@ -45,4 +45,9 @@ API Umbrella bundles the recommended database versions inside its package. Using
     ```
 
 - Elasticsearch 2
-  - API Umbrella is not yet comptabile with Elasticsearch 2+.
+  - API Umbrella can be used with an Elasticsearch 2 instance by setting the following config option in `/etc/api-umbrella/api-umbrella.yml`:
+
+    ```yaml
+    elasticsearch:
+       api_version: 2
+    ```

src/api-umbrella/cli/read_config.lua

@@ -151,10 +151,18 @@ end
 -- Read the /etc/api-umbrella/api-umbrella.yml config file that provides
 -- server-specific overrides for API Umbrella configuration.
 local function read_system_config()
-  local content = file.read(os.getenv("API_UMBRELLA_CONFIG") or "/etc/api-umbrella/api-umbrella.yml", true)
-  if content then
-    local overrides = lyaml.load(content)
-    deep_merge_overwrite_arrays(config, overrides)
+  local config_paths = os.getenv("API_UMBRELLA_CONFIG") or "/etc/api-umbrella/api-umbrella.yml"
+  config_paths = split(config_paths, ":", true)
+  for _, config_path in ipairs(config_paths) do
+    if path.exists(config_path) then
+      local content = file.read(config_path, true)
+      if content then
+        local overrides = lyaml.load(content)
+        deep_merge_overwrite_arrays(config, overrides)
+      end
+    else
+      print("WARNING: Config file does not exist: ", config_path)
+    end
   end
 
   nillify_yaml_nulls(config)

src/api-umbrella/cli/setup.lua

@@ -32,6 +32,7 @@ local function set_template_config()
     _elasticsearch_yaml = lyaml.dump({deep_merge_overwrite_arrays({
       path = {
         conf = path.join(config["etc_dir"], "elasticsearch"),
+        scripts = path.join(config["etc_dir"], "elasticsearch_scripts"),
         data = path.join(config["db_dir"], "elasticsearch"),
         logs = config["log_dir"],
       },

src/api-umbrella/proxy/log_utils.lua

@@ -107,14 +107,36 @@ local function set_url_hierarchy(data)
   end
 end
 
-local function recursive_escape_uri_non_ascii(data)
+local function recursive_elasticsearch_sanitize(data)
   if not data then return end
 
   for key, value in pairs(data) do
     if type(value) == "string" then
+      -- Escaping any non-ASCII chars to prevent invalid or wonky UTF-8
+      -- sequences from generating invalid JSON that will prevent ElasticSearch
+      -- from indexing the request.
       data[key] = escape_uri_non_ascii(value)
     elseif type(value) == "table" then
-      recursive_escape_uri_non_ascii(value)
+      recursive_elasticsearch_sanitize(value)
+    end
+
+    -- As of ElasticSearch 2, field names cannot contain dots. This affects our
+    -- nested hash of query parameters, since incoming query parameters may
+    -- contain dots. For storage purposes, replace these dots with underscores
+    -- (the same approach LogStash's de_dot plugin takes).
+    --
+    -- See:
+    -- https://www.elastic.co/guide/en/elasticsearch/reference/2.0/breaking_20_mapping_changes.html#_field_names_may_not_contain_dots
+    --
+    -- However, dots look like they'll be allowed again (although, treated as
+    -- nested objects) in ElasticSearch 5:
+    -- https://github.com/elastic/elasticsearch/issues/15951
+    -- https://github.com/elastic/elasticsearch/pull/18106
+    -- https://www.elastic.co/blog/elasticsearch-5-0-0-alpha3-released#_dots_in_field_names
+    local sanitized_key = ngx.re.gsub(key, "\\.", "_", "jo")
+    if key ~= sanitized_key then
+      data[sanitized_key] = data[key]
+      data[key] = nil
     end
   end
 end
@@ -139,10 +161,8 @@ function _M.set_url_fields(data)
     data["legacy_request_url_query_hash"] = ngx.decode_args(data["request_url_query"])
 
     -- Since we decoded the argument string to construct the table of
-    -- arguments, we now might have invalid or wonky characters that will cause
-    -- invalid JSON and prevent ElasticSearch from indexing the request. So run
-    -- through all the arguments and escape them.
-    recursive_escape_uri_non_ascii(data["legacy_request_url_query_hash"])
+    -- arguments, we now must recursively prepare it for ElasticSearch storage.
+    recursive_elasticsearch_sanitize(data["legacy_request_url_query_hash"])
   end
 
   data["legacy_request_url"] = data["request_url_scheme"] .. "://" .. data["request_url_host"] .. data["request_url_path"]

src/api-umbrella/web-app/Gemfile

@@ -106,9 +106,6 @@ gem "psych", "~> 2.0.13", :platforms => [:ruby]
 # For user-inputted YAML.
 gem "safe_yaml", "~> 1.0.4", :require => "safe_yaml/load"
 
-# YAML configuration files.
-gem "config", "~> 1.1.0"
-
 # Delayed jobs and background tasks
 gem "delayed_job_mongoid", "~> 2.1.0"
 gem "daemons", "~> 1.2.2"

src/api-umbrella/web-app/Gemfile.lock

@@ -155,9 +155,6 @@ GEM
     compass-rails (1.1.7)
       compass (>= 0.12.2)
       sprockets (<= 2.11.0)
-    config (1.1.0)
-      activesupport (>= 3.0)
-      deep_merge (~> 1.0.1)
     countries (0.11.3)
       currencies (~> 0.4.2)
       i18n_data (~> 0.6.0)
@@ -170,7 +167,6 @@ GEM
     currencies (0.4.2)
     daemons (1.2.2)
     database_cleaner (1.4.1)
-    deep_merge (1.0.1)
     delayed_job (4.0.6)
       activesupport (>= 3.0, < 5.0)
     delayed_job_mongoid (2.1.0)
@@ -560,7 +556,6 @@ DEPENDENCIES
   capybara-screenshot (~> 1.0.11)
   childprocess (~> 0.5.9)
   compass-rails (~> 1.1.7)
-  config (~> 1.1.0)
   countries (~> 0.11.3)
   csv_builder (~> 2.1.1)
   daemons (~> 1.2.2)
@@ -658,4 +653,4 @@ DEPENDENCIES
   webmock (~> 1.21.0)
 
 BUNDLED WITH
-   1.12.4
+   1.12.5

src/api-umbrella/web-app/app/models/log_search/elastic_search.rb

@@ -1,3 +1,6 @@
+
+require('json')
+
 class LogSearch::ElasticSearch < LogSearch::Base
   attr_reader :client
 
@@ -48,9 +51,7 @@ def result
     if query_options[:body][:aggregations] && query_options[:body][:aggregations].blank?
       query_options[:body].delete(:aggregations)
     end
-
     raw_result = @client.search(query_options)
-
     @result = LogResult.factory(self, raw_result)
   end
 
@@ -266,7 +267,7 @@ def aggregate_by_drilldown!(prefix, size = 0)
       :terms => {
         :field => "request_hierarchy",
         :size => size,
-        :include => "^#{Regexp.escape(prefix)}.*",
+        :include => "#{Regexp.escape(prefix)}.*",
       },
     }
   end
@@ -282,7 +283,7 @@ def aggregate_by_drilldown_over_time!(prefix)
       :terms => {
         :field => "request_hierarchy",
         :size => 10,
-        :include => "^#{Regexp.escape(prefix)}.*",
+        :include => "#{Regexp.escape(prefix)}.*",
       },
       :aggregations => {
         :drilldown_over_time => {
@@ -314,6 +315,11 @@ def aggregate_by_drilldown_over_time!(prefix)
         },
       },
     }
+
+    if(ApiUmbrellaConfig[:elasticsearch][:api_version] >= 2)
+      @query[:aggregations][:top_path_hits_over_time][:aggregations][:drilldown_over_time][:date_histogram].delete(:pre_zone_adjust_large_interval)
+      @query[:aggregations][:hits_over_time][:date_histogram].delete(:pre_zone_adjust_large_interval)
+    end
   end
 
   def aggregate_by_interval!
@@ -330,6 +336,10 @@ def aggregate_by_interval!
         },
       },
     }
+
+    if(ApiUmbrellaConfig[:elasticsearch][:api_version] >= 2)
+      @query[:aggregations][:hits_over_time][:date_histogram].delete(:pre_zone_adjust_large_interval)
+    end
   end
 
   def aggregate_by_region!

src/api-umbrella/web-app/config/application.rb

@@ -18,7 +18,13 @@
 module ApiUmbrella
   class Application < Rails::Application
     config.before_configuration do
+      require "symbolize_helper"
+
+      config_files = []
       config_file = ENV["API_UMBRELLA_RUNTIME_CONFIG"]
+      if(config_file.present?)
+        config_files << config_file
+      end
 
       # In non-test environments, load the system-wide runtime_config.yml file
       # if it exists and the API_UMBRELLA_RUNTIME_CONFIG environment variable
@@ -27,23 +33,63 @@ class Application < Rails::Application
       # in the test environment, since we don't want development environment
       # config to be used in test (assuming you're testing from the same
       # machine you're developing on).
-      if(config_file.blank? && Rails.env != "test")
+      if(config_files.blank? && Rails.env != "test")
         default_runtime_config_file = "/opt/api-umbrella/var/run/runtime_config.yml"
         if(File.exist?(default_runtime_config_file) && File.readable?(default_runtime_config_file))
-          config_file = default_runtime_config_file
+          config_files << default_runtime_config_file
         end
       end
 
       # If no config environment variable is set and we're not using the
       # default runtime config file, then fall back to the default.yml file at
       # the top-level of the api-umbrella repo.
-      if(config_file.blank?)
-        config_file = File.expand_path("../../../../../config/default.yml", __FILE__)
+      if(config_files.blank?)
+        config_files << File.expand_path("../../../../../config/default.yml", __FILE__)
+
+        if(Rails.env == "test")
+          if(ENV["API_UMBRELLA_CONFIG"].present?)
+            config_files += ENV["API_UMBRELLA_CONFIG"].split(":")
+          else
+            config_files << File.expand_path("../../../../../test/config/test.yml", __FILE__)
+          end
+        end
       end
 
       # Load the YAML config in.
-      ApiUmbrellaConfig.prepend_source!(config_file)
-      ApiUmbrellaConfig.reload!
+      config = {}
+      config_files.each do |file|
+        data = SymbolizeHelper.symbolize_recursive(YAML.load_file(file))
+        config.deep_merge!(data)
+      end
+
+      if(Rails.env == "test")
+        # When running as part of the integration test suite, where we run all
+        # the API Umbrella processes separately, ensure we connect to those
+        # ports.
+        if(ENV["INTEGRATION_TEST_SUITE"])
+          config[:mongodb][:url] = "mongodb://127.0.0.1:13001/api_umbrella_test"
+
+          # Don't override the Elasticsearch v2 connection tests.
+          if(config[:elasticsearch][:hosts] != ["http://127.0.0.1:9200"])
+            config[:elasticsearch][:hosts] = ["http://127.0.0.1:13002"]
+          end
+
+        # If not running as part of the integration test suite, then we assume
+        # a developer is just running the rails tests a standalone command. In
+        # that case, we'll connect to the default API Umbrella ports for
+        # databases that we assume are running in the development environment.
+        # The only difference is MongoDB, where we want to make sure we connect
+        # to a separate test database so tests don't interfere with
+        # development.
+        elsif(!ENV["FULL_STACK_TEST"])
+          config[:mongodb][:url] = "mongodb://127.0.0.1:14001/api_umbrella_test"
+
+          # Don't override the Elasticsearch v2 connection tests.
+          if(config[:elasticsearch][:hosts] != ["http://127.0.0.1:9200"])
+            config[:elasticsearch][:hosts] = ["http://127.0.0.1:14002"]
+          end
+        end
+      end
 
       # Set the default host used for web application links (for mailers,
       # contact URLs, etc).
@@ -52,17 +98,21 @@ class Application < Rails::Application
       # been set to true (this gets put on `_default_hostname` for easier
       # access). But still allow the web host to be explicitly set via
       # `web.default_host`.
-      if(ApiUmbrellaConfig[:web][:default_host].blank?)
-        ApiUmbrellaConfig[:web][:default_host] = ApiUmbrellaConfig[:_default_hostname]
+      if(config[:web][:default_host].blank?)
+        config[:web][:default_host] = config[:_default_hostname]
 
         # Fallback to something that will at least generate valid URLs if
         # there's no default, or the default is "*" (since in this context, a
         # wildcard doesn't make sense for generating URLs).
-        if(ApiUmbrellaConfig[:web][:default_host].blank? || ApiUmbrellaConfig[:web][:default_host] == "*")
-          ApiUmbrellaConfig[:web][:default_host] = "localhost"
+        if(config[:web][:default_host].blank? || config[:web][:default_host] == "*")
+          config[:web][:default_host] = "localhost"
         end
       end
 
+      # rubocop:disable Style/ConstantName
+      ::ApiUmbrellaConfig = config
+      # rubocop:enable Style/ConstantName
+
       require "js_locale_helper"
     end