Move attribute that drives custom cfgs to elkstack

This moves the node attribute that can configure additional custom configurations to elkstack. This really makes more sense than platformstack, as this is the only treal place that is aware of what is being deployed with elkstack (or not being deployed).

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 3.2.0
+
+- Allow wrappers and other cookbooks to supply additional logstash_config template files.
+
 # 3.1.4
 
 - Don't just raise an error, actually disable backups when cloud account credentials aren't present

README.md

@@ -170,6 +170,18 @@ CentOS 6.5
     <td>Data bag name for lumberjack key and certificate</td>
     <td><tt>lumberjack</tt></td>
   </tr>
+  <tr>
+    <td><tt>['elkstack']['config']['custom_logstash']['name']</tt></td>
+    <td>Array of strings</td>
+    <td>See `attributes/logstash.rb` for an explanation of how to use this attribute to populate additional logstash configuration file templates</td>
+    <td><tt>[]</tt></td>
+  </tr>
+  <tr>
+    <td><tt>['elkstack']['config']['restart_logstash_service']</tt></td>
+    <td>Boolean</td>
+    <td>Restart logstash if we deploy a custom config file</td>
+    <td><tt>true</tt></td>
+  </tr>
 </table>
 
 ## Customizing the stack

attributes/default.rb

@@ -31,3 +31,6 @@
 
 # data bag for lumerjack certificate and key
 default['elkstack']['config']['lumberjack_data_bag'] = 'lumberjack'
+
+# should I restart logstash after applying a custom config file?
+default['elkstack']['config']['restart_logstash_service'] = true

attributes/logstash.rb

@@ -35,3 +35,13 @@
 config_templates_variables['elasticsearch_ip'] = server['elasticsearch_ip']
 config_templates_variables['elasticsearch_protocol'] = server['elasticsearch_protocol']
 server['config_templates_variables'] = config_templates_variables
+
+# arbitrary data structure for any arbitrary logstash config
+default['elkstack']['config']['custom_logstash']['name'] = []
+# Currently for arbitrary logstash configs, the recipe that sets up the logstash file should add:
+# node.default['elkstack']['config']['custom_logstash']['name'].push('<service_name>')
+# and then populate node['elkstack']['config']['custom_logstash'][service_name][setting] with your values
+# default['elkstack']['config']['custom_logstash'][<name>]['name'] = 'my_logstashconfig'
+# default['elkstack']['config']['custom_logstash'][<name>]['source'] = 'my_logstashconfig.conf.erb'
+# default['elkstack']['config']['custom_logstash'][<name>]['cookbook'] = 'your_cookbook'
+# default['elkstack']['config']['custom_logstash'][<name>]['variables'] = { :warning => 'foo' }

definitions/logstash_custom_config.rb

@@ -0,0 +1,28 @@
+# This defintion is designed to remove the repetition used throughout this cookbook.
+
+define :logstash_custom_config, variables: {}, service_name: nil, instance_name: nil, template_name: nil, template_source_file: nil, template_source_cookbook: nil, action: nil do
+
+  params[:action] ||= :create
+  params[:instance_name] ||= 'default'
+  params[:template_name] ||= "#{params[:name]}.conf"
+  params[:template_source_file] ||= "#{params[:name]}.conf.erb"
+  params[:template_source_cookbook] ||= 'elkstack'
+  params[:service_name] ||= 'default'
+
+  logstash_config params[:instance_name] do
+    templates_cookbook params[:template_source_cookbook]
+    templates(params[:template_name] => params[:template_source_file])
+    variables params[:variables]
+    # this is a trick to ensure the notification doesn't hurt us, if the logstash
+    # cookbook is not currently available/included on this node
+    begin
+      resources("logstash_service[#{params[:service_name]}]");
+      if node['elkstack']['config']['restart_logstash_service']
+        notifies :restart, "logstash_service[#{params[:service_name]}]", :delayed
+      end
+    rescue Chef::Exceptions::ResourceNotFound
+      Chef::Log.warn("Could not find logstash_service[#{params[:service_name]}], will not notify it to restart")
+    end
+    action params[:action]
+  end
+end

metadata.rb

@@ -5,7 +5,7 @@
 license          'Apache 2.0'
 description      'Installs/Configures elkstack'
 long_description IO.read(File.join(File.dirname(__FILE__), 'README.md'))
-version          '3.1.4'
+version          '3.2.0'
 
 depends 'apt'
 depends 'yum'

recipes/agent.rb

@@ -67,5 +67,22 @@
   only_if { logging_enabled }
 end
 
+# install additional stacks logstash configuration
+node['elkstack']['config']['custom_logstash']['name'].each do |logcfg|
+
+  logcfg_name = node['elkstack']['config']['custom_logstash'][logcfg]['name']
+  logcfg_source = node['elkstack']['config']['custom_logstash'][logcfg]['source']
+  logcfg_cookbook = node['elkstack']['config']['custom_logstash'][logcfg]['cookbook']
+  logcfg_variables = node['elkstack']['config']['custom_logstash'][logcfg]['variables']
+
+  # add one more config for our additional logs
+  logstash_custom_config logcfg_name do
+    service_name agent_name
+    template_source_file logcfg_source
+    template_source_cookbook logcfg_cookbook
+    variables(logcfg_variables)
+  end
+end
+
 # see attributes, will forward to logstash agent on localhost
 include_recipe 'rsyslog::client'

recipes/logstash.rb

@@ -55,4 +55,21 @@
   notifies :restart, "logstash_service[#{instance_name}]", :delayed
 end
 
+# install additional stacks logstash configuration
+node['elkstack']['config']['custom_logstash']['name'].each do |logcfg|
+
+  logcfg_name = node['elkstack']['config']['custom_logstash'][logcfg]['name']
+  logcfg_source = node['elkstack']['config']['custom_logstash'][logcfg]['source']
+  logcfg_cookbook = node['elkstack']['config']['custom_logstash'][logcfg]['cookbook']
+  logcfg_variables = node['elkstack']['config']['custom_logstash'][logcfg]['variables']
+
+  # add one more config for our additional logs
+  logstash_custom_config logcfg_name do
+    service_name instance_name
+    template_source_file logcfg_source
+    template_source_cookbook logcfg_cookbook
+    variables(logcfg_variables)
+  end
+end
+
 include_recipe 'elkstack::logstash_monitoring'

test/unit/spec/addl_logstash_templates.rb

@@ -0,0 +1,53 @@
+# Encoding: utf-8
+
+require_relative 'spec_helper'
+
+describe 'elkstack::agent' do
+  let(:chef_run) do
+    stub_resources
+    ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '12.04') do |node|
+      node.set['cpu']['total'] = 8
+      node.set['memory']['total'] = 4096
+      node.set['public_info']['remote_ip'] = '127.0.0.1'
+      node.set['filesystem'] = []
+      node.set['platformstack']['elkstack_logging']['enabled'] = true
+
+      # stub an additional template
+      node.set['elkstack']['config']['custom_logstash']['name'] = ['foo']
+      node.set['elkstack']['config']['custom_logstash']['foo']['name'] = 'my_logstashconfig'
+      node.set['elkstack']['config']['custom_logstash']['foo']['source'] = 'my_logstashconfig.conf.erb'
+      node.set['elkstack']['config']['custom_logstash']['foo']['cookbook'] = 'your_cookbook'
+      node.set['elkstack']['config']['custom_logstash']['foo']['variables'] = { :warning => 'foo' }
+    end.converge(described_recipe)
+  end
+
+  it 'creates additional custom config files' do
+    expect(chef_run).to create_logstash_config('agent')
+  end
+
+end
+
+describe 'elkstack::logstash' do
+  let(:chef_run) do
+    stub_resources
+    ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '12.04') do |node|
+      node.set['cpu']['total'] = 8
+      node.set['memory']['total'] = 4096
+      node.set['public_info']['remote_ip'] = '127.0.0.1'
+      node.set['filesystem'] = []
+      node.set['platformstack']['elkstack_logging']['enabled'] = true
+
+      # stub an additional template
+      node.set['elkstack']['config']['custom_logstash']['name'] = ['foo']
+      node.set['elkstack']['config']['custom_logstash']['foo']['name'] = 'my_logstashconfig'
+      node.set['elkstack']['config']['custom_logstash']['foo']['source'] = 'my_logstashconfig.conf.erb'
+      node.set['elkstack']['config']['custom_logstash']['foo']['cookbook'] = 'your_cookbook'
+      node.set['elkstack']['config']['custom_logstash']['foo']['variables'] = { :warning => 'foo' }
+    end.converge(described_recipe)
+  end
+
+  it 'creates additional custom config files' do
+    expect(chef_run).to create_logstash_config('default')
+  end
+
+end

test/unit/spec/agent_spec.rb

@@ -1,11 +1,11 @@
 # Encoding: utf-8
 
-require_relative 'helpers/spec_helper'
+require_relative 'spec_helper'
 
 describe 'elkstack::agent' do
   let(:chef_run) do
     stub_resources
-    ChefSpec::Runner.new(platform: 'ubuntu', version: '12.04') do |node|
+    ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '12.04') do |node|
       node.set['cpu']['total'] = 8
       node.set['memory']['total'] = 4096
       node.set['public_info']['remote_ip'] = '127.0.0.1'

test/unit/spec/default_spec.rb

@@ -1,11 +1,11 @@
 # Encoding: utf-8
 
-require_relative 'helpers/spec_helper'
+require_relative 'spec_helper'
 
 describe 'elkstack::cluster' do
   let(:chef_run) do
     stub_resources
-    ChefSpec::Runner.new(platform: 'redhat', version: '6.5') do |node|
+    ChefSpec::SoloRunner.new(platform: 'redhat', version: '6.5') do |node|
       node.set['cpu']['total'] = 8
       node.set['memory']['total'] = 4096
       node.set['public_info']['remote_ip'] = '127.0.0.1'

test/unit/spec/forwarder_spec.rb

@@ -1,11 +1,11 @@
 # Encoding: utf-8
 
-require_relative 'helpers/spec_helper'
+require_relative 'spec_helper'
 
 describe 'elkstack::forwarder' do
   let(:chef_run) do
     stub_resources
-    ChefSpec::Runner.new(platform: 'ubuntu', version: '12.04') do |node|
+    ChefSpec::SoloRunner.new(platform: 'ubuntu', version: '12.04') do |node|
       node.set['cpu']['total'] = 8
       node.set['memory']['total'] = 4096
       node.set['public_info']['remote_ip'] = '127.0.0.1'

test/unit/spec/helpers/spec_helper.rb → test/unit/spec/spec_helper.rb

@@ -5,6 +5,8 @@
 require 'chefspec/berkshelf'
 require 'chef/application'
 
+Dir['./test/unit/spec/support/**/*.rb'].sort.each { |f| require f }
+
 def stub_resources
   # overall search stub
   chef_search_query = double('Chef::Search::Query', test_search: { foo: false })