Remove lumberjack as default for logstash

- The lumberjack protocol will no longer be the default for Logstash to communicate, due to hosted chef changes and golang runtime SSL changes (fixes #56, #14)
- Don't test the lumberjack protocol with the server install by default, removes lumberjack test suite in .kitchen.yml
- Default to tcp/udp for communication between logstash instances, add flag:
  `node['elkstack']['config']['agent_protocol'] = 'tcp_udp' # could also be lumberjack`
- Don't try to load lumberjack secrets by default for agent, don't fail if they aren't present (rename recipes/_secrets.rb -> recipes/_lumberjack_secrets.rb)
- elkstack::acl recipe changes to support tcp/udp communication
- add logstash configs for tcp/udp communication
- remove tests for lumberjack.key/crt now from default test suites
- remove extra data bags with lumberjack keypairs

.kitchen.yml

@@ -64,8 +64,6 @@ platforms:
 
 suites:
   - name: default # server
-    data_bags_path: "test/integration/default/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/default/encrypted_data_bag_secret"
     run_list:
       - recipe[elkstack::java]
       - recipe[elkstack::cluster] # not testing single, it's practically the same
@@ -75,31 +73,16 @@ suites:
           zen:
             minimum_master_nodes: 1 # since search returns more than one, but they are fake
 
-  - name: lumberjack # server with lumberjack disabled
-    data_bags_path: "test/integration/default/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/default/encrypted_data_bag_secret"
-    run_list:
-      - recipe[elkstack::java]
-      - recipe[elkstack::cluster] # not testing single, it's practically the same
-    attributes:
-      elkstack:
-        config:
-          lumberjack_data_bag: false
-      elasticsearch:
-        discovery:
-          zen:
-            minimum_master_nodes: 1 # since search returns more than one, but they are fake
-
-  - name: agent
-    data_bags_path: "test/integration/agent/data_bags"
-    encrypted_data_bag_secret_key_path: "test/integration/agent/encrypted_data_bag_secret"
+  - name: agent # java agent with server
     run_list:
       - recipe[wrapper::logstash_override]
       - recipe[elkstack::java]
       - recipe[elkstack::agent]
 
-  - name: forwarder
+  - name: forwarder # alternative golang agent with server
     data_bags_path: "test/integration/agent/data_bags"
     encrypted_data_bag_secret_key_path: "test/integration/agent/encrypted_data_bag_secret"
     run_list:
+      - recipe[elkstack::java]
+      - recipe[elkstack::cluster] # not testing single, it's practically the same
       - recipe[elkstack::forwarder]

attributes/default.rb

@@ -7,6 +7,9 @@
 # default to not running the cluster search recipe
 default['elkstack']['config']['cluster'] = false
 
+# attempt to use lumberjack protocol for java agents?
+default['elkstack']['config']['agent_protocol'] = 'tcp_udp' # could also be lumberjack
+
 # attempt to use performance cloud data disk
 default['elkstack']['config']['data_disk']['disk_config_type'] = false
 

recipes/_secrets.rb → recipes/_lumberjack_secrets.rb

@@ -35,61 +35,16 @@
   end
 end
 
-# generate our own keypair since we don't seem to have one
-if lumberjack_secrets.nil?
-  Chef::Log.warn("Generating a new lumberjack keypair and data bag item #{lumberjack_data_bag}/secrets")
-  cert_file = "#{Chef::Config[:file_cache_path]}/lumberjack.crt"
-  key_file = "#{Chef::Config[:file_cache_path]}/lumberjack.key"
-  openssl_x509 cert_file do
-    common_name 'elkstack'
-    org 'elkstack'
-    org_unit 'elkstack'
-    country 'US'
-    key_file key_file
-    action :nothing
-  end.run_action(:create) # do it at compilation
-
-  ruby_block 'read generated keypair from disk' do
-    block do
-      key_file_contents = IO.read(key_file)
-      cert_file_contents = IO.read(cert_file)
-      node.run_state['lumberjack_decoded_key_tmp'] = Base64.encode64(key_file_contents).tr("\n", '')
-      node.run_state['lumberjack_decoded_certificate_tmp'] = Base64.encode64(cert_file_contents).tr("\n", '')
-    end
-    action :nothing
-  end.run_action(:run) # do it at compilation
-
-  key_contents = node.run_state['lumberjack_decoded_key_tmp']
-  certificate_contents = node.run_state['lumberjack_decoded_certificate_tmp']
-
-  # try to create a data bag and put a random keypair in it next
-  secrets = {
-    'id' => 'secrets',
-    'key' => key_contents,
-    'certificate' => certificate_contents
-  }
-
-  # unencrypted data bag if we just need a shared secret for ourselves
-  lumberjack_secrets_bag = Chef::DataBag.new
-  lumberjack_secrets_bag.name(lumberjack_data_bag)
-  lumberjack_secrets_bag.save
-
-  lumberjack_secrets = Chef::DataBagItem.new
-  lumberjack_secrets.data_bag(lumberjack_data_bag)
-  lumberjack_secrets.raw_data = secrets
-  lumberjack_secrets.save
-end
-
 # now try to use the data bag
 if !lumberjack_secrets.nil? && lumberjack_secrets['key'] && lumberjack_secrets['certificate']
   node.run_state['lumberjack_decoded_key'] = Base64.decode64(lumberjack_secrets['key'])
   node.run_state['lumberjack_decoded_certificate'] = Base64.decode64(lumberjack_secrets['certificate'])
 elsif !lumberjack_secrets.nil?
-  fail 'Found a data bag for lumberjack secrets, but it was missing \'key\' and \'certificate\' data bag items'
+  Chef::Log.warn('Found a data bag for lumberjack secrets, but it was missing \'key\' and \'certificate\' data bag items')
 elsif lumberjack_secrets.nil?
-  fail 'Could not find an encrypted or unencrypted data bag to use as a lumberjack keypair, and could not generate a keypair either'
+  Chef::Log.warn('Could not find an encrypted or unencrypted data bag to use as a lumberjack keypair')
 else
-  fail 'Unable to complete lumberjack keypair configuration'
+  Chef::Log.warn('Unable to complete lumberjack keypair configuration')
 end
 
 logstash_basedir = node.deep_fetch('logstash', 'instance_default', 'basedir')

recipes/acl.rb

@@ -11,6 +11,8 @@
 # main point of elkstack, open syslog and lumberjack ports
 add_iptables_rule('INPUT', '-p tcp --dport 5959 -j ACCEPT', 9997, 'allow syslog entries inbound')
 add_iptables_rule('INPUT', '-p tcp --dport 5960 -j ACCEPT', 9997, 'allow lumberjack protocol inbound')
+add_iptables_rule('INPUT', '-p tcp --dport 5961 -j ACCEPT', 9997, 'allow tcp protocol inbound')
+add_iptables_rule('INPUT', '-p tcp --dport 5962 -j ACCEPT', 9997, 'allow udp protocol inbound')
 
 should_cluster = node.deep_fetch('elkstack', 'config', 'cluster')
 if !should_cluster.nil? && should_cluster

recipes/agent.rb

@@ -61,8 +61,24 @@
   chef_environment: node.chef_environment
 }
 
-include_recipe 'elkstack::_secrets'
-unless node.run_state['lumberjack_decoded_certificate'].nil? || node.run_state['lumberjack_decoded_certificate'].nil?
+# preload any lumberjack key or cert that might be available
+include_recipe 'elkstack::_lumberjack_secrets'
+lumberjack_keypair = node.run_state['lumberjack_decoded_key'] && node.run_state['lumberjack_decoded_certificate']
+
+# default is 'tcp_udp'
+if node['elkstack']['config']['agent_protocol'] == 'tcp_udp'
+    # TODO: udp and tcp senders
+
+    my_templates['output_tcp'] = 'logstash/output_tcp.conf.erb'
+    my_templates['output_udp'] = 'logstash/output_udp.conf.erb'
+
+    template_variables[:output_tcp_host] = elk_nodes.split(',').first
+    template_variables[:output_tcp_port] = 5961
+    template_variables[:output_udp_host] = elk_nodes.split(',').first
+    template_variables[:output_udp_port] = 5962
+
+# if flag is set *and* key & cert are available
+elsif node['elkstack']['config']['agent_protocol'] == 'lumberjack' && lumberjack_keypair
   my_templates['output_lumberjack'] = 'logstash/output_lumberjack.conf.erb'
   template_variables['output_lumberjack_ssl_certificate'] = "#{node['logstash']['instance_default']['basedir']}/lumberjack.crt"
   # template_variables['output_lumberjack_ssl_key'] = "#{node['logstash']['instance_default']['basedir']}/lumberjack.key"

recipes/logstash.rb

@@ -27,8 +27,11 @@
 end
 
 # by default, these are the inputs and outputs on the server
+# we receive anything from any protocol we might know about
 my_templates = {
   'input_syslog'         => 'logstash/input_syslog.conf.erb',
+  'input_tcp'            => 'logstash/input_tcp.conf.erb',
+  'input_udp'            => 'logstash/input_udp.conf.erb',
   'output_stdout'        => 'logstash/output_stdout.conf.erb',
   'output_elasticsearch' => 'logstash/output_elasticsearch.conf.erb'
 }
@@ -38,10 +41,15 @@
   input_lumberjack_port: 5960,
   input_syslog_host: '0.0.0.0',
   input_syslog_port: 5959,
+  input_tcp_host: '0.0.0.0',
+  input_tcp_port: 5961,
+  input_udp_host: '0.0.0.0',
+  input_udp_port: 5962,
   chef_environment: node.chef_environment
 }
 
-include_recipe 'elkstack::_secrets'
+# also receive lumberjack if a keypair is available
+include_recipe 'elkstack::_lumberjack_secrets'
 unless node.run_state['lumberjack_decoded_certificate'].nil? || node.run_state['lumberjack_decoded_certificate'].nil?
   my_templates['input_lumberjack'] = 'logstash/input_lumberjack.conf.erb'
   template_variables['input_lumberjack_ssl_certificate'] = "#{node['logstash']['instance_default']['basedir']}/lumberjack.crt"

templates/default/logstash/input_tcp.conf.erb

@@ -0,0 +1,12 @@
+input {
+  tcp {
+    port => "<%= @input_tcp_port %>"
+    host => "<%= @input_tcp_host %>"
+  }
+}
+
+filter {
+  mutate {
+    replace => [ "input_chef_environment", "<%= @chef_environment %>" ]
+  }
+}

templates/default/logstash/input_udp.conf.erb

@@ -0,0 +1,12 @@
+input {
+  udp {
+    port => "<%= @input_udp_port %>"
+    host => "<%= @input_udp_host %>"
+  }
+}
+
+filter {
+  mutate {
+    replace => [ "input_chef_environment", "<%= @chef_environment %>" ]
+  }
+}

templates/default/logstash/output_tcp.conf.erb

@@ -0,0 +1,12 @@
+filter {
+  mutate {
+    replace => [ "output_chef_environment", "<%= @chef_environment %>" ]
+  }
+}
+
+output {
+  tcp {
+    port => "<%= @output_tcp_port %>"
+    host => "<%= @output_tcp_host %>"
+  }
+}

templates/default/logstash/output_udp.conf.erb

@@ -0,0 +1,12 @@
+filter {
+  mutate {
+    replace => [ "output_chef_environment", "<%= @chef_environment %>" ]
+  }
+}
+
+output {
+  udp {
+    port => "<%= @output_udp_port %>"
+    host => "<%= @output_udp_host %>"
+  }
+}