pullu

.gitignore

@@ -23,3 +23,5 @@ etc/jira-output.conf
 coverage/*
 .VERSION.mk
 .idea/*
+spec/reports
+rspec.xml

Makefile

@@ -21,6 +21,12 @@ GEOIP_ASN=vendor/geoip/GeoIPASNum.dat
 GEOIP_ASN_URL=http://logstash.objects.dreamhost.com/maxmind/GeoIPASNum-2014-02-12.dat.gz
 KIBANA_URL=https://download.elasticsearch.org/kibana/kibana/kibana-3.0.1.tar.gz
 PLUGIN_FILES=$(shell find lib -type f| egrep '^lib/logstash/(inputs|outputs|filters|codecs)/[^/]+$$' | egrep -v '/(base|threadable).rb$$|/inputs/ganglia/')
+SCALA_VERSION?=2.9.2
+
+KAFKA_VERSION?=0.8.1.1
+KAFKA_URL=https://archive.apache.org/dist/kafka
+KAFKA=vendor/jar/kafka_$(SCALA_VERSION)-$(KAFKA_VERSION)
+
 QUIET=@
 ifeq (@,$(QUIET))
 	QUIET_OUTPUT=> /dev/null 2>&1
@@ -149,6 +155,11 @@ vendor/jar/graphtastic-rmiclient.jar: | wget-or-curl vendor/jar
 	@echo "=> Fetching graphtastic rmi client jar"
 	$(QUIET)$(DOWNLOAD_COMMAND) $@ http://cloud.github.com/downloads/NickPadilla/GraphTastic/graphtastic-rmiclient.jar
 
+vendor/jar/kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz: | wget-or-curl vendor/jar
+	@echo "=> Fetching kafka $(SCALA_VERSION)-$(KAFKA_VERSION)"
+	$(QUIET)$(DOWNLOAD_COMMAND) $@ $(KAFKA_URL)/$(KAFKA_VERSION)/kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz
+
+
 .PHONY: vendor-elasticsearch
 vendor-elasticsearch: $(ELASTICSEARCH)
 $(ELASTICSEARCH): $(ELASTICSEARCH).tar.gz | vendor/jar
@@ -183,6 +194,15 @@ $(TYPESDB): | vendor/collectd
 	$(QUIET)tar zxf [email protected] -O "collectd-$(COLLECTD_VERSION)/src/types.db" > $@
 	$(QUIET)rm [email protected]
 
+.PHONY: vendor-kafka
+vendor-kafka: $(KAFKA)
+$(KAFKA): $(KAFKA).tgz | vendor/jar
+	@echo "=> Pulling the jars out of $<"
+	$(QUIET)tar -C $(shell dirname $@) -xf $< $(TAR_OPTS) \
+		'kafka_$(SCALA_VERSION)-$(KAFKA_VERSION)/libs/*.jar'
+	$(QUIET)tar -C $(shell dirname $@) -xf $< $(TAR_OPTS) \
+		'kafka_$(SCALA_VERSION)-$(KAFKA_VERSION)/*.jar'
+
 # Always run vendor/bundle
 .PHONY: fix-bundler
 fix-bundler:
@@ -219,7 +239,7 @@ vendor/ua-parser/regexes.yaml: | vendor/ua-parser/
 
 .PHONY: test
 test: QUIET_OUTPUT=
-test: | $(JRUBY) vendor-elasticsearch vendor-geoip vendor-collectd vendor-gems
+test: | $(JRUBY) vendor-elasticsearch vendor-geoip vendor-collectd vendor-kafka vendor-gems
 	$(SPEC_ENV) bin/logstash rspec $(SPEC_OPTS) --order rand --fail-fast $(TESTS)
 
 .PHONY: reporting-test
@@ -355,12 +375,12 @@ show:
 
 .PHONY: prepare-tarball
 prepare-tarball tarball zip: WORKDIR=build/tarball/logstash-$(VERSION)
-prepare-tarball: vendor/kibana $(ELASTICSEARCH) $(JRUBY) $(GEOIP) $(TYPESDB) vendor-gems
+prepare-tarball: vendor/kibana $(ELASTICSEARCH) $(JRUBY) vendor-geoip $(TYPESDB) $(KAFKA) vendor-gems
 prepare-tarball: vendor/ua-parser/regexes.yaml
 prepare-tarball:
 	@echo "=> Preparing tarball"
 	$(QUIET)$(MAKE) $(WORKDIR)
-	$(QUIET)rsync -a --relative bin lib spec locales patterns vendor/bundle/jruby vendor/geoip vendor/jar vendor/kibana vendor/ua-parser vendor/collectd LICENSE README.md --exclude 'vendor/bundle/jruby/1.9/cache' --exclude 'vendor/bundle/jruby/1.9/gems/*/doc' --exclude 'vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION).tar.gz'  $(WORKDIR)
+	$(QUIET)rsync -a --relative bin lib spec locales patterns vendor/bundle/jruby vendor/geoip vendor/jar vendor/kibana vendor/ua-parser vendor/collectd LICENSE README.md --exclude 'vendor/bundle/jruby/1.9/cache' --exclude 'vendor/bundle/jruby/1.9/gems/*/doc' --exclude 'vendor/jar/elasticsearch-$(ELASTICSEARCH_VERSION).tar.gz' --exclude 'vendor/jar/kafka_$(SCALA_VERSION)-$(KAFKA_VERSION).tgz' $(WORKDIR)
 	$(QUIET)sed -i -e 's/^LOGSTASH_VERSION = .*/LOGSTASH_VERSION = "$(VERSION)"/' $(WORKDIR)/lib/logstash/version.rb
 	$(QUIET)sed -i -e 's/%JRUBY_VERSION%/$(JRUBY_VERSION)/' $(WORKDIR)/bin/logstash.bat
 

docs/tutorials/getting-started-with-logstash.asciidoc

@@ -48,7 +48,7 @@ hello world
 
 OK, that's interesting... We ran Logstash with an input called "stdin", and an output named "stdout", and Logstash basically echoed back whatever we typed in some sort of structured format. Note that specifying the *-e* command line flag allows Logstash to accept a configuration directly from the command line. This is especially useful for quickly testing configurations without having to edit a file between iterations.
 
-Let's try a slightly fancier example. First, you should exit Logstash by issuing a 'CTRL-C' command in the shell in which it is running. Now run Logstash again with the following command:
+Let's try a slightly fancier example. First, you should exit Logstash by issuing a 'CTRL-D' command (or 'CTRL-C Enter') in the shell in which it is running. Now run Logstash again with the following command:
 ----
 bin/logstash -e 'input { stdin { } } output { stdout { codec => rubydebug } }'
 ----
@@ -67,7 +67,7 @@ goodnight moon
 So, by re-configuring the "stdout" output (adding a "codec"), we can change the output of Logstash. By adding inputs, outputs and filters to your configuration, it's possible to massage the log data in many ways, in order to maximize flexibility of the stored data when you are querying it.
 
 == Storing logs with Elasticsearch
-Now, you're probably saying, "that's all fine and dandy, but typing all my logs into Logstash isn't really an option, and merely seeing them spit to STDOUT isn't very useful." Good point. First, let's set up Elasticsearch to store the messages we send into Logstash. If you don't have Elasticearch already installed, you can http://www.elasticsearch.org/download/[download the RPM or DEB package], or install manually by downloading the current release tarball, by issuing the following four commands:
+Now, you're probably saying, "that's all fine and dandy, but typing all my logs into Logstash isn't really an option, and merely seeing them spit to STDOUT isn't very useful." Good point. First, let's set up Elasticsearch to store the messages we send into Logstash. If you don't have Elasticsearch already installed, you can http://www.elasticsearch.org/download/[download the RPM or DEB package], or install manually by downloading the current release tarball, by issuing the following four commands:
 ----
 curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-%ELASTICSEARCH_VERSION%.tar.gz
 tar zxvf elasticsearch-%ELASTICSEARCH_VERSION%.tar.gz
@@ -120,7 +120,7 @@ which should return something like this:
 Congratulations! You've successfully stashed logs in Elasticsearch via Logstash.
 
 === Elasticsearch Plugins (an aside)
-Another very useful tool for querying your Logstash data (and Elasticsearch in general) is the Elasticearch-kopf plugin. Here is more information on http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-plugins.html[Elasticsearch plugins]. To install elasticsearch-kopf, simply issue the following command in your Elasticsearch directory (the same one in which you ran Elasticsearch earlier):
+Another very useful tool for querying your Logstash data (and Elasticsearch in general) is the Elasticsearch-kopf plugin. Here is more information on http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/modules-plugins.html[Elasticsearch plugins]. To install elasticsearch-kopf, simply issue the following command in your Elasticsearch directory (the same one in which you ran Elasticsearch earlier):
 ----
 bin/plugin -install lmenezes/elasticsearch-kopf
 ----
@@ -261,7 +261,7 @@ Now, let's configure something actually *useful*... apache2 access log files! We
 input {
   file {
     path => "/tmp/access_log"
-    start_position => beginning
+    start_position => "beginning"
   }
 }
 

gembag.rb

@@ -43,7 +43,7 @@ def install_gem(name, requirement, target)
 module Bundler
   module SharedHelpers
     def default_lockfile
-      ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.ruby_abi_version}"
+      ruby = "#{LogStash::Environment.ruby_engine}-#{LogStash::Environment.gem_ruby_version}"
       return Pathname.new("#{default_gemfile}.#{ruby}.lock")
     end
   end