KAFKA-55: Create SBT sub projects for perf, examples, contrib that are currently using ant

Author: nehanarkhede

bin/kafka-run-class.sh

@@ -8,7 +8,7 @@ fi
 
 base_dir=$(dirname $0)/..
 
-for file in $base_dir/target/scala_2.8.0/*.jar;
+for file in $base_dir/core/target/scala_2.8.0/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done
@@ -20,7 +20,7 @@ do
   fi
 done
 
-for file in $base_dir/lib_managed/scala_2.8.0/compile/*.jar;
+for file in $base_dir/core/lib_managed/scala_2.8.0/compile/*.jar;
 do
   if [ ${file##*/} != "sbt-launch.jar" ]; then
     CLASSPATH=$CLASSPATH:$file

contrib/hadoop-consumer/README

@@ -27,8 +27,10 @@ SimpleKafkaETLJob sets up job properties and files Hadoop job.
 SimpleKafkaETLMapper dumps kafka data into hdfs. 
 
 HOW TO RUN:
-1. Complile using "ant" and you will see kafka-etl-contrib-<version>.jar
-under $KAFKA_ROOT/dist.
+1. Complile using "sbt" to create a package for hadoop consumer code.
+./sbt
+project hadoop consumer
+package-all
 
 2. Produce test events in server and generate offset files
   1) Start kafka server [ Follow the quick start - 

contrib/hadoop-consumer/build.xml

@@ -9,12 +9,12 @@ fi
 base_dir=$(dirname $0)/../..
 
 # include kafka jars
-for file in $base_dir/dist/*.jar;
+for file in $base_dir/core/target/scala_2.8.0/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done
 
-for file in $base_dir/lib/*.jar;
+for file in $base_dir/contrib/hadoop-consumer/lib_managed/scala_2.8.0/compile/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done
@@ -23,11 +23,13 @@ done
 local_dir=$(dirname $0)
 
 # include hadoop-consumer jars
-for file in $local_dir/lib/*.jar;
+for file in $base_dir/contrib/hadoop-consumer/target/scala_2.8.0/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done
 
+echo $CLASSPATH
+
 CLASSPATH=dist:$CLASSPATH:${HADOOP_HOME}/conf
 
 #if [ -z "$KAFKA_OPTS" ]; then

contrib/hadoop-consumer/lib/commons-httpclient-3.1.jar

@@ -108,17 +108,19 @@ public KafkaETLContext(JobConf job, Props props, Reporter reporter,
 
     public boolean hasMore () {
         return _messageIt != null && _messageIt.hasNext() 
-                || _response != null && _response.hasNext() 
+                || _response != null && _response.iterator().hasNext()
                 || _offset < _offsetRange[1]; 
     }
 
     public boolean getNext(KafkaETLKey key, BytesWritable value) throws IOException {
         if ( !hasMore() ) return false;
 
         boolean gotNext = get(key, value);
-        
-        while ( !gotNext && _response != null && _response.hasNext()) {
-            ByteBufferMessageSet msgSet = _response.next();
+
+        Iterator<ByteBufferMessageSet> iter = _response.iterator();
+
+        while ( !gotNext && _response != null && iter.hasNext()) {
+            ByteBufferMessageSet msgSet = iter.next();
             if ( hasError(msgSet)) return false;
             _messageIt =  (Iterator<Message>) msgSet.iterator();
             gotNext = get(key, value);

contrib/hadoop-consumer/lib/commons-logging-1.0.4.jar

@@ -5,4 +5,8 @@ messages from Kafka server.
 
 In order to run demo:
    1. Start Zookeeper and the Kafka server
-   2. Run 'ant' from inside the java directory
+   2. ./sbt from top-level kafka directory
+   3. Switch to the kafka java examples project -> project Kafka Java Examples
+   4. execute run -> run
+   5. For unlimited producer-consumer run, select option 1
+      For simple consumer demo, select option 2

contrib/hadoop-consumer/lib/hadoop-0.20.2-core.jar

@@ -15,15 +15,15 @@ do
       CLASSPATH=$CLASSPATH:$file
   fi
 done
-for file in $base_dir/lib_managed/scala_2.8.0/compile/*.jar;
+for file in $base_dir/core/lib_managed/scala_2.8.0/compile/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done
-for file in $base_dir/target/scala_2.8.0/*.jar;
+for file in $base_dir/core/target/scala_2.8.0/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done
-for file in $base_dir/dist/*.jar;
+for file in $base_dir/perf/target/scala_2.8.0/*.jar;
 do
   CLASSPATH=$CLASSPATH:$file
 done