From bf07279f033734acd50956d77d20a600ae3bb185 Mon Sep 17 00:00:00 2001
From: Geoff Anderson <geoff@confluent.io>
Date: Wed, 9 Sep 2015 00:53:59 -0700
Subject: [PATCH 1/5] Added tools to simplify Vagrant setup when running system
 tests, and updated testing README accordingly.

---
 tests/README.md             | 43 ++++++--------------
 tests/bootstrap-test-env.sh | 79 +++++++++++++++++++++++++++++++++++++
 vagrant/package-base-box.sh | 50 +++++++++++++++++++++++
 3 files changed, 140 insertions(+), 32 deletions(-)
 create mode 100755 tests/bootstrap-test-env.sh
 create mode 100755 vagrant/package-base-box.sh

diff --git a/tests/README.md b/tests/README.md
index bfeacbbe592f1..207043cda5a70 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -14,51 +14,30 @@ https://cwiki.apache.org/confluence/display/KAFKA/tutorial+-+set+up+and+run+Kafk
 
 * Install Virtual Box from [https://www.virtualbox.org/](https://www.virtualbox.org/) (run `$ vboxmanage --version` to check if it's installed).
 * Install Vagrant >= 1.6.4 from [http://www.vagrantup.com/](http://www.vagrantup.com/) (run `vagrant --version` to check if it's installed).
-* Install Vagrant Plugins:
+* Install ducktape (system testing command-line tool and library)
 
-        # Required
-        # Note that vagrant-hostmanager v1.6.0 and up breaks our Vagrant scripts
-        $ vagrant plugin install vagrant-hostmanager --plugin-version 1.5.0
-        $ vagrant plugin install vagrant-cachier
+        $ pip install ducktape
 
-* Build a specific branch of Kafka
-       
-        $ cd kafka
-        $ git checkout $BRANCH
-        $ gradle
-        $ ./gradlew jar
-      
-* Setup a testing cluster with Vagrant. Configure your Vagrant setup by creating the file 
-   `Vagrantfile.local` in the directory of your Kafka checkout. For testing purposes,
-  `num_brokers` and `num_kafka` should be 0, and `num_workers` should be set high enough
-  to run all of you tests. An example resides in kafka/vagrant/system-test-Vagrantfile.local
+* Run the bootstrap script to set up Vagrant for testing
 
-        # Example Vagrantfile.local for use on local machine
-        # Vagrantfile.local should reside in the base Kafka directory
-        num_zookeepers = 0
-        num_kafka = 0
-        num_workers = 9
+        $ tests/bootstrap-test-env.sh
 
-* Bring up the cluster (note that the initial provisioning process can be slow since it involves
-installing dependencies and updates on every vm.):
+* Bring up the test cluster
 
         $ vagrant up
 
-* Install ducktape:
+* Build the desired branch of Kafka
        
-        $ pip install ducktape
-
+        $ cd kafka
+        $ git checkout $BRANCH
+        $ gradle
+        $ ./gradlew jar
+      
 * Run the system tests using ducktape:
 
         $ cd tests
         $ ducktape kafkatest/tests
 
-* If you make changes to your Kafka checkout, you'll need to rebuild and resync to your Vagrant cluster:
-
-        $ cd kafka
-        $ ./gradlew jar
-        $ vagrant rsync # Re-syncs build output to cluster
-        
 EC2 Quickstart
 --------------
 This quickstart will help you run the Kafka system tests on EC2. In this setup, all logic is run
diff --git a/tests/bootstrap-test-env.sh b/tests/bootstrap-test-env.sh
new file mode 100755
index 0000000000000..c43fc3abe2f5a
--- /dev/null
+++ b/tests/bootstrap-test-env.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+# This script automates the process of setting up a local machine for running Kafka system tests
+
+# Helper function which prints version numbers so they can be compared lexically or numerically
+function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d%03d\n", $1,$2,$3,$4); }'; }
+
+base_dir=`dirname $0`/..
+cd $base_dir
+
+echo "Checking Virtual Box installation..."
+bad_vb=false
+if [ -z `vboxmanage --version` ]; then
+    echo "It appears that Virtual Box is not installed. Please install and try again (see https://www.virtualbox.org/ for details)"
+    bad_vb=true   
+else
+    echo "Virtual Box looks good."
+fi
+
+echo "Checking Vagrant installation..."
+vagrant_version=`vagrant --version | egrep -o "\d+\.\d+\.\d+"`
+bad_vagrant=false
+if [ "$(version $vagrant_version)" -lt "$(version 1.6.4)" ]; then
+    echo "Found Vagrant version $vagrant_version. Please upgrade to 1.6.4 or higher (see http://www.vagrantup.com for details)"
+    bad_vagrant=false
+else
+    echo "Vagrant installation looks good."
+fi
+
+if [ "x$bad_vagrant" == "xtrue" -o "x$bad_vb" == "xtrue" ]; then
+    exit 1
+fi
+
+echo "Checking for necessary Vagrant plugins..."
+install_hostmanager=false
+hostmanager_version=`vagrant plugin list | grep vagrant-hostmanager | egrep -o "\d+\.\d+\.\d+"`
+if [ -z "$hostmanager_version"  ]; then
+    install_hostmanager=true
+elif [ "$hostmanager_version" != "1.5.0" ]; then
+    echo "You have the wrong version of vagrant plugin vagrant-hostmanager. Uninstalling..."
+    vagrant plugin uninstall vagrant-hostmanager
+    install_hostmanager=true
+fi
+if [ "x$install_hostmanager" == "xtrue" ]; then
+    vagrant plugin install vagrant-hostmanager --plugin-version 1.5.0
+fi
+
+echo "Creating and packaging a reusable base box for Vagrant..."
+vagrant/package-base-box.sh
+
+if [ ! -e Vagrantfile.local ]; then
+    echo "Creating Vagrantfile.local..."
+    cp vagrant/system-test-Vagrantfile.local Vagrantfile.local
+else
+    echo "Found an existing Vagrantfile.local. Keeping without overwriting..."
+fi
+
+# Sanity check contents of Vagrantfile.local
+echo "Checking Vagrantfile.local..."
+vagrantfile_ok=true
+num_brokers=`egrep -o "num_brokers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs`
+num_zookeepers=`egrep -o "num_zookeepers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs`
+num_workers=`egrep -o "num_workers\s*=\s*\d+" Vagrantfile.local | cut -d '=' -f 2 | xargs`
+if [ "x$num_brokers" == "x" -o "$num_brokers" != 0 ]; then
+    echo "Vagrantfile.local: bad num_brokers. Update to: num_brokers = 0"
+    vagrantfile_ok=false
+fi
+if [ "x$num_zookeepers" == "x" -o "$num_zookeepers" != 0 ]; then
+    echo "Vagrantfile.local: bad num_zookeepers. Update to: num_zookeepers = 0"
+    vagrantfile_ok=false
+fi
+if [ "x$num_workers" == "x" -o "$num_workers" == 0 ]; then
+    echo "Vagrantfile.local: bad num_workers (size of test cluster). Set num_workers high enough to run your tests."
+    vagrantfile_ok=false
+fi
+
+if [ "$vagrantfile_ok" == "true" ]; then
+    echo "Vagrantfile.local looks good."
+fi
diff --git a/vagrant/package-base-box.sh b/vagrant/package-base-box.sh
new file mode 100755
index 0000000000000..0c2cc89118f86
--- /dev/null
+++ b/vagrant/package-base-box.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# This script automates the process of creating and packaging 
+# a new vagrant base_box. For use locally (not aws).
+
+# Name of the new base box
+base_box="kafkatest-worker"
+
+# vagrant VM name
+worker_name="worker1"
+
+base_dir=`dirname $0`/..
+cd $base_dir
+
+echo "Destroying vagrant machines..."
+vagrant destroy -f
+
+echo "Removing $base_box from vagrant..."
+vagrant box remove $base_box
+
+echo "Bringing up a single vagrant machine from scratch..."
+backup_vagrantfile=backup_Vagrantfile.local
+local_vagrantfile=Vagrantfile.local
+if [ -e $local_vagrantfile ]; then
+    mv $local_vagrantfile $backup_vagrantfile
+fi
+echo "num_workers = 1" > $local_vagrantfile
+echo "num_brokers = 0" >> $local_vagrantfile
+echo "num_zookeepers = 0" >> $local_vagrantfile
+vagrant up
+up_success=$?
+if [ $up_sucess != 0 ]; then
+    echo "Failed to bring up a template vm, please try running again."
+    exit $up_success
+fi
+
+echo "Packaging $worker_name..."
+vagrant package $worker_name
+
+echo "Adding new base box $base_box to vagrant..."
+vagrant box add $base_box package.box
+
+echo "Cleaning up..."
+vagrant destroy -f
+rm -f package.box
+# restore the original vagrantfile.local
+rm -f $local_vagrantfile
+if [ -e $backup_vagrantfile ]; then
+    mv $backup_vagrantfile $local_vagrantfile
+fi

From bb0be1e1f54a69bdb1a2015915f063192b149be8 Mon Sep 17 00:00:00 2001
From: Geoff Anderson <geoff@confluent.io>
Date: Wed, 9 Sep 2015 10:29:35 -0700
Subject: [PATCH 2/5] Fixed check on Vagrant installation. Also, restore
 original Vagrantfile.local if vagrant up fails during box packaging.

---
 tests/bootstrap-test-env.sh | 3 ++-
 vagrant/package-base-box.sh | 7 ++++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/bootstrap-test-env.sh b/tests/bootstrap-test-env.sh
index c43fc3abe2f5a..44b6538b2ccf2 100755
--- a/tests/bootstrap-test-env.sh
+++ b/tests/bootstrap-test-env.sh
@@ -22,7 +22,7 @@ vagrant_version=`vagrant --version | egrep -o "\d+\.\d+\.\d+"`
 bad_vagrant=false
 if [ "$(version $vagrant_version)" -lt "$(version 1.6.4)" ]; then
     echo "Found Vagrant version $vagrant_version. Please upgrade to 1.6.4 or higher (see http://www.vagrantup.com for details)"
-    bad_vagrant=false
+    bad_vagrant=true
 else
     echo "Vagrant installation looks good."
 fi
@@ -48,6 +48,7 @@ fi
 echo "Creating and packaging a reusable base box for Vagrant..."
 vagrant/package-base-box.sh
 
+# Set up Vagrantfile.local if necessary
 if [ ! -e Vagrantfile.local ]; then
     echo "Creating Vagrantfile.local..."
     cp vagrant/system-test-Vagrantfile.local Vagrantfile.local
diff --git a/vagrant/package-base-box.sh b/vagrant/package-base-box.sh
index 0c2cc89118f86..f83c621ee5e3c 100755
--- a/vagrant/package-base-box.sh
+++ b/vagrant/package-base-box.sh
@@ -31,6 +31,11 @@ vagrant up
 up_success=$?
 if [ $up_sucess != 0 ]; then
     echo "Failed to bring up a template vm, please try running again."
+    # restore original Vagrantfile.local
+    rm -f $local_vagrantfile
+    if [ -e $backup_vagrantfile ]; then
+        mv $backup_vagrantfile $local_vagrantfile
+    fi
     exit $up_success
 fi
 
@@ -43,7 +48,7 @@ vagrant box add $base_box package.box
 echo "Cleaning up..."
 vagrant destroy -f
 rm -f package.box
-# restore the original vagrantfile.local
+# restore the original Vagrantfile.local
 rm -f $local_vagrantfile
 if [ -e $backup_vagrantfile ]; then
     mv $backup_vagrantfile $local_vagrantfile

From b2b45a0756d6c2342eeb207d0c7c43204a802351 Mon Sep 17 00:00:00 2001
From: Geoff Anderson <geoff@confluent.io>
Date: Thu, 10 Sep 2015 15:58:40 -0700
Subject: [PATCH 3/5] Minor cleanup in package-base-box.sh

---
 vagrant/package-base-box.sh | 48 +++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 21 deletions(-)

diff --git a/vagrant/package-base-box.sh b/vagrant/package-base-box.sh
index f83c621ee5e3c..9ea134f6d88e7 100755
--- a/vagrant/package-base-box.sh
+++ b/vagrant/package-base-box.sh
@@ -3,15 +3,33 @@
 # This script automates the process of creating and packaging 
 # a new vagrant base_box. For use locally (not aws).
 
+base_dir=`dirname $0`/..
+cd $base_dir
+
+backup_vagrantfile=backup_Vagrantfile.local
+local_vagrantfile=Vagrantfile.local
+
+# Restore original Vagrantfile.local, if it exists
+function revert_vagrantfile {
+    rm -f $local_vagrantfile
+    if [ -e $backup_vagrantfile ]; then
+        mv $backup_vagrantfile $local_vagrantfile
+    fi
+}
+
+function clean_up {
+    echo "Cleaning up..."
+    vagrant destroy -f
+    rm -f package.box
+    revert_vagrantfile
+}
+
 # Name of the new base box
 base_box="kafkatest-worker"
 
 # vagrant VM name
 worker_name="worker1"
 
-base_dir=`dirname $0`/..
-cd $base_dir
-
 echo "Destroying vagrant machines..."
 vagrant destroy -f
 
@@ -19,8 +37,6 @@ echo "Removing $base_box from vagrant..."
 vagrant box remove $base_box
 
 echo "Bringing up a single vagrant machine from scratch..."
-backup_vagrantfile=backup_Vagrantfile.local
-local_vagrantfile=Vagrantfile.local
 if [ -e $local_vagrantfile ]; then
     mv $local_vagrantfile $backup_vagrantfile
 fi
@@ -28,15 +44,11 @@ echo "num_workers = 1" > $local_vagrantfile
 echo "num_brokers = 0" >> $local_vagrantfile
 echo "num_zookeepers = 0" >> $local_vagrantfile
 vagrant up
-up_success=$?
-if [ $up_sucess != 0 ]; then
+up_status=$?
+if [ $up_status != 0 ]; then
     echo "Failed to bring up a template vm, please try running again."
-    # restore original Vagrantfile.local
-    rm -f $local_vagrantfile
-    if [ -e $backup_vagrantfile ]; then
-        mv $backup_vagrantfile $local_vagrantfile
-    fi
-    exit $up_success
+    clean_up
+    exit $up_status
 fi
 
 echo "Packaging $worker_name..."
@@ -45,11 +57,5 @@ vagrant package $worker_name
 echo "Adding new base box $base_box to vagrant..."
 vagrant box add $base_box package.box
 
-echo "Cleaning up..."
-vagrant destroy -f
-rm -f package.box
-# restore the original Vagrantfile.local
-rm -f $local_vagrantfile
-if [ -e $backup_vagrantfile ]; then
-    mv $backup_vagrantfile $local_vagrantfile
-fi
+clean_up
+

From 2ff89e2af4906ae8df126f1bf8e613a1cbae1767 Mon Sep 17 00:00:00 2001
From: Geoff Anderson <geoff@confluent.io>
Date: Sat, 12 Sep 2015 16:58:42 -0700
Subject: [PATCH 4/5] Updated system test README - add instructions for
 installing proper versions of various dependencies.

---
 tests/README.md                       | 9 ++++-----
 vagrant/system-test-Vagrantfile.local | 1 +
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/README.md b/tests/README.md
index 207043cda5a70..82d31daad5bae 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -14,9 +14,9 @@ https://cwiki.apache.org/confluence/display/KAFKA/tutorial+-+set+up+and+run+Kafk
 
 * Install Virtual Box from [https://www.virtualbox.org/](https://www.virtualbox.org/) (run `$ vboxmanage --version` to check if it's installed).
 * Install Vagrant >= 1.6.4 from [http://www.vagrantup.com/](http://www.vagrantup.com/) (run `vagrant --version` to check if it's installed).
-* Install ducktape (system testing command-line tool and library)
+* Install system test dependiences, including ducktape, a command-line tool and library for testing distributed systems.
 
-        $ pip install ducktape
+        $ python tests/setup.py develop
 
 * Run the bootstrap script to set up Vagrant for testing
 
@@ -28,11 +28,10 @@ https://cwiki.apache.org/confluence/display/KAFKA/tutorial+-+set+up+and+run+Kafk
 
 * Build the desired branch of Kafka
        
-        $ cd kafka
         $ git checkout $BRANCH
-        $ gradle
+        $ gradle  # (only if necessary)
         $ ./gradlew jar
-      
+     
 * Run the system tests using ducktape:
 
         $ cd tests
diff --git a/vagrant/system-test-Vagrantfile.local b/vagrant/system-test-Vagrantfile.local
index 7f280a4653c6b..0ec04af06fb78 100644
--- a/vagrant/system-test-Vagrantfile.local
+++ b/vagrant/system-test-Vagrantfile.local
@@ -20,3 +20,4 @@
 num_zookeepers = 0
 num_brokers = 0
 num_workers = 9
+base_box = "kafkatest-worker"

From 535cff9f32421a211b6e6570a6144d3fbc6d8635 Mon Sep 17 00:00:00 2001
From: Geoff Anderson <geoff@confluent.io>
Date: Sat, 12 Sep 2015 17:38:07 -0700
Subject: [PATCH 5/5] Added licenses to new scripts.

---
 tests/bootstrap-test-env.sh | 14 ++++++++++++++
 vagrant/package-base-box.sh | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/tests/bootstrap-test-env.sh b/tests/bootstrap-test-env.sh
index 44b6538b2ccf2..e12643a506053 100755
--- a/tests/bootstrap-test-env.sh
+++ b/tests/bootstrap-test-env.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # This script automates the process of setting up a local machine for running Kafka system tests
 
diff --git a/vagrant/package-base-box.sh b/vagrant/package-base-box.sh
index 9ea134f6d88e7..5ac7f0e5356d8 100755
--- a/vagrant/package-base-box.sh
+++ b/vagrant/package-base-box.sh
@@ -1,4 +1,18 @@
 #!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # This script automates the process of creating and packaging 
 # a new vagrant base_box. For use locally (not aws).