improvements build (apache#204)

Author: skonto

Makefile

@@ -5,32 +5,33 @@ CLI_DIST_DIR := $(BUILD_DIR)/cli_dist
 DIST_DIR := $(BUILD_DIR)/dist
 GIT_COMMIT := $(shell git rev-parse HEAD)
 
-S3_BUCKET := infinity-artifacts
-S3_PREFIX := autodelete7d
+S3_BUCKET ?= infinity-artifacts
+S3_PREFIX ?= autodelete7d
+SPARK_REPO_URL ?= https://github.com/mesosphere/spark
 
 .ONESHELL:
 SHELL := /bin/bash
 .SHELLFLAGS = -ec
 
 # This image can be used to build spark dist and run tests
-DOCKER_BUILD_IMAGE := mesosphere/spark-build:$(GIT_COMMIT)
+DOCKER_BUILD_IMAGE ?= mesosphere/spark-build:$(GIT_COMMIT)
 docker-build:
 	docker build -t $(DOCKER_BUILD_IMAGE) .
 	echo $(DOCKER_BUILD_IMAGE) > $@
 
 # Pulls the spark distribution listed in the manifest as default
-SPARK_DIST_URI := $(shell jq ".default_spark_dist.uri" "$(ROOT_DIR)/manifest.json")
+SPARK_DIST_URI ?= $(shell jq ".default_spark_dist.uri" "$(ROOT_DIR)/manifest.json")
 manifest-dist:
 	mkdir -p $(DIST_DIR)
 	pushd $(DIST_DIR)
 	wget $(SPARK_DIST_URI)
 	popd
 
-HADOOP_VERSION := $(shell jq ".default_spark_dist.hadoop_version" "$(ROOT_DIR)/manifest.json")
+HADOOP_VERSION ?= $(shell jq ".default_spark_dist.hadoop_version" "$(ROOT_DIR)/manifest.json")
 
-SPARK_DIR := $(ROOT_DIR)/spark
+SPARK_DIR ?= $(ROOT_DIR)/spark
 $(SPARK_DIR):
-	git clone https://github.com/mesosphere/spark $(SPARK_DIR)
+	git clone $(SPARK_REPO_URL) $(SPARK_DIR)
 
 # Builds a quick dev version of spark from the mesosphere fork
 dev-dist: $(SPARK_DIR)
@@ -85,7 +86,7 @@ clean-dist:
 docker-login:
 	docker login --email="$(DOCKER_EMAIL)" --username="$(DOCKER_USERNAME)" --password="$(DOCKER_PASSWORD)"
 
-DOCKER_DIST_IMAGE := mesosphere/spark-dev:$(GIT_COMMIT)
+DOCKER_DIST_IMAGE ?= mesosphere/spark-dev:$(GIT_COMMIT)
 docker-dist: $(DIST_DIR)
 	tar xvf $(DIST_DIR)/spark-*.tgz -C $(DIST_DIR)
 	rm -rf $(BUILD_DIR)/docker
@@ -99,7 +100,7 @@ docker-dist: $(DIST_DIR)
 	docker push $(DOCKER_DIST_IMAGE)
 	echo "$(DOCKER_DIST_IMAGE)" > $@
 
-CLI_VERSION := $(shell jq -r ".cli_version" "$(ROOT_DIR)/manifest.json")
+CLI_VERSION ?= $(shell jq -r ".cli_version" "$(ROOT_DIR)/manifest.json")
 $(CLI_DIST_DIR):
 	$(MAKE) --directory=cli all
 	mkdir -p $@
@@ -110,7 +111,7 @@ $(CLI_DIST_DIR):
 
 cli: $(CLI_DIST_DIR)
 
-UNIVERSE_URL_PATH := stub-universe-url
+UNIVERSE_URL_PATH ?= stub-universe-url
 $(UNIVERSE_URL_PATH): $(CLI_DIST_DIR) docker-dist
 	UNIVERSE_URL_PATH=$(UNIVERSE_URL_PATH) \
 	TEMPLATE_CLI_VERSION=$(CLI_VERSION) \
@@ -123,7 +124,7 @@ $(UNIVERSE_URL_PATH): $(CLI_DIST_DIR) docker-dist
         $(CLI_DIST_DIR)/dcos-spark.exe \
         $(CLI_DIST_DIR)/*.whl;
 
-DCOS_SPARK_TEST_JAR_PATH := $(ROOT_DIR)/dcos-spark-scala-tests-assembly-0.1-SNAPSHOT.jar
+DCOS_SPARK_TEST_JAR_PATH ?= $(ROOT_DIR)/dcos-spark-scala-tests-assembly-0.1-SNAPSHOT.jar
 $(DCOS_SPARK_TEST_JAR_PATH):
 	cd tests/jobs/scala
 	sbt assembly
@@ -154,15 +155,15 @@ clean-cluster:
 mesos-spark-integration-tests:
 	git clone https://github.com/typesafehub/mesos-spark-integration-tests $(ROOT_DIR)/mesos-spark-integration-tests
 
-MESOS_SPARK_TEST_JAR_PATH := $(ROOT_DIR)/mesos-spark-integration-tests-assembly-0.1.0.jar
+MESOS_SPARK_TEST_JAR_PATH ?= $(ROOT_DIR)/mesos-spark-integration-tests-assembly-0.1.0.jar
 $(MESOS_SPARK_TEST_JAR_PATH): mesos-spark-integration-tests
 	cd $(ROOT_DIR)/mesos-spark-integration-tests/test-runner
 	sbt assembly
 	cd ..
 	sbt clean compile test
 	cp test-runner/target/scala-2.11/mesos-spark-integration-tests-assembly-0.1.0.jar $(MESOS_SPARK_TEST_JAR_PATH)
 
-PYTEST_ARGS := -s -vv -m sanity
+PYTEST_ARGS ?= -s -vv -m sanity
 test: test-env $(DCOS_SPARK_TEST_JAR_PATH) $(MESOS_SPARK_TEST_JAR_PATH) $(UNIVERSE_URL_PATH) cluster-url
 	source $(ROOT_DIR)/test-env/bin/activate
 	if [ -z $(CLUSTER_URL) ]; then \

tools/build_cli.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # exit immediately on failure
 set -e

tools/build_framework.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Prevent jenkins from immediately killing the script when a step fails, allowing us to notify github:
 set +e

tools/build_publishable.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 source ./tools/init_paths.sh

tools/release.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 #script to publish the tools dir to s3 for use in other jobs
 

tools/release_artifacts.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 set -e
 source ${TOOLS_DIR}/init_paths.sh

tools/setup_permissions.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 LINUX_USER=$1
 ROLE=$2