Merge pull request apache#50 from mesosphere/new-cosmos

new cosmos

Author: mgummelt

bin/jenkins-dist-test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-export S3_BUCKET=spark-build
-export S3_PREFIX=
+export S3_BUCKET=infinity-artifacts
+export S3_PREFIX=spark
 export AWS_ACCESS_KEY_ID=${DEV_AWS_ACCESS_KEY_ID}
 export AWS_SECRET_ACCESS_KEY=${DEV_SECRET_ACCESS_KEY}
 export DOCKER_IMAGE=mesosphere/spark-dev:${GIT_COMMIT}

bin/jenkins-package-publish.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-export S3_BUCKET=spark-build
-export S3_PREFIX=
+export S3_BUCKET=infinity-artifacts
+export S3_PREFIX=spark
 export DOCKER_IMAGE=mesosphere/spark:${GIT_BRANCH#refs/tags/}
 
 source spark-build/bin/jenkins.sh

bin/jenkins-package-test.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-export S3_BUCKET=spark-build
-export S3_PREFIX=
+export S3_BUCKET=infinity-artifacts
+export S3_PREFIX=spark
 export DOCKER_IMAGE=mesosphere/spark-dev:${GIT_COMMIT}
 
 source spark-build/bin/jenkins.sh

bin/jenkins.sh

@@ -72,6 +72,9 @@ function install_cli {
 
     # hack because the installer forces an old CLI version
     pip install -U dcoscli
+
+    # needed in `make test`
+    pip3 install jsonschema
 }
 
 function docker_login {

bin/make-universe.sh

@@ -1,35 +1,36 @@
 #!/usr/bin/env bash
 
-# creates
-# - build/spark-universe
-# - build/spark-universe.zip
+# 1) publishes universe docker image at DOCKER_IMAGE:DOCKER_TAG
+# 2) creates universe app at ./build/spark-universe/docker/server/target/marathon.json
 
 set -x -e
 
 rm -rf build/spark-universe*
 
-# make spark package
-# TODO(mgummelt): remove this after some time
-# ./bin/make-package.py
-
 # download universe
-wget -O build/spark-universe.zip https://github.com/mesosphere/universe/archive/version-2.x.zip
+UNIVERSE_BRANCH=version-3.x
+wget -O build/spark-universe.zip "https://github.com/mesosphere/universe/archive/${UNIVERSE_BRANCH}.zip"
 unzip -d build build/spark-universe.zip
-mv build/universe-version-2.x build/spark-universe
+mv "build/universe-${UNIVERSE_BRANCH}" build/spark-universe
 rm build/spark-universe.zip
 
 # make new universe
 SPARK_DIR=build/spark-universe/repo/packages/S/spark
-rm -rf ${SPARK_DIR}/*
-cp -r build/package ${SPARK_DIR}/0
+rm -rf "${SPARK_DIR}"/*
+cp -r build/package "${SPARK_DIR}/0"
 
+# build universe docker image
 pushd build/spark-universe
 ./scripts/build.sh
+DOCKER_TAG=spark-$(openssl rand -hex 8)
+DOCKER_TAG="${DOCKER_TAG}" ./docker/server/build.bash
+DOCKER_TAG="${DOCKER_TAG}" ./docker/server/build.bash publish
 popd
 
-if [ -x "$(command -v zip)" ]; then
-    (cd build && zip -r spark-universe.zip spark-universe)
-else
-    # TODO: remove the docker wrapper once `zip` is available on TC
-    docker run -v $(pwd)/build/:/build/ ubuntu:latest sh -c "apt-get update && apt-get install -y zip && cd /build/ && zip -r spark-universe.zip spark-universe"
-fi
+
+# if [ -x "$(command -v zip)" ]; then
+#     (cd build && zip -r spark-universe.zip spark-universe)
+# else
+#     # TODO: remove the docker wrapper once `zip` is available on TC
+#     docker run -v $(pwd)/build/:/build/ ubuntu:latest sh -c "apt-get update && apt-get install -y zip && cd /build/ && zip -r spark-universe.zip spark-universe"
+# fi

bin/test.sh

@@ -30,16 +30,17 @@ set -o pipefail
 
 
 build_universe() {
-    ./bin/make-package.py
+    make package
     (cd build && tar czf package.tgz package)
-    ./bin/make-universe.sh
+
+    # temporarily unset DOCKER_IMAGE so it doesn't conflict with universe's build.bash
+    (unset DOCKER_IMAGE && make universe)
 }
 
 start_cluster() {
     if [ -z "${DCOS_URL}" ]; then
         DCOS_URL=http://$(./bin/launch-cluster.sh)
     fi
-    #TOKEN=$(python -c "import requests;js={'token':'"${DCOS_OAUTH_TOKEN}"'};r=requests.post('"${DCOS_URL}"/acs/api/v1/auth/login',json=js);print(r.json()['token'])")
     TOKEN=$(python -c "import requests;js={'uid':'"${DCOS_USERNAME}"', 'password': '"${DCOS_PASSWORD}"'};r=requests.post('"${DCOS_URL}"/acs/api/v1/auth/login',json=js);print(r.json()['token'])")
     dcos config set core.dcos_acs_token ${TOKEN}
 }
@@ -48,9 +49,14 @@ configure_cli() {
     dcos config set core.dcos_url "${DCOS_URL}"
 
     # add universe
-    local S3_FILENAME="${S3_PREFIX}spark-universe-${BUILD_ID}.zip"
-    aws s3 cp ./build/spark-universe.zip "s3://${S3_BUCKET}/${S3_FILENAME}" --acl public-read
-    dcos package repo add --index=0 spark-test "http://${S3_BUCKET}.s3.amazonaws.com/${S3_FILENAME}"
+    # local S3_FILENAME="${S3_PREFIX}spark-universe-${BUILD_ID}.zip"
+    # aws s3 cp ./build/spark-universe.zip "s3://${S3_BUCKET}/${S3_FILENAME}" --acl public-read
+    # dcos package repo add --index=0 spark-test "http://${S3_BUCKET}.s3.amazonaws.com/${S3_FILENAME}"
+    dcos marathon app add ./build/spark-universe/docker/server/target/marathon.json
+    dcos package repo add --index=0 spark-test http://universe.marathon.mesos:8085/repo-1.7
+
+    # wait for universe server to come up
+    sleep 45
 }
 
 install_spark() {

docker/Dockerfile

@@ -57,4 +57,3 @@ RUN ln -s /var/lib/runit/service/spark /etc/service/spark
 RUN ln -s /var/lib/runit/service/nginx /etc/service/nginx
 
 WORKDIR /opt/spark/dist
-