Initial keytar commit.

test/cluster/keytar/Dockerfile

@@ -0,0 +1,43 @@
+# Dockerfile for generating the keytar image. See README.md for more information.
+FROM debian:jessie
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update -y \
+ && apt-get install --no-install-recommends -y -q \
+    apt-utils \
+    apt-transport-https \
+    build-essential \
+    curl \
+    python2.7 \
+    python2.7-dev \
+    python-pip \
+    git \
+    wget \
+ && pip install -U pip \
+ && pip install virtualenv
+
+RUN echo "deb https://packages.cloud.google.com/apt cloud-sdk-jessie main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
+RUN apt-get update -y && apt-get install -y google-cloud-sdk && apt-get install -y kubectl
+
+WORKDIR /app
+RUN virtualenv /env
+ADD requirements.txt /app/requirements.txt
+RUN /env/bin/pip install -r /app/requirements.txt
+ADD keytar.py test_runner.py /app/
+ADD static /app/static
+
+ENV USER keytar
+
+ENV PYTHONPATH /env/lib/python2.7/site-packages
+ENV CLOUDSDK_PYTHON_SITEPACKAGES $PYTHONPATH
+
+RUN /bin/bash -c "source ~/.bashrc"
+
+EXPOSE 8080
+CMD []
+ENTRYPOINT ["/env/bin/python", "keytar.py"]
+
+ENV PATH /env/bin:$PATH
+

test/cluster/keytar/README.md

@@ -0,0 +1,43 @@
+# Keytar
+
+Keytar is an internally used Vitess system for continuous execution of cluster tests on Kubernetes/Google Cloud. It monitors docker images on [Docker Hub](https://hub.docker.com). When a new image is uploaded to Docker Hub, Keytar starts a cluster on Google Compute Engine (GKE) and runs Kubernetes applications for the purpose of executing cluster tests. It will then locally run tests against the cluster. It exposes a simple web status page showing test results.
+
+## Setup
+
+How to set up Keytar for Vitess:
+
+* Create service account keys with GKE credentials on the account to run the tests on. Follow [step 1 from the GKE developers page](https://developers.google.com/identity/protocols/application-default-credentials?hl=en_US#howtheywork).
+* Move the generated keyfile to `$VTTOP/test/cluster/keytar/config`.
+* Create or modify the test configuration file (`$VTTOP/test/cluster/keytar/config/vitess_config.yaml`).
+* Ensure the configuration has the correct values for GKE project name and keyfile:
+  ```
+  cluster_setup:
+  - type: gke
+    project_name: <your_gke_project_name>
+    keyfile: /config/<your_keyfile_name>
+  ```
+* Then run the following commands:
+  ```
+  > cd $VTTOP/test/cluster/keytar
+  > KEYTAR_PASSWORD=<desired password> KEYTAR_PORT=<desired port, default 8080> KEYTAR_CONFIG=<desired configuration, default vitess_config.yaml> ./keytar-up.sh
+  ```
+* Add a Docker Hub webhook pointing to the Keytar service. The webhook URL should be in the form:
+  ```
+  http://<keytar-service-IP>:80/test_request?password=<KEYTAR_PASSWORD>
+  ```
+
+## Dashboard
+
+The script to start Keytar should output a web address to view the current status. If not, the following command can also be run:
+```shell
+> kubectl get service keytar -o template --template '{{if ge (len .status.loadBalancer) 1}}{{index (index .status.loadBalancer.ingress 0) "ip"}}{{end}}'
+```
+
+## Limitations
+
+Currently, Keytar has the following limitations:
+
+* Only one configuration file allowed at a time.
+* Configuration cannot be updated dynamically.
+* Test results are saved in memory and are not durable.
+* Results are only shown on the dashboard, there is no notification mechanism.

test/cluster/keytar/config/vitess_config.yaml

@@ -0,0 +1,45 @@
+install:
+  dependencies:
+    - python-mysqldb
+  extra:
+    - apt-get update
+    - wget https://storage.googleapis.com/golang/go1.7.4.linux-amd64.tar.gz
+    - tar -C /usr/local -xzf go1.7.4.linux-amd64.tar.gz
+    - wget https://storage.googleapis.com/kubernetes-helm/helm-v2.1.3-linux-amd64.tar.gz
+    - tar -zxvf helm-v2.1.3-linux-amd64.tar.gz
+    - pip install numpy
+    - pip install selenium
+    - pip install --upgrade grpcio==1.0.4
+  path:
+    - /usr/local/go/bin
+    - /app/linux-amd64/
+  cluster_setup:
+    - type: gke
+      keyfile: /config/keyfile.json
+config:
+  - docker_image: vitess/root
+    github:
+      repo: youtube/vitess
+      repo_prefix: src/github.com/youtube/vitess
+    environment:
+      sandbox: test/cluster/sandbox/vitess_kubernetes_sandbox.py
+      config: test/cluster/sandbox/example_sandbox.yaml
+      cluster_type: gke
+      application_type: k8s
+    before_test:
+      - export VTTOP=$(pwd)
+      - export VTROOT="${VTROOT:-${VTTOP/\/src\/github.com\/youtube\/vitess/}}"
+      - export GOPATH=$VTROOT
+      - export PYTHONPATH=$VTTOP/py:$VTTOP/test:$VTTOP/test/cluster/sandbox:/usr/lib/python2.7/dist-packages:/env/lib/python2.7/site-packages
+      - go get github.com/youtube/vitess/go/cmd/vtctlclient
+      - export PATH=$GOPATH/bin:$PATH
+    tests:
+      - file: test/cluster/drain_test.py
+        params:
+          num_drains: 1
+      - file: test/cluster/backup_test.py
+        params:
+          num_backups: 1
+      - file: test/cluster/reparent_test.py
+        params:
+          num_reparents: 1

test/cluster/keytar/dummy_test.py

@@ -0,0 +1,18 @@
+#!/usr/bin/env python
+"""Dummy no-op test to be used in the webdriver test."""
+
+import logging
+import sys
+import unittest
+
+
+class DummyTest(unittest.TestCase):
+
+  def test_dummy(self):
+    logging.info('Dummy output.')
+
+
+if __name__ == '__main__':
+  logging.getLogger().setLevel(logging.INFO)
+  del sys.argv[1:]
+  unittest.main()

test/cluster/keytar/keytar-controller-template.yaml

@@ -0,0 +1,30 @@
+kind: ReplicationController
+apiVersion: v1
+metadata:
+  name: keytar
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        component: frontend
+        app: keytar
+    spec:
+      containers:
+        - name: keytar
+          image: vitess/keytar
+          ports:
+            - name: http-server
+              containerPort: {{port}}
+          resources:
+            limits:
+              memory: "4Gi"
+              cpu: "500m"
+          args: ["--config_file", "{{config}}", "--port", "{{port}}", "--password", "{{password}}"]
+          volumeMounts:
+            - name: config
+              mountPath: /config
+      volumes:
+        - name: config
+          configMap:
+            name: config

test/cluster/keytar/keytar-down.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+KUBECTL=${KUBECTL:-kubectl}
+
+$KUBECTL delete replicationcontroller keytar
+$KUBECTL delete service keytar
+$KUBECTL delete configmap config
+gcloud container clusters delete keytar -z us-central1-b -q
+gcloud compute firewall-rules delete keytar -q

test/cluster/keytar/keytar-service.yaml

@@ -0,0 +1,15 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: keytar
+  labels:
+    component: frontend
+    app: keytar
+spec:
+  ports:
+    - port: 80
+      targetPort: http-server
+  selector:
+    component: frontend
+    app: keytar
+  type: LoadBalancer

test/cluster/keytar/keytar-up.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+set -e
+
+KUBECTL=${KUBECTL:-kubectl}
+
+config_path=${KEYTAR_CONFIG_PATH:-"./config"}
+port=${KEYTAR_PORT:-8080}
+password=${KEYTAR_PASSWORD:-"defaultkey"}
+config=${KEYTAR_CONFIG:-"/config/vitess_config.yaml"}
+
+sed_script=""
+for var in config_path port config password; do
+  sed_script+="s,{{$var}},${!var},g;"
+done
+
+gcloud container clusters create keytar --machine-type n1-standard-4 --num-nodes 1 --scopes cloud-platform --zone us-central1-b
+
+echo "Creating keytar configmap"
+$KUBECTL create configmap --from-file=$config_path config
+
+echo "Creating keytar service"
+$KUBECTL create -f keytar-service.yaml
+
+echo "Creating keytar controller"
+cat keytar-controller-template.yaml | sed -e "$sed_script" | $KUBECTL create -f -
+
+echo "Creating firewall-rule"
+gcloud compute firewall-rules create keytar --allow tcp:80
+
+for i in `seq 1 20`; do
+  ip=`$KUBECTL get service keytar -o template --template '{{if ge (len .status.loadBalancer) 1}}{{index (index .status.loadBalancer.ingress 0) "ip"}}{{end}}'`
+  if [[ -n "$ip" ]]; then
+    echo "Keytar address: http://${ip}:80"
+    break
+  fi
+  echo "Waiting for keytar external IP"
+  sleep 10
+done