progress with aws & skaffold

providers/aws/Dockerfile

@@ -1,5 +1,13 @@
 FROM alpine:3.10
 
+# Set dependencies versions.
+ARG EKSCTL_VERSION=0.7.0
+ARG KUBECTL_VERSION=1.16.2
+ARG SKAFFOLD_VERSION=0.40.0
+ARG ISTIO_VERSION=1.3.3
+ARG HELM_VERSION=2.15.0
+
+# Install base apps.
 RUN apk add --update-cache --no-cache \
   bash \
   build-base \
@@ -14,35 +22,40 @@ RUN apk add --update-cache --no-cache \
 RUN pip install awscli
 
 # Install eksctl
+ARG EKSCTL_HOST="https://github.com/weaveworks/eksctl/releases/download"
 RUN curl --silent \
-  --location "https://github.com/weaveworks/eksctl/releases/download/latest_release/eksctl_$(uname -s)_amd64.tar.gz" \
+  --location "${EKSCTL_HOST}/${EKSCTL_VERSION}/eksctl_$(uname -s)_amd64.tar.gz" \
   | tar xz -C /tmp
 RUN mv /tmp/eksctl /usr/local/bin
 
 # Install kubectl
-RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
-    chmod +x ./kubectl && \
-    mv ./kubectl /usr/local/bin/kubectl
+ARG KUBECTL_HOST="https://storage.googleapis.com/kubernetes-release/release"
+RUN curl -LO "${KUBECTL_HOST}/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl" && \
+  chmod +x ./kubectl && \
+  mv ./kubectl /usr/local/bin/kubectl
 
 # Install Helm
-RUN curl -L https://git.io/get_helm.sh | bash
-
-# Install Ksync.
-RUN curl https://vapor-ware.github.io/gimme-that/gimme.sh | bash
+RUN curl -L https://git.io/get_helm.sh | DESIRED_VERSION="v${HELM_VERSION}" bash
 
 # Download Istio CRDs
-RUN curl -L https://git.io/getLatestIstio | sh -
+RUN curl -L https://git.io/getLatestIstio | ISTIO_VERSION="${ISTIO_VERSION}" sh -
+
+# Install Skaffold.
+ARG SKAFFOLD_HOST="https://storage.googleapis.com/skaffold/releases"
+# RUN curl -Lo skaffold "${SKAFFOLD_HOST}/v${SKAFFOLD_VERSION}/skaffold-linux-amd64" && \
+RUN curl -Lo skaffold "https://storage.googleapis.com/artifacts.gpu-sh.appspot.com/releases/latest/skaffold-linux-amd64" && \
+  chmod +x skaffold && \
+  mv skaffold /usr/local/bin
 
 # Copy the provider's commands.
 COPY . /kuda_cmd
 ENV KUDA_CMD_DIR /kuda_cmd
 RUN chmod +x /kuda_cmd/*.sh
 RUN ln -s /kuda_cmd/app_deploy.sh /usr/local/bin/kuda_app_deploy && \
   ln -s /kuda_cmd/app_delete.sh /usr/local/bin/kuda_app_delete && \
+  ln -s /kuda_cmd/app_dev.sh /usr/local/bin/kuda_app_dev && \
   ln -s /kuda_cmd/setup.sh /usr/local/bin/kuda_setup && \
   ln -s /kuda_cmd/delete.sh /usr/local/bin/kuda_delete && \
-  ln -s /kuda_cmd/dev_start.sh /usr/local/bin/kuda_dev_start && \
-  ln -s /kuda_cmd/dev_stop.sh /usr/local/bin/kuda_dev_stop && \
   ln -s /kuda_cmd/get.sh /usr/local/bin/kuda_get
 
 # Go to the app home.

providers/aws/README.md

@@ -1,6 +1,7 @@
 ## Amazon Web Service Provider
 
 Hacky & bare implementation with shell scripts.
+It uses [ECR](https://aws.amazon.com/ecr) to privately store the app images.
 
 # Status
 
@@ -9,23 +10,23 @@ Hacky & bare implementation with shell scripts.
 | setup | ✔ |
 | delete | ✔ |
 | get | Not Started |
+| app dev | WIP |
 | app deploy | WIP |
 | app delete | ✔ |
-| dev start | Not Started |
-| dev stop | Not Started |
+| dev start | ✗ |
+| dev stop | ✗ |
 
-# Configuration
+# Prerequisites
 
-**Prerequisites:**
 - You must have subscribed to [EKS-optimize AMI with GPU support](https://aws.amazon.com/marketplace/pp/B07GRHFXGM)
 - You must have an increased limit of at least 1 instance of type p2.xlarge. You can make requests [here](http://aws.amazon.com/contact-us/ec2-request)
 - You must have an [aws configuration](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) on your local machine in `~/.aws/` with credentials that have authorization for:
     - cloudformation
     - ec2
+    - ec2 autoscaling
     - eks
     - iam
     - api
-    - ec2 autoscaling
     - ecr
     - elb
 

providers/aws/app_dev.sh

@@ -0,0 +1,127 @@
+#!/bin/bash
+
+set -e
+
+source $KUDA_CMD_DIR/.config.sh
+
+aws_account_id="$(aws sts get-caller-identity | jq -r .Account)"
+ecr_domain="$aws_account_id.dkr.ecr.$KUDA_AWS_CLUSTER_REGION.amazonaws.com"
+
+app_name=$1
+app_registry="$ecr_domain/$app_name"
+app_image="$app_registry:$app_version"
+namespace="default"
+
+app_cache_name=$app_name-cache
+
+echo $app_image
+
+# Create Container Registry if it doesn't exists.
+if [ -z "$(aws ecr describe-repositories --region $KUDA_AWS_CLUSTER_REGION | grep $app_name)" ]; then
+  aws ecr create-repository \
+    --repository-name $app_name \
+    --region $KUDA_AWS_CLUSTER_REGION
+else
+  echo "Container Registry $app_registry already exists"
+fi
+
+# Create the cache registry if it doesn't exists.
+if [ -z "$(aws ecr describe-repositories --region $KUDA_AWS_CLUSTER_REGION | grep $app_cache_name)" ]; then
+  aws ecr create-repository \
+    --repository-name $app_cache_name \
+    --region $KUDA_AWS_CLUSTER_REGION
+else
+  echo "Container Registry $app_registry-cache already exists"
+fi
+
+# Retrieve cluster token.
+aws eks update-kubeconfig \
+  --name $KUDA_AWS_CLUSTER_NAME \
+  --region $KUDA_AWS_CLUSTER_REGION
+
+# Login Container Registry.
+# aws ecr get-login --region $KUDA_AWS_CLUSTER_REGION --no-include-email | bash
+
+#TODO: Build & Push image using Kaniko.
+
+# Write Knative service config.
+# cat <<EOF | kubectl apply -f -
+# apiVersion: serving.knative.dev/v1alpha1
+# kind: Service
+# metadata:
+#   name: $app_name
+#   namespace: default
+# spec:
+#   template:
+#     spec:
+#       nodeSelector:
+#         nvidia.com/gpu: "true"
+#       tolerations:
+#         - key: "nvidia.com/gpu"
+#           operator: "Exists"
+#           effect: "NoSchedule"
+#       containers:
+#         - image: $app_image
+#           resources:
+#             limits:
+#               nvidia.com/gpu: 1
+# EOF
+
+# Write Knative service config.
+echo "
+apiVersion: serving.knative.dev/v1alpha1
+kind: Service
+metadata:
+  name: $app_name
+  namespace: $namespace
+spec:
+  template:
+    spec:
+      # nodeSelector:
+      #   nvidia.com/gpu: true
+      # tolerations:
+      #   - key: nvidia.com/gpu
+      #     operator: Exists
+      #     effect: NoSchedule
+      containers:
+        - image: $app_registry
+          resources:
+            limits:
+              nvidia.com/gpu: 1
+" >.kuda-app.k8.yaml
+
+# cat <<EOF | skaffold dev -n $namespace -f -
+cat <<EOF | skaffold run -v debug -n $namespace -f -
+apiVersion: skaffold/v1beta17
+kind: Config
+build:
+  artifacts:
+    - image: $app_registry
+      sync:
+        manual:
+          - src: './**/*'
+            dest: .
+      kaniko:
+        buildArgs:
+          verbosity: debug
+        buildContext:
+          localDir: {}
+        cache:
+          repo: $app_registry-cache
+        env:
+          - name: AWS_REGION
+            value: eu-west-1
+  cluster:
+    pullSecretName: aws-secret
+    pullSecretMountPath: /root/.aws/
+    dockerConfig: 
+      secretName: docker-kaniko-secret
+    namespace: $namespace
+  tagPolicy:
+    dateTime:
+      format: "2006-01-02-15-04-05"
+deploy:
+  kubectl:
+    manifests:
+      - .kuda-app.k8.yaml
+EOF

providers/aws/setup.sh

@@ -33,7 +33,7 @@ function create_cluster() {
         k8s.io/cluster-autoscaler/enabled: 'true'
       labels:
         lifecycle: Ec2Spot
-        nvidia.com/gpu: 'true'
+        nvidia.com/gpu: true
         k8s.amazonaws.com/accelerator: nvidia-tesla
       taints:
         nvidia.com/gpu: "true:NoSchedule"
@@ -42,8 +42,10 @@ EOF
 }
 
 function install_nvidia_drivers() {
+  nvidia_driver_version=1.0.0-beta3
+  nvidia_driver_host="https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin"
   kubectl create \
-    -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/1.0.0-beta3/nvidia-device-plugin.yml
+    -f $nvidia_driver_host/$nvidia_driver_version/nvidia-device-plugin.yml
 }
 
 function install_istio() {
@@ -67,12 +69,13 @@ function install_istio() {
       --name istio-init \
       --namespace istio-system \
       $istio_folder/install/kubernetes/helm/istio-init
+
+      # Dirty hack to let the pods install.
+      sleep 10
   else
     echo "Istio prerequisites already installed."
   fi
 
-  sleep 3
-
   # Install Istio.
   echo "Installing Istio..."
   helm install \
@@ -127,18 +130,41 @@ else
 fi
 
 # Install Knative.
-if [ -z "$(kubectl -n knative-serving | grep 'webhook')" ]; then
+if [ -z "$(kubectl -n knative-serving get pods | grep 'webhook')" ]; then
   install_knative
 else
   echo "Knative is already installed."
 fi
 
+# Create credentials for skaffold.
+# https://github.com/GoogleContainerTools/skaffold/issues/1719
+if [ -z "$(kubectl get secret | grep aws-secret)" ]; then
+  kubectl create secret generic aws-secret --from-file /aws-credentials/credentials
+else
+  echo "aws-secret already configured."
+fi
+
+# Setup credential helpers in cluster for kaniko.
+if [ -z "$(kubectl get secret | grep docker-kaniko-secret)" ]; then
+  aws_account_id="$(aws sts get-caller-identity | jq -r .Account)"
+  ecr_domain="$aws_account_id.dkr.ecr.$KUDA_AWS_CLUSTER_REGION.amazonaws.com"
+  tmp_config_file="/tmp/config.json"
+  echo "{ \"credHelpers\": { \"$ecr_domain\": \"ecr-login\" }}" > $tmp_config_file
+  kubectl create secret generic docker-kaniko-secret --from-file $tmp_config_file
+  rm $tmp_config_file
+else
+  echo "docker-kaniko-secret already configured."
+fi
+
 # Install Cluster Autoscaler
 # echo "Installing cluster autoscaler"
 # kubectl apply -f /kuda_cmd/config/cluster-autoscaler.yaml
 
-echo "Retrieving hostname:"
-kubectl -n istio-system get service istio-ingressgateway -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'
+echo
+echo "Hostname:"
+kubectl -n istio-system get service istio-ingressgateway \
+  -o jsonpath='{.status.loadBalancer.ingress[0].hostname}'
+echo
 
 echo
 echo "Cluster $KUDA_AWS_CLUSTER_NAME is ready!"