Start work on AWS provider

README.md

@@ -36,10 +36,13 @@
 - The `kuda dev` command lets you spawn a remote development session with GPU inside the cluster.
 - It uses [Ksync](https://github.com/vapor-ware/ksync) to synchronise your working directory with the remote session so you can code from your workstation while running the app on the remote session.
 
-**Compatibility**
+**Cloud provider Compatibility**
 
-- [GCP](https://cloud.google.com) provider is already implemented.
-- [AWS](https://aws.amazon.com) and [Azure](https://azure.microsoft.com) should follow soon.
+| Provider | Status |
+| - | - |
+| [GCP](providers/gcp) | ✔ |
+| [AWS](providers/aws) | In progress |
+| Azure | Not started |
 
 ## Ready?
 

cmd/setup-aws.go

@@ -0,0 +1,73 @@
+/*
+Package cmd -
+
+Copyright © 2019 Cyril Diagne <[email protected]>.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/cyrildiagne/kuda/pkg/docker"
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// awsCmd represents the `setup gcp` command
+var awsCmd = &cobra.Command{
+	Use:   "aws",
+	Short: "Setup Kuda on AWS.",
+	Long:  "Setup Kuda on AWS.",
+	Run: func(cmd *cobra.Command, args []string) {
+		viper.BindPFlags(cmd.Flags())
+		if err := setupAWS(); err != nil {
+			fmt.Println("ERROR:", err)
+		}
+	},
+}
+
+func init() {
+	setupCmd.AddCommand(awsCmd)
+
+	// viper.BindPFlags(gcpCmd.Flags())
+}
+
+func setupAWS() error {
+	const provider = "aws"
+	const providerVersion = "0.1.0"
+
+	// Set provider config.
+	viper.Set("provider", provider)
+
+	// Setup the provider's image.
+	image := "gcr.io/kuda-project/provider-" + provider + ":" + providerVersion
+	viper.Set("image", image)
+
+	// Setup the volume mounting for the credentials.
+	home, err := homedir.Dir()
+	if err != nil {
+		fmt.Println(err)
+	}
+	volumeSecret := docker.VolumeMapping{
+		From: home + "/.aws",
+		To:   "/aws-credentials/",
+	}
+	viper.Set("volumes", []docker.VolumeMapping{volumeSecret})
+
+	Setup()
+
+	return nil
+}

providers/aws/.config.sh

@@ -0,0 +1,13 @@
+set -e
+
+echo
+echo -e "\e[1m \e[34mKuda AWS provider \e[36mv$(cat /kuda_cmd/VERSION) \e[0m"
+echo
+
+# Set custom path to AWS config file
+export AWS_CONFIG_FILE='/aws-credentials/config'
+export AWS_SHARED_CREDENTIALS_FILE='/aws-credentials/credentials'
+
+# Set KUDA envs.
+export KUDA_AWS_CLUSTER_NAME="${KUDA_CLUSTER_NAME:-kuda}"
+export KUDA_AWS_CLUSTER_REGION="${KUDA_CLUSTER_REGION:-eu-west-1}"

providers/aws/Dockerfile

@@ -0,0 +1,51 @@
+FROM alpine:3.10
+
+RUN apk add --update-cache --no-cache \
+  bash \
+  build-base \
+  curl \
+  python \
+  python-dev \
+  py-pip \
+  openssl \
+  jq
+
+# Install AWS CLI
+RUN pip install awscli
+
+# Install eksctl
+RUN curl --silent \
+  --location "https://github.com/weaveworks/eksctl/releases/download/latest_release/eksctl_$(uname -s)_amd64.tar.gz" \
+  | tar xz -C /tmp
+RUN mv /tmp/eksctl /usr/local/bin
+
+# Install kubectl
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \
+    chmod +x ./kubectl && \
+    mv ./kubectl /usr/local/bin/kubectl
+
+# Install Helm
+RUN curl -L https://git.io/get_helm.sh | bash
+
+# Install Ksync.
+RUN curl https://vapor-ware.github.io/gimme-that/gimme.sh | bash
+
+# Download Istio CRDs
+RUN curl -L https://git.io/getLatestIstio | sh -
+
+# Copy the provider's commands.
+COPY . /kuda_cmd
+ENV KUDA_CMD_DIR /kuda_cmd
+RUN chmod +x /kuda_cmd/*.sh
+RUN ln -s /kuda_cmd/app_deploy.sh /usr/local/bin/kuda_app_deploy && \
+  ln -s /kuda_cmd/app_delete.sh /usr/local/bin/kuda_app_delete && \
+  ln -s /kuda_cmd/setup.sh /usr/local/bin/kuda_setup && \
+  ln -s /kuda_cmd/delete.sh /usr/local/bin/kuda_delete && \
+  ln -s /kuda_cmd/dev_start.sh /usr/local/bin/kuda_dev_start && \
+  ln -s /kuda_cmd/dev_stop.sh /usr/local/bin/kuda_dev_stop && \
+  ln -s /kuda_cmd/get.sh /usr/local/bin/kuda_get
+
+# Go to the app home.
+WORKDIR /app_home
+
+ENTRYPOINT ["/bin/bash"]

providers/aws/README.md

@@ -0,0 +1,40 @@
+## Amazon Web Service Provider
+
+Hacky & bare implementation with shell scripts.
+
+# Status
+
+| Command | Status |
+| - | - |
+| setup | ✔ |
+| delete | ✔ |
+| get | Not Started |
+| app deploy | WIP |
+| app delete | ✔ |
+| dev start | Not Started |
+| dev stop | Not Started |
+
+# Configuration
+
+**Prerequisites:**
+- You must have subscribed to [EKS-optimize AMI with GPU support](https://aws.amazon.com/marketplace/pp/B07GRHFXGM)
+- You must have an increased limit of at least 1 instance of type p2.xlarge. You can make requests [here](http://aws.amazon.com/contact-us/ec2-request)
+- You must have an [aws configuration](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html) on your local machine in `~/.aws/` with credentials that have authorization for:
+    - cloudformation
+    - ec2
+    - eks
+    - iam
+    - api
+    - ec2 autoscaling
+    - ecr
+
+# Configuration
+
+You can override the following settings by adding them as flags of the `kuda setup` command (ex: `kuda setup aws ... --aws_...=mycluster`).
+
+| Parameter | Default | Description |
+| - | - | - |
+| | | |
+
+
+# Limitations

providers/aws/VERSION

@@ -0,0 +1 @@
+0.1.0

providers/aws/app_delete.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+source $KUDA_CMD_DIR/.config.sh
+
+aws_account_id="$(aws sts get-caller-identity | jq -r .Account)"
+app_name=$(echo $1 | cut -f1 -d':')
+
+# Retrieve cluster token.
+aws eks update-kubeconfig --name $KUDA_AWS_CLUSTER_NAME --region $KUDA_AWS_CLUSTER_REGION
+
+# Delete Repository.
+kubectl delete ksvc $app_name
+
+# Delete Repository.
+aws ecr delete-repository \
+  --repository-name $app_name \
+  --region $KUDA_AWS_CLUSTER_REGION \
+  --force

providers/aws/app_deploy.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+
+source $KUDA_CMD_DIR/.config.sh
+
+aws_account_id="$(aws sts get-caller-identity | jq -r .Account)"
+app_name=$(echo $1 | cut -f1 -d':')
+app_version=$(echo $1 | cut -f2 -d':')
+app_registry="$aws_account_id.dkr.ecr.$KUDA_AWS_CLUSTER_REGION.amazonaws.com/$app_name"
+app_image="$app_registry:$app_version"
+
+echo $app_image
+
+# Create Container Registry if it doesn't exists.
+if [ -z "$(aws ecr describe-repositories --region $KUDA_AWS_CLUSTER_REGION --repository-name $app_name)" ]; then
+  aws ecr create-repository \
+    --repository-name $app_name \
+    --region $KUDA_AWS_CLUSTER_REGION
+else
+  echo "Container Registry $app_registry already exists"
+fi
+
+#TODO: Build image.
+
+
+# Login Docker.
+aws ecr get-login --region $KUDA_AWS_CLUSTER_REGION --no-include-email | bash
+
+#TODO: Push image.
+
+
+# Launch.
+cat <<EOF | kubectl apply -f -
+apiVersion: serving.knative.dev/v1alpha1
+kind: Service
+metadata:
+  name: $app_name
+  namespace: default
+spec:
+  template:
+    spec:
+      containers:
+        - image: $app_image
+          resources:
+            limits:
+              nvidia.com/gpu: 1
+EOF

providers/aws/delete.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+source $KUDA_CMD_DIR/.config.sh
+
+eksctl delete cluster \
+--name=$KUDA_CLUSTER_NAME \
+--region=$KUDA_CLUSTER_REGION \
+--wait

providers/aws/dev_start.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+source $KUDA_CMD_DIR/.config.sh

providers/aws/dev_stop.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+source $KUDA_CMD_DIR/.config.sh

providers/aws/get.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -e
+
+source $KUDA_CMD_DIR/.config.sh