Skip to content

Commit

Permalink
feat(inspector): Integrated proxy with kubernetes
Browse files Browse the repository at this point in the history
  • Loading branch information
@isala404
isala404 committed Mar 30, 2022
1 parent 6b645bb commit db10b66
Show file tree
Hide file tree
Showing 7 changed files with 185 additions and 54 deletions.
41 changes: 3 additions & 38 deletions control-plane/config/samples/lazykoala_v1alpha1_inspector.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,10 @@
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-1
name: sample-service
namespace: default
spec:
deploymentRef: service-1-a0b7dd98
serviceRef: service-1-a0b7dd98
deploymentRef: sample-service
serviceRef: sample-service
modelName: sample_model
namespace: default
---
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-2
namespace: default
spec:
deploymentRef: service-2-a0b7dd98
serviceRef: service-2-a0b7dd98
modelName: sample_model
namespace: default
---
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-3
namespace: default
spec:
deploymentRef: service-3-a0b7dd98
serviceRef: service-3-a0b7dd98
modelName: sample_model
namespace: default
---
apiVersion: lazykoala.isala.me/v1alpha1
kind: Inspector
metadata:
name: service-4
namespace: default
spec:
deploymentRef: service-4-a0b7dd98
serviceRef: service-4-a0b7dd98
modelName: sample_model
namespace: default
---

22 changes: 22 additions & 0 deletions control-plane/config/samples/loads.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
apiVersion: microsim.isala.me/v1alpha1
kind: LoadGenerator
metadata:
name: lazy-koala-load
namespace: default
spec:
simulationRef:
name: lazy-koala-simulation
namespace: default
# requestCount: 1
replicas: 50
# timeout: 15m
betweenDelay: 1s
requests:
- |
{"designation":"service_4","probability":97,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_6","probability":64,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_7","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_8","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_3","probability":53,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_1","probability":77,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_8","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_2","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_9","probability":90,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_3","probability":64,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_7","probability":56,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_4","probability":52,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_8","probability":94,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_3","probability":76,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_6","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":[{"designation":"service_4","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":208}}],"after":[{"type":"latency","args":{"delay":225}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]}
- |
{"designation":"service_4","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_2","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_3","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_2","probability":66,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_6","probability":64,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_8","probability":72,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_3","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_5","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_8","probability":68,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_2","probability":61,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_6","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_4","probability":50,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_5","probability":82,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_8","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_4","probability":95,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":[{"designation":"service_9","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":371}}],"after":[{"type":"latency","args":{"delay":153}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]}
- |
{"designation":"service_9","probability":60,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_2","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_1","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_8","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_6","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_8","probability":65,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_4","probability":98,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_5","probability":71,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_9","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_3","probability":51,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_2","probability":63,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_5","probability":56,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_4","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_2","probability":93,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_1","probability":59,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":[{"designation":"service_4","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":138}}],"after":[{"type":"latency","args":{"delay":214}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]}
- |
{"designation":"service_9","probability":79,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_5","probability":73,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_4","probability":92,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":88,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_7","probability":95,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_3","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_2","probability":74,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":75,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_8","probability":54,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_4","probability":90,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_6","probability":78,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_1","probability":69,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_5","probability":80,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_8","probability":87,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":[{"designation":"service_1","probability":85,"faults":{"before":[{"type":"latency","args":{"delay":400}}],"after":[{"type":"latency","args":{"delay":384}}]},"routes":null}]}]}]}]}]}]}]}]}]}]}]}]}]}]}]}
38 changes: 38 additions & 0 deletions control-plane/config/samples/simulation.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
apiVersion: microsim.isala.me/v1alpha1
kind: Simulation
metadata:
name: lazy-koala-simulation
namespace: default
spec:
name:
services:
service_1:
language: go
framework: gorilla
service_2:
language: go
framework: gorilla
service_3:
language: go
framework: gorilla
service_4:
language: go
framework: gorilla
service_5:
language: go
framework: gorilla
service_6:
language: go
framework: gorilla
service_7:
language: go
framework: gorilla
service_8:
language: go
framework: gorilla
service_9:
language: node
framework: express
service_10:
language: python
framework: flask
12 changes: 12 additions & 0 deletions gazer/deployment.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,18 @@ rules:
- get
- list
- watch
- apiGroups:
- lazykoala.isala.me
resources:
- inspectors
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
Expand Down
76 changes: 60 additions & 16 deletions inspector/main.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,14 @@
package main

import (
"crypto/tls"
"crypto/x509"
"embed"
"fmt"
"io/fs"
"io/ioutil"
"log"
"net"
"net/http"
"net/http/httputil"
"net/url"
Expand All @@ -12,7 +17,11 @@ import (
"strings"

"github.com/gorilla/mux"
"github.com/rs/cors"
)

const (
tokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
rootCAFile = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
)

//go:embed ui/*
Expand Down Expand Up @@ -44,19 +53,60 @@ func clientHandler() http.Handler {
}

func main() {
kube_endpoint, _ := url.Parse("http://127.0.0.1:8001/")
prom_endpoint, _ := url.Parse("http://127.0.0.1:9090/")

// Setup proxy data
token, err := ioutil.ReadFile(tokenFile)
if err != nil {
log.Println("failed to read the token from kubernetes secrets")
}
CAData, err := ioutil.ReadFile(rootCAFile)
if err != nil {
log.Println("failed to read the CA Data from kubernetes secrets")
}

// Get the SystemCertPool, continue with an empty pool on error
rootCAs, _ := x509.SystemCertPool()
if rootCAs == nil {
rootCAs = x509.NewCertPool()
}

// Append our cert to the system pool
if ok := rootCAs.AppendCertsFromPEM(CAData); !ok {
log.Println("failed to append k8s custom CA, using system certs only")
}

if err != nil {
log.Println("it seems like operator is not running in side cluster, kube-api proxy will not operate as intended")
}

proxyTransport := &http.Transport{TLSClientConfig: &tls.Config{
RootCAs: rootCAs,
}}

kube_endpoint, _ := url.Parse("https://" + net.JoinHostPort(os.Getenv("KUBERNETES_SERVICE_HOST"), os.Getenv("KUBERNETES_SERVICE_PORT")))

prom_endpoint, _ := url.Parse(os.Getenv("PROMETHEUS_ENDPOINT"))
r := mux.NewRouter()

r.PathPrefix("/k8s").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
log.Printf("Proxying %s Request from %s to KubeAPI on %s\n", r.Method, r.RemoteAddr, r.URL.Path)

w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Private-Network", "true")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
w.Header().Set("Access-Control-Allow-Methods", "OPTIONS, GET, POST, DELETE")
if r.Method == http.MethodOptions {
w.WriteHeader(204)
return
}
proxy := httputil.NewSingleHostReverseProxy(kube_endpoint)
proxy.Transport = proxyTransport
r.Header.Add("Authorization", fmt.Sprintf("Bearer %s", string(token)))

r.URL.Path = "/" + strings.Join(strings.Split(r.URL.Path, "/")[2:], "/")

log.Printf("Proxying Request from %s to KubeAPI on %s\n", r.RemoteAddr, r.URL.Path)

proxy.ServeHTTP(w, r)
})
}).Methods(http.MethodGet, http.MethodPost, http.MethodDelete, http.MethodOptions)

r.PathPrefix("/prom").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
proxy := httputil.NewSingleHostReverseProxy(prom_endpoint)
Expand All @@ -66,22 +116,16 @@ func main() {
log.Printf("Proxying Request from %s to prometheus on %s\n", r.RemoteAddr, r.URL.Path)

proxy.ServeHTTP(w, r)
})
// fmt.Printf("proxy.ModifyResponse: %v\n", proxy.ModifyResponse)
}).Methods(http.MethodGet, http.MethodOptions)

r.PathPrefix("/").Handler(clientHandler())

corsOpts := cors.New(cors.Options{
AllowedOrigins: []string{"*"},
AllowedMethods: []string{
http.MethodGet,
http.MethodDelete,
http.MethodPost,
},
})
r.Use(mux.CORSMethodMiddleware(r))

log.Println("Client app started on port :8090")

if err := http.ListenAndServe(":8090", corsOpts.Handler(r)); err != http.ErrServerClosed {
if err := http.ListenAndServe(":8090", r); err != http.ErrServerClosed {
log.Fatal("Receiver webserver crashed: %s", err)
os.Exit(1)
}
Expand Down
1 change: 1 addition & 0 deletions scripts/.gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
routes.json
49 changes: 49 additions & 0 deletions scripts/loadTemplete.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import json
import random

services = list(range(1,10))

random.shuffle(services)

templete = {
"designation": None,
"probability": 50,
"faults": {
"before": [{"type": "latency", "args": { "delay": 600 }}],
"after": [{"type": "latency", "args": { "delay": 600 }}]
},
"routes": None
}


def create_route(k, last_designation=None):
path = None

if k <= 15:
k+=1
route = templete.copy()
route['designation'] = f"service_{services[random.randint(0,len(services)-1)]}"
while route['designation'] == last_designation:
route['designation'] = f"service_{services[random.randint(0,len(services)-1)]}"
route['probability'] = random.randint(50, 100)
route['faults']['before'][0]['args']['delay'] = random.randint(100, 400)
route['faults']['after'][0]['args']['delay'] = random.randint(100, 400)
route['routes'] = [create_route(k, route['designation'])]
if route['routes'][0] == None:
route['routes'] = None
# for i in range(random.randint(0,2)):
# k+=1
# sub_route = create_route(k)
# if sub_route:
# route['routes'].append(sub_route)
# else:
# route['routes'] = None
# break

path = route

return path

out_file = open("routes.json", "w")

json.dump(create_route(0), out_file, separators=(',', ':'))

0 comments on commit db10b66

Please sign in to comment.