Merge pull request #36 from OSS-MLOPS-PLATFORM/21-modular-install

21 modular install

.gitignore

@@ -62,4 +62,5 @@ tutorials/openstack/secure.yaml
 # Others
 old/
 istio*
-temp/
+temp/
+.platform/

CONTRIBUTORS.md

@@ -2,7 +2,7 @@
 
 This file lists all the individuals who have contributed to this project. Thanks to each and every one of you for your valuable contributions!
 
-### Silo AI:
+### [Silo AI](https://www.silo.ai/):
 
 Original project leads and main developers:
 
@@ -40,12 +40,14 @@ Other developers and testers of the platform:
 - Kristian Sikiric
 - Kaustav Tamuly
 - Jonathan Burdge
+- Ammar Aldhahyani
 
-### IML4E project and other contributors:
+### [IML4E](https://itea4.org/project/iml4e.html) project and other contributors:
 
-Univerwity of Helsinki:
+[University of Helsinki](https://www.helsinki.fi/en/researchgroups/empirical-software-engineering):
 
 - Niila Siilasjoki
+- Dennis Muiruri
 
 Fraunhofer Institute:
 

config.env

@@ -1,4 +1,2 @@
 HOST_IP="127.0.0.1"
-CLUSTER_NAME="kind-ep"
-INSTALL_LOCAL_REGISTRY="true"
-INSTALL_RAY="false"
+CLUSTER_NAME="mlops-platform"

deployment/README.md

@@ -1,7 +1,19 @@
 ## Deploy the stack
 
+Choose the deployment option that best fits your needs:
+1. `kubeflow-monitoring`: Full Kubeflow deployment with all components.
+2. `kubeflow`: Full Kubeflow deployment without monitoring components (prometheus, grafana).
+3. `standalone-kfp-monitoring`: Standalone KFP deployment.
+4. `standalone-kfp`: Standalone KFP deployment without monitoring components (prometheus, grafana).
+5. `standalone-kfp-kserve-monitoring`: Standalone KFP and Kserve deployment.
+6. `standalone-kfp-kserve`: Standalone KFP and Kserve deployment without monitoring components (prometheus, grafana).
+
+```bash
+export DEPLOYMENT_OPTION=kubeflow-monitoring
+```
+
 Deploy to your kubernetes cluster with the following command:
 
 ```bash
-while ! kustomize build deployment | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 10; done
+while ! kustomize build "deployment/envs/$DEPLOYMENT_OPTION" | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 10; done
 ```

deployment/kubeflow-custom/kserve-sa.yaml → deployment/custom/kserve-custom/base/kserve-sa.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: mysecret
-  namespace: kubeflow-user-example-com
   annotations:
      serving.kserve.io/s3-endpoint: mlflow-minio-service.mlflow.svc.cluster.local:9000
      serving.kserve.io/s3-usehttps: "0"
@@ -15,6 +14,5 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: kserve-sa
-  namespace: kubeflow-user-example-com
 secrets:
 - name: mysecret

deployment/kubeflow-custom/kustomization.yaml → deployment/custom/kserve-custom/base/kustomization.yaml

@@ -2,5 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-- aws-secret.yaml
 - kserve-sa.yaml

deployment/custom/kserve-custom/env/kubeflow/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow-user-example-com
+
+resources:
+- ../../base

deployment/custom/kserve-custom/env/standalone-kfp/kserve-inference-namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kserve-inference

deployment/custom/kserve-custom/env/standalone-kfp/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kserve-inference
+
+resources:
+  - ../../base
+  - kserve-inference-namespace.yaml

deployment/kubeflow-custom/aws-secret.yaml → deployment/custom/kubeflow-custom/base/aws-secret.yaml

@@ -2,7 +2,6 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: aws-secret
-  namespace: kubeflow-user-example-com
 type: Opaque
 data:
   # your BASE64 encoded AWS_ACCESS_KEY_ID

deployment/custom/kubeflow-custom/base/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- aws-secret.yaml

deployment/custom/kubeflow-custom/env/kubeflow/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow-user-example-com
+
+resources:
+- ../../base

deployment/custom/kubeflow-custom/env/standalone-kfp-kserve/kserve-deployer.yaml

@@ -0,0 +1,110 @@
+# Required for deploy model to have the necessery permissions to create inference services
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: kserve-deployer
+rules:
+  - verbs:
+      - '*'
+    apiGroups:
+      - ''
+    resources:
+      - secrets
+      - serviceaccounts
+  - verbs:
+      - get
+      - watch
+      - list
+    apiGroups:
+      - ''
+    resources:
+      - configmaps
+  - verbs:
+      - '*'
+    apiGroups:
+      - ''
+    resources:
+      - persistentvolumes
+      - persistentvolumeclaims
+  - verbs:
+      - create
+      - delete
+      - get
+    apiGroups:
+      - snapshot.storage.k8s.io
+    resources:
+      - volumesnapshots
+  - verbs:
+      - get
+      - list
+      - watch
+      - update
+      - patch
+    apiGroups:
+      - argoproj.io
+    resources:
+      - workflows
+  - verbs:
+      - '*'
+    apiGroups:
+      - ''
+    resources:
+      - pods
+      - pods/exec
+      - pods/log
+      - services
+  - verbs:
+      - '*'
+    apiGroups:
+      - ''
+      - apps
+      - extensions
+    resources:
+      - deployments
+      - replicasets
+  - verbs:
+      - '*'
+    apiGroups:
+      - kubeflow.org
+    resources:
+      - '*'
+  - verbs:
+      - '*'
+    apiGroups:
+      - batch
+    resources:
+      - jobs
+  - verbs:
+      - '*'
+    apiGroups:
+      - machinelearning.seldon.io
+    resources:
+      - seldondeployments
+  - verbs:
+      - '*'
+    apiGroups:
+      - serving.kserve.io
+    resources:
+      - '*'
+  - verbs:
+      - '*'
+    apiGroups:
+      - networking.istio.io
+    resources:
+      - '*'
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: pipeline-runner-binding-cluster
+  labels:
+    application-crd-id: kubeflow-pipelines
+subjects:
+  - kind: ServiceAccount
+    name: pipeline-runner
+    namespace: kubeflow
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kserve-deployer

deployment/custom/kubeflow-custom/env/standalone-kfp-kserve/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow
+
+resources:
+  - ../../base
+  - kserve-deployer.yaml

deployment/custom/kubeflow-custom/env/standalone-kfp/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kubeflow
+
+resources:
+  - ../../base

deployment/envs/kubeflow-monitoring/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../kubeflow/manifests/in-cluster-setup/kubeflow
+- ../../custom/kubeflow-custom/env/kubeflow
+- ../../custom/kserve-custom/env/kubeflow
+- ../../mlflow/env/local
+- ../../monitoring

deployment/envs/kubeflow/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../kubeflow/manifests/in-cluster-setup/kubeflow
+- ../../custom/kubeflow-custom/env/kubeflow
+- ../../custom/kserve-custom/env/kubeflow
+- ../../mlflow/env/local

deployment/envs/standalone-kfp-kserve-monitoring/kustomization.yaml

@@ -0,0 +1,9 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../kubeflow/manifests/in-cluster-setup/standalone-kfp-kserve
+- ../../custom/kubeflow-custom/env/standalone-kfp-kserve
+- ../../custom/kserve-custom/env/standalone-kfp
+- ../../mlflow/env/local
+- ../../monitoring

deployment/envs/standalone-kfp-kserve/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../kubeflow/manifests/in-cluster-setup/standalone-kfp-kserve
+- ../../custom/kubeflow-custom/env/standalone-kfp-kserve
+- ../../custom/kserve-custom/env/standalone-kfp
+- ../../mlflow/env/local

deployment/envs/standalone-kfp-monitoring/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../kubeflow/manifests/in-cluster-setup/standalone-kfp
+- ../../custom/kubeflow-custom/env/standalone-kfp
+- ../../mlflow/env/local
+- ../../monitoring

deployment/envs/standalone-kfp/kustomization.yaml

@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- ../../kubeflow/manifests/in-cluster-setup/standalone-kfp
+- ../../custom/kubeflow-custom/env/standalone-kfp
+- ../../mlflow/env/local

deployment/kubeflow/manifests/in-cluster-setup/kubeflow/README.md

@@ -0,0 +1,10 @@
+# Kubeflow
+
+Components:
+- Multiuser isolation
+- Central Dashboard
+- Jupyter Notebooks
+- Kubeflow Pipelines (KFP)
+- Kserve
+- Katib
+- TensorBoard