From b6e604ea9159b605e757f4c28086a3a62a9baceb Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Thu, 4 May 2023 16:49:28 +0300 Subject: [PATCH 1/3] Remove deployment dir Signed-off-by: Yury Kulazhenkov --- deployment/helm/TODO | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 deployment/helm/TODO diff --git a/deployment/helm/TODO b/deployment/helm/TODO deleted file mode 100644 index e69de29..0000000 From ee428a78ae7597fc1cd2b11a9410f56d063e0aa9 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Thu, 4 May 2023 16:49:50 +0300 Subject: [PATCH 2/3] Add single yaml with all resources This file defines all required objects for IPAM deployment Signed-off-by: Yury Kulazhenkov --- deploy/nv-ipam-node.yaml | 107 ------------------ deploy/nv-ipam.yaml | 234 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 234 insertions(+), 107 deletions(-) delete mode 100644 deploy/nv-ipam-node.yaml create mode 100644 deploy/nv-ipam.yaml diff --git a/deploy/nv-ipam-node.yaml b/deploy/nv-ipam-node.yaml deleted file mode 100644 index 8de5129..0000000 --- a/deploy/nv-ipam-node.yaml +++ /dev/null @@ -1,107 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-node -rules: - - apiGroups: - - "" - resources: - - nodes - verbs: - - get ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: nv-ipam-node -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nv-ipam-node -subjects: -- kind: ServiceAccount - name: nv-ipam-node - namespace: kube-system ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nv-ipam-node - namespace: kube-system ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: kube-nv-ipam-node-ds - namespace: kube-system - labels: - tier: node - app: nv-ipam-node - name: nv-ipam-node -spec: - selector: - matchLabels: - name: nv-ipam-node - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - tier: node - app: nv-ipam-node - name: nv-ipam-node - spec: - hostNetwork: true - tolerations: - - operator: Exists - effect: NoSchedule - - operator: Exists - effect: NoExecute - serviceAccountName: nv-ipam-node - containers: - - name: kube-nv-ipam-node - image: ghcr.io/mellanox/nvidia-k8s-ipam:latest - imagePullPolicy: IfNotPresent - env: - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - command: ["/ipam-node"] - resources: - requests: - cpu: "100m" - memory: "50Mi" - limits: - cpu: "100m" - memory: "50Mi" - securityContext: - privileged: true - volumeMounts: - - name: cnibin - mountPath: /host/opt/cni/bin - - name: cni - mountPath: /host/etc/cni/net.d - - name: hostlocalcnibin - mountPath: /host/var/lib/cni/nv-ipam/bin - - name: hostlocalcnistate - mountPath: /host/var/lib/cni/nv-ipam/state/host-local - terminationGracePeriodSeconds: 10 - volumes: - - name: cnibin - hostPath: - path: /opt/cni/bin - type: DirectoryOrCreate - - name: cni - hostPath: - path: /etc/cni/net.d - type: DirectoryOrCreate - - name: hostlocalcnibin - hostPath: - path: /var/lib/cni/nv-ipam/bin - type: DirectoryOrCreate - - name: hostlocalcnistate - hostPath: - path: /var/lib/cni/nv-ipam/state/host-local - type: DirectoryOrCreate diff --git a/deploy/nv-ipam.yaml b/deploy/nv-ipam.yaml new file mode 100644 index 0000000..bba20db --- /dev/null +++ b/deploy/nv-ipam.yaml @@ -0,0 +1,234 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nv-ipam-node +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nv-ipam-node +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nv-ipam-node +subjects: + - kind: ServiceAccount + name: nv-ipam-node + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nv-ipam-node + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-nv-ipam-node-ds + namespace: kube-system + labels: + tier: node + app: nv-ipam-node + name: nv-ipam-node +spec: + selector: + matchLabels: + name: nv-ipam-node + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + tier: node + app: nv-ipam-node + name: nv-ipam-node + spec: + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + - operator: Exists + effect: NoExecute + serviceAccountName: nv-ipam-node + containers: + - name: kube-nv-ipam-node + image: ghcr.io/mellanox/nvidia-k8s-ipam:latest + imagePullPolicy: IfNotPresent + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: [ "/ipam-node" ] + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + volumeMounts: + - name: cnibin + mountPath: /host/opt/cni/bin + - name: cni + mountPath: /host/etc/cni/net.d + - name: hostlocalcnibin + mountPath: /host/var/lib/cni/nv-ipam/bin + - name: hostlocalcnistate + mountPath: /host/var/lib/cni/nv-ipam/state/host-local + terminationGracePeriodSeconds: 10 + volumes: + - name: cnibin + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + - name: hostlocalcnibin + hostPath: + path: /var/lib/cni/nv-ipam/bin + type: DirectoryOrCreate + - name: hostlocalcnistate + hostPath: + path: /var/lib/cni/nv-ipam/state/host-local + type: DirectoryOrCreate +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nv-ipam-controller +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nv-ipam-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: nv-ipam-controller +subjects: + - kind: ServiceAccount + name: nv-ipam-controller + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nv-ipam-controller + namespace: kube-system +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nv-ipam-controller + namespace: kube-system + annotations: + kubernetes.io/description: | + This deployment launches the nv-ipam controller for nv-ipam. +spec: + strategy: + type: RollingUpdate + replicas: 1 + selector: + matchLabels: + name: nv-ipam-controller + template: + metadata: + labels: + name: nv-ipam-controller + spec: + priorityClassName: system-cluster-critical + serviceAccountName: nv-ipam-controller + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + - weight: 1 + preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: nv-ipam-controller + image: ghcr.io/mellanox/nvidia-k8s-ipam:latest + imagePullPolicy: IfNotPresent + command: [ "/ipam-controller" ] + args: + - --config-name=nvidia-k8s-ipam-config + - --config-namespace=$(POD_NAMESPACE) + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + requests: + cpu: 100m + memory: 300Mi From e676dbee06ad8e1c1dc54b1e7d29d1681df60e77 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Thu, 4 May 2023 16:50:05 +0300 Subject: [PATCH 3/3] Add example configuration file for IPAM contrller Signed-off-by: Yury Kulazhenkov --- examples/configmap.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 examples/configmap.yaml diff --git a/examples/configmap.yaml b/examples/configmap.yaml new file mode 100644 index 0000000..bb96fcd --- /dev/null +++ b/examples/configmap.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nvidia-k8s-ipam-config + namespace: kube-system +data: + config: | + { + "pools": { + "pool1": { "subnet": "192.168.0.0/16", "perNodeBlockSize": 100 , "gateway": "192.168.0.1"}, + "pool2": { "subnet": "172.16.0.0/16", "perNodeBlockSize": 50 , "gateway": "172.16.0.1"} + }, + "nodeSelector": {"kubernetes.io/os": "linux"} + }