Add a parameter to helm to use prelaunched vineyar DaemonSet. (#204)

Signed-off-by: Tao He <[email protected]>

Author: sighingnow

charts/graphscope/templates/coordinator.yaml

@@ -108,6 +108,8 @@ spec:
         - {{ .Values.engines.gie.graph_manager.resources.requests.cpu | quote }}
         - "--k8s_gie_graph_manager_mem"
         - {{ .Values.engines.gie.graph_manager.resources.requests.memory }}
+        - "--k8s_vineyard_daemonset"
+        - {{ .Values.vineyard.daemonset }}
         - "--k8s_vineyard_cpu"
         - {{ .Values.vineyard.resources.requests.cpu | quote }}
         - "--k8s_vineyard_mem"

charts/graphscope/values.yaml

@@ -84,6 +84,12 @@ engines:
 
 
 vineyard:
+  # When `vineyard.daemonset` is set to the Helm release name, the coordinator will
+  # tries to discover the vineyard DaemonSet in current namespace, then use it if
+  # found, and fallback to bundled vineyard container otherwise.
+  #
+  # The vineyard IPC socket is placed on host at /var/run/vineyard-{namespace}-{release}.
+  daemonset: ""
   image:
     name: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope
     # Overrides the image tag whose default is the chart appVersion.

coordinator/gscoordinator/cluster.py

@@ -170,6 +170,7 @@ def __init__(
         gie_graph_manager_mem=None,
         engine_cpu=None,
         engine_mem=None,
+        vineyard_daemonset=None,
         vineyard_cpu=None,
         vineyard_mem=None,
         vineyard_shared_mem=None,
@@ -203,12 +204,6 @@ def __init__(
         self._gie_graph_manager_name = (
             self._gie_graph_manager_name_prefix + self._instance_id
         )
-        self._gie_graph_manager_service_name = (
-            self._gie_graph_manager_service_name_prefix + self._instance_id
-        )
-        self._vineyard_service_name = (
-            self._vineyard_service_name_prefix + self._instance_id
-        )
 
         self._namespace = namespace
         self._service_type = service_type
@@ -226,6 +221,7 @@ def __init__(
         self._engine_mem = engine_mem
 
         # vineyard container info
+        self._vineyard_daemonset = vineyard_daemonset
         self._vineyard_cpu = vineyard_cpu
         self._vineyard_mem = vineyard_mem
         self._vineyard_shared_mem = vineyard_shared_mem
@@ -278,6 +274,17 @@ def __init__(
         self._graphlearn_services = dict()
         self._learning_instance_processes = {}
 
+        # component service name
+        self._gie_graph_manager_service_name = (
+            self._gie_graph_manager_service_name_prefix + self._instance_id
+        )
+        if self._exists_vineyard_daemonset(self._vineyard_daemonset):
+            self._vineyard_service_name = self._vineyard_daemonset + "-rpc"
+        else:
+            self._vineyard_service_name = (
+                self._vineyard_service_name_prefix + self._instance_id
+            )
+
     def __del__(self):
         self.stop()
 
@@ -318,11 +325,21 @@ def _create_engine_replicaset(self):
         )
         # volume1 is for vineyard ipc socket
         # MaxGraph: /home/maxgraph/data/vineyard
+        if self._exists_vineyard_daemonset(self._vineyard_daemonset):
+            vineyard_socket_volume_type = "hostPath"
+            vineyard_socket_volume_fields = {
+                "type": "Directory",
+                "path": "/var/run/vineyard-%s-%s"
+                % (self._namespace, self._vineyard_daemonset),
+            }
+        else:
+            vineyard_socket_volume_type = "emptyDir"
+            vineyard_socket_volume_fields = {}
         engine_builder.add_volume(
             VolumeBuilder(
                 name="vineyard-ipc-volume",
-                type="emptyDir",
-                field={},
+                type=vineyard_socket_volume_type,
+                field=vineyard_socket_volume_fields,
                 mounts_list=[
                     {"mountPath": "/tmp/vineyard_workspace"},
                     {"mountPath": "/home/maxgraph/data/vineyard"},
@@ -347,16 +364,17 @@ def _create_engine_replicaset(self):
         # add env
         engine_builder.add_simple_envs({"GLOG_v": str(self._glog_level)})
         # add vineyard container
-        engine_builder.add_vineyard_container(
-            name=self._vineyard_container_name,
-            image=self._gs_image,
-            cpu=self._vineyard_cpu,
-            mem=self._vineyard_mem,
-            shared_mem=self._vineyard_shared_mem,
-            preemptive=self._preemptive,
-            etcd_endpoint=self._etcd_endpoint,
-            port=self._vineyard_service_port,
-        )
+        if not self._exists_vineyard_daemonset(self._vineyard_daemonset):
+            engine_builder.add_vineyard_container(
+                name=self._vineyard_container_name,
+                image=self._gs_image,
+                cpu=self._vineyard_cpu,
+                mem=self._vineyard_mem,
+                shared_mem=self._vineyard_shared_mem,
+                preemptive=self._preemptive,
+                etcd_endpoint=self._etcd_endpoint,
+                port=self._vineyard_service_port,
+            )
         # add engine container
         engine_builder.add_engine_container(
             name=self._engine_container_name,
@@ -606,7 +624,8 @@ def _create_services(self):
         logger.info("Etcd is ready, endpoint is {}".format(self._etcd_endpoint))
 
         self._create_engine_replicaset()
-        self._create_vineyard_service()
+        if not self._exists_vineyard_daemonset(self._vineyard_daemonset):
+            self._create_vineyard_service()
 
     def _waiting_for_services_ready(self):
         start_time = time.time()
@@ -796,6 +815,17 @@ def _delete_dangling_coordinator(self):
                             )
                         )
 
+    def _exists_vineyard_daemonset(self, release):
+        # check if vineyard daemonset exists.
+        if not release:
+            return False
+        try:
+            self._app_api.read_namespaced_daemon_set(release, self._namespace)
+        except K8SApiException:
+            return False
+        else:
+            return True
+
     def start(self):
         try:
             self._create_services()

coordinator/gscoordinator/coordinator.py

@@ -818,6 +818,12 @@ def parse_sys_args():
         default="graphscope",
         help="A list of secret name, comma separated.",
     )
+    parser.add_argument(
+        "--k8s_vineyard_daemonset",
+        type=str,
+        default="",
+        help="Try to use the existing vineyard DaemonSet with name 'k8s_vineyard_daemonset'.",
+    )
     parser.add_argument(
         "--k8s_vineyard_cpu",
         type=float,
@@ -950,6 +956,7 @@ def launch_graphscope():
             gie_graph_manager_mem=args.k8s_gie_graph_manager_mem,
             engine_cpu=args.k8s_engine_cpu,
             engine_mem=args.k8s_engine_mem,
+            vineyard_daemonset=args.k8s_vineyard_daemonset,
             vineyard_cpu=args.k8s_vineyard_cpu,
             vineyard_mem=args.k8s_vineyard_mem,
             vineyard_shared_mem=args.k8s_vineyard_shared_mem,

python/graphscope/client/session.py

@@ -168,6 +168,7 @@ def __init__(
         k8s_zookeeper_mem=gs_config.k8s_zookeeper_mem,
         k8s_gie_graph_manager_cpu=gs_config.k8s_gie_graph_manager_cpu,
         k8s_gie_graph_manager_mem=gs_config.k8s_gie_graph_manager_mem,
+        k8s_vineyard_daemonset=gs_config.k8s_vineyard_daemonset,
         k8s_vineyard_cpu=gs_config.k8s_vineyard_cpu,
         k8s_vineyard_mem=gs_config.k8s_vineyard_mem,
         k8s_vineyard_shared_mem=gs_config.k8s_vineyard_shared_mem,
@@ -218,6 +219,10 @@ def __init__(
 
             k8s_zookeeper_image (str, optional): The image of zookeeper, which used by GIE graph manager.
 
+            k8s_vineyard_daemonset (str, optional): The name of vineyard Helm deployment to use. GraphScope will try to
+                discovery the daemonset from kubernetes cluster, then use it if exists, and fallback to launching
+                a bundled vineyard container otherwise.
+
             k8s_vineyard_cpu (float, optional): Minimum number of CPU cores request for vineyard container. Defaults to 0.5.
 
             k8s_vineyard_mem (str, optional): Minimum number of memory request for vineyard container. Defaults to '512Mi'.
@@ -355,6 +360,7 @@ def __init__(
             "k8s_zookeeper_mem",
             "k8s_gie_graph_manager_cpu",
             "k8s_gie_graph_manager_mem",
+            "k8s_vineyard_daemonset",
             "k8s_vineyard_cpu",
             "k8s_vineyard_mem",
             "k8s_vineyard_shared_mem",
@@ -729,6 +735,7 @@ def _connect(self):
                 zookeeper_image=self._config_params["k8s_zookeeper_image"],
                 image_pull_policy=self._config_params["k8s_image_pull_policy"],
                 image_pull_secrets=self._config_params["k8s_image_pull_secrets"],
+                vineyard_daemonset=self._config_params["k8s_vineyard_daemonset"],
                 vineyard_cpu=self._config_params["k8s_vineyard_cpu"],
                 vineyard_mem=self._config_params["k8s_vineyard_mem"],
                 vineyard_shared_mem=self._config_params["k8s_vineyard_shared_mem"],

python/graphscope/config.py

@@ -64,6 +64,7 @@ class GSConfig(object):
     k8s_gie_gremlin_server_mem = "1Gi"
 
     # vineyard resource configuration
+    k8s_vineyard_daemonset = ""
     k8s_vineyard_cpu = 0.5
     k8s_vineyard_mem = "512Mi"
     k8s_vineyard_shared_mem = "4Gi"

python/graphscope/deploy/kubernetes/cluster.py

@@ -110,6 +110,9 @@ class KubernetesCluster(object):
         gie_graph_manager_mem: str
             Minimum number of memory request for graph manager container.
 
+        vineyard_daemonset: str
+            The name of Helm deployment for vineyard DaemonSet.
+
         vineyard_cpu: float
             Minimum number of CPU cores request for vineyard container.
 
@@ -174,6 +177,7 @@ def __init__(
         zookeeper_image=None,
         image_pull_policy=None,
         image_pull_secrets=None,
+        vineyard_daemonset=None,
         vineyard_cpu=None,
         vineyard_mem=None,
         vineyard_shared_mem=None,
@@ -223,6 +227,7 @@ def __init__(
         self._gie_graph_manager_cpu = gie_graph_manager_cpu
         self._gie_graph_manager_mem = gie_graph_manager_mem
 
+        self._vineyard_daemonset = vineyard_daemonset
         self._vineyard_cpu = vineyard_cpu
         self._vineyard_mem = vineyard_mem
         self._vineyard_shared_mem = vineyard_shared_mem
@@ -464,6 +469,7 @@ def _create_coordinator(self):
             zookeeper_mem=self._zookeeper_mem,
             gie_graph_manager_cpu=self._gie_graph_manager_cpu,
             gie_graph_manager_mem=self._gie_graph_manager_mem,
+            vineyard_daemonset=self._vineyard_daemonset,
             vineyard_cpu=self._vineyard_cpu,
             vineyard_mem=self._vineyard_mem,
             vineyard_shared_mem=self._vineyard_shared_mem,

python/graphscope/deploy/kubernetes/resource_builder.py

@@ -1056,6 +1056,7 @@ def add_coordinator_container(
         zookeeper_mem,
         gie_graph_manager_cpu,
         gie_graph_manager_mem,
+        vineyard_daemonset,
         vineyard_cpu,
         vineyard_mem,
         vineyard_shared_mem,
@@ -1091,6 +1092,7 @@ def add_coordinator_container(
         self._zookeeper_mem = zookeeper_mem
         self._gie_graph_manager_cpu = gie_graph_manager_cpu
         self._gie_graph_manager_mem = gie_graph_manager_mem
+        self._vineyard_daemonset = vineyard_daemonset
         self._vineyard_cpu = vineyard_cpu
         self._vineyard_mem = vineyard_mem
         self._vineyard_shared_mem = vineyard_shared_mem
@@ -1200,6 +1202,8 @@ def build_container_command(self):
             str(self._gie_graph_manager_cpu),
             "--k8s_gie_graph_manager_mem",
             self._gie_graph_manager_mem,
+            "--k8s_vineyard_daemonset",
+            str(self._vineyard_daemonset),
             "--k8s_vineyard_cpu",
             str(self._vineyard_cpu),
             "--k8s_vineyard_mem",