Deprecated 'k8s_minikube_vm_driver' parameter in session. (#129)

* Deprecated support minikube deployed with vm-driver

* Remove all show_log=True in session

* GraphScope require protobuf>=3.12.0

interactive_engine/tests/function_test.sh

@@ -32,8 +32,9 @@ function _start {
     cd $curdir/../deploy/testing && python3 maxgraph_test_server.py ${_port} &
     sleep 5s
     curl -XPOST http://localhost:${_port} -d 'import graphscope'
+    curl -XPOST http://localhost:${_port} -d 'graphscope.set_option(show_log=True)'
     curl -XPOST http://localhost:${_port} -d 'from graphscope.framework.loader import Loader'
-    curl_sess="curl -XPOST http://localhost:${_port} -d 'session = graphscope.session(num_workers=${workers}, show_log=True, k8s_volumes={\"data\": {\"type\": \"hostPath\", \"field\": {\"path\": \"${GS_TEST_DIR}\", \"type\": \"Directory\"}, \"mounts\": {\"mountPath\": \"/testingdata\"}}}, k8s_coordinator_cpu=1.0, k8s_coordinator_mem='\''4Gi'\'', k8s_vineyard_cpu=1.0, k8s_vineyard_mem='\''4Gi'\'', k8s_vineyard_shared_mem='\''4Gi'\'', k8s_engine_cpu=1.0, k8s_engine_mem='\''4Gi'\'', k8s_gie_graph_manager_image='\''${gie_manager_image}'\'', k8s_gs_image='\''${gs_image}'\'')' --write-out %{http_code} --silent --output ./curl.tmp"
+    curl_sess="curl -XPOST http://localhost:${_port} -d 'session = graphscope.session(num_workers=${workers}, k8s_volumes={\"data\": {\"type\": \"hostPath\", \"field\": {\"path\": \"${GS_TEST_DIR}\", \"type\": \"Directory\"}, \"mounts\": {\"mountPath\": \"/testingdata\"}}}, k8s_coordinator_cpu=1.0, k8s_coordinator_mem='\''4Gi'\'', k8s_vineyard_cpu=1.0, k8s_vineyard_mem='\''4Gi'\'', k8s_vineyard_shared_mem='\''4Gi'\'', k8s_engine_cpu=1.0, k8s_engine_mem='\''4Gi'\'', k8s_gie_graph_manager_image='\''${gie_manager_image}'\'', k8s_gs_image='\''${gs_image}'\'')' --write-out %{http_code} --silent --output ./curl.tmp"
 
     echo $curl_sess
     code=`sh -c "$curl_sess"`

python/graphscope/client/session.py

@@ -271,10 +271,7 @@ def __init__(
             k8s_waiting_for_delete (bool, optional): Waiting for service delete or not. Defaults to False.
 
             **kw (dict, optional): Other optional parameters will be put to :code:`**kw`.
-                - k8s_minikube_vm_driver (bool, optional):
-                    If your kubernetes cluster deployed on inner virtual machine
-                    (such as minikube with param --vm-driver is not None), you can specify
-                    :code:`k8s_minikube_vm_driver` is :code:`True`.
+                - k8s_minikube_vm_driver: Deprecated.
 
                 - k8s_client_config (dict, optional):
                     Provide configurable parameters for connecting to remote k8s,
@@ -364,11 +361,6 @@ def __init__(
                 category=DeprecationWarning,
             )
 
-        # deploy minikube on virtual machine
-        self._config_params["k8s_minikube_vm_driver"] = kw.pop(
-            "k8s_minikube_vm_driver", False
-        )
-
         # update k8s_client_config params
         self._config_params["k8s_client_config"] = kw.pop("k8s_client_config", {})
 
@@ -678,7 +670,6 @@ def _connect(self):
                 api_client=api_client,
                 namespace=self._config_params["k8s_namespace"],
                 service_type=self._config_params["k8s_service_type"],
-                minikube_vm_driver=self._config_params["k8s_minikube_vm_driver"],
                 num_workers=self._config_params["num_workers"],
                 gs_image=self._config_params["k8s_gs_image"],
                 etcd_image=self._config_params["k8s_etcd_image"],
@@ -742,10 +733,6 @@ def _connect(self):
                     pass
             raise
 
-        if self._config_params["enable_k8s"]:
-            # minikube service
-            self._k8s_cluster.check_and_set_vineyard_rpc_endpoint(self._engine_config)
-
         return proc, endpoint
 
     def get_config(self):

python/graphscope/deploy/kubernetes/cluster.py

@@ -44,7 +44,6 @@
 from graphscope.deploy.kubernetes.utils import KubernetesPodWatcher
 from graphscope.deploy.kubernetes.utils import delete_kubernetes_object
 from graphscope.deploy.kubernetes.utils import get_service_endpoints
-from graphscope.deploy.kubernetes.utils import is_minikube_cluster
 from graphscope.deploy.kubernetes.utils import try_to_read_namespace_from_context
 from graphscope.deploy.kubernetes.utils import wait_for_deployment_complete
 from graphscope.framework.errors import K8sError
@@ -66,9 +65,6 @@ class KubernetesCluster(object):
         service_type: str, optional
             Type determines how the GraphScope service is exposed.
 
-        minikube_vm_driver: bool, optional
-            True if minikube cluster :code:`--vm-driver` is not :code:`None`
-
         num_workers: int
             Number of workers to launch graphscope engine.
 
@@ -161,7 +157,6 @@ def __init__(
         api_client=None,
         namespace=None,
         service_type=None,
-        minikube_vm_driver=None,
         num_workers=None,
         gs_image=None,
         etcd_image=None,
@@ -194,7 +189,6 @@ def __init__(
 
         self._namespace = namespace
         self._service_type = service_type
-        self._minikube_vm_driver = minikube_vm_driver
         self._gs_image = gs_image
         self._num_workers = num_workers
         self._etcd_image = etcd_image
@@ -262,12 +256,6 @@ def get_namespace(self):
         """
         return self._namespace
 
-    def check_and_set_vineyard_rpc_endpoint(self, engine_config):
-        if is_minikube_cluster() and self._minikube_vm_driver:
-            engine_config["vineyard_rpc_endpoint"] = self._get_minikube_service(
-                self._namespace, engine_config["vineyard_service_name"]
-            )
-
     def _get_free_namespace(self):
         while True:
             namespace = "gs-" + random_string(6)
@@ -515,53 +503,7 @@ def _waiting_for_services_ready(self):
             for pod_watcher in self._coordinator_pods_watcher:
                 pod_watcher.stop()
 
-    def _get_minikube_service(self, namespace, service_name):
-        def minikube_get_service_url(process, rlt, messages):
-            for line in process.stdout:
-                line = line.decode("utf-8")
-                messages.append(line)
-                for match in re.finditer(self._url_pattern, line):
-                    rlt.append(match.group())
-                    return
-
-        cmd = ["minikube", "service", service_name, "-n", namespace, "--url"]
-        process = subprocess.Popen(
-            cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
-        )
-        rlt = []
-        messages = []
-        t = threading.Thread(
-            target=minikube_get_service_url,
-            args=(
-                process,
-                rlt,
-                messages,
-            ),
-        )
-        t.start()
-        try:
-            # 10 seconds is enough
-            t.join(timeout=10)
-        except:  # noqa: E722
-            pass
-        process.terminate()
-
-        if not rlt:
-            raise RuntimeError(
-                "Minikube get service error: {}".format("".join(messages))
-            )
-
-        endpoint_match = re.search(self._endpoint_pattern, rlt[0])
-        return "{}:{}".format(
-            endpoint_match.group("host"), endpoint_match.group("port")
-        )
-
     def _get_coordinator_endpoint(self):
-        if is_minikube_cluster() and self._minikube_vm_driver:
-            return self._get_minikube_service(
-                self._namespace, self._coordinator_service_name
-            )
-
         # Always len(endpoints) >= 1
         endpoints = get_service_endpoints(
             api_client=self._api_client,

python/graphscope/deploy/kubernetes/utils.py

@@ -48,12 +48,6 @@ def parse_readable_memory(value):
     return value
 
 
-def is_minikube_cluster():
-    contexts, active_context = kube_config.list_kube_config_contexts()
-    if contexts:
-        return active_context["context"]["cluster"] == "minikube"
-
-
 def try_to_read_namespace_from_context():
     contexts, active_context = kube_config.list_kube_config_contexts()
     if contexts and "namespace" in active_context["context"]:

python/graphscope/deploy/tests/test_demo_script.py

@@ -268,7 +268,6 @@ def test_traversal_modern_graph(modern_graph_data_dir):
 
     gs_image, gie_manager_image = get_gs_image_on_ci_env()
     sess = graphscope.session(
-        show_log=True,
         num_workers=1,
         k8s_gs_image=gs_image,
         k8s_gie_graph_manager_image=gie_manager_image,

python/requirements.txt

@@ -7,7 +7,7 @@ nest_asyncio
 networkx
 numpy
 pandas
-protobuf
+protobuf>=3.12.0
 PyYAML
 scipy
 vineyard==0.1.8

python/tests/conftest.py

@@ -33,7 +33,8 @@
 
 @pytest.fixture(scope="module")
 def graphscope_session():
-    sess = graphscope.session(run_on_local=True, show_log=True)
+    graphscope.set_option(show_log=True)
+    sess = graphscope.session(run_on_local=True)
     yield sess
     sess.close()
 

python/tests/test_scalability.py

@@ -28,7 +28,8 @@
 
 def p2p_property_graph(num_workers, directed=True):
     data_dir = os.path.expandvars("${GS_TEST_DIR}/property")
-    sess = graphscope.session(show_log=True, num_workers=num_workers, run_on_local=True)
+    graphscope.set_option(show_log=True)
+    sess = graphscope.session(num_workers=num_workers, run_on_local=True)
 
     g = sess.load_from(
         edges={

python/tests/test_session.py

@@ -33,6 +33,10 @@
 new_data_dir = os.path.expandvars("${GS_TEST_DIR}/new_property/v2_e2")
 
 
+def setUpModule():
+    graphscope.set_option(show_log=True)
+
+
 @pytest.fixture
 def invalid_config_file():
     with tempfile.TemporaryDirectory() as dir_name:
@@ -120,15 +124,15 @@ def load_graph(session):
 
 
 def test_default_session():
-    s = graphscope.session(run_on_local=True, show_log=True)
+    s = graphscope.session(run_on_local=True)
 
     info = s.info
     assert info["status"] == "active"
     s.close()
 
 
 def test_launch_cluster_on_local(local_config_file):
-    s = graphscope.session(run_on_local=True, show_log=True, config=local_config_file)
+    s = graphscope.session(run_on_local=True, config=local_config_file)
     info = s.info
     assert info["status"] == "active"
     s.close()
@@ -138,7 +142,7 @@ def test_launch_session_from_config(local_config_file):
     saved = os.environ.get("GS_CONFIG_PATH", "")
     try:
         os.environ["GS_CONFIG_PATH"] = local_config_file
-        s = graphscope.session(run_on_local=True, show_log=True)
+        s = graphscope.session(run_on_local=True)
 
         info = s.info
         assert info["status"] == "active"
@@ -149,17 +153,15 @@ def test_launch_session_from_config(local_config_file):
 
 def test_launch_session_from_dict():
     conf_dict = {"num_workers": 4}
-    s = graphscope.session(run_on_local=True, show_log=True, config=conf_dict)
+    s = graphscope.session(run_on_local=True, config=conf_dict)
 
     info = s.info
     assert info["status"] == "active"
     s.close()
 
 
 def test_config_dict_has_highest_priority(local_config_file):
-    s = graphscope.session(
-        run_on_local=True, show_log=True, config=local_config_file, num_workers=2
-    )
+    s = graphscope.session(run_on_local=True, config=local_config_file, num_workers=2)
 
     info = s.info
     assert info["status"] == "active"
@@ -168,20 +170,18 @@ def test_config_dict_has_highest_priority(local_config_file):
 
 def test_error_on_config_file_not_exist():
     with pytest.raises(FileNotFoundError, match="No such file or directory"):
-        graphscope.session(
-            run_on_local=True, show_log=True, config="~/non_existing_filename.txt"
-        )
+        graphscope.session(run_on_local=True, config="~/non_existing_filename.txt")
 
 
 def test_error_on_invalid_config_file(invalid_config_file):
     # invalid config file (example json format incorrect)
     with pytest.raises(json.decoder.JSONDecodeError):
-        graphscope.session(run_on_local=True, show_log=True, config=invalid_config_file)
+        graphscope.session(run_on_local=True, config=invalid_config_file)
 
 
 def test_error_on_used_after_close():
     # use after session close
-    s1 = graphscope.session(run_on_local=True, show_log=True)
+    s1 = graphscope.session(run_on_local=True)
 
     s1.close()
     with pytest.raises(RuntimeError, match="Attempted to use a closed Session."):
@@ -204,7 +204,7 @@ def test_error_on_used_after_close():
         )
 
     # close after close
-    s2 = graphscope.session(run_on_local=True, show_log=True)
+    s2 = graphscope.session(run_on_local=True)
 
     s2.close()
     assert s2.info["status"] == "closed"
@@ -214,7 +214,7 @@ def test_error_on_used_after_close():
 
 
 def test_correct_closing_on_hosts():
-    s1 = graphscope.session(run_on_local=True, show_log=True)
+    s1 = graphscope.session(run_on_local=True)
 
     s1.close()
     # check, launched coordinator and graphscope-engines on local are correctly closed.
@@ -223,9 +223,9 @@ def test_correct_closing_on_hosts():
 
 
 def test_border_cases():
-    s1 = graphscope.session(run_on_local=True, show_log=True)
-    s2 = graphscope.session(run_on_local=True, show_log=True)
-    s3 = graphscope.session(run_on_local=True, show_log=True)
+    s1 = graphscope.session(run_on_local=True)
+    s2 = graphscope.session(run_on_local=True)
+    s3 = graphscope.session(run_on_local=True)
 
     with pytest.raises(RuntimeError, match="No default session found."):
         g = graphscope.load_from(

tutorials/6_unsupervised_learning_with_graphsage.ipynb

@@ -54,7 +54,7 @@
     "\n",
     "# create session\n",
     "graphscope.set_option(show_log=True)\n",
-    "sess = graphscope.session(show_log=True, k8s_volumes=k8s_volumes)\n",
+    "sess = graphscope.session(k8s_volumes=k8s_volumes)\n",
     "\n",
     "# loading ppi graph\n",
     "graph = sess.load_from(\n",

tutorials/7_supervised_learning_with_gcn.ipynb

@@ -55,7 +55,7 @@
     "\n",
     "# create session\n",
     "graphscope.set_option(show_log=True)\n",
-    "sess = graphscope.session(show_log=True, k8s_volumes=k8s_volumes)\n",
+    "sess = graphscope.session(k8s_volumes=k8s_volumes)\n",
     "\n",
     "# loading cora graph\n",
     "graph = sess.load_from(\n",

tutorials/zh/6_unsupervised_learning_with_graphsage.ipynb

@@ -54,7 +54,7 @@
     "\n",
     "# 建立会话\n",
     "graphscope.set_option(show_log=True)\n",
-    "sess = graphscope.session(show_log=True, k8s_volumes=k8s_volumes)\n",
+    "sess = graphscope.session(k8s_volumes=k8s_volumes)\n",
     "\n",
     "# 加载PPI图数据\n",
     "graph = sess.load_from(\n",

tutorials/zh/7_supervised_learning_with_gcn.ipynb

@@ -52,7 +52,7 @@
     "\n",
     "# 开启会话\n",
     "graphscope.set_option(show_log=True)\n",
-    "sess = graphscope.session(show_log=True, k8s_volumes=k8s_volumes)\n",
+    "sess = graphscope.session(k8s_volumes=k8s_volumes)\n",
     "\n",
     "# 载入cora图数据\n",
     "graph = sess.load_from(\n",