From e8c7fdd0afc9db6c7c5bfb17fbb67003320975bc Mon Sep 17 00:00:00 2001 From: Xiwen Zhang Date: Thu, 4 Feb 2021 00:03:25 -0500 Subject: [PATCH 1/2] FEAT#27, add env variable for MODIN-GPU. Signed-off-by: Xiwen Zhang --- modin/config/__main__.py | 1 + modin/config/envvars.py | 25 +++++++++++++++++++++++-- modin/pandas/__init__.py | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/modin/config/__main__.py b/modin/config/__main__.py index 9a2f4f7f39f..9cf1852a36f 100644 --- a/modin/config/__main__.py +++ b/modin/config/__main__.py @@ -19,6 +19,7 @@ def print_config_help(): for objname in sorted(globals()): obj = globals()[objname] if isinstance(obj, type) and issubclass(obj, Parameter) and not obj.is_abstract: + pring("print_config_help()") print(f"{obj.get_help()}\n\tCurrent value: {obj.get()}") # noqa: T001 diff --git a/modin/config/envvars.py b/modin/config/envvars.py index 245619273ef..39ae9e6e52f 100644 --- a/modin/config/envvars.py +++ b/modin/config/envvars.py @@ -94,7 +94,7 @@ class Backend(EnvironmentVariable, type=str): varname = "MODIN_BACKEND" default = "Pandas" - choices = ("Pandas", "OmniSci", "Pyarrow") + choices = ("Pandas", "OmniSci", "Pyarrow", "cuDF") class IsExperimental(EnvironmentVariable, type=bool): @@ -134,6 +134,24 @@ def _get_default(cls): return multiprocessing.cpu_count() +class GpuCount(EnvironmentVariable, type=int): + """ + How may GPU devices to utilize across the whole distribution + """ + + varname = "MODIN_GPUS" + + # @classmethod + # def _get_default(cls): + # import ray + # if ray.is_initialized(): + # return len(ray.get_gpu_ids()) + # else: + # print( + # f"Ray has not been initialized yet. " + # f"MODIN-GPU doesn't know the number of available GPUs. " + # ) + # return 1 class Memory(EnvironmentVariable, type=int): """ @@ -152,7 +170,10 @@ class NPartitions(EnvironmentVariable, type=int): @classmethod def _get_default(cls): - return CpuCount.get() + if Backend.get() == "cuDF": + return GpuCount.get() + else: + return CpuCount.get() class RayPlasmaDir(EnvironmentVariable, type=ExactStr): diff --git a/modin/pandas/__init__.py b/modin/pandas/__init__.py index cc8433928bb..0f7c14c8dca 100644 --- a/modin/pandas/__init__.py +++ b/modin/pandas/__init__.py @@ -100,7 +100,7 @@ def _update_engine(publisher: Parameter): global dask_client - from modin.config import Backend, CpuCount + from modin.config import Backend, CpuCount, GpuCount if publisher.get() == "Ray": from modin.engines.ray.utils import initialize_ray From 0c8dd1fd91d76d7c41d0d19eca7f002e38c9e42e Mon Sep 17 00:00:00 2001 From: Xiwen Zhang Date: Thu, 4 Feb 2021 00:18:21 -0500 Subject: [PATCH 2/2] REFACTOR#27: code formatting with black Signed-off-by: Xiwen Zhang --- modin/config/envvars.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modin/config/envvars.py b/modin/config/envvars.py index 39ae9e6e52f..010184924e5 100644 --- a/modin/config/envvars.py +++ b/modin/config/envvars.py @@ -134,6 +134,7 @@ def _get_default(cls): return multiprocessing.cpu_count() + class GpuCount(EnvironmentVariable, type=int): """ How may GPU devices to utilize across the whole distribution @@ -153,6 +154,7 @@ class GpuCount(EnvironmentVariable, type=int): # ) # return 1 + class Memory(EnvironmentVariable, type=int): """ How much memory give to each Ray worker (in bytes)