Skip to content

Commit

Permalink
Support for site-provided dependencies (CrayLabs#182)
Browse files Browse the repository at this point in the history 
Support for site-provided dependencies

Include an optional argument`--only_python_packages` to
to `smart build`. This will also issue a warning for a
case where a user might accidentally omit the device flag (which
defaults to cpu) and not install the GPU variant of torch.

The config can now also detect whether dependencies are available
elsewhere on the platform. If so, those libraries and executables are
not built installed within the user's conda environment.

Reviewed by @Spartee
  • Loading branch information
@ashao @al-rigazzi
ashao authored and al-rigazzi committed May 16, 2022
1 parent 82209d0 commit f0d584b
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 68 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ clean:

# help: clobber - clean, remove deps, builds, (be careful)
.PHONY: clobber
clobber: clean clean-deps
clobber: clean


# help:
Expand Down
132 changes: 80 additions & 52 deletions smartsim/_core/_cli/build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,21 @@
# NOTE: all smartsim modules need full paths as the smart cli
# may be installed into a different directory.

def _install_torch_from_pip(versions, device="cpu", verbose=False):
packages = []
end_point = None
# if we are on linux cpu, use the torch without CUDA
if sys.platform == "linux" and device == "cpu":
packages.append(f"torch=={versions.TORCH}+cpu")
packages.append(f"torchvision=={versions.TORCHVISION}+cpu")
end_point = "https://download.pytorch.org/whl/torch_stable.html"

# otherwise just use the version downloaded by pip
else:
packages.append(f"torch=={versions.TORCH}")
packages.append(f"torchvision=={versions.TORCHVISION}")

pip_install(packages, end_point=end_point, verbose=verbose)

class Build:
def __init__(self):
Expand Down Expand Up @@ -67,6 +82,12 @@ def __init__(self):
type=str,
help="Path to custom libtensorflow directory (ONLY USED IF NEEDED)",
)
parser.add_argument(
"--only_python_packages",
action="store_true",
default=False,
help="If true, only install the python packages (i.e. skip backend builds)"
)
parser.add_argument(
"--keydb",
action="store_true",
Expand All @@ -84,47 +105,57 @@ def __init__(self):

logger.info("Running SmartSim build process...")
try:
logger.info("Checking for build tools...")
self.build_env = BuildEnv()

if self.verbose:
logger.info("Build Environment:")
env = self.build_env.as_dict()
print(tabulate(env, headers=env.keys(), tablefmt="github"), "\n")

logger.info("Checking requested versions...")
self.versions = Versioner()

if self.keydb:
self.versions.REDIS = Version_("6.2.0")
self.versions.REDIS_URL = "https://github.com/EQ-Alpha/KeyDB"
self.versions.REDIS_BRANCH = "v6.2.0"
CONFIG.conf_path = Path(CONFIG.core_path, "config", "keydb.conf")
if not CONFIG.conf_path.resolve().is_file():
raise SSConfigError(
"Database configuration file at REDIS_CONF could not be found"
)

if self.verbose:
db_name = "KEYDB" if self.keydb else "REDIS"
logger.info("Version Information:")
vers = self.versions.as_dict(db_name=db_name)
print(tabulate(vers, headers=vers.keys(), tablefmt="github"), "\n")

# REDIS/KeyDB
self.build_database()

# REDISAI
self.build_redis_ai(
str(args.device), pt, tf, onnx, args.torch_dir, args.libtensorflow_dir
)
if args.only_python_packages:
logger.info("Only installing Python packages...skipping build")
self.build_env = BuildEnv(checks=False)
if not args.no_pt:
self.install_torch(device=args.device)
else:
logger.info("Checking for build tools...")
self.build_env = BuildEnv()

if self.verbose:
logger.info("Build Environment:")
env = self.build_env.as_dict()
print(tabulate(env, headers=env.keys(), tablefmt="github"), "\n")
if self.keydb:
self.versions.REDIS = Version_("6.2.0")
self.versions.REDIS_URL = "https://github.com/EQ-Alpha/KeyDB"
self.versions.REDIS_BRANCH = "v6.2.0"
CONFIG.conf_path = Path(CONFIG.core_path, "config", "keydb.conf")
if not CONFIG.conf_path.resolve().is_file():
raise SSConfigError(
"Database configuration file at REDIS_CONF could not be found"
)

if self.verbose:
db_name = "KEYDB" if self.keydb else "REDIS"
logger.info("Version Information:")
vers = self.versions.as_dict(db_name=db_name)
print(tabulate(vers, headers=vers.keys(), tablefmt="github"), "\n")

# REDIS/KeyDB
self.build_database()

if self.verbose:
logger.info("Version Information:")
vers = self.versions.as_dict()
print(tabulate(vers, headers=vers.keys(), tablefmt="github"), "\n")

# REDISAI
self.build_redis_ai(
str(args.device), pt, tf, onnx, args.torch_dir, args.libtensorflow_dir
)

backends = [
backend.capitalize() for backend in installed_redisai_backends()
]
logger.info(
(", ".join(backends) if backends else "No") + " backend(s) built"
)
backends = [
backend.capitalize() for backend in installed_redisai_backends()
]
logger.info(
(", ".join(backends) if backends else "No") + " backend(s) built"
)

except (SetupError, BuildError) as e:
logger.error(str(e))
Expand Down Expand Up @@ -251,26 +282,23 @@ def build_redis_ai(
)
logger.info("ML Backends and RedisAI build complete!")

def infer_torch_device(self):
backend_torch_path = f"{CONFIG.lib_path}/backends/redisai_torch"
device = "cpu"
if (Path(f"{backend_torch_path}/lib/libtorch_cuda.so").is_file()):
device = "gpu"
return device

def install_torch(self, device="cpu"):
"""Torch shared libraries installed by pip are used in the build
for SmartSim backends so we download them here.
"""
packages = []
end_point = None
if not self.build_env.check_installed("torch", self.versions.TORCH):
# if we are on linux cpu, use the torch without CUDA
if sys.platform == "linux" and device == "cpu":
packages.append(f"torch=={self.versions.TORCH}+cpu")
packages.append(f"torchvision=={self.versions.TORCHVISION}+cpu")
end_point = "https://download.pytorch.org/whl/torch_stable.html"

# otherwise just use the version downloaded by pip
else:
packages.append(f"torch=={self.versions.TORCH}")
packages.append(f"torchvision=={self.versions.TORCHVISION}")

pip_install(packages, end_point=end_point, verbose=self.verbose)

if not self.build_env.check_installed("torch", self.versions.TORCH):
inferred_device = self.infer_torch_device()
if (inferred_device == "gpu") and (device == "cpu"):
logger.warning("CPU requested, but GPU backend is available")
_install_torch_from_pip(self.versions, device, self.verbose)
# if torch already installed, check the versions to make sure correct
# torch version is downloaded for that particular device
else:
Expand Down
12 changes: 6 additions & 6 deletions smartsim/_core/_cli/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,19 @@
from smartsim._core._cli.build import Build
from smartsim._core._cli.clean import Clean
from smartsim._core._cli.utils import get_install_path
from smartsim._core._install.buildenv import Versioner


def _usage():
usage = [
"smart <command> [<args>]\n",
"Commands:",
"\tbuild Build SmartSim dependencies (Redis, RedisAI, ML runtimes)",
"\tclean Remove previous ML runtime installation",
"\tclobber Remove all previous dependency installations",
"\tbuild Build SmartSim dependencies (Redis, RedisAI, ML runtimes)",
"\tclean Remove previous ML runtime installation",
"\tclobber Remove all previous dependency installations",
"\nDeveloper:",
"\tsite Print the installation site of SmartSim",
"\tdbcli Print the path to the redis-cli binary" "\n\n",
"\tsite Print the installation site of SmartSim",
"\tdbcli Print the path to the redis-cli binary" "\n\n",
]
return "\n".join(usage)

Expand Down Expand Up @@ -66,6 +67,5 @@ def dbcli(self):
print("Database (Redis or KeyDB) dependencies not found")
exit(1)


def main():
SmartCli()
19 changes: 13 additions & 6 deletions smartsim/_core/_install/builder.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,15 @@ def __init__(self, env, jobs=1, verbose=False):

# Find _core directory and set up paths
_core_dir = Path(os.path.abspath(__file__)).parent.parent

dependency_path = _core_dir
if os.getenv('SMARTSIM_DEP_PATH'):
dependency_path = Path(os.environ['SMARTSIM_DEP_PATH'])

self.build_dir = _core_dir / ".third-party"
self.bin_path = _core_dir / "bin"
self.lib_path = _core_dir / "lib"

self.bin_path = dependency_path / "bin"
self.lib_path = dependency_path / "lib"

# Set wether build process will output to std output
self.out = subprocess.DEVNULL
Expand All @@ -52,10 +58,11 @@ def __init__(self, env, jobs=1, verbose=False):
# make build directory "SmartSim/smartsim/_core/.third-party"
if not self.build_dir.is_dir():
self.build_dir.mkdir()
if not self.bin_path.is_dir():
self.bin_path.mkdir()
if not self.lib_path.is_dir():
self.lib_path.mkdir()
if dependency_path == _core_dir:
if not self.bin_path.is_dir():
self.bin_path.mkdir()
if not self.lib_path.is_dir():
self.lib_path.mkdir()

self.jobs = jobs

Expand Down
9 changes: 6 additions & 3 deletions smartsim/_core/config/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,12 @@ class Config:
def __init__(self):
# SmartSim/smartsim/_core
self.core_path = Path(os.path.abspath(__file__)).parent.parent
self.lib_path = Path(self.core_path, "lib").resolve()
self.bin_path = Path(self.core_path, "bin").resolve()
self.conf_path = Path(self.core_path, "config", "redis6.conf")

dependency_path = os.environ.get('SMARTSIM_DEP_INSTALL_PATH', self.core_path)

self.lib_path = Path(dependency_path, "lib").resolve()
self.bin_path = Path(dependency_path, "bin").resolve()
self.conf_path = Path(dependency_path, "config", "redis6.conf")

@property
def redisai(self) -> str:
Expand Down

0 comments on commit f0d584b

Please sign in to comment.