From 7480eec72be3f815ffb510061519c75263501572 Mon Sep 17 00:00:00 2001 From: Robin Andersson Date: Tue, 6 Dec 2022 15:00:33 +0100 Subject: [PATCH] [HWORKS-299] Add enable_jupyter_python_kernel_non_k8s flag (#1210) (#1247) --- .../hopsworks/api/jupyter/JupyterService.java | 7 +--- .../config/JupyterConfigFilesGenerator.java | 33 +++++++++++-------- .../hops/hopsworks/common/util/Settings.java | 26 ++++++++++----- 3 files changed, 37 insertions(+), 29 deletions(-) diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java index aff7ff1a00..ed01032229 100644 --- a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java +++ b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java @@ -254,13 +254,8 @@ public Response settings(@Context SecurityContext sc) { Users user = jWTHelper.getUserPrincipal(sc); JupyterSettings js = jupyterSettingsFacade.findByProjectUser(project, user.getEmail()); - - if (settings.isPythonKernelEnabled()) { - js.setPrivateDir(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret()); - } - + js.setPrivateDir(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret()); js.setMode(JupyterMode.JUPYTER_LAB); - return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(js).build(); } diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java index 5fd2307e57..37eb07d38f 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java @@ -222,7 +222,7 @@ public void createJupyterNotebookConfig(Writer out, Project project, int port, DockerJobConfiguration dockerJobConfiguration = (DockerJobConfiguration)js.getDockerConfig(); JupyterContentsManager jcm = JupyterContentsManager.HDFS_CONTENTS_MANAGER; - JupyterNotebookConfigTemplate template = JupyterNotebookConfigTemplateBuilder.newBuilder() + JupyterNotebookConfigTemplateBuilder builder = JupyterNotebookConfigTemplateBuilder.newBuilder() .setProject(project) .setNamenodeIp(namenode.getAddress()) .setNamenodePort(String.valueOf(namenode.getPort())) @@ -232,8 +232,6 @@ public void createJupyterNotebookConfig(Writer out, Project project, int port, .setPort(port) .setBaseDirectory(js.getBaseDir()) .setHdfsUser(hdfsUser) - .setWhiteListedKernels("'" + pythonKernelName(project.getPythonEnvironment().getPythonVersion()) + - "', 'pysparkkernel', 'sparkkernel', 'sparkrkernel'") .setHadoopHome(settings.getHadoopSymbolicLinkDir()) .setJupyterCertsDirectory(certsDir) .setSecretDirectory(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret()) @@ -246,8 +244,17 @@ public void createJupyterNotebookConfig(Writer out, Project project, int port, .setKafkaBrokers(kafkaBrokers.getKafkaBrokersString()) .setHopsworksPublicHost(settings.getHopsworksPublicHost()) .setAllocatedNotebookMBs(dockerJobConfiguration.getResourceConfig().getMemory()) - .setAllocatedNotebookCores(dockerJobConfiguration.getResourceConfig().getCores()) - .build(); + .setAllocatedNotebookCores(dockerJobConfiguration.getResourceConfig().getCores()); + + if(settings.isPythonKernelEnabled()) { + builder.setWhiteListedKernels("'" + pythonKernelName(project.getPythonEnvironment().getPythonVersion()) + + "', 'pysparkkernel', 'sparkkernel', 'sparkrkernel'"); + } else { + builder.setWhiteListedKernels("'pysparkkernel', 'sparkkernel', 'sparkrkernel'"); + } + + JupyterNotebookConfigTemplate template = builder.build(); + Map dataModel = new HashMap<>(1); dataModel.put("conf", template); try { @@ -340,15 +347,13 @@ private void createConfigFiles(JupyterPaths jp, String hdfsUser, Users hopsworks if (!jupyter_config_file.exists()) { String pythonKernelName = pythonKernelName(project.getPythonEnvironment().getPythonVersion()); - if (settings.isPythonKernelEnabled()) { - String pythonKernelPath = pythonKernelPath(kernelsDir, pythonKernelName); - File pythonKernelFile = new File(pythonKernelPath, KernelTemplate.FILE_NAME); - - new File(pythonKernelPath).mkdir(); - // Create the python kernel - try (Writer out = new FileWriter(pythonKernelFile, false)) { - createJupyterKernelConfig(out, project, js, hdfsUser); - } + String pythonKernelPath = pythonKernelPath(kernelsDir, pythonKernelName); + File pythonKernelFile = new File(pythonKernelPath, KernelTemplate.FILE_NAME); + + new File(pythonKernelPath).mkdir(); + // Create the python kernel + try (Writer out = new FileWriter(pythonKernelFile, false)) { + createJupyterKernelConfig(out, project, js, hdfsUser); } try (Writer out = new FileWriter(jupyter_config_file, false)) { diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java index 69ab22688d..13c7d04056 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java @@ -126,7 +126,6 @@ public class Settings implements Serializable { private static final String VARIABLE_PYPI_SIMPLE_ENDPOINT = "pypi_simple_endpoint"; private static final String VARIABLE_PYTHON_LIBRARY_UPDATES_MONITOR_INTERVAL = "python_library_updates_monitor_interval"; - private static final String VARIABLE_PYTHON_KERNEL = "python_kernel"; private static final String VARIABLE_HADOOP_VERSION = "hadoop_version"; private static final String VARIABLE_KIBANA_IP = "kibana_ip"; private static final String VARIABLE_LOCALHOST = "localhost"; @@ -288,6 +287,9 @@ public class Settings implements Serializable { private static final String VARIABLE_JUPYTER_HOST = "jupyter_host"; private static final String VARIABLE_JUPYTER_ORIGIN_SCHEME = "jupyter_origin_scheme"; + private static final String VARIABLE_ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES = + "enable_jupyter_python_kernel_non_kubernetes"; + // JWT Variables private static final String VARIABLE_JWT_SIGNATURE_ALGORITHM = "jwt_signature_algorithm"; private static final String VARIABLE_JWT_LIFETIME_MS = "jwt_lifetime_ms"; @@ -571,7 +573,6 @@ private void populateCache() { LOCALHOST = setBoolVar(VARIABLE_LOCALHOST, LOCALHOST); CLOUD = setStrVar(VARIABLE_CLOUD, CLOUD); REQUESTS_VERIFY = setBoolVar(VARIABLE_REQUESTS_VERIFY, REQUESTS_VERIFY); - PYTHON_KERNEL = setBoolVar(VARIABLE_PYTHON_KERNEL, PYTHON_KERNEL); TWOFACTOR_AUTH = setVar(VARIABLE_TWOFACTOR_AUTH, TWOFACTOR_AUTH); TWOFACTOR_EXCLUDE = setVar(VARIABLE_TWOFACTOR_EXCLUD, TWOFACTOR_EXCLUDE); HOPSWORKS_USER = setVar(VARIABLE_HOPSWORKS_USER, HOPSWORKS_USER); @@ -864,6 +865,10 @@ private void populateCache() { QUOTAS_MAX_PARALLEL_EXECUTIONS); QUOTAS_MAX_PARALLEL_EXECUTIONS = setLongVar(VARIABLE_QUOTAS_MAX_PARALLEL_EXECUTIONS, QUOTAS_MAX_PARALLEL_EXECUTIONS); + + ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES = setBoolVar(VARIABLE_ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES, + ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES); + cached = true; } } @@ -2419,13 +2424,6 @@ public synchronized String getZipState(String hdfsPath) { return state; } - private boolean PYTHON_KERNEL = true; - - public synchronized boolean isPythonKernelEnabled() { - checkCache(); - return PYTHON_KERNEL; - } - private String PYPI_REST_ENDPOINT = "https://pypi.org/pypi/{package}/json"; public synchronized String getPyPiRESTEndpoint() { @@ -3356,6 +3354,16 @@ public synchronized String getJupyterHost() { return JUPYTER_HOST; } + private boolean ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES = false; + + public synchronized boolean isPythonKernelEnabled() { + checkCache(); + if(getKubeInstalled()) { + return true; + } + return ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES; + } + //These dependencies were collected by installing jupyterlab in a new environment public static final List JUPYTER_DEPENDENCIES = Arrays.asList("urllib3", "chardet", "idna", "requests", "attrs", "zipp", "importlib-metadata", "pyrsistent", "six", "jsonschema", "prometheus-client", "pycparser",