[HWORKS-299] Add enable_jupyter_python_kernel_non_k8s flag (#1210) (#…

…1247)
logicalclocks · Dec 6, 2022 · 7480eec · 7480eec
1 parent 448c95d
commit 7480eec
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 29 deletions.
diff --git a/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java b/hopsworks-api/src/main/java/io/hops/hopsworks/api/jupyter/JupyterService.java
@@ -254,13 +254,8 @@ public Response settings(@Context SecurityContext sc) {
 
     Users user = jWTHelper.getUserPrincipal(sc);
     JupyterSettings js = jupyterSettingsFacade.findByProjectUser(project, user.getEmail());
-
-    if (settings.isPythonKernelEnabled()) {
-      js.setPrivateDir(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret());
-    }
-
+    js.setPrivateDir(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret());
     js.setMode(JupyterMode.JUPYTER_LAB);
-
     return noCacheResponse.getNoCacheResponseBuilder(Response.Status.OK).entity(js).build();
   }
 

diff --git a/...rc/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java b/...rc/main/java/io/hops/hopsworks/common/dao/jupyter/config/JupyterConfigFilesGenerator.java
@@ -222,7 +222,7 @@ public void createJupyterNotebookConfig(Writer out, Project project, int port,
     DockerJobConfiguration dockerJobConfiguration = (DockerJobConfiguration)js.getDockerConfig();
 
     JupyterContentsManager jcm = JupyterContentsManager.HDFS_CONTENTS_MANAGER;
-    JupyterNotebookConfigTemplate template = JupyterNotebookConfigTemplateBuilder.newBuilder()
+    JupyterNotebookConfigTemplateBuilder builder = JupyterNotebookConfigTemplateBuilder.newBuilder()
         .setProject(project)
         .setNamenodeIp(namenode.getAddress())
         .setNamenodePort(String.valueOf(namenode.getPort()))
@@ -232,8 +232,6 @@ public void createJupyterNotebookConfig(Writer out, Project project, int port,
         .setPort(port)
         .setBaseDirectory(js.getBaseDir())
         .setHdfsUser(hdfsUser)
-        .setWhiteListedKernels("'" + pythonKernelName(project.getPythonEnvironment().getPythonVersion()) +
-            "', 'pysparkkernel', 'sparkkernel', 'sparkrkernel'")
         .setHadoopHome(settings.getHadoopSymbolicLinkDir())
         .setJupyterCertsDirectory(certsDir)
         .setSecretDirectory(settings.getStagingDir() + Settings.PRIVATE_DIRS + js.getSecret())
@@ -246,8 +244,17 @@ public void createJupyterNotebookConfig(Writer out, Project project, int port,
         .setKafkaBrokers(kafkaBrokers.getKafkaBrokersString())
         .setHopsworksPublicHost(settings.getHopsworksPublicHost())
         .setAllocatedNotebookMBs(dockerJobConfiguration.getResourceConfig().getMemory())
-        .setAllocatedNotebookCores(dockerJobConfiguration.getResourceConfig().getCores())
-        .build();
+        .setAllocatedNotebookCores(dockerJobConfiguration.getResourceConfig().getCores());
+
+    if(settings.isPythonKernelEnabled()) {
+      builder.setWhiteListedKernels("'" + pythonKernelName(project.getPythonEnvironment().getPythonVersion()) +
+        "', 'pysparkkernel', 'sparkkernel', 'sparkrkernel'");
+    } else {
+      builder.setWhiteListedKernels("'pysparkkernel', 'sparkkernel', 'sparkrkernel'");
+    }
+
+    JupyterNotebookConfigTemplate template = builder.build();
+
     Map<String, Object> dataModel = new HashMap<>(1);
     dataModel.put("conf", template);
     try {
@@ -340,15 +347,13 @@ private void createConfigFiles(JupyterPaths jp, String hdfsUser, Users hopsworks
 
     if (!jupyter_config_file.exists()) {
       String pythonKernelName = pythonKernelName(project.getPythonEnvironment().getPythonVersion());
-      if (settings.isPythonKernelEnabled()) {
-        String pythonKernelPath = pythonKernelPath(kernelsDir, pythonKernelName);
-        File pythonKernelFile = new File(pythonKernelPath, KernelTemplate.FILE_NAME);
-
-        new File(pythonKernelPath).mkdir();
-        // Create the python kernel
-        try (Writer out = new FileWriter(pythonKernelFile, false)) {
-          createJupyterKernelConfig(out, project, js, hdfsUser);
-        }
+      String pythonKernelPath = pythonKernelPath(kernelsDir, pythonKernelName);
+      File pythonKernelFile = new File(pythonKernelPath, KernelTemplate.FILE_NAME);
+
+      new File(pythonKernelPath).mkdir();
+      // Create the python kernel
+      try (Writer out = new FileWriter(pythonKernelFile, false)) {
+        createJupyterKernelConfig(out, project, js, hdfsUser);
       }
 
       try (Writer out = new FileWriter(jupyter_config_file, false)) {

diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/util/Settings.java
@@ -126,7 +126,6 @@ public class Settings implements Serializable {
   private static final String VARIABLE_PYPI_SIMPLE_ENDPOINT = "pypi_simple_endpoint";
   private static final String VARIABLE_PYTHON_LIBRARY_UPDATES_MONITOR_INTERVAL =
     "python_library_updates_monitor_interval";
-  private static final String VARIABLE_PYTHON_KERNEL = "python_kernel";
   private static final String VARIABLE_HADOOP_VERSION = "hadoop_version";
   private static final String VARIABLE_KIBANA_IP = "kibana_ip";
   private static final String VARIABLE_LOCALHOST = "localhost";
@@ -288,6 +287,9 @@ public class Settings implements Serializable {
   private static final String VARIABLE_JUPYTER_HOST = "jupyter_host";
   private static final String VARIABLE_JUPYTER_ORIGIN_SCHEME = "jupyter_origin_scheme";
 
+  private static final String VARIABLE_ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES =
+    "enable_jupyter_python_kernel_non_kubernetes";
+
   // JWT Variables
   private static final String VARIABLE_JWT_SIGNATURE_ALGORITHM = "jwt_signature_algorithm";
   private static final String VARIABLE_JWT_LIFETIME_MS = "jwt_lifetime_ms";
@@ -571,7 +573,6 @@ private void populateCache() {
       LOCALHOST = setBoolVar(VARIABLE_LOCALHOST, LOCALHOST);
       CLOUD = setStrVar(VARIABLE_CLOUD, CLOUD);
       REQUESTS_VERIFY = setBoolVar(VARIABLE_REQUESTS_VERIFY, REQUESTS_VERIFY);
-      PYTHON_KERNEL = setBoolVar(VARIABLE_PYTHON_KERNEL, PYTHON_KERNEL);
       TWOFACTOR_AUTH = setVar(VARIABLE_TWOFACTOR_AUTH, TWOFACTOR_AUTH);
       TWOFACTOR_EXCLUDE = setVar(VARIABLE_TWOFACTOR_EXCLUD, TWOFACTOR_EXCLUDE);
       HOPSWORKS_USER = setVar(VARIABLE_HOPSWORKS_USER, HOPSWORKS_USER);
@@ -864,6 +865,10 @@ private void populateCache() {
           QUOTAS_MAX_PARALLEL_EXECUTIONS);
       QUOTAS_MAX_PARALLEL_EXECUTIONS = setLongVar(VARIABLE_QUOTAS_MAX_PARALLEL_EXECUTIONS,
           QUOTAS_MAX_PARALLEL_EXECUTIONS);
+
+      ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES = setBoolVar(VARIABLE_ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES,
+        ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES);
+
       cached = true;
     }
   }
@@ -2419,13 +2424,6 @@ public synchronized String getZipState(String hdfsPath) {
     return state;
   }
 
-  private boolean PYTHON_KERNEL = true;
-
-  public synchronized boolean isPythonKernelEnabled() {
-    checkCache();
-    return PYTHON_KERNEL;
-  }
-
   private String PYPI_REST_ENDPOINT = "https://pypi.org/pypi/{package}/json";
 
   public synchronized String getPyPiRESTEndpoint() {
@@ -3356,6 +3354,16 @@ public synchronized String getJupyterHost() {
     return JUPYTER_HOST;
   }
 
+  private boolean ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES = false;
+
+  public synchronized boolean isPythonKernelEnabled() {
+    checkCache();
+    if(getKubeInstalled()) {
+      return true;
+    }
+    return ENABLE_JUPYTER_PYTHON_KERNEL_NON_KUBERNETES;
+  }
+
   //These dependencies were collected by installing jupyterlab in a new environment
   public  static final List<String> JUPYTER_DEPENDENCIES = Arrays.asList("urllib3", "chardet", "idna", "requests",
       "attrs", "zipp", "importlib-metadata", "pyrsistent", "six", "jsonschema", "prometheus-client", "pycparser",