diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java index 9af0a60cd94..8f5a44178fa 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/InterpreterRestApi.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Map; import java.util.Properties; - import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -32,36 +31,35 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import com.google.gson.Gson; import org.apache.commons.lang.exception.ExceptionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.RemoteRepository; + import org.apache.zeppelin.annotation.ZeppelinApi; import org.apache.zeppelin.dep.Repository; -import org.apache.zeppelin.interpreter.*; -import org.apache.zeppelin.interpreter.Interpreter.RegisteredInterpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.rest.message.NewInterpreterSettingRequest; import org.apache.zeppelin.rest.message.UpdateInterpreterSettingRequest; import org.apache.zeppelin.server.JsonResponse; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.Gson; -import org.sonatype.aether.RepositoryException; -import org.sonatype.aether.repository.RemoteRepository; /** * Interpreter Rest API - * */ @Path("/interpreter") @Produces("application/json") public class InterpreterRestApi { - Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class); + private static final Logger logger = LoggerFactory.getLogger(InterpreterRestApi.class); private InterpreterFactory interpreterFactory; Gson gson = new Gson(); public InterpreterRestApi() { - } public InterpreterRestApi(InterpreterFactory interpreterFactory) { @@ -70,52 +68,39 @@ public InterpreterRestApi(InterpreterFactory interpreterFactory) { /** * List all interpreter settings - * @return */ @GET @Path("setting") @ZeppelinApi public Response listSettings() { - List interpreterSettings = null; + List interpreterSettings; interpreterSettings = interpreterFactory.get(); - return new JsonResponse(Status.OK, "", interpreterSettings).build(); + return new JsonResponse<>(Status.OK, "", interpreterSettings).build(); } /** * Add new interpreter setting - * @param message - * @return - * @throws IOException - * @throws InterpreterException + * + * @param message NewInterpreterSettingRequest */ @POST @Path("setting") @ZeppelinApi public Response newSettings(String message) { try { - NewInterpreterSettingRequest request = gson.fromJson(message, - NewInterpreterSettingRequest.class); + NewInterpreterSettingRequest request = + gson.fromJson(message, NewInterpreterSettingRequest.class); Properties p = new Properties(); p.putAll(request.getProperties()); - InterpreterSetting interpreterSetting = interpreterFactory.add(request.getName(), - request.getGroup(), - request.getDependencies(), - request.getOption(), - p); - logger.info("new setting created with {}", interpreterSetting.id()); - return new JsonResponse(Status.CREATED, "", interpreterSetting).build(); - } catch (InterpreterException e) { + InterpreterSetting interpreterSetting = interpreterFactory + .createNewSetting(request.getName(), request.getGroup(), request.getDependencies(), + request.getOption(), p); + logger.info("new setting created with {}", interpreterSetting.getId()); + return new JsonResponse<>(Status.CREATED, "", interpreterSetting).build(); + } catch (InterpreterException | IOException e) { logger.error("Exception in InterpreterRestApi while creating ", e); - return new JsonResponse( - Status.NOT_FOUND, - e.getMessage(), - ExceptionUtils.getStackTrace(e)).build(); - } catch (IOException | RepositoryException e) { - logger.error("Exception in InterpreterRestApi while creating ", e); - return new JsonResponse( - Status.INTERNAL_SERVER_ERROR, - e.getMessage(), - ExceptionUtils.getStackTrace(e)).build(); + return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) + .build(); } } @@ -126,26 +111,25 @@ public Response updateSetting(String message, @PathParam("settingId") String set logger.info("Update interpreterSetting {}", settingId); try { - UpdateInterpreterSettingRequest request = gson.fromJson(message, - UpdateInterpreterSettingRequest.class); - interpreterFactory.setPropertyAndRestart(settingId, - request.getOption(), - request.getProperties(), - request.getDependencies()); + UpdateInterpreterSettingRequest request = + gson.fromJson(message, UpdateInterpreterSettingRequest.class); + interpreterFactory + .setPropertyAndRestart(settingId, request.getOption(), request.getProperties(), + request.getDependencies()); } catch (InterpreterException e) { logger.error("Exception in InterpreterRestApi while updateSetting ", e); - return new JsonResponse( - Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) + .build(); } catch (IOException | RepositoryException e) { logger.error("Exception in InterpreterRestApi while updateSetting ", e); - return new JsonResponse( - Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), + ExceptionUtils.getStackTrace(e)).build(); } InterpreterSetting setting = interpreterFactory.get(settingId); if (setting == null) { - return new JsonResponse(Status.NOT_FOUND, "", settingId).build(); + return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build(); } - return new JsonResponse(Status.OK, "", setting).build(); + return new JsonResponse<>(Status.OK, "", setting).build(); } /** @@ -172,14 +156,14 @@ public Response restartSetting(@PathParam("settingId") String settingId) { interpreterFactory.restart(settingId); } catch (InterpreterException e) { logger.error("Exception in InterpreterRestApi while restartSetting ", e); - return new JsonResponse( - Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + return new JsonResponse<>(Status.NOT_FOUND, e.getMessage(), ExceptionUtils.getStackTrace(e)) + .build(); } InterpreterSetting setting = interpreterFactory.get(settingId); if (setting == null) { - return new JsonResponse(Status.NOT_FOUND, "", settingId).build(); + return new JsonResponse<>(Status.NOT_FOUND, "", settingId).build(); } - return new JsonResponse(Status.OK, "", setting).build(); + return new JsonResponse<>(Status.OK, "", setting).build(); } /** @@ -188,27 +172,25 @@ public Response restartSetting(@PathParam("settingId") String settingId) { @GET @ZeppelinApi public Response listInterpreter(String message) { - Map m = Interpreter.registeredInterpreters; - return new JsonResponse(Status.OK, "", m).build(); + Map m = interpreterFactory.getAvailableInterpreterSettings(); + return new JsonResponse<>(Status.OK, "", m).build(); } /** * List of dependency resolving repositories - * @return */ @GET @Path("repository") @ZeppelinApi public Response listRepositories() { - List interpreterRepositories = null; - interpreterRepositories = interpreterFactory.getRepositories(); - return new JsonResponse(Status.OK, "", interpreterRepositories).build(); + List interpreterRepositories = interpreterFactory.getRepositories(); + return new JsonResponse<>(Status.OK, "", interpreterRepositories).build(); } /** * Add new repository - * @param message - * @return + * + * @param message Repository */ @POST @Path("repository") @@ -216,24 +198,21 @@ public Response listRepositories() { public Response addRepository(String message) { try { Repository request = gson.fromJson(message, Repository.class); - interpreterFactory.addRepository( - request.getId(), - request.getUrl(), - request.isSnapshot(), + interpreterFactory.addRepository(request.getId(), request.getUrl(), request.isSnapshot(), request.getAuthentication()); logger.info("New repository {} added", request.getId()); } catch (Exception e) { logger.error("Exception in InterpreterRestApi while adding repository ", e); - return new JsonResponse( - Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), + ExceptionUtils.getStackTrace(e)).build(); } return new JsonResponse(Status.CREATED).build(); } /** * Delete repository - * @param repoId - * @return + * + * @param repoId ID of repository */ @DELETE @Path("repository/{repoId}") @@ -244,8 +223,8 @@ public Response removeRepository(@PathParam("repoId") String repoId) { interpreterFactory.removeRepository(repoId); } catch (Exception e) { logger.error("Exception in InterpreterRestApi while removing repository ", e); - return new JsonResponse( - Status.INTERNAL_SERVER_ERROR, e.getMessage(), ExceptionUtils.getStackTrace(e)).build(); + return new JsonResponse<>(Status.INTERNAL_SERVER_ERROR, e.getMessage(), + ExceptionUtils.getStackTrace(e)).build(); } return new JsonResponse(Status.OK).build(); } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java index c10dee87551..700fe1a1c2d 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/NotebookRestApi.java @@ -18,8 +18,12 @@ package org.apache.zeppelin.rest; import java.io.IOException; -import java.util.*; - +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.POST; @@ -31,7 +35,14 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.Status; +import com.google.common.collect.Sets; +import com.google.common.reflect.TypeToken; +import com.google.gson.Gson; import org.apache.commons.lang3.StringUtils; +import org.quartz.CronExpression; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.zeppelin.annotation.ZeppelinApi; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; @@ -48,16 +59,7 @@ import org.apache.zeppelin.socket.NotebookServer; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.utils.SecurityUtils; -import org.quartz.CronExpression; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.google.common.collect.Sets; -import com.google.gson.Gson; -import com.google.gson.reflect.TypeToken; -import com.google.gson.GsonBuilder; -import com.google.gson.stream.JsonReader; -import java.io.StringReader; /** * Rest api endpoint for the noteBook. */ @@ -71,7 +73,8 @@ public class NotebookRestApi { private SearchService notebookIndex; private NotebookAuthorization notebookAuthorization; - public NotebookRestApi() {} + public NotebookRestApi() { + } public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchService search) { this.notebook = notebook; @@ -87,21 +90,19 @@ public NotebookRestApi(Notebook notebook, NotebookServer notebookServer, SearchS @Path("{noteId}/permissions") @ZeppelinApi public Response getNotePermissions(@PathParam("noteId") String noteId) { - Note note = notebook.getNote(noteId); - HashMap> permissionsMap = new HashMap(); + HashMap> permissionsMap = new HashMap<>(); permissionsMap.put("owners", notebookAuthorization.getOwners(noteId)); permissionsMap.put("readers", notebookAuthorization.getReaders(noteId)); permissionsMap.put("writers", notebookAuthorization.getWriters(noteId)); return new JsonResponse<>(Status.OK, "", permissionsMap).build(); } - String ownerPermissionError(Set current, - Set allowed) throws IOException { + private String ownerPermissionError(Set current, Set allowed) throws IOException { LOG.info("Cannot change permissions. Connection owners {}. Allowed owners {}", - current.toString(), allowed.toString()); + current.toString(), allowed.toString()); return "Insufficient privileges to change permissions.\n\n" + - "Allowed owners: " + allowed.toString() + "\n\n" + - "User belongs to: " + current.toString(); + "Allowed owners: " + allowed.toString() + "\n\n" + + "User belongs to: " + current.toString(); } /** @@ -112,25 +113,25 @@ String ownerPermissionError(Set current, @ZeppelinApi public Response putNotePermissions(@PathParam("noteId") String noteId, String req) throws IOException { - HashMap permMap = gson.fromJson(req, - new TypeToken>(){}.getType()); + /** + * TODO(jl): Fixed the type of HashSet + * https://issues.apache.org/jira/browse/ZEPPELIN-1162 + */ + HashMap permMap = + gson.fromJson(req, new TypeToken>() { + }.getType()); Note note = notebook.getNote(noteId); String principal = SecurityUtils.getPrincipal(); HashSet roles = SecurityUtils.getRoles(); - LOG.info("Set permissions {} {} {} {} {}", - noteId, - principal, - permMap.get("owners"), - permMap.get("readers"), - permMap.get("writers") - ); - - HashSet userAndRoles = new HashSet(); + LOG.info("Set permissions {} {} {} {} {}", noteId, principal, permMap.get("owners"), + permMap.get("readers"), permMap.get("writers")); + + HashSet userAndRoles = new HashSet<>(); userAndRoles.add(principal); userAndRoles.addAll(roles); if (!notebookAuthorization.isOwner(noteId, userAndRoles)) { - return new JsonResponse<>(Status.FORBIDDEN, ownerPermissionError(userAndRoles, - notebookAuthorization.getOwners(noteId))).build(); + return new JsonResponse<>(Status.FORBIDDEN, + ownerPermissionError(userAndRoles, notebookAuthorization.getOwners(noteId))).build(); } HashSet readers = permMap.get("readers"); @@ -146,7 +147,7 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re } } // Set writers, if owners is empty -> set to user requesting the change - if ( writers != null && !writers.isEmpty()) { + if (writers != null && !writers.isEmpty()) { if (owners.isEmpty()) { owners = Sets.newHashSet(SecurityUtils.getPrincipal()); } @@ -155,10 +156,8 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re notebookAuthorization.setReaders(noteId, readers); notebookAuthorization.setWriters(noteId, writers); notebookAuthorization.setOwners(noteId, owners); - LOG.debug("After set permissions {} {} {}", - notebookAuthorization.getOwners(noteId), - notebookAuthorization.getReaders(noteId), - notebookAuthorization.getWriters(noteId)); + LOG.debug("After set permissions {} {} {}", notebookAuthorization.getOwners(noteId), + notebookAuthorization.getReaders(noteId), notebookAuthorization.getWriters(noteId)); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.persist(subject); notebookServer.broadcastNote(note); @@ -167,13 +166,15 @@ public Response putNotePermissions(@PathParam("noteId") String noteId, String re /** * bind a setting to note + * * @throws IOException */ @PUT @Path("interpreter/bind/{noteId}") @ZeppelinApi public Response bind(@PathParam("noteId") String noteId, String req) throws IOException { - List settingIdList = gson.fromJson(req, new TypeToken>(){}.getType()); + List settingIdList = gson.fromJson(req, new TypeToken>() { + }.getType()); notebook.bindInterpretersToNote(noteId, settingIdList); return new JsonResponse<>(Status.OK).build(); } @@ -185,38 +186,27 @@ public Response bind(@PathParam("noteId") String noteId, String req) throws IOEx @Path("interpreter/bind/{noteId}") @ZeppelinApi public Response bind(@PathParam("noteId") String noteId) { - List settingList - = new LinkedList(); + List settingList = new LinkedList<>(); List selectedSettings = notebook.getBindedInterpreterSettings(noteId); for (InterpreterSetting setting : selectedSettings) { - settingList.add(new InterpreterSettingListForNoteBind( - setting.id(), - setting.getName(), - setting.getGroup(), - setting.getInterpreterInfos(), - true) - ); + settingList.add(new InterpreterSettingListForNoteBind(setting.getId(), setting.getName(), + setting.getInterpreterInfos(), true)); } List availableSettings = notebook.getInterpreterFactory().get(); for (InterpreterSetting setting : availableSettings) { boolean selected = false; for (InterpreterSetting selectedSetting : selectedSettings) { - if (selectedSetting.id().equals(setting.id())) { + if (selectedSetting.getId().equals(setting.getId())) { selected = true; break; } } if (!selected) { - settingList.add(new InterpreterSettingListForNoteBind( - setting.id(), - setting.getName(), - setting.getGroup(), - setting.getInterpreterInfos(), - false) - ); + settingList.add(new InterpreterSettingListForNoteBind(setting.getId(), setting.getName(), + setting.getInterpreterInfos(), false)); } } return new JsonResponse<>(Status.OK, "", settingList).build(); @@ -228,7 +218,7 @@ public Response bind(@PathParam("noteId") String noteId) { public Response getNotebookList() throws IOException { AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); List> notesInfo = notebookServer.generateNotebooksInfo(false, subject); - return new JsonResponse<>(Status.OK, "", notesInfo ).build(); + return new JsonResponse<>(Status.OK, "", notesInfo).build(); } @GET @@ -245,8 +235,8 @@ public Response getNotebook(@PathParam("notebookId") String notebookId) throws I /** * export note REST API - * - * @param + * + * @param noteId ID of Note * @return note JSON with status.OK * @throws IOException */ @@ -255,12 +245,12 @@ public Response getNotebook(@PathParam("notebookId") String notebookId) throws I @ZeppelinApi public Response exportNoteBook(@PathParam("id") String noteId) throws IOException { String exportJson = notebook.exportNote(noteId); - return new JsonResponse(Status.OK, "", exportJson).build(); + return new JsonResponse<>(Status.OK, "", exportJson).build(); } /** * import new note REST API - * + * * @param req - notebook Json * @return JSON with new note ID * @throws IOException @@ -273,9 +263,10 @@ public Response importNotebook(String req) throws IOException { Note newNote = notebook.importNote(req, null, subject); return new JsonResponse<>(Status.CREATED, "", newNote.getId()).build(); } - + /** * Create new note REST API + * * @param message - JSON with new note name * @return JSON with new note ID * @throws IOException @@ -284,9 +275,8 @@ public Response importNotebook(String req) throws IOException { @Path("/") @ZeppelinApi public Response createNote(String message) throws IOException { - LOG.info("Create new notebook by JSON {}" , message); - NewNotebookRequest request = gson.fromJson(message, - NewNotebookRequest.class); + LOG.info("Create new notebook by JSON {}", message); + NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note note = notebook.createNote(subject); List initialParagraphs = request.getParagraphs(); @@ -307,12 +297,13 @@ public Response createNote(String message) throws IOException { note.persist(subject); notebookServer.broadcastNote(note); notebookServer.broadcastNoteList(subject); - return new JsonResponse<>(Status.CREATED, "", note.getId() ).build(); + return new JsonResponse<>(Status.CREATED, "", note.getId()).build(); } /** * Delete note REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException */ @@ -332,21 +323,21 @@ public Response deleteNote(@PathParam("notebookId") String notebookId) throws IO notebookServer.broadcastNoteList(subject); return new JsonResponse<>(Status.OK, "").build(); } - + /** * Clone note REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.CREATED * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ @POST @Path("{notebookId}") @ZeppelinApi - public Response cloneNote(@PathParam("notebookId") String notebookId, String message) throws - IOException, CloneNotSupportedException, IllegalArgumentException { - LOG.info("clone notebook by JSON {}" , message); - NewNotebookRequest request = gson.fromJson(message, - NewNotebookRequest.class); + public Response cloneNote(@PathParam("notebookId") String notebookId, String message) + throws IOException, CloneNotSupportedException, IllegalArgumentException { + LOG.info("clone notebook by JSON {}", message); + NewNotebookRequest request = gson.fromJson(message, NewNotebookRequest.class); String newNoteName = request.getName(); AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); Note newNote = notebook.cloneNote(notebookId, newNoteName, subject); @@ -357,6 +348,7 @@ public Response cloneNote(@PathParam("notebookId") String notebookId, String mes /** * Insert paragraph REST API + * * @param message - JSON containing paragraph's information * @return JSON with status.OK * @throws IOException @@ -370,7 +362,7 @@ public Response insertParagraph(@PathParam("notebookId") String notebookId, Stri Note note = notebook.getNote(notebookId); if (note == null) { - return new JsonResponse(Status.NOT_FOUND, "note not found.").build(); + return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } NewParagraphRequest request = gson.fromJson(message, NewParagraphRequest.class); @@ -388,12 +380,13 @@ public Response insertParagraph(@PathParam("notebookId") String notebookId, Stri AuthenticationInfo subject = new AuthenticationInfo(SecurityUtils.getPrincipal()); note.persist(subject); notebookServer.broadcastNote(note); - return new JsonResponse(Status.CREATED, "", p.getId()).build(); + return new JsonResponse<>(Status.CREATED, "", p.getId()).build(); } /** * Get paragraph REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with information of the paragraph * @throws IOException */ @@ -401,7 +394,7 @@ public Response insertParagraph(@PathParam("notebookId") String notebookId, Stri @Path("{notebookId}/paragraph/{paragraphId}") @ZeppelinApi public Response getParagraph(@PathParam("notebookId") String notebookId, - @PathParam("paragraphId") String paragraphId) throws IOException { + @PathParam("paragraphId") String paragraphId) throws IOException { LOG.info("get paragraph {} {}", notebookId, paragraphId); Note note = notebook.getNote(notebookId); @@ -414,11 +407,12 @@ public Response getParagraph(@PathParam("notebookId") String notebookId, return new JsonResponse(Status.NOT_FOUND, "paragraph not found.").build(); } - return new JsonResponse(Status.OK, "", p).build(); + return new JsonResponse<>(Status.OK, "", p).build(); } /** * Move paragraph REST API + * * @param newIndex - new index to move * @return JSON with status.OK * @throws IOException @@ -427,8 +421,8 @@ public Response getParagraph(@PathParam("notebookId") String notebookId, @Path("{notebookId}/paragraph/{paragraphId}/move/{newIndex}") @ZeppelinApi public Response moveParagraph(@PathParam("notebookId") String notebookId, - @PathParam("paragraphId") String paragraphId, - @PathParam("newIndex") String newIndex) throws IOException { + @PathParam("paragraphId") String paragraphId, @PathParam("newIndex") String newIndex) + throws IOException { LOG.info("move paragraph {} {} {}", notebookId, paragraphId, newIndex); Note note = notebook.getNote(notebookId); @@ -456,7 +450,8 @@ public Response moveParagraph(@PathParam("notebookId") String notebookId, /** * Delete paragraph REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException */ @@ -464,7 +459,7 @@ public Response moveParagraph(@PathParam("notebookId") String notebookId, @Path("{notebookId}/paragraph/{paragraphId}") @ZeppelinApi public Response deleteParagraph(@PathParam("notebookId") String notebookId, - @PathParam("paragraphId") String paragraphId) throws IOException { + @PathParam("paragraphId") String paragraphId) throws IOException { LOG.info("delete paragraph {} {}", notebookId, paragraphId); Note note = notebook.getNote(notebookId); @@ -487,36 +482,38 @@ public Response deleteParagraph(@PathParam("notebookId") String notebookId, /** * Run notebook jobs REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @POST @Path("job/{notebookId}") @ZeppelinApi - public Response runNoteJobs(@PathParam("notebookId") String notebookId) throws - IOException, IllegalArgumentException { + public Response runNoteJobs(@PathParam("notebookId") String notebookId) + throws IOException, IllegalArgumentException { LOG.info("run notebook jobs {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - + note.runAll(); return new JsonResponse<>(Status.OK).build(); } /** * Stop(delete) notebook jobs REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @DELETE @Path("job/{notebookId}") @ZeppelinApi - public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws - IOException, IllegalArgumentException { + public Response stopNoteJobs(@PathParam("notebookId") String notebookId) + throws IOException, IllegalArgumentException { LOG.info("stop notebook jobs {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { @@ -530,18 +527,19 @@ public Response stopNoteJobs(@PathParam("notebookId") String notebookId) throws } return new JsonResponse<>(Status.OK).build(); } - + /** * Get notebook job status REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @GET @Path("job/{notebookId}") @ZeppelinApi - public Response getNoteJobStatus(@PathParam("notebookId") String notebookId) throws - IOException, IllegalArgumentException { + public Response getNoteJobStatus(@PathParam("notebookId") String notebookId) + throws IOException, IllegalArgumentException { LOG.info("get notebook job status."); Note note = notebook.getNote(notebookId); if (note == null) { @@ -550,23 +548,21 @@ public Response getNoteJobStatus(@PathParam("notebookId") String notebookId) thr return new JsonResponse<>(Status.OK, null, note.generateParagraphsInfo()).build(); } - + /** * Run paragraph job REST API - * + * * @param message - JSON with params if user wants to update dynamic form's value * null, empty string, empty json if user doesn't want to update - * * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @POST @Path("job/{notebookId}/{paragraphId}") @ZeppelinApi - public Response runParagraph(@PathParam("notebookId") String notebookId, - @PathParam("paragraphId") String paragraphId, - String message) throws - IOException, IllegalArgumentException { + public Response runParagraph(@PathParam("notebookId") String notebookId, + @PathParam("paragraphId") String paragraphId, String message) + throws IOException, IllegalArgumentException { LOG.info("run paragraph job {} {} {}", notebookId, paragraphId, message); Note note = notebook.getNote(notebookId); @@ -581,8 +577,8 @@ public Response runParagraph(@PathParam("notebookId") String notebookId, // handle params if presented if (!StringUtils.isEmpty(message)) { - RunParagraphWithParametersRequest request = gson.fromJson(message, - RunParagraphWithParametersRequest.class); + RunParagraphWithParametersRequest request = + gson.fromJson(message, RunParagraphWithParametersRequest.class); Map paramsForUpdating = request.getParams(); if (paramsForUpdating != null) { paragraph.settings.getParams().putAll(paramsForUpdating); @@ -598,16 +594,21 @@ public Response runParagraph(@PathParam("notebookId") String notebookId, /** * Stop(delete) paragraph job REST API - * @param + * + * @param notebookId ID of Notebook + * @param paragraphId ID of Paragraph * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @DELETE @Path("job/{notebookId}/{paragraphId}") @ZeppelinApi - public Response stopParagraph(@PathParam("notebookId") String notebookId, - @PathParam("paragraphId") String paragraphId) throws - IOException, IllegalArgumentException { + public Response stopParagraph(@PathParam("notebookId") String notebookId, + @PathParam("paragraphId") String paragraphId) throws IOException, IllegalArgumentException { + /** + * TODO(jl): Fixed notebookId to noteId + * https://issues.apache.org/jira/browse/ZEPPELIN-1163 + */ LOG.info("stop paragraph job {} ", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { @@ -621,9 +622,10 @@ public Response stopParagraph(@PathParam("notebookId") String notebookId, p.abort(); return new JsonResponse<>(Status.OK).build(); } - + /** * Register cron job REST API + * * @param message - JSON with cron expressions. * @return JSON with status.OK * @throws IOException, IllegalArgumentException @@ -631,18 +633,18 @@ public Response stopParagraph(@PathParam("notebookId") String notebookId, @POST @Path("cron/{notebookId}") @ZeppelinApi - public Response registerCronJob(@PathParam("notebookId") String notebookId, String message) throws - IOException, IllegalArgumentException { + public Response registerCronJob(@PathParam("notebookId") String notebookId, String message) + throws IOException, IllegalArgumentException { + // TODO(jl): Fixed notebookId to noteId LOG.info("Register cron job note={} request cron msg={}", notebookId, message); - CronRequest request = gson.fromJson(message, - CronRequest.class); - + CronRequest request = gson.fromJson(message, CronRequest.class); + Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - + if (!CronExpression.isValidExpression(request.getCronString())) { return new JsonResponse<>(Status.BAD_REQUEST, "wrong cron expressions.").build(); } @@ -651,60 +653,64 @@ public Response registerCronJob(@PathParam("notebookId") String notebookId, Stri config.put("cron", request.getCronString()); note.setConfig(config); notebook.refreshCron(note.id()); - + return new JsonResponse<>(Status.OK).build(); } - + /** * Remove cron job REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @DELETE @Path("cron/{notebookId}") @ZeppelinApi - public Response removeCronJob(@PathParam("notebookId") String notebookId) throws - IOException, IllegalArgumentException { + public Response removeCronJob(@PathParam("notebookId") String notebookId) + throws IOException, IllegalArgumentException { + // TODO(jl): Fixed notebookId to noteId LOG.info("Remove cron job note {}", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - + Map config = note.getConfig(); config.put("cron", null); note.setConfig(config); notebook.refreshCron(note.id()); - + return new JsonResponse<>(Status.OK).build(); - } - + } + /** * Get cron job REST API - * @param + * + * @param notebookId ID of Notebook * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @GET @Path("cron/{notebookId}") @ZeppelinApi - public Response getCronJob(@PathParam("notebookId") String notebookId) throws - IOException, IllegalArgumentException { + public Response getCronJob(@PathParam("notebookId") String notebookId) + throws IOException, IllegalArgumentException { + // TODO(jl): Fixed notebookId to noteId LOG.info("Get cron job note {}", notebookId); Note note = notebook.getNote(notebookId); if (note == null) { return new JsonResponse<>(Status.NOT_FOUND, "note not found.").build(); } - + return new JsonResponse<>(Status.OK, note.getConfig().get("cron")).build(); } /** * Get notebook jobs for job manager - * @param + * * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @@ -726,7 +732,7 @@ public Response getJobListforNotebook() throws IOException, IllegalArgumentExcep /** * Get updated notebook jobs for job manager - * @param + * * @return JSON with status.OK * @throws IOException, IllegalArgumentException */ @@ -734,8 +740,8 @@ public Response getJobListforNotebook() throws IOException, IllegalArgumentExcep @Path("jobmanager/{lastUpdateUnixtime}/") @ZeppelinApi public Response getUpdatedJobListforNotebook( - @PathParam("lastUpdateUnixtime") long lastUpdateUnixTime) throws - IOException, IllegalArgumentException { + @PathParam("lastUpdateUnixtime") long lastUpdateUnixTime) + throws IOException, IllegalArgumentException { LOG.info("Get updated notebook jobs lastUpdateTime {}", lastUpdateUnixTime); List> notebookJobs; @@ -759,7 +765,7 @@ public Response search(@QueryParam("q") String queryTerm) { LOG.info("Searching notebooks for: {}", queryTerm); String principal = SecurityUtils.getPrincipal(); HashSet roles = SecurityUtils.getRoles(); - HashSet userAndRoles = new HashSet(); + HashSet userAndRoles = new HashSet<>(); userAndRoles.add(principal); userAndRoles.addAll(roles); List> notebooksFound = notebookIndex.query(queryTerm); @@ -767,8 +773,8 @@ public Response search(@QueryParam("q") String queryTerm) { String[] Id = notebooksFound.get(i).get("id").split("/", 2); String noteId = Id[0]; if (!notebookAuthorization.isOwner(noteId, userAndRoles) && - !notebookAuthorization.isReader(noteId, userAndRoles) && - !notebookAuthorization.isWriter(noteId, userAndRoles)) { + !notebookAuthorization.isReader(noteId, userAndRoles) && + !notebookAuthorization.isWriter(noteId, userAndRoles)) { notebooksFound.remove(i); i--; } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/InterpreterSettingListForNoteBind.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/InterpreterSettingListForNoteBind.java index e0ddacbbf92..6ec5a54a3ac 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/InterpreterSettingListForNoteBind.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/InterpreterSettingListForNoteBind.java @@ -19,69 +19,22 @@ import java.util.List; -import org.apache.zeppelin.interpreter.Interpreter; -import org.apache.zeppelin.interpreter.InterpreterSetting; +import org.apache.zeppelin.interpreter.InterpreterInfo; /** * InterpreterSetting information for binding */ public class InterpreterSettingListForNoteBind { - String id; - String name; - String group; + private String id; + private String name; private boolean selected; - private List interpreters; + private List interpreters; public InterpreterSettingListForNoteBind(String id, String name, - String group, - List interpreters, - boolean selected) { - super(); + List interpreters, boolean selected) { this.id = id; this.name = name; - this.group = group; this.interpreters = interpreters; this.selected = selected; } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getGroup() { - return group; - } - - public void setGroup(String group) { - this.group = group; - } - - public List getInterpreterNames() { - return interpreters; - } - - public void setInterpreterNames(List interpreters) { - this.interpreters = interpreters; - } - - public boolean isSelected() { - return selected; - } - - public void setSelected(boolean selected) { - this.selected = selected; - } - } diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java index a559fb53e87..7e3a4147233 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/rest/message/NewInterpreterSettingRequest.java @@ -24,16 +24,15 @@ import org.apache.zeppelin.interpreter.InterpreterOption; /** - * NewInterpreterSetting rest api request message - * + * NewInterpreterSetting rest api request message */ public class NewInterpreterSettingRequest { - String name; - String group; + private String name; + private String group; - Map properties; - List dependencies; - InterpreterOption option; + private Map properties; + private List dependencies; + private InterpreterOption option; public NewInterpreterSettingRequest() { diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java index f2d4c99ca49..78de7738927 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/AbstractTestRestApi.java @@ -22,9 +22,7 @@ import java.lang.ref.WeakReference; import java.net.InetAddress; import java.net.UnknownHostException; -import java.util.LinkedList; import java.util.List; -import java.util.Properties; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -37,9 +35,6 @@ import org.apache.commons.httpclient.methods.PutMethod; import org.apache.commons.httpclient.methods.RequestEntity; import org.apache.zeppelin.conf.ZeppelinConfiguration; -import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.interpreter.InterpreterGroup; -import org.apache.zeppelin.interpreter.InterpreterOption; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.server.ZeppelinServer; import org.hamcrest.Description; @@ -51,7 +46,6 @@ import com.google.gson.JsonElement; import com.google.gson.JsonParseException; import com.google.gson.JsonParser; -import org.sonatype.aether.RepositoryException; public abstract class AbstractTestRestApi { @@ -125,7 +119,7 @@ protected static void startUp() throws Exception { // assume first one is spark InterpreterSetting sparkIntpSetting = null; for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { - if (intpSetting.getGroup().equals("spark")) { + if (intpSetting.getName().equals("spark")) { sparkIntpSetting = intpSetting; } } @@ -138,12 +132,12 @@ protected static void startUp() throws Exception { sparkIntpSetting.getProperties().setProperty("spark.home", getSparkHome()); pySpark = true; sparkR = true; - ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id()); + ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId()); } else { // assume first one is spark InterpreterSetting sparkIntpSetting = null; for(InterpreterSetting intpSetting : ZeppelinServer.notebook.getInterpreterFactory().get()) { - if (intpSetting.getGroup().equals("spark")) { + if (intpSetting.getName().equals("spark")) { sparkIntpSetting = intpSetting; } } @@ -156,7 +150,7 @@ protected static void startUp() throws Exception { sparkR = true; } - ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.id()); + ZeppelinServer.notebook.getInterpreterFactory().restart(sparkIntpSetting.getId()); } } } @@ -375,18 +369,6 @@ protected void describeMismatchSafely(String item, Description description) { }; } - //Create new Setting and return Setting ID - protected String createTempSetting(String tempName) - throws IOException, RepositoryException { - InterpreterSetting setting = ZeppelinServer.notebook.getInterpreterFactory() - .add(tempName, - "newGroup", - new LinkedList(), - new InterpreterOption(false), - new Properties()); - return setting.id(); - } - protected TypeSafeMatcher hasRootElementNamed(final String memberName) { return new TypeSafeMatcher() { @Override diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java index 4d7caf16c9e..e92432fefaa 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/InterpreterRestApiTest.java @@ -25,7 +25,6 @@ import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.methods.PostMethod; import org.apache.commons.httpclient.methods.PutMethod; -import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; @@ -69,7 +68,7 @@ public void getAvailableInterpreters() throws IOException { Map resp = gson.fromJson(get.getResponseBodyAsString(), new TypeToken>() { }.getType()); Map body = (Map) resp.get("body"); - assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getRegisteredInterpreterList().size(), body.size()); + assertEquals(ZeppelinServer.notebook.getInterpreterFactory().getAvailableInterpreterSettings().size(), body.size()); get.releaseConnection(); } @@ -162,7 +161,7 @@ public void testInterpreterRestart() throws IOException, InterruptedException { for (InterpreterSetting setting : ZeppelinServer.notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { if (setting.getName().equals("md")) { // Call Restart Interpreter REST API - PutMethod put = httpPut("/interpreter/setting/restart/" + setting.id(), ""); + PutMethod put = httpPut("/interpreter/setting/restart/" + setting.getId(), ""); assertThat("test interpreter restart:", put, isAllowed()); put.releaseConnection(); break; diff --git a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java index 3c77b45e71f..24a1b906df9 100644 --- a/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java +++ b/zeppelin-server/src/test/java/org/apache/zeppelin/rest/ZeppelinSparkClusterTest.java @@ -20,16 +20,10 @@ import java.io.File; import java.io.IOException; -import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.PumpStreamHandler; import org.apache.commons.io.FileUtils; -import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.notebook.Note; import org.apache.zeppelin.notebook.Paragraph; @@ -204,8 +198,8 @@ public void pySparkDepLoaderTest() throws IOException { ZeppelinServer.notebook.getBindedInterpreterSettings(note.id()); for (InterpreterSetting setting : settings) { - if (setting.getGroup().equals("spark")) { - ZeppelinServer.notebook.getInterpreterFactory().restart(setting.id()); + if (setting.getName().equals("spark")) { + ZeppelinServer.notebook.getInterpreterFactory().restart(setting.getId()); break; } } diff --git a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html index 97e3103dbc4..599248b2e56 100644 --- a/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html +++ b/zeppelin-web/src/app/interpreter/interpreter-create/interpreter-create.html @@ -19,7 +19,7 @@

Create new interpreter

- Name + Interpreter Name
@@ -29,8 +29,8 @@

Create new interpreter

style="width:180px"> diff --git a/zeppelin-web/src/app/interpreter/interpreter.controller.js b/zeppelin-web/src/app/interpreter/interpreter.controller.js index 5a90b79cd26..a6499443190 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.controller.js +++ b/zeppelin-web/src/app/interpreter/interpreter.controller.js @@ -184,15 +184,14 @@ angular.module('zeppelinWebApp').controller('InterpreterCtrl', }; $scope.newInterpreterGroupChange = function() { - var el = _.pluck(_.filter($scope.availableInterpreters, {'group': $scope.newInterpreterSetting.group}), + var el = _.pluck(_.filter($scope.availableInterpreters, {'name': $scope.newInterpreterSetting.group}), 'properties'); - var properties = {}; for (var i = 0; i < el.length; i++) { var intpInfo = el[i]; for (var key in intpInfo) { properties[key] = { - value: intpInfo[key].defaultValue, + value: intpInfo[key], description: intpInfo[key].description }; } diff --git a/zeppelin-web/src/app/interpreter/interpreter.html b/zeppelin-web/src/app/interpreter/interpreter.html index af3b8446229..9e07019657b 100644 --- a/zeppelin-web/src/app/interpreter/interpreter.html +++ b/zeppelin-web/src/app/interpreter/interpreter.html @@ -87,7 +87,7 @@

Repositories

+ ng-repeat="setting in interpreterSettings | orderBy: 'name' | filter: searchInterpreter">
@@ -96,7 +96,7 @@

{{setting.name}} , - %{{setting.group}}{{setting.name}}.{{interpreter.name}} (default) diff --git a/zeppelin-web/src/app/notebook/notebook.controller.js b/zeppelin-web/src/app/notebook/notebook.controller.js index 97b35f0be00..9513e7df60f 100644 --- a/zeppelin-web/src/app/notebook/notebook.controller.js +++ b/zeppelin-web/src/app/notebook/notebook.controller.js @@ -584,9 +584,9 @@ angular.module('zeppelinWebApp').controller('NotebookCtrl', function($scope, $ro var selectedIntp = {}; for (key in $scope.interpreterBindings) { setting = $scope.interpreterBindings[key]; - if (!selectedIntp[setting.group]) { + if (!selectedIntp[setting.name]) { setting.selected = true; - selectedIntp[setting.group] = true; + selectedIntp[setting.name] = true; } } $scope.showSetting = true; diff --git a/zeppelin-web/src/app/notebook/notebook.html b/zeppelin-web/src/app/notebook/notebook.html index f0e87a7250b..00fcacdffc3 100644 --- a/zeppelin-web/src/app/notebook/notebook.html +++ b/zeppelin-web/src/app/notebook/notebook.html @@ -40,7 +40,7 @@

Interpreter binding
, - %{{item.group}}{{item.name}}.{{intp.name}} (default) diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java index 6759f979d13..e0514881a0e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/helium/HeliumApplicationFactory.java @@ -20,6 +20,7 @@ import org.apache.thrift.TException; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterInfo; import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcess; @@ -442,8 +443,8 @@ public void onNoteCreate(Note note) { public void onUnbindInterpreter(Note note, InterpreterSetting setting) { for (Paragraph p : note.getParagraphs()) { Interpreter currentInterpreter = p.getCurrentRepl(); - List infos = setting.getInterpreterInfos(); - for (InterpreterSetting.InterpreterInfo info : infos) { + List infos = setting.getInterpreterInfos(); + for (InterpreterInfo info : infos) { if (info.getClassName().equals(currentInterpreter.getClassName())) { onParagraphRemove(p); break; diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index 3fed07f7c54..c02c882f922 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -17,13 +17,50 @@ package org.apache.zeppelin.interpreter; +import java.io.BufferedReader; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStreamWriter; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Type; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLClassLoader; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import com.google.common.base.Preconditions; import com.google.gson.Gson; import com.google.gson.GsonBuilder; - import com.google.gson.reflect.TypeToken; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.NullArgumentException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.sonatype.aether.RepositoryException; +import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.RemoteRepository; + import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.dep.Dependency; @@ -39,24 +76,6 @@ import org.apache.zeppelin.interpreter.remote.RemoteInterpreterProcessListener; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.sonatype.aether.RepositoryException; -import org.sonatype.aether.repository.Authentication; -import org.sonatype.aether.repository.RemoteRepository; - -import java.io.*; -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Type; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLClassLoader; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.*; /** * Manage interpreters. @@ -70,10 +89,19 @@ public class InterpreterFactory implements InterpreterGroupFactory { Collections.synchronizedMap(new HashMap()); private ZeppelinConfiguration conf; - @Deprecated - private String[] interpreterClassList; + @Deprecated private String[] interpreterClassList; private String[] interpreterGroupOrderList; + /** + * This is only references with default settings, name and properties + * key: InterpreterSetting.name + */ + private Map interpreterSettingsRef = new HashMap<>(); + + /** + * This is used by creating and running Interpreters + * key: InterpreterSetting.id <- This is becuase backward compatibility + */ private Map interpreterSettings = new HashMap<>(); private Map> interpreterBindings = new HashMap<>(); @@ -96,8 +124,7 @@ public class InterpreterFactory implements InterpreterGroupFactory { public InterpreterFactory(ZeppelinConfiguration conf, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener, - DependencyResolver depResolver) + ApplicationEventListener appEventListener, DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this(conf, new InterpreterOption(true), angularObjectRegistryListener, remoteInterpreterProcessListener, appEventListener, depResolver); @@ -107,8 +134,7 @@ public InterpreterFactory(ZeppelinConfiguration conf, public InterpreterFactory(ZeppelinConfiguration conf, InterpreterOption defaultOption, AngularObjectRegistryListener angularObjectRegistryListener, RemoteInterpreterProcessListener remoteInterpreterProcessListener, - ApplicationEventListener appEventListener, - DependencyResolver depResolver) + ApplicationEventListener appEventListener, DependencyResolver depResolver) throws InterpreterException, IOException, RepositoryException { this.conf = conf; this.defaultOption = defaultOption; @@ -135,8 +161,8 @@ private void init() throws InterpreterException, IOException, RepositoryExceptio Path interpretersDir = Paths.get(conf.getInterpreterDir()); if (Files.exists(interpretersDir)) { - for (Path interpreterDir : Files.newDirectoryStream(interpretersDir, - new DirectoryStream.Filter() { + for (Path interpreterDir : Files + .newDirectoryStream(interpretersDir, new DirectoryStream.Filter() { @Override public boolean accept(Path entry) throws IOException { return Files.exists(entry) && Files.isDirectory(entry); @@ -160,10 +186,10 @@ public boolean accept(Path entry) throws IOException { Class.forName(className, true, ccl); Set interpreterKeys = Interpreter.registeredInterpreters.keySet(); for (String interpreterKey : interpreterKeys) { - if (className.equals( - Interpreter.registeredInterpreters.get(interpreterKey).getClassName())) { - Interpreter.registeredInterpreters.get(interpreterKey).setPath( - interpreterDirString); + if (className + .equals(Interpreter.registeredInterpreters.get(interpreterKey).getClassName())) { + Interpreter.registeredInterpreters.get(interpreterKey) + .setPath(interpreterDirString); logger.info("Interpreter " + interpreterKey + " found. class=" + className); cleanCl.put(interpreterDirString, ccl); } @@ -175,78 +201,83 @@ public boolean accept(Path entry) throws IOException { } } - for (RegisteredInterpreter registeredInterpreter : - Interpreter.registeredInterpreters.values()) { - logger.debug("Registered: {} -> {}. Properties: {}", - registeredInterpreter.getInterpreterKey(), registeredInterpreter.getClassName(), - registeredInterpreter.getProperties()); + for (RegisteredInterpreter registeredInterpreter : Interpreter.registeredInterpreters + .values()) { + logger + .debug("Registered: {} -> {}. Properties: {}", registeredInterpreter.getInterpreterKey(), + registeredInterpreter.getClassName(), registeredInterpreter.getProperties()); + } + + // RegisteredInterpreters -> interpreterSettingRef + InterpreterInfo interpreterInfo; + for (RegisteredInterpreter r : Interpreter.registeredInterpreters.values()) { + interpreterInfo = + new InterpreterInfo(r.getClassName(), r.getName(), r.isDefaultInterpreter()); + add(r.getGroup(), interpreterInfo, convertInterpreterProperties(r.getProperties()), + r.getPath()); + } + + for (String settingId : interpreterSettingsRef.keySet()) { + InterpreterSetting setting = interpreterSettingsRef.get(settingId); + logger.info("InterpreterSettingRef name {}", setting.getName()); } loadFromFile(); // if no interpreter settings are loaded, create default set - synchronized (interpreterSettings) { - if (interpreterSettings.size() == 0) { - HashMap> groupClassNameMap = new HashMap<>(); - - for (String k : Interpreter.registeredInterpreters.keySet()) { - RegisteredInterpreter info = Interpreter.registeredInterpreters.get(k); - String group = info.getGroup(); - - if (!groupClassNameMap.containsKey(group)) { - groupClassNameMap.put(group, new LinkedList()); - groupClassNameMap.get(group).add(info); - } else { - if (info.isDefaultInterpreter()) { - groupClassNameMap.get(group).add(0, info); - } else { - groupClassNameMap.get(group).add(info); - } - } - } + if (0 == interpreterSettings.size()) { + Map temp = new HashMap<>(); + InterpreterSetting interpreterSetting; + for (InterpreterSetting setting : interpreterSettingsRef.values()) { + interpreterSetting = createFromInterpreterSettingRef(setting); + temp.put(setting.getName(), interpreterSetting); + } - for (String groupName : interpreterGroupOrderList) { - List infos = groupClassNameMap.remove(groupName); - if (null != infos) { - Properties p = new Properties(); - for (RegisteredInterpreter info : infos) { - Map interpreterProperties = info.getProperties(); - if (null != interpreterProperties) { - for (String key : info.getProperties().keySet()) { - p.put(key, info.getProperties().get(key).getValue()); - } - } - } - add(groupName, groupName, new LinkedList(), defaultOption, p); - } + for (String group : interpreterGroupOrderList) { + if (null != (interpreterSetting = temp.remove(group))) { + interpreterSettings.put(interpreterSetting.getId(), interpreterSetting); } + } - for (String groupName : groupClassNameMap.keySet()) { - List infos = groupClassNameMap.get(groupName); - Properties p = new Properties(); - for (RegisteredInterpreter info : infos) { - Map interpreterProperties = info.getProperties(); - if (null != interpreterProperties) { - for (String key : info.getProperties().keySet()) { - p.put(key, info.getProperties().get(key).getValue()); - } - } - } - add(groupName, groupName, new LinkedList(), defaultOption, p); - } + for (InterpreterSetting setting : temp.values()) { + interpreterSettings.put(setting.getId(), setting); } + + saveToFile(); } + for (String settingId : interpreterSettings.keySet()) { InterpreterSetting setting = interpreterSettings.get(settingId); - logger.info("Interpreter setting group {} : id={}, name={}", setting.getGroup(), settingId, + logger.info("InterpreterSetting group {} : id={}, name={}", setting.getGroup(), settingId, setting.getName()); } } + private InterpreterSetting createFromInterpreterSettingRef(String name) { + Preconditions.checkNotNull(name, "reference name should be not null"); + InterpreterSetting settingRef = interpreterSettingsRef.get(name); + return createFromInterpreterSettingRef(settingRef); + } + + private InterpreterSetting createFromInterpreterSettingRef(InterpreterSetting o) { + InterpreterSetting setting = + new InterpreterSetting(o.getName(), o.getName(), o.getInterpreterInfos(), o.getProperties(), + o.getDependencies(), o.getOption(), o.getPath()); + setting.setInterpreterGroupFactory(this); + return setting; + } + + private Properties convertInterpreterProperties(Map p) { + Properties properties = new Properties(); + for (String key : p.keySet()) { + properties.put(key, p.get(key).getValue()); + } + return properties; + } + private void registerInterpreterFromResource(ClassLoader cl, String interpreterDir, - String interpreterJson) - throws MalformedURLException { + String interpreterJson) throws IOException, RepositoryException { URL[] urls = recursiveBuildLibList(new File(interpreterDir)); ClassLoader tempClassLoader = new URLClassLoader(urls, cl); @@ -254,20 +285,20 @@ private void registerInterpreterFromResource(ClassLoader cl, String interpreterD if (null != inputStream) { logger.debug("Reading {} from resources in {}", interpreterJson, interpreterDir); - List registeredInterpreterList = getInterpreterListFromJson( - inputStream); + List registeredInterpreterList = + getInterpreterListFromJson(inputStream); registerInterpreters(registeredInterpreterList, interpreterDir); } } - private void registerInterpreterFromPath(String interpreterDir, - String interpreterJson) throws IOException { + private void registerInterpreterFromPath(String interpreterDir, String interpreterJson) + throws IOException, RepositoryException { Path interpreterJsonPath = Paths.get(interpreterDir, interpreterJson); if (Files.exists(interpreterJsonPath)) { logger.debug("Reading {}", interpreterJsonPath); - List registeredInterpreterList = getInterpreterListFromJson( - interpreterJsonPath); + List registeredInterpreterList = + getInterpreterListFromJson(interpreterJsonPath); registerInterpreters(registeredInterpreterList, interpreterDir); } } @@ -284,23 +315,24 @@ private List getInterpreterListFromJson(InputStream strea } private void registerInterpreters(List registeredInterpreters, - String absolutePath) { + String absolutePath) throws IOException, RepositoryException { + for (RegisteredInterpreter registeredInterpreter : registeredInterpreters) { - String className = registeredInterpreter.getClassName(); - if (validateRegisterInterpreter(registeredInterpreter) && - null == Interpreter.findRegisteredInterpreterByClassName(className)) { - registeredInterpreter.setPath(absolutePath); - Interpreter.register(registeredInterpreter); - logger.debug("Registered. key: {}, className: {}, path: {}", - registeredInterpreter.getInterpreterKey(), registeredInterpreter.getClassName(), - registeredInterpreter.getProperties()); + InterpreterInfo interpreterInfo = + new InterpreterInfo(registeredInterpreter.getClassName(), registeredInterpreter.getName(), + registeredInterpreter.isDefaultInterpreter()); + Properties properties = new Properties(); + Map p = registeredInterpreter.getProperties(); + + if (null != p) { + for (String key : p.keySet()) { + properties.setProperty(key, p.get(key).getValue()); + } } + + add(registeredInterpreter.getGroup(), interpreterInfo, properties, absolutePath); } - } - private boolean validateRegisterInterpreter(RegisteredInterpreter registeredInterpreter) { - return null != registeredInterpreter.getGroup() && null != registeredInterpreter.getName() && - null != registeredInterpreter.getClassName(); } private void loadFromFile() throws IOException { @@ -332,12 +364,13 @@ private void loadFromFile() throws IOException { // previously created setting should turn this feature on here. setting.getOption().setRemote(true); - InterpreterSetting intpSetting = new InterpreterSetting(setting.id(), setting.getName(), - setting.getGroup(), setting.getInterpreterInfos(), setting.getProperties(), - setting.getDependencies(), setting.getOption()); + // Update transient information from InterpreterSettingRef + // TODO(jl): Check if reference of setting is null + + setting.setPath(interpreterSettingsRef.get(setting.getGroup()).getPath()); - intpSetting.setInterpreterGroupFactory(this); - interpreterSettings.put(k, intpSetting); + setting.setInterpreterGroupFactory(this); + interpreterSettings.put(k, setting); } this.interpreterBindings = info.interpreterBindings; @@ -354,7 +387,7 @@ private void loadFromFile() throws IOException { private void loadInterpreterDependencies(InterpreterSetting intSetting) throws IOException, RepositoryException { // dependencies to prevent library conflict - File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + intSetting.id()); + File localRepoDir = new File(conf.getInterpreterLocalRepoPath() + "/" + intSetting.getId()); if (localRepoDir.exists()) { FileUtils.cleanDirectory(localRepoDir); } @@ -367,9 +400,9 @@ private void loadInterpreterDependencies(InterpreterSetting intSetting) if (d.getExclusions() != null) { depResolver.load(d.getGroupArtifactVersion(), d.getExclusions(), - new File(destDir, intSetting.id())); + new File(destDir, intSetting.getId())); } else { - depResolver.load(d.getGroupArtifactVersion(), new File(destDir, intSetting.id())); + depResolver.load(d.getGroupArtifactVersion(), new File(destDir, intSetting.getId())); } } } @@ -416,62 +449,120 @@ public List getDefaultInterpreterSettingList() { List sortedSettings = get(); for (InterpreterSetting setting : sortedSettings) { - if (defaultSettings.contains(setting.id())) { + if (defaultSettings.contains(setting.getId())) { continue; } - if (!interpreterGroupCheck.containsKey(setting.getGroup())) { - defaultSettings.add(setting.id()); - interpreterGroupCheck.put(setting.getGroup(), true); + if (!interpreterGroupCheck.containsKey(setting.getName())) { + defaultSettings.add(setting.getId()); + interpreterGroupCheck.put(setting.getName(), true); } } return defaultSettings; } - public List getRegisteredInterpreterList() { + List getRegisteredInterpreterList() { return new ArrayList<>(Interpreter.registeredInterpreters.values()); } + + private boolean findDefaultInterpreter(List infos) { + for (InterpreterInfo interpreterInfo : infos) { + if (interpreterInfo.isDefaultInterpreter()) + return true; + } + return false; + } + + public InterpreterSetting createNewSetting(String name, String group, + List dependencies, InterpreterOption option, Properties p) throws IOException { + InterpreterSetting setting = createFromInterpreterSettingRef(group); + setting.setName(name); + setting.setGroup(group); + setting.appendDependencies(dependencies); + setting.setInterpreterOption(option); + setting.updateProperties(p); + setting.setInterpreterGroupFactory(this); + interpreterSettings.put(setting.getId(), setting); + saveToFile(); + return setting; + } + + private InterpreterSetting add(String group, InterpreterInfo interpreterInfo, + Properties properties, String path) + throws InterpreterException, IOException, RepositoryException { + ArrayList infos = new ArrayList<>(); + infos.add(interpreterInfo); + return add(group, infos, new ArrayList(), defaultOption, properties, path); + } + /** - * @param name user defined name - * @param groupName interpreter group name to instantiate + * @param group InterpreterSetting reference name * @param properties * @return * @throws InterpreterException * @throws IOException */ - public InterpreterSetting add(String name, String groupName, List dependencies, - InterpreterOption option, Properties properties) + public InterpreterSetting add(String group, ArrayList interpreterInfos, + List dependencies, InterpreterOption option, Properties properties, String path) throws InterpreterException, IOException, RepositoryException { - synchronized (interpreterSettings) { - List interpreterInfos = new ArrayList<>(); - - for (RegisteredInterpreter registeredInterpreter : - Interpreter.registeredInterpreters.values()) { - if (registeredInterpreter.getGroup().equals(groupName)) { - if (registeredInterpreter.isDefaultInterpreter()) { - interpreterInfos.add(0, - new InterpreterSetting.InterpreterInfo( - registeredInterpreter.getClassName(), registeredInterpreter.getName())); - } else { - interpreterInfos.add(new InterpreterSetting.InterpreterInfo( - registeredInterpreter.getClassName(), registeredInterpreter.getName())); + Preconditions.checkNotNull(group, "name should not be null"); + Preconditions.checkNotNull(interpreterInfos, "interpreterInfos should not be null"); + Preconditions.checkNotNull(dependencies, "dependencies should not be null"); + Preconditions.checkNotNull(option, "option should not be null"); + Preconditions.checkNotNull(properties, "properties should not be null"); + + InterpreterSetting interpreterSetting; + + synchronized (interpreterSettingsRef) { + if (interpreterSettingsRef.containsKey(group)) { + interpreterSetting = interpreterSettingsRef.get(group); + + // Append InterpreterInfo + List infos = interpreterSetting.getInterpreterInfos(); + boolean hasDefaultInterpreter = findDefaultInterpreter(infos); + for (InterpreterInfo interpreterInfo : interpreterInfos) { + if (!infos.contains(interpreterInfo)) { + if (!hasDefaultInterpreter && interpreterInfo.isDefaultInterpreter()) { + hasDefaultInterpreter = true; + infos.add(0, interpreterInfo); + } else { + infos.add(interpreterInfo); + } } } - } - InterpreterSetting intpSetting = new InterpreterSetting(name, groupName, interpreterInfos, - properties, dependencies, option); + // Append dependencies + List dependencyList = interpreterSetting.getDependencies(); + for (Dependency dependency : dependencies) { + if (!dependencyList.contains(dependency)) { + dependencyList.add(dependency); + } + } - if (dependencies.size() > 0) { - loadInterpreterDependencies(intpSetting); + // Append properties + Properties interpreterProperties = interpreterSetting.getProperties(); + for (String key : properties.stringPropertyNames()) { + if (!interpreterProperties.containsKey(key)) { + interpreterProperties.setProperty(key, properties.getProperty(key)); + } + } + + } else { + interpreterSetting = + new InterpreterSetting(group, null, interpreterInfos, properties, dependencies, + option, path); + interpreterSettingsRef.put(group, interpreterSetting); } + } - intpSetting.setInterpreterGroupFactory(this); - interpreterSettings.put(intpSetting.id(), intpSetting); - saveToFile(); - return intpSetting; + + if (dependencies.size() > 0) { + loadInterpreterDependencies(interpreterSetting); } + + interpreterSetting.setInterpreterGroupFactory(this); + return interpreterSetting; } @Override @@ -486,8 +577,8 @@ public InterpreterGroup createInterpreterGroup(String id, InterpreterOption opti InterpreterGroup interpreterGroup = new InterpreterGroup(id); if (option.isRemote()) { - angularObjectRegistry = new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, - interpreterGroup); + angularObjectRegistry = + new RemoteAngularObjectRegistry(id, angularObjectRegistryListener, interpreterGroup); } else { angularObjectRegistry = new AngularObjectRegistry(id, angularObjectRegistryListener); @@ -510,8 +601,7 @@ public void removeInterpretersForNote(InterpreterSetting interpreterSetting, Str interpreterGroup.remove(noteId); interpreterGroup.notifyAll(); // notify createInterpreterForNote() } - logger.info("Interpreter instance {} for note {} is removed", - interpreterSetting.getName(), + logger.info("Interpreter instance {} for note {} is removed", interpreterSetting.getName(), noteId); } } @@ -519,7 +609,6 @@ public void removeInterpretersForNote(InterpreterSetting interpreterSetting, Str public void createInterpretersForNote(InterpreterSetting interpreterSetting, String noteId, String key) { InterpreterGroup interpreterGroup = interpreterSetting.getInterpreterGroup(noteId); - String groupName = interpreterSetting.getGroup(); InterpreterOption option = interpreterSetting.getOption(); Properties properties = interpreterSetting.getProperties(); if (option.isExistingProcess) { @@ -549,44 +638,37 @@ public void createInterpretersForNote(InterpreterSetting interpreterSetting, Str logger.info("Create interpreter instance {} for note {}", interpreterSetting.getName(), noteId); - Set keys = Interpreter.registeredInterpreters.keySet(); - for (String intName : keys) { - RegisteredInterpreter info = Interpreter.registeredInterpreters.get(intName); - if (info.getGroup().equals(groupName)) { - Interpreter intp; - - if (option.isRemote()) { - if (option.isConnectExistingProcess()) { - intp = connectToRemoteRepl( - noteId, - info.getClassName(), - option.getHost(), option.getPort(), properties); - } else { - intp = createRemoteRepl(info.getPath(), - key, - info.getClassName(), - properties, - interpreterSetting.id()); - } + List interpreterInfos = interpreterSetting.getInterpreterInfos(); + String path = interpreterSetting.getPath(); + Interpreter interpreter; + for (InterpreterInfo info : interpreterInfos) { + if (option.isRemote()) { + if (option.isConnectExistingProcess()) { + interpreter = + connectToRemoteRepl(noteId, info.getClassName(), option.getHost(), option.getPort(), + properties); } else { - intp = createRepl(info.getPath(), info.getClassName(), properties); + interpreter = createRemoteRepl(path, key, info.getClassName(), properties, + interpreterSetting.getId()); } + } else { + interpreter = createRepl(interpreterSetting.getPath(), info.getClassName(), properties); + } - synchronized (interpreterGroup) { - List interpreters = interpreterGroup.get(key); - if (interpreters == null) { - interpreters = new LinkedList<>(); - interpreterGroup.put(key, interpreters); - } - if (info.isDefaultInterpreter()) { - interpreters.add(0, intp); - } else { - interpreters.add(intp); - } + synchronized (interpreterGroup) { + List interpreters = interpreterGroup.get(key); + if (null == interpreters) { + interpreters = new ArrayList<>(); + interpreterGroup.put(key, interpreters); + } + if (info.isDefaultInterpreter()) { + interpreters.add(0, interpreter); + } else { + interpreters.add(interpreter); } - logger.info("Interpreter " + intp.getClassName() + " " + intp.hashCode() + " created"); - intp.setInterpreterGroup(interpreterGroup); } + logger.info("Interpreter {} {} created", interpreter.getClassName(), interpreter.hashCode()); + interpreter.setInterpreterGroup(interpreterGroup); } } @@ -624,18 +706,18 @@ public List get() { synchronized (interpreterSettings) { List orderedSettings = new LinkedList<>(); - Map> groupNameInterpreterSettingMap = new HashMap<>(); + Map> nameInterpreterSettingMap = new HashMap<>(); for (InterpreterSetting interpreterSetting : interpreterSettings.values()) { - String groupName = interpreterSetting.getGroup(); - if (!groupNameInterpreterSettingMap.containsKey(groupName)) { - groupNameInterpreterSettingMap.put(groupName, new ArrayList()); + String name = interpreterSetting.getName(); + if (!nameInterpreterSettingMap.containsKey(name)) { + nameInterpreterSettingMap.put(name, new ArrayList()); } - groupNameInterpreterSettingMap.get(groupName).add(interpreterSetting); + nameInterpreterSettingMap.get(name).add(interpreterSetting); } for (String groupName : interpreterGroupOrderList) { List interpreterSettingList = - groupNameInterpreterSettingMap.remove(groupName); + nameInterpreterSettingMap.remove(groupName); if (null != interpreterSettingList) { for (InterpreterSetting interpreterSetting : interpreterSettingList) { orderedSettings.add(interpreterSetting); @@ -645,8 +727,7 @@ public List get() { List settings = new ArrayList<>(); - for (List interpreterSettingList : - groupNameInterpreterSettingMap.values()) { + for (List interpreterSettingList : nameInterpreterSettingMap.values()) { for (InterpreterSetting interpreterSetting : interpreterSettingList) { settings.add(interpreterSetting); } @@ -697,7 +778,8 @@ private void putNoteInterpreterSettingBinding(String noteId, List settin public void removeNoteInterpreterSettingBinding(String noteId) { synchronized (interpreterSettings) { List settingIds = (interpreterBindings.containsKey(noteId) ? - interpreterBindings.remove(noteId) : Collections.emptyList()); + interpreterBindings.remove(noteId) : + Collections.emptyList()); for (String settingId : settingIds) { this.removeInterpretersForNote(get(settingId), noteId); } @@ -734,14 +816,13 @@ public void setPropertyAndRestart(String id, InterpreterOption option, Propertie intpsetting.closeAndRmoveAllInterpreterGroups(); intpsetting.setOption(option); - intpsetting.setProperties(properties); + intpsetting.updateProperties(properties); intpsetting.setDependencies(dependencies); loadInterpreterDependencies(intpsetting); saveToFile(); } else { - throw new InterpreterException("Interpreter setting id " + id - + " not found"); + throw new InterpreterException("Interpreter setting id " + id + " not found"); } } } @@ -756,8 +837,7 @@ public void restart(String id) { intpsetting.closeAndRmoveAllInterpreterGroups(); } else { - throw new InterpreterException("Interpreter setting id " + id - + " not found"); + throw new InterpreterException("Interpreter setting id " + id + " not found"); } } } @@ -807,8 +887,7 @@ public void run() { } } - private Interpreter createRepl(String dirName, String className, - Properties property) + private Interpreter createRepl(String dirName, String className, Properties property) throws InterpreterException { logger.info("Create repl {} from {}", className, dirName); @@ -820,7 +899,7 @@ private Interpreter createRepl(String dirName, String className, URLClassLoader ccl = cleanCl.get(dirName); if (ccl == null) { // classloader fallback - ccl = URLClassLoader.newInstance(new URL[]{}, oldcl); + ccl = URLClassLoader.newInstance(new URL[] {}, oldcl); } boolean separateCL = true; @@ -836,7 +915,7 @@ private Interpreter createRepl(String dirName, String className, URLClassLoader cl; if (separateCL == true) { - cl = URLClassLoader.newInstance(new URL[]{}, ccl); + cl = URLClassLoader.newInstance(new URL[] {}, ccl); } else { cl = ccl; } @@ -844,11 +923,10 @@ private Interpreter createRepl(String dirName, String className, Class replClass = (Class) cl.loadClass(className); Constructor constructor = - replClass.getConstructor(new Class[]{Properties.class}); + replClass.getConstructor(new Class[] {Properties.class}); Interpreter repl = constructor.newInstance(property); repl.setClassloaderUrls(ccl.getURLs()); - LazyOpenInterpreter intp = new LazyOpenInterpreter( - new ClassloaderInterpreter(repl, cl)); + LazyOpenInterpreter intp = new LazyOpenInterpreter(new ClassloaderInterpreter(repl, cl)); return intp; } catch (SecurityException e) { throw new InterpreterException(e); @@ -869,24 +947,13 @@ private Interpreter createRepl(String dirName, String className, } } - private Interpreter connectToRemoteRepl(String noteId, - String className, - String host, - int port, + private Interpreter connectToRemoteRepl(String noteId, String className, String host, int port, Properties property) { int connectTimeout = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT); int maxPoolSize = conf.getInt(ConfVars.ZEPPELIN_INTERPRETER_MAX_POOL_SIZE); LazyOpenInterpreter intp = new LazyOpenInterpreter( - new RemoteInterpreter( - property, - noteId, - className, - host, - port, - connectTimeout, - maxPoolSize, - remoteInterpreterProcessListener, - appEventListener)); + new RemoteInterpreter(property, noteId, className, host, port, connectTimeout, maxPoolSize, + remoteInterpreterProcessListener, appEventListener)); return intp; } @@ -899,10 +966,10 @@ private Interpreter createRemoteRepl(String interpreterPath, String noteId, Stri updatePropertiesFromRegisteredInterpreter(property, className); - RemoteInterpreter remoteInterpreter = new RemoteInterpreter( - property, noteId, className, conf.getInterpreterRemoteRunnerPath(), - interpreterPath, localRepoPath, connectTimeout, - maxPoolSize, remoteInterpreterProcessListener, appEventListener); + RemoteInterpreter remoteInterpreter = + new RemoteInterpreter(property, noteId, className, conf.getInterpreterRemoteRunnerPath(), + interpreterPath, localRepoPath, connectTimeout, maxPoolSize, + remoteInterpreterProcessListener, appEventListener); remoteInterpreter.setEnv(env); return new LazyOpenInterpreter(remoteInterpreter); @@ -910,8 +977,8 @@ private Interpreter createRemoteRepl(String interpreterPath, String noteId, Stri private Properties updatePropertiesFromRegisteredInterpreter(Properties properties, String className) { - RegisteredInterpreter registeredInterpreter = Interpreter.findRegisteredInterpreterByClassName( - className); + RegisteredInterpreter registeredInterpreter = + Interpreter.findRegisteredInterpreterByClassName(className); if (null != registeredInterpreter) { Map defaultProperties = registeredInterpreter.getProperties(); for (String key : defaultProperties.keySet()) { @@ -1001,8 +1068,53 @@ public InterpreterSetting getDefaultInterpreterSetting(String noteId) { return getDefaultInterpreterSetting(getInterpreterSettings(noteId)); } + + private InterpreterSetting getInterpreterSettingByGroup(List settings, + String group) { + Preconditions.checkNotNull(group, "group should be not null"); + + for (InterpreterSetting setting : settings) { + if (group.equals(setting.getName())) { + return setting; + } + } + return null; + } + + private String getInterpreterClassFromInterpreterSetting(InterpreterSetting setting, + String name) { + Preconditions.checkNotNull(name, "name should be not null"); + + for (InterpreterInfo info : setting.getInterpreterInfos()) { + String infoName = info.getName(); + if (null != info.getName() && name.equals(infoName)) { + return info.getClassName(); + } + } + return null; + } + + private Interpreter getInterpreter(String noteId, InterpreterSetting setting, String name) { + Preconditions.checkNotNull(noteId, "noteId should be not null"); + Preconditions.checkNotNull(setting, "setting should be not null"); + Preconditions.checkNotNull(name, "name should be not null"); + + String className; + if (null != (className = getInterpreterClassFromInterpreterSetting(setting, name))) { + List interpreterGroup = createOrGetInterpreterList(noteId, setting); + for (Interpreter interpreter : interpreterGroup) { + if (className.equals(interpreter.getClassName())) { + return interpreter; + } + } + } + return null; + } + public Interpreter getInterpreter(String noteId, String replName) { List settings = getInterpreterSettings(noteId); + InterpreterSetting setting; + Interpreter interpreter; if (settings == null || settings.size() == 0) { return null; @@ -1015,66 +1127,44 @@ public Interpreter getInterpreter(String noteId, String replName) { return createOrGetInterpreterList(noteId, defaultSettings).get(0); } - if (Interpreter.registeredInterpreters == null) { - return null; - } - String[] replNameSplit = replName.split("\\."); - String group = null; - String name = null; if (replNameSplit.length == 2) { + String group = null; + String name = null; group = replNameSplit[0]; name = replNameSplit[1]; - Interpreter.RegisteredInterpreter registeredInterpreter = Interpreter.registeredInterpreters - .get(group + "." + name); - if (registeredInterpreter == null - || registeredInterpreter.getClassName() == null) { - throw new InterpreterException(replName + " interpreter not found"); - } - String interpreterClassName = registeredInterpreter.getClassName(); - - for (InterpreterSetting setting : settings) { - if (registeredInterpreter.getGroup().equals(setting.getGroup())) { - List intpGroup = createOrGetInterpreterList(noteId, setting); - for (Interpreter interpreter : intpGroup) { - if (interpreterClassName.equals(interpreter.getClassName())) { - return interpreter; - } - } + setting = getInterpreterSettingByGroup(settings, group); + + if (null != setting) { + interpreter = getInterpreter(noteId, setting, name); + + if (null != interpreter) { + return interpreter; } } + throw new InterpreterException(replName + " interpreter not found"); + } else { // first assume replName is 'name' of interpreter. ('groupName' is ommitted) // search 'name' from first (default) interpreter group - InterpreterSetting defaultSetting = getDefaultInterpreterSetting(settings); - Interpreter.RegisteredInterpreter registeredInterpreter = - Interpreter.registeredInterpreters.get(defaultSetting.getGroup() + "." + replName); - if (registeredInterpreter != null) { - List interpreters = createOrGetInterpreterList(noteId, defaultSetting); - for (Interpreter interpreter : interpreters) { - - RegisteredInterpreter intp = - Interpreter.findRegisteredInterpreterByClassName(interpreter.getClassName()); - if (intp == null) { - continue; - } + // TODO(jl): Handle with noteId to support defaultInterpreter per note. + setting = getDefaultInterpreterSetting(settings); - if (intp.getName().equals(replName)) { - return interpreter; - } - } + interpreter = getInterpreter(noteId, setting, replName); - throw new InterpreterException( - defaultSetting.getGroup() + "." + replName + " interpreter not found"); + if (null != interpreter) { + return interpreter; } // next, assume replName is 'group' of interpreter ('name' is ommitted) // search interpreter group and return first interpreter. - for (InterpreterSetting setting : settings) { - if (setting.getGroup().equals(replName)) { - List interpreters = createOrGetInterpreterList(noteId, setting); + setting = getInterpreterSettingByGroup(settings, replName); + + if (null != setting) { + List interpreters = createOrGetInterpreterList(noteId, setting); + if (null != interpreters) { return interpreters.get(0); } } @@ -1088,6 +1178,10 @@ public Interpreter getInterpreter(String noteId, String replName) { return null; } + public Map getAvailableInterpreterSettings() { + return interpreterSettingsRef; + } + private URL[] recursiveBuildLibList(File path) throws MalformedURLException { URL[] urls = new URL[0]; if (path == null || !path.exists()) { @@ -1103,7 +1197,7 @@ private URL[] recursiveBuildLibList(File path) throws MalformedURLException { } return urls; } else { - return new URL[]{path.toURI().toURL()}; + return new URL[] {path.toURI().toURL()}; } } @@ -1131,17 +1225,15 @@ public void setEnv(Map env) { } - public Interpreter getDevInterpreter() { + private Interpreter getDevInterpreter() { if (devInterpreter == null) { InterpreterOption option = new InterpreterOption(); option.setRemote(true); InterpreterGroup interpreterGroup = createInterpreterGroup("dev", option); - devInterpreter = connectToRemoteRepl("dev", DevInterpreter.class.getName(), - "localhost", - ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT, - new Properties()); + devInterpreter = connectToRemoteRepl("dev", DevInterpreter.class.getName(), "localhost", + ZeppelinDevServer.DEFAULT_TEST_INTERPRETER_PORT, new Properties()); LinkedList intpList = new LinkedList<>(); intpList.add(devInterpreter); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java new file mode 100644 index 00000000000..c104b9d7eea --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterInfo.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zeppelin.interpreter; + +import com.google.gson.annotations.SerializedName; + +/** + * Information of interpreters in this interpreter setting. + * this will be serialized for conf/interpreter.json and REST api response. + */ +public class InterpreterInfo { + private String name; + @SerializedName("class") private String className; + private boolean defaultInterpreter = false; + + InterpreterInfo(String className, String name, boolean defaultInterpreter) { + this.className = className; + this.name = name; + this.defaultInterpreter = defaultInterpreter; + } + + public String getName() { + return name; + } + + public String getClassName() { + return className; + } + + public void setName(String name) { + this.name = name; + } + + boolean isDefaultInterpreter() { + return defaultInterpreter; + } + + @Override + public boolean equals(Object obj) { + if (!(obj instanceof InterpreterInfo)) { + return false; + } + InterpreterInfo other = (InterpreterInfo) obj; + + boolean sameName = + null == getName() ? null == other.getName() : getName().equals(other.getName()); + boolean sameClassName = null == getClassName() ? + null == other.getClassName() : + getClassName().equals(other.getClassName()); + boolean sameIsDefaultInterpreter = defaultInterpreter == other.isDefaultInterpreter(); + + return sameName && sameClassName && sameIsDefaultInterpreter; + } +} diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java index cb6c752d545..5f340e9ede5 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSetting.java @@ -17,11 +17,19 @@ package org.apache.zeppelin.interpreter; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Properties; import com.google.gson.annotations.SerializedName; + import org.apache.zeppelin.dep.Dependency; -import org.apache.zeppelin.notebook.utility.IdHashes; + +import static org.apache.zeppelin.notebook.utility.IdHashes.generateId; /** * Interpreter settings @@ -30,87 +38,61 @@ public class InterpreterSetting { private static final String SHARED_PROCESS = "shared_process"; private String id; private String name; - private String group; - private String description; + private String group; // always be null in case of InterpreterSettingRef private Properties properties; - private transient InterpreterGroupFactory interpreterGroupFactory; - // use 'interpreterGroup' as a field name to keep backward compatibility of - // conf/interpreter.json file format - private List interpreterGroup; - private transient Map interpreterGroupRef = new HashMap<>(); + @SerializedName("interpreterGroup") private List interpreterInfos; + private final transient Map interpreterGroupRef = new HashMap<>(); private List dependencies; private InterpreterOption option; + private transient String path; + + @Deprecated private transient InterpreterGroupFactory interpreterGroupFactory; + + public InterpreterSetting() { + + } public InterpreterSetting(String id, String name, String group, List interpreterInfos, Properties properties, List dependencies, - InterpreterOption option) { + InterpreterOption option, String path) { this.id = id; this.name = name; this.group = group; - this.interpreterGroup = interpreterInfos; + this.interpreterInfos = interpreterInfos; this.properties = properties; this.dependencies = dependencies; this.option = option; + this.path = path; } public InterpreterSetting(String name, String group, List interpreterInfos, - Properties properties, List dependencies, InterpreterOption option) { - this(generateId(), name, group, interpreterInfos, properties, dependencies, option); + Properties properties, List dependencies, InterpreterOption option, String path) { + this(generateId(), name, group, interpreterInfos, properties, dependencies, option, path); } /** - * Information of interpreters in this interpreter setting. - * this will be serialized for conf/interpreter.json and REST api response. + * Create interpreter from interpreterSettingRef + * + * @param o interpreterSetting from interpreterSettingRef */ - public static class InterpreterInfo { - private final String name; - @SerializedName("class") - private final String className; - - public InterpreterInfo(String className, String name) { - this.className = className; - this.name = name; - } - - public String getName() { - return name; - } - - public String getClassName() { - return className; - } + public InterpreterSetting(InterpreterSetting o) { + this(generateId(), o.getName(), o.getGroup(), o.getInterpreterInfos(), o.getProperties(), + o.getDependencies(), o.getOption(), o.getPath()); } - public String id() { + public String getId() { return id; } - private static String generateId() { - return IdHashes.encode(System.currentTimeMillis() + new Random().nextInt()); - } - public String getName() { return name; } - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String desc) { - this.description = desc; - } - - public String getGroup() { + String getGroup() { return group; } - private String getInterpreterProcessKey(String noteId) { if (getOption().isExistingProcess) { return Constants.EXISTING_PROCESS; @@ -125,7 +107,7 @@ public InterpreterGroup getInterpreterGroup(String noteId) { String key = getInterpreterProcessKey(noteId); synchronized (interpreterGroupRef) { if (!interpreterGroupRef.containsKey(key)) { - String interpreterGroupId = id() + ":" + key; + String interpreterGroupId = getId() + ":" + key; InterpreterGroup intpGroup = interpreterGroupFactory.createInterpreterGroup(interpreterGroupId, getOption()); interpreterGroupRef.put(key, intpGroup); @@ -140,7 +122,7 @@ public Collection getAllInterpreterGroups() { } } - public void closeAndRemoveInterpreterGroup(String noteId) { + void closeAndRemoveInterpreterGroup(String noteId) { String key = getInterpreterProcessKey(noteId); InterpreterGroup groupToRemove; synchronized (interpreterGroupRef) { @@ -153,7 +135,7 @@ public void closeAndRemoveInterpreterGroup(String noteId) { } } - public void closeAndRmoveAllInterpreterGroups() { + void closeAndRmoveAllInterpreterGroups() { synchronized (interpreterGroupRef) { HashSet groupsToRemove = new HashSet<>(interpreterGroupRef.keySet()); for (String key : groupsToRemove) { @@ -166,10 +148,6 @@ public Properties getProperties() { return properties; } - public void setProperties(Properties properties) { - this.properties = properties; - } - public List getDependencies() { if (dependencies == null) { return new LinkedList<>(); @@ -193,11 +171,43 @@ public void setOption(InterpreterOption option) { this.option = option; } + public String getPath() { + return path; + } + + public void setPath(String path) { + this.path = path; + } + public List getInterpreterInfos() { - return interpreterGroup; + return interpreterInfos; } - public void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) { + void setInterpreterGroupFactory(InterpreterGroupFactory interpreterGroupFactory) { this.interpreterGroupFactory = interpreterGroupFactory; } + + void appendDependencies(List dependencies) { + for (Dependency dependency : dependencies) { + if (!this.dependencies.contains(dependency)) { + this.dependencies.add(dependency); + } + } + } + + void setInterpreterOption(InterpreterOption interpreterOption) { + this.option = interpreterOption; + } + + void updateProperties(Properties p) { + this.properties.putAll(p); + } + + void setGroup(String group) { + this.group = group; + } + + void setName(String name) { + this.name = name; + } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java index 285bcedbbe8..9106cf5f17c 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Note.java @@ -19,19 +19,32 @@ import java.io.IOException; import java.io.Serializable; -import java.util.*; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; +import com.google.gson.Gson; import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.display.AngularObject; import org.apache.zeppelin.display.AngularObjectRegistry; import org.apache.zeppelin.display.Input; -import org.apache.zeppelin.helium.HeliumApplicationFactory; -import org.apache.zeppelin.interpreter.*; +import org.apache.zeppelin.interpreter.Interpreter; +import org.apache.zeppelin.interpreter.InterpreterException; +import org.apache.zeppelin.interpreter.InterpreterFactory; +import org.apache.zeppelin.interpreter.InterpreterGroup; +import org.apache.zeppelin.interpreter.InterpreterOutput; +import org.apache.zeppelin.interpreter.InterpreterResult; +import org.apache.zeppelin.interpreter.InterpreterSetting; import org.apache.zeppelin.interpreter.remote.RemoteAngularObjectRegistry; import org.apache.zeppelin.interpreter.thrift.InterpreterCompletion; import org.apache.zeppelin.notebook.repo.NotebookRepo; @@ -39,26 +52,21 @@ import org.apache.zeppelin.resource.ResourcePoolUtils; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.Job.Status; -import org.apache.zeppelin.scheduler.JobListener; import org.apache.zeppelin.search.SearchService; - -import com.google.common.base.Optional; -import com.google.gson.Gson; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * Binded interpreters for a note */ public class Note implements Serializable, ParagraphJobListener { - static Logger logger = LoggerFactory.getLogger(Note.class); + private static final Logger logger = LoggerFactory.getLogger(Note.class); private static final long serialVersionUID = 7920699076577612429L; // threadpool for delayed persist of note private static final ScheduledThreadPoolExecutor delayedPersistThreadPool = - new ScheduledThreadPoolExecutor(0); + new ScheduledThreadPoolExecutor(0); + static { delayedPersistThreadPool.setRemoveOnCancelPolicy(true); } @@ -71,8 +79,7 @@ public class Note implements Serializable, ParagraphJobListener { private AtomicReference lastReplName = new AtomicReference<>(StringUtils.EMPTY); private transient ZeppelinConfiguration conf = ZeppelinConfiguration.create(); - @SuppressWarnings("rawtypes") - Map> angularObjects = new HashMap<>(); + private Map> angularObjects = new HashMap<>(); private transient InterpreterFactory factory; private transient JobListenerFactory jobListenerFactory; @@ -82,26 +89,24 @@ public class Note implements Serializable, ParagraphJobListener { private transient NoteEventListener noteEventListener; private transient Credentials credentials; - /** + /* * note configurations. - * * - looknfeel - cron */ private Map config = new HashMap<>(); - /** + /* * note information. - * * - cron : cron expression validity. */ private Map info = new HashMap<>(); - public Note() {} + public Note() { + } - public Note(NotebookRepo repo, InterpreterFactory factory, - JobListenerFactory jlFactory, SearchService noteIndex, Credentials credentials, - NoteEventListener noteEventListener) { + public Note(NotebookRepo repo, InterpreterFactory factory, JobListenerFactory jlFactory, + SearchService noteIndex, Credentials credentials, NoteEventListener noteEventListener) { this.repo = repo; this.factory = factory; this.jobListenerFactory = jlFactory; @@ -112,12 +117,12 @@ public Note(NotebookRepo repo, InterpreterFactory factory, } private void generateId() { - id = IdHashes.encode(System.currentTimeMillis() + new Random().nextInt()); + id = IdHashes.generateId(); } private String getDefaultInterpreterName() { InterpreterSetting setting = factory.getDefaultInterpreterSetting(getId()); - return null != setting ? setting.getGroup() : StringUtils.EMPTY; + return null != setting ? setting.getName() : StringUtils.EMPTY; } void putDefaultReplName() { @@ -138,10 +143,10 @@ public String getName() { return name; } - private String normalizeNoteName(String name){ + private String normalizeNoteName(String name) { name = name.trim(); name = name.replace("\\", "/"); - while (name.indexOf("///") >= 0) { + while (name.contains("///")) { name = name.replaceAll("///", "/"); } name = name.replaceAll("//", "/"); @@ -158,7 +163,7 @@ public void setName(String name) { this.name = name; } - public void setInterpreterFactory(InterpreterFactory factory) { + void setInterpreterFactory(InterpreterFactory factory) { this.factory = factory; synchronized (paragraphs) { for (Paragraph p : paragraphs) { @@ -167,19 +172,11 @@ public void setInterpreterFactory(InterpreterFactory factory) { } } - public JobListenerFactory getJobListenerFactory() { - return jobListenerFactory; - } - - public void setJobListenerFactory(JobListenerFactory jobListenerFactory) { + void setJobListenerFactory(JobListenerFactory jobListenerFactory) { this.jobListenerFactory = jobListenerFactory; } - public NotebookRepo getNotebookRepo() { - return repo; - } - - public void setNotebookRepo(NotebookRepo repo) { + void setNotebookRepo(NotebookRepo repo) { this.repo = repo; } @@ -189,22 +186,20 @@ public void setIndex(SearchService index) { public Credentials getCredentials() { return credentials; - }; + } public void setCredentials(Credentials credentials) { this.credentials = credentials; } - @SuppressWarnings("rawtypes") - public Map> getAngularObjects() { + Map> getAngularObjects() { return angularObjects; } /** * Add paragraph last. */ - public Paragraph addParagraph() { Paragraph p = new Paragraph(this, this, factory); addLastReplNameIfEmptyText(p); @@ -220,9 +215,9 @@ public Paragraph addParagraph() { /** * Clone paragraph and add it to note. * - * @param srcParagraph + * @param srcParagraph source paragraph */ - public void addCloneParagraph(Paragraph srcParagraph) { + void addCloneParagraph(Paragraph srcParagraph) { // Keep paragraph original ID final Paragraph newParagraph = new Paragraph(srcParagraph.getId(), this, this, factory); @@ -244,8 +239,8 @@ public void addCloneParagraph(Paragraph srcParagraph) { newParagraph.setReturn(result, null); } catch (Exception e) { // 'result' part of Note consists of exception, instead of actual interpreter results - logger.warn("Paragraph " + srcParagraph.getId() + " has a result with exception. " - + e.getMessage()); + logger.warn( + "Paragraph " + srcParagraph.getId() + " has a result with exception. " + e.getMessage()); } synchronized (paragraphs) { @@ -259,7 +254,7 @@ public void addCloneParagraph(Paragraph srcParagraph) { /** * Insert paragraph in given index. * - * @param index + * @param index index of paragraphs */ public Paragraph insertParagraph(int index) { Paragraph p = new Paragraph(this, this, factory); @@ -292,7 +287,7 @@ private String getInterpreterName(String replName) { /** * Remove paragraph by id. * - * @param paragraphId + * @param paragraphId ID of paragraph * @return a paragraph that was deleted, or null otherwise */ public Paragraph removeParagraph(String paragraphId) { @@ -319,13 +314,12 @@ public Paragraph removeParagraph(String paragraphId) { /** * Clear paragraph output by id. * - * @param paragraphId - * @return + * @param paragraphId ID of paragraph + * @return Paragraph */ public Paragraph clearParagraphOutput(String paragraphId) { synchronized (paragraphs) { - for (int i = 0; i < paragraphs.size(); i++) { - Paragraph p = paragraphs.get(i); + for (Paragraph p : paragraphs) { if (p.getId().equals(paragraphId)) { p.setReturn(null, null); return p; @@ -338,8 +332,8 @@ public Paragraph clearParagraphOutput(String paragraphId) { /** * Move paragraph into the new index (order from 0 ~ n-1). * - * @param paragraphId - * @param index new index + * @param paragraphId ID of paragraph + * @param index new index */ public void moveParagraph(String paragraphId, int index) { moveParagraph(paragraphId, index, false); @@ -348,8 +342,8 @@ public void moveParagraph(String paragraphId, int index) { /** * Move paragraph into the new index (order from 0 ~ n-1). * - * @param paragraphId - * @param index new index + * @param paragraphId ID of paragraph + * @param index new index * @param throwWhenIndexIsOutOfBound whether throw IndexOutOfBoundException * when index is out of bound */ @@ -413,7 +407,7 @@ public Paragraph getLastParagraph() { } } - public List> generateParagraphsInfo (){ + public List> generateParagraphsInfo() { List> paragraphsInfo = new LinkedList<>(); synchronized (paragraphs) { for (Paragraph p : paragraphs) { @@ -463,7 +457,7 @@ public void runAll() { /** * Run a single paragraph. * - * @param paragraphId + * @param paragraphId ID of paragraph */ public void run(String paragraphId) { Paragraph p = getParagraph(paragraphId); @@ -488,9 +482,8 @@ public void run(String paragraphId) { /** * Check whether all paragraphs belongs to this note has terminated - * @return */ - public boolean isTerminated() { + boolean isTerminated() { synchronized (paragraphs) { for (Paragraph p : paragraphs) { if (!p.isTerminated()) { @@ -505,14 +498,13 @@ public boolean isTerminated() { public List completion(String paragraphId, String buffer, int cursor) { Paragraph p = getParagraph(paragraphId); p.setListener(jobListenerFactory.getParagraphJobListener(this)); - List completion = p.completion(buffer, cursor); - return completion; + return p.completion(buffer, cursor); } public List getParagraphs() { synchronized (paragraphs) { - return new LinkedList(paragraphs); + return new LinkedList<>(paragraphs); } } @@ -532,7 +524,7 @@ private void snapshotAngularObjectRegistry() { } private void removeAllAngularObjectInParagraph(String paragraphId) { - angularObjects = new HashMap>(); + angularObjects = new HashMap<>(); List settings = factory.getInterpreterSettings(getId()); if (settings == null || settings.size() == 0) { @@ -551,8 +543,8 @@ private void removeAllAngularObjectInParagraph(String paragraphId) { List appStates = getParagraph(paragraphId).getAllApplicationStates(); if (appStates != null) { for (ApplicationState app : appStates) { - ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess( - id, app.getId()); + ((RemoteAngularObjectRegistry) registry) + .removeAllAndNotifyRemoteProcess(id, app.getId()); } } } else { @@ -588,13 +580,12 @@ public void setLastReplName(String paragraphId) { /** * Persist this note with maximum delay. - * @param maxDelaySec */ public void persist(int maxDelaySec, AuthenticationInfo subject) { startDelayedPersistTimer(maxDelaySec, subject); } - public void unpersist(AuthenticationInfo subject) throws IOException { + void unpersist(AuthenticationInfo subject) throws IOException { repo.remove(id(), subject); } @@ -651,7 +642,7 @@ public void setInfo(Map info) { this.info = info; } - public String getLastReplName() { + String getLastReplName() { return lastReplName.get(); } @@ -714,13 +705,7 @@ public void onOutputUpdate(Paragraph paragraph, InterpreterOutput out, String ou } } - - - public NoteEventListener getNoteEventListener() { - return noteEventListener; - } - - public void setNoteEventListener(NoteEventListener noteEventListener) { + void setNoteEventListener(NoteEventListener noteEventListener) { this.noteEventListener = noteEventListener; } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java index 70735209465..c56c4eed36e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/Notebook.java @@ -30,6 +30,22 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.stream.JsonReader; +import org.quartz.CronScheduleBuilder; +import org.quartz.CronTrigger; +import org.quartz.JobBuilder; +import org.quartz.JobDetail; +import org.quartz.JobExecutionContext; +import org.quartz.JobExecutionException; +import org.quartz.JobKey; +import org.quartz.SchedulerException; +import org.quartz.TriggerBuilder; +import org.quartz.impl.StdSchedulerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.zeppelin.conf.ZeppelinConfiguration; import org.apache.zeppelin.conf.ZeppelinConfiguration.ConfVars; import org.apache.zeppelin.display.AngularObject; @@ -46,34 +62,21 @@ import org.apache.zeppelin.search.SearchService; import org.apache.zeppelin.user.AuthenticationInfo; import org.apache.zeppelin.user.Credentials; -import org.quartz.CronScheduleBuilder; -import org.quartz.CronTrigger; -import org.quartz.JobBuilder; -import org.quartz.JobDetail; -import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; -import org.quartz.JobKey; -import org.quartz.SchedulerException; -import org.quartz.TriggerBuilder; -import org.quartz.impl.StdSchedulerFactory; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.stream.JsonReader; /** * Collection of Notes. */ public class Notebook implements NoteEventListener { - static Logger logger = LoggerFactory.getLogger(Notebook.class); + private static final Logger logger = LoggerFactory.getLogger(Notebook.class); @SuppressWarnings("unused") @Deprecated //TODO(bzz): remove unused private SchedulerFactory schedulerFactory; private InterpreterFactory replFactory; - /** Keep the order. */ - Map notes = new LinkedHashMap(); + /** + * Keep the order. + */ + private final Map notes = new LinkedHashMap<>(); private ZeppelinConfiguration conf; private StdSchedulerFactory quertzSchedFact; private org.quartz.Scheduler quartzSched; @@ -88,23 +91,15 @@ public class Notebook implements NoteEventListener { /** * Main constructor \w manual Dependency Injection * - * @param conf - * @param notebookRepo - * @param schedulerFactory - * @param replFactory - * @param jobListenerFactory - * @param notebookIndex - (nullable) for indexing all notebooks on creating. - * @param notebookAuthorization - * + * @param notebookIndex - (nullable) for indexing all notebooks on creating. * @throws IOException * @throws SchedulerException */ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, - SchedulerFactory schedulerFactory, - InterpreterFactory replFactory, JobListenerFactory jobListenerFactory, - SearchService notebookIndex, - NotebookAuthorization notebookAuthorization, - Credentials credentials) throws IOException, SchedulerException { + SchedulerFactory schedulerFactory, InterpreterFactory replFactory, + JobListenerFactory jobListenerFactory, SearchService notebookIndex, + NotebookAuthorization notebookAuthorization, Credentials credentials) + throws IOException, SchedulerException { this.conf = conf; this.notebookRepo = notebookRepo; this.schedulerFactory = schedulerFactory; @@ -132,7 +127,6 @@ public Notebook(ZeppelinConfiguration conf, NotebookRepo notebookRepo, /** * Create new note. * - * @return * @throws IOException */ public Note createNote(AuthenticationInfo subject) throws IOException { @@ -149,18 +143,12 @@ public Note createNote(AuthenticationInfo subject) throws IOException { /** * Create new note. * - * @return * @throws IOException */ public Note createNote(List interpreterIds, AuthenticationInfo subject) throws IOException { - Note note = new Note( - notebookRepo, - replFactory, - jobListenerFactory, - notebookIndex, - credentials, - this); + Note note = + new Note(notebookRepo, replFactory, jobListenerFactory, notebookIndex, credentials, this); synchronized (notes) { notes.put(note.id(), note); } @@ -174,9 +162,10 @@ public Note createNote(List interpreterIds, AuthenticationInfo subject) fireNoteCreateEvent(note); return note; } - + /** * Export existing note. + * * @param noteId - the note ID to clone * @return Note JSON * @throws IOException, IllegalArgumentException @@ -194,8 +183,9 @@ public String exportNote(String noteId) throws IOException, IllegalArgumentExcep /** * import JSON as a new note. + * * @param sourceJson - the note JSON to import - * @param noteName - the name of the new note + * @param noteName - the name of the new note * @return notebook ID * @throws IOException */ @@ -204,8 +194,8 @@ public Note importNote(String sourceJson, String noteName, AuthenticationInfo su GsonBuilder gsonBuilder = new GsonBuilder(); gsonBuilder.setPrettyPrinting(); - Gson gson = gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()) - .create(); + Gson gson = + gsonBuilder.registerTypeAdapter(Date.class, new NotebookImportDeserializer()).create(); JsonReader reader = new JsonReader(new StringReader(sourceJson)); reader.setLenient(true); Note newNote; @@ -232,13 +222,14 @@ public Note importNote(String sourceJson, String noteName, AuthenticationInfo su /** * Clone existing note. + * * @param sourceNoteID - the note ID to clone - * @param newNoteName - the name of the new note + * @param newNoteName - the name of the new note * @return noteId * @throws IOException, CloneNotSupportedException, IllegalArgumentException */ - public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject) throws - IOException, CloneNotSupportedException, IllegalArgumentException { + public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInfo subject) + throws IOException, CloneNotSupportedException, IllegalArgumentException { Note sourceNote = getNote(sourceNoteID); if (sourceNote == null) { @@ -262,13 +253,13 @@ public Note cloneNote(String sourceNoteID, String newNoteName, AuthenticationInf return newNote; } - public void bindInterpretersToNote(String id, - List interpreterSettingIds) throws IOException { + public void bindInterpretersToNote(String id, List interpreterSettingIds) + throws IOException { Note note = getNote(id); if (note != null) { List currentBindings = replFactory.getInterpreterSettings(id); for (InterpreterSetting setting : currentBindings) { - if (!interpreterSettingIds.contains(setting.id())) { + if (!interpreterSettingIds.contains(setting.getId())) { fireUnbindInterpreter(note, setting); } } @@ -279,12 +270,12 @@ public void bindInterpretersToNote(String id, } } - public List getBindedInterpreterSettingsIds(String id) { + List getBindedInterpreterSettingsIds(String id) { Note note = getNote(id); if (note != null) { return getInterpreterFactory().getInterpreters(note.getId()); } else { - return new LinkedList(); + return new LinkedList<>(); } } @@ -293,7 +284,7 @@ public List getBindedInterpreterSettings(String id) { if (note != null) { return replFactory.getInterpreterSettings(note.getId()); } else { - return new LinkedList(); + return new LinkedList<>(); } } @@ -325,8 +316,8 @@ public void removeNote(String id, AuthenticationInfo subject) { List appStates = p.getAllApplicationStates(); if (appStates != null) { for (ApplicationState app : appStates) { - ((RemoteAngularObjectRegistry) registry).removeAllAndNotifyRemoteProcess( - id, app.getId()); + ((RemoteAngularObjectRegistry) registry) + .removeAllAndNotifyRemoteProcess(id, app.getId()); } } } @@ -393,8 +384,7 @@ private Note loadNoteFromRepo(String id, AuthenticationInfo subject) { Date lastUpdatedDate = new Date(0); for (Paragraph p : note.getParagraphs()) { p.setNote(note); - if (p.getDateFinished() != null && - lastUpdatedDate.before(p.getDateFinished())) { + if (p.getDateFinished() != null && lastUpdatedDate.before(p.getDateFinished())) { lastUpdatedDate = p.getDateFinished(); } } @@ -456,7 +446,6 @@ private void loadAllNotes() throws IOException { * Reload all notes from repository after clearing `notes` * to reflect the changes of added/deleted/modified notebooks on file system level. * - * @return * @throws IOException */ public void reloadAllNotes(AuthenticationInfo subject) throws IOException { @@ -477,34 +466,34 @@ public void reloadAllNotes(AuthenticationInfo subject) throws IOException { } } - @SuppressWarnings("rawtypes") - class SnapshotAngularObject { + private class SnapshotAngularObject { String intpGroupId; AngularObject angularObject; Date lastUpdate; - public SnapshotAngularObject(String intpGroupId, - AngularObject angularObject, Date lastUpdate) { + SnapshotAngularObject(String intpGroupId, AngularObject angularObject, Date lastUpdate) { super(); this.intpGroupId = intpGroupId; this.angularObject = angularObject; this.lastUpdate = lastUpdate; } - public String getIntpGroupId() { + String getIntpGroupId() { return intpGroupId; } - public AngularObject getAngularObject() { + + AngularObject getAngularObject() { return angularObject; } - public Date getLastUpdate() { + + Date getLastUpdate() { return lastUpdate; } } public List getAllNotes() { synchronized (notes) { - List noteList = new ArrayList(notes.values()); + List noteList = new ArrayList<>(notes.values()); Collections.sort(noteList, new Comparator() { @Override public int compare(Note note1, Note note2) { @@ -523,14 +512,6 @@ public int compare(Note note1, Note note2) { } } - public JobListenerFactory getJobListenerFactory() { - return jobListenerFactory; - } - - public void setJobListenerFactory(JobListenerFactory jobListenerFactory) { - this.jobListenerFactory = jobListenerFactory; - } - private Map getParagraphForJobManagerItem(Paragraph paragraph) { Map paragraphItem = new HashMap<>(); @@ -553,16 +534,16 @@ private Map getParagraphForJobManagerItem(Paragraph paragraph) { private long getUnixTimeLastRunParagraph(Paragraph paragraph) { - Date lastRunningDate = null; - long lastRunningUnixTime = 0; + Date lastRunningDate; + long lastRunningUnixTime; Date paragaraphDate = paragraph.getDateStarted(); // diff started time <-> finishied time if (paragaraphDate == null) { paragaraphDate = paragraph.getDateFinished(); } else { - if (paragraph.getDateFinished() != null && - paragraph.getDateFinished().after(paragaraphDate)) { + if (paragraph.getDateFinished() != null && paragraph.getDateFinished() + .after(paragaraphDate)) { paragaraphDate = paragraph.getDateFinished(); } } @@ -612,11 +593,10 @@ public List> getJobListforNotebook(boolean needsReload, } // set notebook type ( cron or normal ) - if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) == true && - !note.getConfig().get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { + if (note.getConfig().containsKey(CRON_TYPE_NOTEBOOK_KEYWORD) && !note.getConfig() + .get(CRON_TYPE_NOTEBOOK_KEYWORD).equals("")) { info.put("notebookType", "cron"); - } - else { + } else { info.put("notebookType", "normal"); } @@ -624,7 +604,7 @@ public List> getJobListforNotebook(boolean needsReload, List> paragraphsInfo = new LinkedList<>(); for (Paragraph paragraph : note.getParagraphs()) { // check paragraph's status. - if (paragraph.getStatus().isRunning() == true) { + if (paragraph.getStatus().isRunning()) { isNotebookRunning = true; isUpdateNotebook = true; } @@ -642,13 +622,13 @@ public List> getJobListforNotebook(boolean needsReload, // set interpreter bind type String interpreterGroupName = null; - if (replFactory.getInterpreterSettings(note.getId()) != null && - replFactory.getInterpreterSettings(note.getId()).size() >= 1) { - interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getGroup(); + if (replFactory.getInterpreterSettings(note.getId()) != null + && replFactory.getInterpreterSettings(note.getId()).size() >= 1) { + interpreterGroupName = replFactory.getInterpreterSettings(note.getId()).get(0).getName(); } // not update and not running -> pass - if (isUpdateNotebook == false && isNotebookRunning == false) { + if (!isUpdateNotebook && !isNotebookRunning) { continue; } @@ -675,7 +655,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException { String noteId = context.getJobDetail().getJobDataMap().getString("noteId"); Note note = notebook.getNote(noteId); note.runAll(); - + while (!note.isTerminated()) { try { Thread.sleep(1000); @@ -683,7 +663,7 @@ public void execute(JobExecutionContext context) throws JobExecutionException { logger.error(e.toString(), e); } } - + boolean releaseResource = false; try { Map config = note.getConfig(); @@ -694,11 +674,11 @@ public void execute(JobExecutionContext context) throws JobExecutionException { logger.error(e.getMessage(), e); } if (releaseResource) { - for (InterpreterSetting setting : - notebook.getInterpreterFactory().getInterpreterSettings(note.getId())) { - notebook.getInterpreterFactory().restart(setting.id()); + for (InterpreterSetting setting : notebook.getInterpreterFactory() + .getInterpreterSettings(note.getId())) { + notebook.getInterpreterFactory().restart(setting.getId()); } - } + } } } @@ -723,17 +703,15 @@ public void refreshCron(String id) { JobDetail newJob = JobBuilder.newJob(CronJob.class).withIdentity(id, "note").usingJobData("noteId", id) - .build(); + .build(); Map info = note.getInfo(); info.put("cron", null); CronTrigger trigger = null; try { - trigger = - TriggerBuilder.newTrigger().withIdentity("trigger_" + id, "note") - .withSchedule(CronScheduleBuilder.cronSchedule(cronExpr)).forJob(id, "note") - .build(); + trigger = TriggerBuilder.newTrigger().withIdentity("trigger_" + id, "note") + .withSchedule(CronScheduleBuilder.cronSchedule(cronExpr)).forJob(id, "note").build(); } catch (Exception e) { logger.error("Error", e); info.put("cron", e.getMessage()); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java index 11f9dcc9d2a..04021e7e808 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/utility/IdHashes.java @@ -20,14 +20,15 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.Random; /** * Generate Tiny ID. */ public class IdHashes { - public static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9', - 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', - 'V', 'W', 'X', 'Y', 'Z'}; + private static final char[] DICTIONARY = new char[] {'1', '2', '3', '4', '5', '6', '7', '8', '9', + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'J', 'K', 'M', 'N', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + 'W', 'X', 'Y', 'Z'}; /** * encodes the given string into the base of the dictionary provided in the constructor. @@ -35,7 +36,7 @@ public class IdHashes { * @param value the number to encode. * @return the encoded string. */ - public static String encode(Long value) { + private static String encode(Long value) { List result = new ArrayList(); BigInteger base = new BigInteger("" + DICTIONARY.length); @@ -68,4 +69,8 @@ public static String encode(Long value) { } return sb.toString(); } + + public static String generateId() { + return encode(System.currentTimeMillis() + new Random().nextInt()); + } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java index 602f38425ed..4f6232493d4 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/helium/HeliumApplicationFactoryTest.java @@ -24,7 +24,6 @@ import org.apache.zeppelin.interpreter.mock.MockInterpreter2; import org.apache.zeppelin.notebook.*; import org.apache.zeppelin.notebook.repo.VFSNotebookRepo; -import org.apache.zeppelin.scheduler.ExecutorFactory; import org.apache.zeppelin.scheduler.Job; import org.apache.zeppelin.scheduler.SchedulerFactory; import org.apache.zeppelin.search.SearchService; @@ -37,8 +36,6 @@ import java.util.HashMap; import java.util.LinkedList; import java.util.List; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.atomic.AtomicInteger; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; @@ -259,7 +256,7 @@ public void testUnloadOnInterpreterRestart() throws IOException { String mock1IntpSettingId = null; for (InterpreterSetting setting : notebook.getBindedInterpreterSettings(note1.id())) { if (setting.getName().equals("mock1")) { - mock1IntpSettingId = setting.id(); + mock1IntpSettingId = setting.getId(); break; } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java index 9df494f8519..e28416491ee 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterFactoryTest.java @@ -18,6 +18,7 @@ package org.apache.zeppelin.interpreter; import java.io.*; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import java.util.Properties; @@ -78,6 +79,8 @@ public void testBasic() { } } +// mock1Setting = factory.createNewSetting("mock11", "mock1", new ArrayList(), new InterpreterOption(false), new Properties()); + InterpreterGroup interpreterGroup = mock1Setting.getInterpreterGroup("sharedProcess"); factory.createInterpretersForNote(mock1Setting, "sharedProcess", "session"); @@ -88,7 +91,7 @@ public void testBasic() { assertNull(factory.get("unknown")); // restart interpreter - factory.restart(mock1Setting.id()); + factory.restart(mock1Setting.getId()); assertNull(mock1Setting.getInterpreterGroup("sharedProcess").get("session")); } @@ -104,12 +107,12 @@ public void testExceptions() throws InterpreterException, IOException, Repositor List all = factory.getDefaultInterpreterSettingList(); // add setting with null option & properties expected nullArgumentException.class try { - factory.add("a mock", "mock2", new LinkedList(), null, new Properties()); + factory.add("mock2", new ArrayList(), new LinkedList(), new InterpreterOption(false), new Properties(), ""); } catch(NullArgumentException e) { assertEquals("Test null option" , e.getMessage(),new NullArgumentException("option").getMessage()); } try { - factory.add("a mock", "mock2", new LinkedList(), new InterpreterOption(false), null); + factory.add("mock2", new ArrayList(), new LinkedList(), new InterpreterOption(false), new Properties(), ""); } catch (NullArgumentException e){ assertEquals("Test null properties" , e.getMessage(),new NullArgumentException("properties").getMessage()); } @@ -124,10 +127,33 @@ public void testSaveLoad() throws IOException, RepositoryException { // check if file saved assertTrue(new File(conf.getInterpreterSettingPath()).exists()); - factory.add("newsetting", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); + factory.createNewSetting("new-mock1", "mock1", new LinkedList(), new InterpreterOption(false), new Properties()); assertEquals(numInterpreters + 1, factory.get().size()); InterpreterFactory factory2 = new InterpreterFactory(conf, null, null, null, depResolver); assertEquals(numInterpreters + 1, factory2.get().size()); } + + @Test + public void testInterpreterAliases() throws IOException, RepositoryException { + factory = new InterpreterFactory(conf, null, null, null, depResolver); + final InterpreterInfo info1 = new InterpreterInfo("className1", "name1", true); + final InterpreterInfo info2 = new InterpreterInfo("className2", "name1", true); + factory.add("group1", new ArrayList(){{ + add(info1); + }}, new ArrayList(), new InterpreterOption(true), new Properties(), "/path1"); + factory.add("group2", new ArrayList(){{ + add(info2); + }}, new ArrayList(), new InterpreterOption(true), new Properties(), "/path2"); + + final InterpreterSetting setting1 = factory.createNewSetting("test-group1", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); + final InterpreterSetting setting2 = factory.createNewSetting("test-group2", "group1", new ArrayList(), new InterpreterOption(true), new Properties()); + + factory.setInterpreters("note", new ArrayList() {{ + add(setting1.getId()); + add(setting2.getId()); + }}); + + assertEquals("className1", factory.getInterpreter("note", "test-group1").getClassName()); + } } diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java index 9a737e69347..cff66adc78d 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteTest.java @@ -118,7 +118,7 @@ public void putDefaultReplNameIfInterpreterSettingAbsent() { @Test public void putDefaultReplNameIfInterpreterSettingPresent() { InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class); - when(interpreterSetting.getGroup()).thenReturn("spark"); + when(interpreterSetting.getName()).thenReturn("spark"); when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(interpreterSetting); @@ -132,7 +132,7 @@ public void putDefaultReplNameIfInterpreterSettingPresent() { @Test public void addParagraphWithLastReplName() { InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class); - when(interpreterSetting.getGroup()).thenReturn("spark"); + when(interpreterSetting.getName()).thenReturn("spark"); when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(interpreterSetting); @@ -147,7 +147,7 @@ public void addParagraphWithLastReplName() { @Test public void insertParagraphWithLastReplName() { InterpreterSetting interpreterSetting = Mockito.mock(InterpreterSetting.class); - when(interpreterSetting.getGroup()).thenReturn("spark"); + when(interpreterSetting.getName()).thenReturn("spark"); when(interpreterFactory.getDefaultInterpreterSetting(anyString())) .thenReturn(interpreterSetting); diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java index 67902978abe..bfa97e05de3 100644 --- a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NotebookTest.java @@ -532,7 +532,7 @@ public void testAngularObjectRemovalOnInterpreterRestart() throws InterruptedExc registry.add("o2", "object2", null, null); // restart interpreter - factory.restart(factory.getInterpreterSettings(note.getId()).get(0).id()); + factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId()); registry = factory.getInterpreterSettings(note.getId()).get(0).getInterpreterGroup("sharedProcess") .getAngularObjectRegistry(); @@ -608,7 +608,7 @@ public void testAbortParagraphStatusOnInterpreterRestart() throws InterruptedExc while (paragraphs.get(0).getStatus() != Status.FINISHED) Thread.yield(); - factory.restart(factory.getInterpreterSettings(note.getId()).get(0).id()); + factory.restart(factory.getInterpreterSettings(note.getId()).get(0).getId()); boolean isAborted = false; for (Paragraph p : paragraphs) { @@ -634,7 +634,7 @@ public void testPerSessionInterpreterCloseOnNoteRemoval() throws IOException { // restart interpreter with per note session enabled for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { setting.getOption().setPerNoteSession(true); - notebook.getInterpreterFactory().restart(setting.id()); + notebook.getInterpreterFactory().restart(setting.getId()); } note1.run(p1.getId()); @@ -679,7 +679,7 @@ public void testPerSessionInterpreter() throws IOException { // restart interpreter with per note session enabled for (InterpreterSetting setting : notebook.getInterpreterFactory().getInterpreterSettings(note1.getId())) { setting.getOption().setPerNoteSession(true); - notebook.getInterpreterFactory().restart(setting.id()); + notebook.getInterpreterFactory().restart(setting.getId()); } // run per note session enabled @@ -705,7 +705,7 @@ public void testPerSessionInterpreterCloseOnUnbindInterpreterSetting() throws IO // restart interpreter with per note session enabled for (InterpreterSetting setting : factory.getInterpreterSettings(note1.getId())) { setting.getOption().setPerNoteSession(true); - notebook.getInterpreterFactory().restart(setting.id()); + notebook.getInterpreterFactory().restart(setting.getId()); } note1.run(p1.getId());