From 1e57afe27653fddbd8b0d6387c871eb8fd3f81ea Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Fri, 13 Oct 2017 15:41:01 +0800 Subject: [PATCH 1/2] ZEPPELIN-2990. Matplotlib sometimes fails in IPythonInterpreter --- .../org/apache/zeppelin/python/IPythonClient.java | 2 +- .../apache/zeppelin/python/IPythonInterpreter.java | 1 + .../zeppelin/python/IPythonInterpreterTest.java | 12 ++++++++++++ .../zeppelin/spark/IPySparkInterpreterTest.java | 4 ++-- .../org/apache/zeppelin/socket/NotebookServer.java | 2 +- .../remote/RemoteInterpreterEventPoller.java | 1 - 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java index 05fe4ba47a7..ac1020498bc 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonClient.java @@ -96,7 +96,7 @@ public void onNext(ExecuteResponse executeResponse) { LOGGER.debug("Interpreter Streaming Output: " + executeResponse.getOutput()); if (isPreviousOutputImage) { // add '\n' when switch from image to text - interpreterOutput.write("\n".getBytes()); + interpreterOutput.write("\n%text ".getBytes()); } isPreviousOutputImage = false; interpreterOutput.write(executeResponse.getOutput().getBytes()); diff --git a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java index 1c2ced501b0..77e45b99c2f 100644 --- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java @@ -286,6 +286,7 @@ protected Map setupIPythonEnv() throws IOException { } else { envs.put("PYTHONPATH", additionalPythonPath); } + LOGGER.info("PYTHONPATH:" + envs.get("PYTHONPATH")); return envs; } diff --git a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java index 6a388c2b9b3..104fe1901c8 100644 --- a/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/IPythonInterpreterTest.java @@ -284,12 +284,24 @@ public void run() { // the order of IMAGE and TEXT is not determined // check there must be one IMAGE output boolean hasImageOutput = false; + boolean hasLineText = false; + boolean hasFigureText = false; for (InterpreterResultMessage msg : interpreterResultMessages) { if (msg.getType() == InterpreterResult.Type.IMG) { hasImageOutput = true; } + if (msg.getType() == InterpreterResult.Type.TEXT + && msg.getData().contains("matplotlib.lines.Line2D")) { + hasLineText = true; + } + if (msg.getType() == InterpreterResult.Type.TEXT + && msg.getData().contains("matplotlib.figure.Figure")) { + hasFigureText = true; + } } assertTrue("No Image Output", hasImageOutput); + assertTrue("No Line Text", hasLineText); + assertTrue("No Figure Text", hasFigureText); // bokeh // bokeh initialization diff --git a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java b/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java index faf0473092b..5f6d42c6307 100644 --- a/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java +++ b/spark/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java @@ -146,7 +146,7 @@ public void run() { List interpreterResultMessages = null; try { interpreterResultMessages = context2.out.getInterpreterResultMessages(); - assertTrue(interpreterResultMessages.get(0).getData().contains("cancelled")); + assertTrue(interpreterResultMessages.get(0).getData().contains("KeyboardInterrupt")); } catch (IOException e) { e.printStackTrace(); } @@ -180,7 +180,7 @@ public void run() { "ssc.start()\n" + "time.sleep(6)\n" + "ssc.stop(stopSparkContext=False, stopGraceFully=True)", context); - Thread.sleep(100); + Thread.sleep(1000); assertEquals(InterpreterResult.Code.SUCCESS, result.code()); interpreterResultMessages = context.out.getInterpreterResultMessages(); assertEquals(1, interpreterResultMessages.size()); diff --git a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java index 1e7b74f313e..8c13e21508d 100644 --- a/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java +++ b/zeppelin-server/src/main/java/org/apache/zeppelin/socket/NotebookServer.java @@ -357,7 +357,7 @@ public void onMessage(NotebookSocket conn, String msg) { break; } } catch (Exception e) { - LOG.error("Can't handle message", e); + LOG.error("Can't handle message: " + msg, e); } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java index 35b6b6c84cb..abda81e03c4 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterEventPoller.java @@ -246,7 +246,6 @@ public RemoteInterpreterEvent call(Client client) throws Exception { listener.onParaInfosReceived(noteId, paraId, settingId, paraInfos); } } - logger.debug("Event from remote process {}", event.getType()); } catch (Exception e) { logger.error("Can't handle event " + event, e); } From 55e6f88afa058d5cca229ae5ea6d5fb14ad1d81a Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 16 Oct 2017 11:28:23 +0800 Subject: [PATCH 2/2] fix unit test --- interpreter-parent/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/interpreter-parent/pom.xml b/interpreter-parent/pom.xml index 32bdc570e78..cb089654618 100644 --- a/interpreter-parent/pom.xml +++ b/interpreter-parent/pom.xml @@ -78,6 +78,14 @@ maven-dependency-plugin + + copy-dependencies + none + + true + + + copy-interpreter-dependencies package