From 0eade713f1609c77518e19bec2c8750bc3447367 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 4 Aug 2016 14:05:22 +0800 Subject: [PATCH 1/3] ZEPPELIN-1287. No need to call print to display output in PythonInterpreter --- .../zeppelin/python/PythonInterpreter.java | 5 +-- .../apache/zeppelin/python/PythonProcess.java | 35 ++++++++----------- python/src/main/resources/bootstrap.py | 2 -- 3 files changed, 17 insertions(+), 25 deletions(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index 670dffca39c..676f7ca42b0 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -134,9 +134,9 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr InterpreterResult result; if (pythonErrorIn(output)) { - result = new InterpreterResult(Code.ERROR, output.replaceAll(">>>", "").trim()); + result = new InterpreterResult(Code.ERROR, output.replaceAll(">>> ", "").trim()); } else { - result = new InterpreterResult(Code.SUCCESS, output.replaceAll(">>>", "") + result = new InterpreterResult(Code.SUCCESS, output.replaceAll(">>> ", "") .replaceAll("\\.\\.\\.", "").trim()); } return result; @@ -265,4 +265,5 @@ private int findRandomOpenPortOnAllLocalInterfaces() { public int getMaxResult() { return maxResult; } + } diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java index 348ced68a45..0e6bc2332e5 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java @@ -20,13 +20,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.InputStreamReader; +import java.io.*; import java.lang.reflect.Field; /** @@ -34,11 +28,11 @@ * Python process (REPL) used by python interpreter */ public class PythonProcess { - Logger logger = LoggerFactory.getLogger(PythonProcess.class); - + private static final Logger logger = LoggerFactory.getLogger(PythonProcess.class); + private static final String STATEMENT_END = "*!?flush reader!?*"; InputStream stdout; OutputStream stdin; - BufferedWriter writer; + PrintWriter writer; BufferedReader reader; Process process; @@ -56,7 +50,7 @@ public void open() throws IOException { process = builder.start(); stdout = process.getInputStream(); stdin = process.getOutputStream(); - writer = new BufferedWriter(new OutputStreamWriter(stdin)); + writer = new PrintWriter(stdin, true); reader = new BufferedReader(new InputStreamReader(stdout)); try { pid = findPid(); @@ -85,22 +79,21 @@ public void interrupt() throws IOException { } public String sendAndGetResult(String cmd) throws IOException { - writer.write(cmd + "\n\n"); - writer.write("print (\"*!?flush reader!?*\")\n\n"); - writer.flush(); - - String output = ""; - String line; - while (!(line = reader.readLine()).contains("*!?flush reader!?*")) { + writer.println(cmd); + writer.println(); + writer.println("\"" + STATEMENT_END + "\""); + StringBuilder output = new StringBuilder(); + String line = null; + while (!(line = reader.readLine()).contains(STATEMENT_END)) { logger.debug("Read line from python shell : " + line); if (line.equals("...")) { logger.warn("Syntax error ! "); - output += "Syntax error ! "; + output.append("Syntax error ! "); break; } - output += "\r" + line + "\n"; + output.append(line + "\n"); } - return output; + return output.toString(); } private long findPid() throws NoSuchFieldException, IllegalAccessException { diff --git a/python/src/main/resources/bootstrap.py b/python/src/main/resources/bootstrap.py index 09e51e31cca..4a5b5878dcd 100644 --- a/python/src/main/resources/bootstrap.py +++ b/python/src/main/resources/bootstrap.py @@ -25,8 +25,6 @@ except ImportError: import io as io -sys.displayhook = lambda x: None - def intHandler(signum, frame): # Set the signal handler print ("Paragraph interrupted") raise KeyboardInterrupt() From 3e9f1699ee77b1a3b38ee5fef168d82d470831af Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 4 Aug 2016 20:19:41 +0800 Subject: [PATCH 2/3] address comments --- .../java/org/apache/zeppelin/python/PythonProcess.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java index 0e6bc2332e5..0ab14613101 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonProcess.java @@ -20,7 +20,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.io.OutputStream; import java.lang.reflect.Field; /** From b48b56fbe68dfd3e3a20a71babac72ada52c02d8 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Mon, 8 Aug 2016 08:40:46 +0800 Subject: [PATCH 3/3] fix unit test fail --- .../org/apache/zeppelin/python/PythonInterpreter.java | 7 ++++--- python/src/main/resources/bootstrap.py | 2 ++ .../org/apache/zeppelin/python/PythonInterpreterTest.java | 8 ++++---- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java index 676f7ca42b0..877d6975255 100644 --- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java +++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java @@ -134,10 +134,11 @@ public InterpreterResult interpret(String cmd, InterpreterContext contextInterpr InterpreterResult result; if (pythonErrorIn(output)) { - result = new InterpreterResult(Code.ERROR, output.replaceAll(">>> ", "").trim()); + result = new InterpreterResult(Code.ERROR, output); } else { - result = new InterpreterResult(Code.SUCCESS, output.replaceAll(">>> ", "") - .replaceAll("\\.\\.\\.", "").trim()); + // TODO(zjffdu), we should not do string replacement operation in the result, as it is + // possible that the output contains the kind of pattern itself, e.g. print("...") + result = new InterpreterResult(Code.SUCCESS, output.replaceAll("\\.\\.\\.", "")); } return result; } diff --git a/python/src/main/resources/bootstrap.py b/python/src/main/resources/bootstrap.py index 4a5b5878dcd..889b456b50a 100644 --- a/python/src/main/resources/bootstrap.py +++ b/python/src/main/resources/bootstrap.py @@ -30,6 +30,8 @@ def intHandler(signum, frame): # Set the signal handler raise KeyboardInterrupt() signal.signal(signal.SIGINT, intHandler) +# set prompt as empty string so that java side don't need to remove the prompt. +sys.ps1="" def help(): print("""%html diff --git a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java index a4c80ae8966..8866e6ce1c1 100644 --- a/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java +++ b/python/src/test/java/org/apache/zeppelin/python/PythonInterpreterTest.java @@ -115,7 +115,7 @@ public void testPy4jIsNotInstalled() { */ @Test public void testPy4jInstalled() throws IOException, InterruptedException { - when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn(">>>"); + when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn(""); pythonInterpreter.open(); Integer py4jPort = pythonInterpreter.getPy4jPort(); @@ -137,7 +137,7 @@ public void testPy4jInstalled() throws IOException, InterruptedException { @Test public void testClose() throws IOException, InterruptedException { //given: py4j is installed - when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn(">>>"); + when(mockPythonProcess.sendAndGetResult(eq("\n\nimport py4j\n"))).thenReturn(""); pythonInterpreter.open(); Integer py4jPort = pythonInterpreter.getPy4jPort(); @@ -210,11 +210,11 @@ private String answerFromPythonMock(InvocationOnMock invocationOnMock) { String output = ""; for (int i = 0; i < lines.length; i++) { - output += ">>>" + lines[i]; + output += lines[i]; } return output; } else { - return ">>>"; + return ""; } }