diff --git a/.gitignore b/.gitignore index 502ab0b69e3..38fa11885ba 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ lens/lens-cli-hist.log # conf file conf/zeppelin-env.sh +conf/zeppelin-env.cmd conf/zeppelin-site.xml conf/keystore conf/truststore diff --git a/bin/common.cmd b/bin/common.cmd new file mode 100644 index 00000000000..c84f0778291 --- /dev/null +++ b/bin/common.cmd @@ -0,0 +1,112 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +if not defined ZEPPELIN_HOME ( + for %%d in ("%~dp0..") do ( + set ZEPPELIN_HOME=%%~fd + ) +) + +if not defined ZEPPELIN_CONF_DIR ( + set ZEPPELIN_CONF_DIR=%ZEPPELIN_HOME%\conf +) + +if not defined ZEPPELIN_LOG_DIR ( + set ZEPPELIN_LOG_DIR=%ZEPPELIN_HOME%\logs +) + +if not defined ZEPPELIN_NOTEBOOK_DIR ( + set ZEPPELIN_NOTEBOOK_DIR=%ZEPPELIN_HOME%\notebook +) + +if not defined ZEPPELIN_PID_DIR ( + set ZEPPELIN_PID_DIR=%ZEPPELIN_HOME%\run +) + +if not defined ZEPPELIN_WAR ( + if exist "%ZEPPELIN_HOME%\zeppelin-web\dist" ( + set ZEPPELIN_WAR=%ZEPPELIN_HOME%\zeppelin-web\dist + ) else ( + for %%d in ("%ZEPPELIN_HOME%\zeppelin-web*.war") do ( + set ZEPPELIN_WAR=%%d + ) + ) +) + +if not defined ZEPPELIN_INTERPRETER_DIR ( + set ZEPPELIN_INTERPRETER_DIR=%ZEPPELIN_HOME%\interpreter +) + +if exist "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd" ( + call "%ZEPPELIN_CONF_DIR%\zeppelin-env.cmd" +) + +if not defined ZEPPELIN_CLASSPATH ( + set ZEPPELIN_CLASSPATH="%ZEPPELIN_CONF_DIR%" +) else ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_CONF_DIR%" +) + +if not defined ZEPPELIN_ENCODING ( + set ZEPPELIN_ENCODING=UTF-8 +) + +if not defined ZEPPELIN_MEM ( + set ZEPPELIN_MEM=-Xms1024m -Xmx1024m -XX:MaxPermSize=512m +) + +if not defined ZEPPELIN_JAVA_OPTS ( + set ZEPPELIN_JAVA_OPTS=-Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM% +) else ( + set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM% +) + +if not defined JAVA_OPTS ( + set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% +) else ( + set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS% +) + +if not defined ZEPPELIN_INTP_JAVA_OPTS ( + set ZEPPELIN_INTP_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS% +) + +if not defined ZEPPELIN_INTP_MEM ( + set ZEPPELIN_INTP_MEM=%ZEPPELIN_MEM% +) + +set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS% -Dfile.encoding=%ZEPPELIN_ENCODING% + +if not defined JAVA_HOME ( + set ZEPPELIN_RUNNER=java +) else ( + set ZEPPELIN_RUNNER=%JAVA_HOME%\bin\java +) + +if not defined ZEPPELIN_IDENT_STRING ( + set ZEPPELIN_IDENT_STRING=%USERNAME% +) + +if not defined DEBUG ( + set DEBUG=0 +) + +if not defined ZEPPELIN_INTERPRETER_REMOTE_RUNNER ( + set ZEPPELIN_INTERPRETER_REMOTE_RUNNER=bin\interpreter.cmd +) + +exit /b diff --git a/bin/functions.cmd b/bin/functions.cmd new file mode 100644 index 00000000000..2aa4f7491f2 --- /dev/null +++ b/bin/functions.cmd @@ -0,0 +1,38 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +if not "%1"=="" goto %1 + +exit /b + +:ADDEACHJARINDIR +for %%d in ("%~2\*.jar") do ( + set ZEPPELIN_CLASSPATH="%%d";!ZEPPELIN_CLASSPATH! +) +exit /b + +:ADDEACHJARINDIRRECURSIVE +for /r "%~2" %%d in (*.jar) do ( + set ZEPPELIN_CLASSPATH="%%d";!ZEPPELIN_CLASSPATH! +) +exit /b + +:ADDJARINDIR +if exist "%~2" ( + set ZEPPELIN_CLASSPATH="%~2\*";%ZEPPELIN_CLASSPATH% +) +exit /b diff --git a/bin/interpreter.cmd b/bin/interpreter.cmd new file mode 100644 index 00000000000..4a501f09ab2 --- /dev/null +++ b/bin/interpreter.cmd @@ -0,0 +1,136 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal enableextensions enabledelayedexpansion + +set bin=%~dp0 + +:loop +if "%~1"=="" goto cont +if /I "%~1"=="-h" goto usage +if /I "%~1"=="-d" ( + set INTERPRETER_DIR=%~2 + set INTERPRETER_ID=%~n2 +) +if /I "%~1"=="-p" set PORT=%~2 +if /I "%~1"=="-l" set LOCAL_INTERPRETER_REPO=%~2 +shift +goto loop +:cont + +if "%PORT%"=="" goto usage +if "%INTERPRETER_DIR%"=="" goto usage + +call "%bin%\common.cmd" + +if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" +) else ( + for %%d in ("%ZEPPELIN_HOME%\lib\zeppelin-interpreter*.jar") do ( + set ZEPPELIN_INTERPRETER_JAR=%%d + ) + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"!ZEPPELIN_INTERPRETER_JAR!" +) + +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%" + +set HOSTNAME=%COMPUTERNAME% +set ZEPPELIN_SERVER=org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer + +set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-interpreter-%INTERPRETER_ID%-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log + +if not exist "%ZEPPELIN_LOG_DIR%" ( + echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR% + mkdir "%ZEPPELIN_LOG_DIR%" +) + +if /I "%INTERPRETER_ID%"=="spark" ( + if defined SPARK_HOME ( + set SPARK_SUBMIT=%SPARK_HOME%\bin\spark-submit.cmd + for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\zeppelin-spark*.jar") do ( + set SPARK_APP_JAR=%%d + ) + set ZEPPELIN_CLASSPATH="!SPARK_APP_JAR!" + + for %%d in ("%SPARK_HOME%\python\lib\py4j-*-src.zip") do ( + set py4j=%%d + ) + + if not defined PYTHONPATH ( + set PYTHONPATH=!py4j!;%SPARK_HOME%\python + ) else ( + set PYTHONPATH=!py4j!;%SPARK_HOME%\python;%PYTHONPATH% + ) + ) else ( + if defined HADOOP_HOME if exist "%HADOOP_HOME%\bin\hadoop.cmd" ( + for /f "tokens=*" %%d in ('"%HADOOP_HOME%\bin\hadoop.cmd" classpath') do ( + set LOCAL_HADOOP_CLASSPATH=%%d + ) + set ZEPPELIN_CLASSPATH=!LOCAL_HADOOP_CLASSPATH!;%ZEPPELIN_CLASSPATH% + ) + + call "%bin%\functions.cmd" ADDJARINDIR "%INTERPRETER_DIR%\dep" + + for %%d in ("%ZEPPELIN_HOME%\interpreter\spark\pyspark\py4j-*-src.zip") do ( + set py4j=%%d + ) + + set PYSPARKPATH=%ZEPPELIN_HOME%\interpreter\spark\pyspark\pyspark.zip;!py4j! + + if not defined PYTHONPATH ( + set PYTHONPATH=!PYSPARKPATH! + ) else ( + set PYTHONPATH=%PYTHONPATH%;!PYSPARKPATH! + ) + + set PYSPARKPATH= + + if defined HADOOP_HOME if not defined HADOOP_CONF_DIR ( + if exist "%HADOOP_HOME%\etc\hadoop" ( + set HADOOP_CONF_DIR=%HADOOP_HOME%\etc\hadoop + ) + ) + + if exist "%HADOOP_CONF_DIR%" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%HADOOP_CONF_DIR%" + ) + ) +) + +call "%bin%\functions.cmd" ADDJARINDIR "%LOCAL_INTERPRETER_REPO%" + +if not defined ZEPPELIN_CLASSPATH_OVERRIDES ( + set CLASSPATH=%ZEPPELIN_CLASSPATH% +) else ( + set CLASSPATH=%ZEPPELIN_CLASSPATH_OVERRIDES%;%ZEPPELIN_CLASSPATH% +) + +if defined SPARK_SUBMIT ( + set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file='%ZEPPELIN_LOGFILE%' + + "%SPARK_SUBMIT%" --class %ZEPPELIN_SERVER% --jars %CLASSPATH% --driver-java-options "!JAVA_INTP_OPTS!" %SPARK_SUBMIT_OPTIONS% "%SPARK_APP_JAR%" %PORT% +) else ( + set JAVA_INTP_OPTS=%JAVA_INTP_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%" + + "%ZEPPELIN_RUNNER%" !JAVA_INTP_OPTS! %ZEPPELIN_INTP_MEM% -cp %ZEPPELIN_CLASSPATH_OVERRIDES%;%CLASSPATH% %ZEPPELIN_SERVER% %PORT% +) + +exit /b + +:usage +echo Usage: %~n0 -p ^ -d ^ -l ^ diff --git a/bin/zeppelin.cmd b/bin/zeppelin.cmd new file mode 100644 index 00000000000..a2d5644c276 --- /dev/null +++ b/bin/zeppelin.cmd @@ -0,0 +1,91 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. + +setlocal enableextensions enabledelayedexpansion + +set bin=%~dp0 + +if not "%1"=="--config" goto MAIN + +:SET_CONFIG +shift +set conf_dir=%~f1 +shift +if not exist "%conf_dir%" ( + echo ERROR: %conf_dir% is not a directory + echo Usage: %~n0 [--config ^] + exit /b 1 +) else ( + set ZEPPELIN_CONF_DIR=%conf_dir% +) + +:MAIN +call "%bin%\common.cmd" + +set HOSTNAME=%COMPUTERNAME% +set ZEPPELIN_LOGFILE=%ZEPPELIN_LOG_DIR%\zeppelin-%ZEPPELIN_IDENT_STRING%-%HOSTNAME%.log + +set ZEPPELIN_SERVER=org.apache.zeppelin.server.ZeppelinServer +set JAVA_OPTS=%JAVA_OPTS% -Dzeppelin.log.file="%ZEPPELIN_LOGFILE%" + +if exist "%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-interpreter\target\classes" +) + +if exist "%ZEPPELIN_HOME%\zeppelin-zengine\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-zengine\target\classes" +) + +if exist "%ZEPPELIN_HOME%\zeppelin-server\target\classes" ( + set ZEPPELIN_CLASSPATH=%ZEPPELIN_CLASSPATH%;"%ZEPPELIN_HOME%\zeppelin-server\target\classes" +) + +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-interpreter\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-zengine\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-server\target\lib" +call "%bin%\functions.cmd" ADDJARINDIR "%ZEPPELIN_HOME%\zeppelin-web\target\lib" + +if not defined CLASSPATH ( + set CLASSPATH=%ZEPPELIN_CLASSPATH% +) else ( + set CLASSPATH=%CLASSPATH%;%ZEPPELIN_CLASSPATH% +) + +if not defined ZEPPELIN_CLASSPATH_OVERRIDES ( + set CLASSPATH=%ZEPPELIN_CLASSPATH% +) else ( + set CLASSPATH=%ZEPPELIN_CLASSPATH_OVERRIDES%;%ZEPPELIN_CLASSPATH% +) + +if not exist %ZEPPELIN_LOG_DIR% ( + echo Log dir doesn't exist, create %ZEPPELIN_LOG_DIR% + mkdir "%ZEPPELIN_LOG_DIR%" +) + +if not exist %ZEPPELIN_PID_DIR% ( + echo Pid dir doesn't exist, create %ZEPPELIN_PID_DIR% + mkdir "%ZEPPELIN_PID_DIR%" +) + +if not exist %ZEPPELIN_NOTEBOOK_DIR% ( + echo Notebook dir doesn't exist, create %ZEPPELIN_NOTEBOOK_DIR% + mkdir "%ZEPPELIN_NOTEBOOK_DIR%" +) + +"%ZEPPELIN_RUNNER%" %JAVA_OPTS% -cp %CLASSPATH% %ZEPPELIN_SERVER% "%*" diff --git a/conf/zeppelin-env.cmd.template b/conf/zeppelin-env.cmd.template new file mode 100644 index 00000000000..06799b5fd85 --- /dev/null +++ b/conf/zeppelin-env.cmd.template @@ -0,0 +1,64 @@ +@echo off + +REM Licensed to the Apache Software Foundation (ASF) under one or more +REM contributor license agreements. See the NOTICE file distributed with +REM this work for additional information regarding copyright ownership. +REM The ASF licenses this file to You under the Apache License, Version 2.0 +REM (the "License"); you may not use this file except in compliance with +REM the License. You may obtain a copy of the License at +REM +REM http://www.apache.org/licenses/LICENSE-2.0 +REM +REM Unless required by applicable law or agreed to in writing, software +REM distributed under the License is distributed on an "AS IS" BASIS, +REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +REM See the License for the specific language governing permissions and +REM limitations under the License. +REM + +REM set JAVA_HOME= +REM set MASTER= REM Spark master url. eg. spark://master_addr:7077. Leave empty if you want to use local mode. +REM set ZEPPELIN_JAVA_OPTS REM Additional jvm options. for example, set ZEPPELIN_JAVA_OPTS="-Dspark.executor.memory=8g -Dspark.cores.max=16" +REM set ZEPPELIN_MEM REM Zeppelin jvm mem options Default -Xmx1024m -XX:MaxPermSize=512m +REM set ZEPPELIN_INTP_MEM REM zeppelin interpreter process jvm mem options. Default = ZEPPELIN_MEM +REM set ZEPPELIN_INTP_JAVA_OPTS REM zeppelin interpreter process jvm options. Default = ZEPPELIN_JAVA_OPTS + +REM set ZEPPELIN_LOG_DIR REM Where log files are stored. PWD by default. +REM set ZEPPELIN_PID_DIR REM The pid files are stored. /tmp by default. +REM set ZEPPELIN_WAR_TEMPDIR REM The location of jetty temporary directory. +REM set ZEPPELIN_NOTEBOOK_DIR REM Where notebook saved +REM set ZEPPELIN_NOTEBOOK_HOMESCREEN REM Id of notebook to be displayed in homescreen. ex) 2A94M5J1Z +REM set ZEPPELIN_NOTEBOOK_HOMESCREEN_HIDE REM hide homescreen notebook from list when this value set to "true". default "false" +REM set ZEPPELIN_NOTEBOOK_S3_BUCKET REM Bucket where notebook saved +REM set ZEPPELIN_NOTEBOOK_S3_USER REM User in bucket where notebook saved. For example bucket/user/notebook/2A94M5J1Z/note.json +REM set ZEPPELIN_IDENT_STRING REM A string representing this instance of zeppelin. $USER by default. +REM set ZEPPELIN_NICENESS REM The scheduling priority for daemons. Defaults to 0. +REM set ZEPPELIN_INTERPRETER_LOCALREPO REM Local repository for interpreter's additional dependency loading + + +REM Spark interpreter configuration + +REM Use provided spark installation +REM defining SPARK_HOME makes Zeppelin run spark interpreter process using spark-submit +REM +REM set SPARK_HOME REM (required) When it is defined, load it instead of Zeppelin embedded Spark libraries +REM set SPARK_SUBMIT_OPTIONS REM (optional) extra options to pass to spark submit. eg) "--driver-memory 512M --executor-memory 1G". +REM set SPARK_APP_NAME REM (optional) The name of spark application. + +REM Use embedded spark binaries +REM without SPARK_HOME defined, Zeppelin still able to run spark interpreter process using embedded spark binaries. +REM however, it is not encouraged when you can define SPARK_HOME +REM +REM Options read in YARN client mode +REM set HADOOP_CONF_DIR REM yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. +REM Pyspark (supported with Spark 1.2.1 and above) +REM To configure pyspark, you need to set spark distribution's path to 'spark.home' property in Interpreter setting screen in Zeppelin GUI +REM set PYSPARK_PYTHON REM path to the python command. must be the same path on the driver(Zeppelin) and all workers. +REM set PYTHONPATH + +REM Spark interpreter options +REM +REM set ZEPPELIN_SPARK_USEHIVECONTEXT REM Use HiveContext instead of SQLContext if set true. true by default. +REM set ZEPPELIN_SPARK_CONCURRENTSQL REM Execute multiple SQL concurrently if set true. false by default. +REM set ZEPPELIN_SPARK_MAXRESULT REM Max number of SparkSQL result to display. 1000 by default. + diff --git a/docs/install/install.md b/docs/install/install.md index b86c5bb4599..884b4bf6257 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -45,7 +45,7 @@ If you don't have requirements prepared, please check instructions in [README.md ## Zeppelin Configuration -You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` and **java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will be used priorly. +You can configure Zeppelin with both **environment variables** in `conf/zeppelin-env.sh` (`conf\zeppelin-env.cmd` for Windows) and **Java properties** in `conf/zeppelin-site.xml`. If both are defined, then the **environment variables** will take priority. @@ -278,4 +278,9 @@ chdir /usr/share/zeppelin exec bin/zeppelin-daemon.sh upstart ``` +#### Running on Windows + +``` +bin\zeppelin.cmd +``` diff --git a/docs/interpreter/spark.md b/docs/interpreter/spark.md index 027d4b639f6..b2bc875390c 100644 --- a/docs/interpreter/spark.md +++ b/docs/interpreter/spark.md @@ -58,6 +58,8 @@ export HADOOP_CONF_DIR=/usr/lib/hadoop export SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0" ``` +For Windows, ensure you have `winutils.exe` in `%HADOOP_HOME%\bin`. For more details please see [Problems running Hadoop on Windows](https://wiki.apache.org/hadoop/WindowsProblems) + ### 2. Set master in Interpreter menu After start Zeppelin, go to **Interpreter** menu and edit **master** property in your Spark interpreter setting. The value may vary depending on your Spark cluster deployment type. diff --git a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java index 6133c328c03..e3c4d0c0f31 100644 --- a/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java +++ b/shell/src/main/java/org/apache/zeppelin/shell/ShellInterpreter.java @@ -19,17 +19,14 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Properties; - +import java.util.*; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.DefaultExecutor; import org.apache.commons.exec.ExecuteException; import org.apache.commons.exec.ExecuteWatchdog; import org.apache.commons.exec.Executor; import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.lang3.StringUtils; import org.apache.zeppelin.interpreter.Interpreter; import org.apache.zeppelin.interpreter.InterpreterContext; import org.apache.zeppelin.interpreter.InterpreterPropertyBuilder; @@ -50,6 +47,10 @@ public class ShellInterpreter extends Interpreter { public static final String SHELL_COMMAND_TIMEOUT = "shell.command.timeout.millisecs"; public static final String DEFAULT_COMMAND_TIMEOUT = "600000"; int commandTimeOut; + private static final boolean isWindows = System + .getProperty("os.name") + .startsWith("Windows"); + final String shell = isWindows ? "cmd /c" : "bash -c"; static { Interpreter.register( @@ -83,11 +84,15 @@ public void close() {} @Override public InterpreterResult interpret(String cmd, InterpreterContext contextInterpreter) { logger.debug("Run shell command '" + cmd + "'"); - CommandLine cmdLine = CommandLine.parse("bash"); - cmdLine.addArgument("-c", false); + CommandLine cmdLine = CommandLine.parse(shell); + // the Windows CMD shell doesn't handle multiline statements, + // they need to be delimited by '&&' instead + if (isWindows) { + String[] lines = StringUtils.split(cmd, "\n"); + cmd = StringUtils.join(lines, " && "); + } cmdLine.addArgument(cmd, false); DefaultExecutor executor = new DefaultExecutor(); - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ByteArrayOutputStream errorStream = new ByteArrayOutputStream(); executor.setStreamHandler(new PumpStreamHandler(contextInterpreter.out, errorStream)); executor.setWatchdog(new ExecuteWatchdog(commandTimeOut)); diff --git a/zeppelin-distribution/src/assemble/distribution.xml b/zeppelin-distribution/src/assemble/distribution.xml index 782aee9b8e3..4750a7290e8 100644 --- a/zeppelin-distribution/src/assemble/distribution.xml +++ b/zeppelin-distribution/src/assemble/distribution.xml @@ -69,6 +69,12 @@ ../conf + + interpreter.json + zeppelin-env.cmd + zeppelin-env.sh + zeppelin-site.xml + ../interpreter diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java index 84327dd6c45..7ffa1708710 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteAngularObjectTest.java @@ -35,6 +35,11 @@ import org.junit.Test; public class RemoteAngularObjectTest implements AngularObjectRegistryListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + private InterpreterGroup intpGroup; private HashMap env; private RemoteInterpreter intp; @@ -63,7 +68,7 @@ public void setUp() throws Exception { p, "note", MockInterpreterAngular.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java index 3fbf5bcc8b2..4a473f3f368 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterOutputTestStream.java @@ -38,6 +38,10 @@ * Test for remote interpreter output stream */ public class RemoteInterpreterOutputTestStream implements RemoteInterpreterProcessListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; private InterpreterGroup intpGroup; private HashMap env; @@ -61,7 +65,7 @@ private RemoteInterpreter createMockInterpreter() { new Properties(), "note", MockInterpreterOutputStream.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java index 7beaee111f6..52d38581ec2 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterProcessTest.java @@ -28,12 +28,16 @@ import org.junit.Test; public class RemoteInterpreterProcessTest { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; @Test public void testStartStop() { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( - "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), + INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap(), 10 * 1000, null); assertFalse(rip.isRunning()); assertEquals(0, rip.referenceCount()); @@ -50,7 +54,7 @@ public void testStartStop() { public void testClientFactory() throws Exception { InterpreterGroup intpGroup = new InterpreterGroup(); RemoteInterpreterProcess rip = new RemoteInterpreterProcess( - "../bin/interpreter.sh", "nonexists", "fakeRepo", new HashMap(), + INTERPRETER_SCRIPT, "nonexists", "fakeRepo", new HashMap(), mock(RemoteInterpreterEventPoller.class), 10 * 1000); rip.reference(intpGroup); assertEquals(0, rip.getNumActiveClient()); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java index 182b7a2a9c2..d545a5f4e56 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/interpreter/remote/RemoteInterpreterTest.java @@ -46,6 +46,11 @@ public class RemoteInterpreterTest { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; + private InterpreterGroup intpGroup; private HashMap env; @@ -71,7 +76,7 @@ private RemoteInterpreter createMockInterpreterA(Properties p, String noteId) { p, noteId, MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -88,7 +93,7 @@ private RemoteInterpreter createMockInterpreterB(Properties p, String noteId) { p, noteId, MockInterpreterB.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -186,7 +191,7 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException p, "note", MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -201,7 +206,7 @@ public void testRemoteSchedulerSharing() throws TTransportException, IOException p, "note", MockInterpreterB.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java index a99fde2a4a3..ae94fd7a8e2 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/resource/DistributedResourcePoolTest.java @@ -39,6 +39,10 @@ * Unittest for DistributedResourcePool */ public class DistributedResourcePoolTest { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; private InterpreterGroup intpGroup1; private InterpreterGroup intpGroup2; private HashMap env; @@ -60,7 +64,7 @@ public void setUp() throws Exception { p, "note", MockInterpreterResourcePool.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -77,7 +81,7 @@ public void setUp() throws Exception { p, "note", MockInterpreterResourcePool.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java index 9ce7a65a6a4..40dcef28ab5 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/scheduler/RemoteSchedulerTest.java @@ -46,6 +46,10 @@ public class RemoteSchedulerTest implements RemoteInterpreterProcessListener { + private static final String INTERPRETER_SCRIPT = + System.getProperty("os.name").startsWith("Windows") ? + "../bin/interpreter.cmd" : + "../bin/interpreter.sh"; private SchedulerFactory schedulerSvc; private static final int TICK_WAIT = 100; private static final int MAX_WAIT_CYCLES = 100; @@ -71,7 +75,7 @@ public void test() throws Exception { p, "note", MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, @@ -159,7 +163,7 @@ public void testAbortOnPending() throws Exception { p, "note", MockInterpreterA.class.getName(), - new File("../bin/interpreter.sh").getAbsolutePath(), + new File(INTERPRETER_SCRIPT).getAbsolutePath(), "fake", "fakeRepo", env, diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 109ea49ce4f..8619262944d 100755 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -476,7 +476,9 @@ public static enum ConfVars { ZEPPELIN_NOTEBOOK_AZURE_SHARE("zeppelin.notebook.azure.share", "zeppelin"), ZEPPELIN_NOTEBOOK_AZURE_USER("zeppelin.notebook.azure.user", "user"), ZEPPELIN_NOTEBOOK_STORAGE("zeppelin.notebook.storage", VFSNotebookRepo.class.getName()), - ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner", "bin/interpreter.sh"), + ZEPPELIN_INTERPRETER_REMOTE_RUNNER("zeppelin.interpreter.remoterunner", + System.getProperty("os.name") + .startsWith("Windows") ? "bin/interpreter.cmd" : "bin/interpreter.sh"), // Decide when new note is created, interpreter settings will be binded automatically or not. ZEPPELIN_NOTEBOOK_AUTO_INTERPRETER_BINDING("zeppelin.notebook.autoInterpreterBinding", true), ZEPPELIN_CONF_DIR("zeppelin.conf.dir", "conf"), diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java index 1f9308f6700..036fc0fd062 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/repo/VFSNotebookRepo.java @@ -60,7 +60,11 @@ public VFSNotebookRepo(ZeppelinConfiguration conf) throws IOException { this.conf = conf; try { - filesystemRoot = new URI(conf.getNotebookDir()); + if (conf.isWindowsPath(conf.getNotebookDir())) { + filesystemRoot = new File(conf.getNotebookDir()).toURI(); + } else { + filesystemRoot = new URI(conf.getNotebookDir()); + } } catch (URISyntaxException e1) { throw new IOException(e1); } @@ -72,8 +76,6 @@ public VFSNotebookRepo(ZeppelinConfiguration conf) throws IOException { } catch (URISyntaxException e) { throw new IOException(e); } - } else { - this.filesystemRoot = filesystemRoot; } fsManager = VFS.getManager();