diff --git a/bin/install-interpreter.sh b/bin/install-interpreter.sh new file mode 100755 index 00000000000..06be75cbf44 --- /dev/null +++ b/bin/install-interpreter.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Run Zeppelin +# + +bin=$(dirname "${BASH_SOURCE-$0}") +bin=$(cd "${bin}">/dev/null; pwd) + +. "${bin}/common.sh" + + +ZEPPELIN_INSTALL_INTERPRETER_MAIN=org.apache.zeppelin.interpreter.install.InstallInterpreter +ZEPPELIN_LOGFILE="${ZEPPELIN_LOG_DIR}/install-interpreter.log" +JAVA_OPTS+=" -Dzeppelin.log.file=${ZEPPELIN_LOGFILE}" + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-zengine/target/classes" +fi +addJarInDir "${ZEPPELIN_HOME}/zeppelin-server/target/lib" + +if [[ -d "${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" ]]; then + ZEPPELIN_CLASSPATH+=":${ZEPPELIN_HOME}/zeppelin-interpreter/target/classes" +fi +addJarInDir "${ZEPPELIN_HOME}/zeppelin-interpreter/target/lib" + +addJarInDir "${ZEPPELIN_HOME}/lib" + +CLASSPATH+=":${ZEPPELIN_CLASSPATH}" +$ZEPPELIN_RUNNER $JAVA_OPTS -cp $CLASSPATH $ZEPPELIN_INSTALL_INTERPRETER_MAIN ${@} diff --git a/conf/interpreter-list b/conf/interpreter-list new file mode 100644 index 00000000000..a2f64269633 --- /dev/null +++ b/conf/interpreter-list @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# [name] [maven artifact] [description] + +alluxio org.apache.zeppelin:zeppelin-alluxio:0.6.0 Alluxio interpreter +angular org.apache.zeppelin:zeppelin-angular:0.6.0 HTML and AngularJS view rendering +cassandra org.apache.zeppelin:zeppelin-cassandra:0.6.0 Cassandra interpreter +elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.0 Elasticsearch interpreter +file org.apache.zeppelin:zeppelin-file:0.6.0 HDFS file interpreter +flink org.apache.zeppelin:zeppelin-flink:0.6.0 Flink interpreter +hbase org.apache.zeppelin:zeppelin-hbase:0.6.0 Hbase interpreter +ignite org.apache.zeppelin:zeppelin-ignite:0.6.0 Ignite interpreter +jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.0 Jdbc interpreter +kylin org.apache.zeppelin:zeppelin-kylin:0.6.0 Kylin interpreter +lens org.apache.zeppelin:zeppelin-lens:0.6.0 Lens interpreter +livy org.apache.zeppelin:zeppelin-livy:0.6.0 Livy interpreter +md org.apache.zeppelin:zeppelin-markdown:0.6.0 Markdown support +postgresql org.apache.zeppelin:zeppelin-postgresql:0.6.0 Postgresql interpreter +python org.apache.zeppelin:zeppelin-python:0.6.0 Python interpreter +shell org.apache.zeppelin:zeppelin-shell:0.6.0 Shell command diff --git a/dev/create_release.sh b/dev/create_release.sh index f1bba0dded6..2363d904bcc 100755 --- a/dev/create_release.sh +++ b/dev/create_release.sh @@ -103,6 +103,7 @@ function make_binary_release() { git_clone make_source_package make_binary_release all "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -Psparkr -Pr" +make_binary_release netinst "-Pspark-1.6 -Phadoop-2.4 -Pyarn -Ppyspark -pl !alluxio,!angular,!cassandra,!elasticsearch,!file,!flink,!hbase,!ignite,!jdbc,!kylin,!lens,!livy,!markdown,!postgresql,!python,!shell" # remove non release files and dirs rm -rf "${WORKING_DIR}/zeppelin" diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index 40b9fb6024b..498a2d36782 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -21,8 +21,8 @@
  • What is Apache Zeppelin ?
  • Getting Started
    • -
  • Quick Start
    • -
  • Configuration
    • +
  • Install
    • +
  • Configuration
  • Explore Zeppelin UI
  • Tutorial
    • @@ -42,6 +42,7 @@
  • Overview
  • Usage
    • +
  • Interpreter Installation
  • Dynamic Interpreter Loading
  • Interpreter Dependency Management
    • @@ -110,3 +111,4 @@ + \ No newline at end of file diff --git a/docs/install/install.md b/docs/install/install.md index 93d9febe507..55741dcf1cb 100644 --- a/docs/install/install.md +++ b/docs/install/install.md @@ -60,6 +60,9 @@ Although it can be unstable somehow since it is on development status, you can e ### Downloading Binary Package If you want to install Apache Zeppelin with a stable binary package, please visit [Apache Zeppelin download Page](http://zeppelin.apache.org/download.html). + +If you have downloaded `netinst` binary, [install additional interpreters](../manual/interpreterinstallation.html) before you start Zeppelin. Or simply run `./bin/install-interpreter.sh --all`. + After unpacking, jump to [Starting Apache Zeppelin with Command Line](#starting-apache-zeppelin-with-command-line) section. ### Building from Source @@ -388,4 +391,4 @@ You can configure Apache Zeppelin with both **environment variables** in `conf/z 1024000 Size in characters of the maximum text message to be received by websocket. - + \ No newline at end of file diff --git a/docs/manual/interpreterinstallation.md b/docs/manual/interpreterinstallation.md new file mode 100644 index 00000000000..d522e620e53 --- /dev/null +++ b/docs/manual/interpreterinstallation.md @@ -0,0 +1,164 @@ +--- +layout: page +title: "Interpreter Installation" +description: "" +group: manual +--- + +{% include JB/setup %} + +# Interpreter Installation + +Apache Zeppelin provides **Interpreter Installation** mechanism for whom downloaded Zeppelin `netinst` binary package, or just want to install another 3rd party interpreters. + +## Community managed interpreters +Apache Zeppelin provides several interpreters as [community managed interpreters](#available-community-managed-interpreters). +If you downloaded `netinst` binary package, you need to install by using below commands. + +#### Install all community managed interpreters + +``` +./bin/install-interpreter.sh --all +``` + +#### Install specific interpreters + +``` +./bin/install-interpreter.sh --name md,shell,jdbc,python +``` + +You can get full list of community managed interpreters by running + +``` +./bin/install-interpreter.sh --list +``` + +Once you have installed interpreters, you need to restart Zeppelin. And then [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +## 3rd party interpreters + +You can also install 3rd party interpreters located in the maven repository by using below commands. + +#### Install 3rd party interpreters + +``` +./bin/install-interpreter.sh --name interpreter1 --artifact groupId1:artifact1:version1 +``` + +The above command will download maven artifact `groupId1:artifact1:version1` and all of it's transitive dependencies into `interpreter/interpreter1` directory. + +Once you have installed interpreters, you'll need to add interpreter class name into `zeppelin.interpreters` property in [configuration](../install/install.html#apache-zeppelin-configuration). +And then restart Zeppelin, [create interpreter setting](../manual/interpreters.html#what-is-zeppelin-interpreter) and [bind it with your notebook](../manual/interpreters.html#what-is-zeppelin-interpreter-setting). + + +#### Install multiple 3rd party interpreters at once + +``` +./bin/install-interpreter.sh --name interpreter1,interpreter2 --artifact groupId1:artifact1:version1,groupId2:artifact2:version2 +``` + +`--name` and `--artifact` arguments will recieve comma separated list. + +## Available community managed interpreters + +You can also find the below community managed interpreter list in `conf/interpreter-list` file. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    NameMaven ArtifactDescription
    alluxioorg.apache.zeppelin:zeppelin-alluxio:0.6.0Alluxio interpreter
    angularorg.apache.zeppelin:zeppelin-angular:0.6.0HTML and AngularJS view rendering
    cassandraorg.apache.zeppelin:zeppelin-cassandra:0.6.0Cassandra interpreter
    elasticsearchorg.apache.zeppelin:zeppelin-elasticsearch:0.6.0Elasticsearch interpreter
    fileorg.apache.zeppelin:zeppelin-file:0.6.0HDFS file interpreter
    flinkorg.apache.zeppelin:zeppelin-flink:0.6.0Flink interpreter
    hbaseorg.apache.zeppelin:zeppelin-hbase:0.6.0Hbase interpreter
    igniteorg.apache.zeppelin:zeppelin-ignite:0.6.0Ignite interpreter
    jdbcorg.apache.zeppelin:zeppelin-jdbc:0.6.0Jdbc interpreter
    kylinorg.apache.zeppelin:zeppelin-kylin:0.6.0Kylin interpreter
    lensorg.apache.zeppelin:zeppelin-lens:0.6.0Lens interpreter
    livyorg.apache.zeppelin:zeppelin-livy:0.6.0Livy interpreter
    mdorg.apache.zeppelin:zeppelin-markdown:0.6.0Markdown support
    postgresqlorg.apache.zeppelin:zeppelin-postgresql:0.6.0Postgresql interpreter
    pythonorg.apache.zeppelin:zeppelin-python:0.6.0Python interpreter
    shellorg.apache.zeppelin:zeppelin-shell:0.6.0Shell command
    diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java index b22941ef6b2..1e0844efaac 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/AbstractDependencyResolver.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.dep; +import java.net.URL; import java.util.Collection; import java.util.Iterator; import java.util.LinkedList; @@ -25,6 +26,7 @@ import org.sonatype.aether.RepositorySystem; import org.sonatype.aether.RepositorySystemSession; import org.sonatype.aether.repository.Authentication; +import org.sonatype.aether.repository.Proxy; import org.sonatype.aether.repository.RemoteRepository; import org.sonatype.aether.repository.RepositoryPolicy; import org.sonatype.aether.resolution.ArtifactResult; @@ -44,6 +46,16 @@ public AbstractDependencyResolver(String localRepoPath) { repos.add(Booter.newLocalRepository()); } + public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) { + Authentication auth = new Authentication(proxyUser, proxyPassword); + Proxy proxy = new Proxy(proxyUrl.getProtocol(), proxyUrl.getHost(), proxyUrl.getPort(), auth); + synchronized (repos) { + for (RemoteRepository repo : repos) { + repo.setProxy(proxy); + } + } + } + public List getRepos() { return this.repos; } diff --git a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java index 60ef1f9e6e0..214175a2640 100644 --- a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java +++ b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/dep/DependencyResolver.java @@ -19,6 +19,7 @@ import java.io.File; import java.io.IOException; +import java.net.URL; import java.util.Arrays; import java.util.Collection; import java.util.Iterator; @@ -63,11 +64,6 @@ public List load(String artifact) return load(artifact, new LinkedList()); } - public List load(String artifact, String destPath) - throws RepositoryException, IOException { - return load(artifact, new LinkedList(), destPath); - } - public synchronized List load(String artifact, Collection excludes) throws RepositoryException, IOException { if (StringUtils.isBlank(artifact)) { @@ -85,25 +81,20 @@ public synchronized List load(String artifact, Collection excludes return libs; } } - - public List load(String artifact, Collection excludes, String destPath) + + public List load(String artifact, File destPath) throws IOException, RepositoryException { + return load(artifact, new LinkedList(), destPath); + } + + public List load(String artifact, Collection excludes, File destPath) throws RepositoryException, IOException { List libs = new LinkedList(); if (StringUtils.isNotBlank(artifact)) { libs = load(artifact, excludes); - // find home dir - String home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } - for (File srcFile : libs) { - File destFile = new File(home + "/" + destPath, srcFile.getName()); + File destFile = new File(destPath, srcFile.getName()); if (!destFile.exists() || !FileUtils.contentEquals(srcFile, destFile)) { FileUtils.copyFile(srcFile, destFile); logger.info("copy {} to {}", srcFile.getAbsolutePath(), destPath); diff --git a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java index a8664ef67fe..af2c7ff6963 100644 --- a/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java +++ b/zeppelin-interpreter/src/test/java/org/apache/zeppelin/dep/DependencyResolverTest.java @@ -33,27 +33,20 @@ public class DependencyResolverTest { private static DependencyResolver resolver; private static String testPath; - private static String testCopyPath; - private static String home; - + private static File testCopyPath; + private static File tmpDir; + @BeforeClass public static void setUp() throws Exception { - testPath = "test-repo"; - testCopyPath = "test-copy-repo"; + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + testPath = tmpDir.getAbsolutePath() + "/test-repo"; + testCopyPath = new File(tmpDir, "test-copy-repo"); resolver = new DependencyResolver(testPath); - home = System.getenv("ZEPPELIN_HOME"); - if (home == null) { - home = System.getProperty("zeppelin.home"); - } - if (home == null) { - home = ".."; - } } @AfterClass public static void tearDown() throws Exception { - FileUtils.deleteDirectory(new File(home + "/" + testPath)); - FileUtils.deleteDirectory(new File(home + "/" + testCopyPath)); + FileUtils.deleteDirectory(tmpDir); } @Rule @@ -78,19 +71,19 @@ public void testDelRepo() { public void testLoad() throws Exception { // basic load resolver.load("com.databricks:spark-csv_2.10:1.3.0", testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 4); - FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + assertEquals(testCopyPath.list().length, 4); + FileUtils.cleanDirectory(testCopyPath); // load with exclusions parameter resolver.load("com.databricks:spark-csv_2.10:1.3.0", Collections.singletonList("org.scala-lang:scala-library"), testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 3); - FileUtils.cleanDirectory(new File(home + "/" + testCopyPath)); + assertEquals(testCopyPath.list().length, 3); + FileUtils.cleanDirectory(testCopyPath); // load from added repository resolver.addRepo("sonatype", "https://oss.sonatype.org/content/repositories/agimatec-releases/", false); resolver.load("com.agimatec:agimatec-validation:0.9.3", testCopyPath); - assertEquals(new File(home + "/" + testCopyPath).list().length, 8); + assertEquals(testCopyPath.list().length, 8); // load invalid artifact resolver.delRepo("sonatype"); diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java index 45fbba4d57d..0a7b8c0121e 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java @@ -346,6 +346,10 @@ public String getS3EncryptionMaterialsProviderClass() { return getString(ConfVars.ZEPPELIN_NOTEBOOK_S3_EMP); } + public String getInterpreterListPath() { + return getRelativeDir(String.format("%s/interpreter-list", getConfDir())); + } + public String getInterpreterDir() { return getRelativeDir(ConfVars.ZEPPELIN_INTERPRETER_DIR); } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java index bad18c02347..aeb78187777 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterFactory.java @@ -350,15 +350,17 @@ private void loadInterpreterDependencies(InterpreterSetting intSetting) List deps = intSetting.getDependencies(); if (deps != null) { for (Dependency d: deps) { + File destDir = new File(conf.getRelativeDir(ConfVars.ZEPPELIN_DEP_LOCALREPO)); + if (d.getExclusions() != null) { depResolver.load( d.getGroupArtifactVersion(), d.getExclusions(), - conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + new File(destDir, intSetting.id())); } else { depResolver.load( d.getGroupArtifactVersion(), - conf.getString(ConfVars.ZEPPELIN_DEP_LOCALREPO) + "/" + intSetting.id()); + new File(destDir, intSetting.id())); } } } diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java new file mode 100644 index 00000000000..da67d9f1249 --- /dev/null +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/install/InstallInterpreter.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.zeppelin.interpreter.install; + +import org.apache.commons.io.FileUtils; +import org.apache.log4j.ConsoleAppender; +import org.apache.log4j.Logger; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.apache.zeppelin.dep.DependencyResolver; +import org.apache.zeppelin.util.Util; +import org.sonatype.aether.RepositoryException; +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Commandline utility to install interpreter from maven repository + */ +public class InstallInterpreter { + private final File interpreterListFile; + private final File interpreterBaseDir; + private final List availableInterpreters; + private final String localRepoDir; + private URL proxyUrl; + private String proxyUser; + private String proxyPassword; + + /** + * + * @param interpreterListFile + * @param interpreterBaseDir interpreter directory for installing binaries + * @throws IOException + */ + public InstallInterpreter(File interpreterListFile, File interpreterBaseDir, String localRepoDir) + throws IOException { + this.interpreterListFile = interpreterListFile; + this.interpreterBaseDir = interpreterBaseDir; + this.localRepoDir = localRepoDir; + availableInterpreters = new LinkedList(); + readAvailableInterpreters(); + } + + + /** + * Information for available informations + */ + private static class AvailableInterpreterInfo { + public final String name; + public final String artifact; + public final String description; + + public AvailableInterpreterInfo(String name, String artifact, String description) { + this.name = name; + this.artifact = artifact; + this.description = description; + } + } + + private void readAvailableInterpreters() throws IOException { + if (!interpreterListFile.isFile()) { + System.err.println("Can't find interpreter list " + interpreterListFile.getAbsolutePath()); + return; + } + String text = FileUtils.readFileToString(interpreterListFile); + String[] lines = text.split("\n"); + + Pattern pattern = Pattern.compile("(\\S+)\\s+(\\S+)\\s+(.*)"); + + int lineNo = 0; + for (String line : lines) { + lineNo++; + if (line == null || line.length() == 0 || line.startsWith("#")) { + continue; + } + + Matcher match = pattern.matcher(line); + if (match.groupCount() != 3) { + System.err.println("Error on line " + lineNo + ", " + line); + continue; + } + + match.find(); + + String name = match.group(1); + String artifact = match.group(2); + String description = match.group(3); + + availableInterpreters.add(new AvailableInterpreterInfo(name, artifact, description)); + } + } + + public List list() { + for (AvailableInterpreterInfo info : availableInterpreters) { + System.out.println(info.name + "\t\t\t" + info.description); + } + + return availableInterpreters; + } + + public void installAll() { + for (AvailableInterpreterInfo info : availableInterpreters) { + install(info.name, info.artifact); + } + } + + public void install(String [] names) { + for (String name : names) { + install(name); + } + } + + public void install(String name) { + // find artifact name + for (AvailableInterpreterInfo info : availableInterpreters) { + if (name.equals(info.name)) { + install(name, info.artifact); + return; + } + } + + throw new RuntimeException("Can't find interpreter '" + name + "'"); + } + + public void install(String [] names, String [] artifacts) { + if (names.length != artifacts.length) { + throw new RuntimeException("Length of given names and artifacts are different"); + } + + for (int i = 0; i < names.length; i++) { + install(names[i], artifacts[i]); + } + } + + public void install(String name, String artifact) { + DependencyResolver depResolver = new DependencyResolver(localRepoDir); + if (proxyUrl != null) { + depResolver.setProxy(proxyUrl, proxyUser, proxyPassword); + } + + File installDir = new File(interpreterBaseDir, name); + if (installDir.exists()) { + System.err.println("Directory " + installDir.getAbsolutePath() + " already exists. Skipping"); + return; + } + + System.out.println("Install " + name + "(" + artifact + ") to " + + installDir.getAbsolutePath() + " ... "); + + try { + depResolver.load(artifact, installDir); + System.out.println("Interpreter " + name + " installed under " + + installDir.getAbsolutePath() + "."); + } catch (RepositoryException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + public void setProxy(URL proxyUrl, String proxyUser, String proxyPassword) { + this.proxyUrl = proxyUrl; + this.proxyUser = proxyUser; + this.proxyPassword = proxyPassword; + } + + public static void usage() { + System.out.println("Options"); + System.out.println(" -l, --list List available interpreters"); + System.out.println(" -a, --all Install all available interpreters"); + System.out.println(" -n, --name [NAMES] Install interpreters (comma separated " + + "list)" + + "e.g. md,shell,jdbc,python,angular"); + System.out.println(" -t, --artifact [ARTIFACTS] (Optional with -n) custom artifact names" + + ". " + + "(comma separated list correspond to --name) " + + "e.g. customGroup:customArtifact:customVersion"); + System.out.println(" --proxy-url [url] (Optional) proxy url. http(s)://host:port"); + System.out.println(" --proxy-user [user] (Optional) proxy user"); + System.out.println(" --proxy-password [password] (Optional) proxy password"); + } + + public static void main(String [] args) throws IOException { + if (args.length == 0) { + usage(); + return; + } + + ZeppelinConfiguration conf = ZeppelinConfiguration.create(); + InstallInterpreter installer = new InstallInterpreter( + new File(conf.getInterpreterListPath()), + new File(conf.getInterpreterDir()), + conf.getInterpreterLocalRepoPath()); + + String names = null; + String artifacts = null; + URL proxyUrl = null; + String proxyUser = null; + String proxyPassword = null; + boolean all = false; + + for (int i = 0; i < args.length; i++) { + String arg = args[i].toLowerCase(Locale.US); + switch (arg) { + case "--list": + case "-l": + installer.list(); + System.exit(0); + break; + case "--all": + case "-a": + all = true; + break; + case "--name": + case "-n": + names = args[++i]; + break; + case "--artifact": + case "-t": + artifacts = args[++i]; + break; + case "--version": + case "-v": + Util.getVersion(); + break; + case "--proxy-url": + proxyUrl = new URL(args[++i]); + break; + case "--proxy-user": + proxyUser = args[++i]; + break; + case "--proxy-password": + proxyPassword = args[++i]; + break; + case "--help": + case "-h": + usage(); + System.exit(0); + break; + default: + System.out.println("Unknown option " + arg); + } + } + + if (proxyUrl != null) { + installer.setProxy(proxyUrl, proxyUser, proxyPassword); + } + + if (all) { + installer.installAll(); + System.exit(0); + } + + if (names != null) { + if (artifacts != null) { + installer.install(names.split(","), artifacts.split(",")); + startTip(); + configurationTip(); + interpreterSettingTip(); + } else { + installer.install(names.split(",")); + startTip(); + interpreterSettingTip(); + } + } + } + + private static void startTip() { + System.out.println(""); + } + + private static void configurationTip() { + System.out.println("Add interpreter class name to '" + + ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETERS.getVarName() + "' property " + + "in your conf/zeppelin-site.xml file"); + } + + private static void interpreterSettingTip() { + System.out.println("Create interpreter setting in 'Interpreter' menu on GUI." + + " And then you can bind interpreter on your notebook"); + } +} diff --git a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java new file mode 100644 index 00000000000..e934f1a0a1c --- /dev/null +++ b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/install/InstallInterpreterTest.java @@ -0,0 +1,86 @@ +package org.apache.zeppelin.interpreter.install; + +import org.apache.commons.io.FileUtils; +import org.apache.zeppelin.conf.ZeppelinConfiguration; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +public class InstallInterpreterTest { + private File tmpDir; + private InstallInterpreter installer; + private File interpreterBaseDir; + + @Before + public void setUp() throws IOException { + tmpDir = new File(System.getProperty("java.io.tmpdir")+"/ZeppelinLTest_"+System.currentTimeMillis()); + new File(tmpDir, "conf").mkdirs(); + interpreterBaseDir = new File(tmpDir, "interpreter"); + File localRepoDir = new File(tmpDir, "local-repo"); + interpreterBaseDir.mkdir(); + localRepoDir.mkdir(); + + File interpreterListFile = new File(tmpDir, "conf/interpreter-list"); + + + // create interpreter list file + System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_HOME.getVarName(), tmpDir.getAbsolutePath()); + + String interpreterList = ""; + interpreterList += "intp1 org.apache.commons:commons-csv:1.1 test interpreter 1\n"; + interpreterList += "intp2 org.apache.commons:commons-math3:3.6.1 test interpreter 2\n"; + + FileUtils.writeStringToFile(new File(tmpDir, "conf/interpreter-list"), interpreterList); + + installer = new InstallInterpreter(interpreterListFile, interpreterBaseDir, localRepoDir + .getAbsolutePath()); + } + + @After + public void tearDown() throws IOException { + FileUtils.deleteDirectory(tmpDir); + } + + + @Test + public void testList() { + assertEquals(2, installer.list().size()); + } + + @Test + public void install() { + assertEquals(0, interpreterBaseDir.listFiles().length); + + installer.install("intp1"); + assertTrue(new File(interpreterBaseDir, "intp1").isDirectory()); + } + + @Test + public void installAll() { + installer.installAll(); + assertTrue(new File(interpreterBaseDir, "intp1").isDirectory()); + assertTrue(new File(interpreterBaseDir, "intp2").isDirectory()); + } +}