diff --git a/docs/_includes/themes/zeppelin/_navigation.html b/docs/_includes/themes/zeppelin/_navigation.html index a396d562c34..4bbe64ead0a 100644 --- a/docs/_includes/themes/zeppelin/_navigation.html +++ b/docs/_includes/themes/zeppelin/_navigation.html @@ -105,6 +105,7 @@
  • Advanced
  • Zeppelin on Vagrant VM
  • Zeppelin on Spark Cluster Mode (Standalone)
  • +
  • Zeppelin on Spark Cluster Mode (YARN)
  • Contibute
  • Writing Zeppelin Interpreter
  • diff --git a/docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png b/docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png new file mode 100644 index 00000000000..06c5296ad63 Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/yarn_applications.png differ diff --git a/docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png b/docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png new file mode 100644 index 00000000000..435193ac15e Binary files /dev/null and b/docs/assets/themes/zeppelin/img/docs-img/zeppelin_yarn_conf.png differ diff --git a/docs/index.md b/docs/index.md index 70931e59a0f..bff5253816d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -170,6 +170,7 @@ Join to our [Mailing list](https://zeppelin.apache.org/community.html) and repor * Advanced * [Apache Zeppelin on Vagrant VM](./install/virtual_machine.html) * [Zeppelin on Spark Cluster Mode (Standalone via Docker)](./install/spark_cluster_mode.html#spark-standalone-mode) + * [Zeppelin on Spark Cluster Mode (YARN via Docker)](./install/spark_cluster_mode.html#spark-yarn-mode) * Contribute * [Writing Zeppelin Interpreter](./development/writingzeppelininterpreter.html) * [Writing Zeppelin Application (Experimental)](./development/writingzeppelinapplication.html) diff --git a/docs/install/spark_cluster_mode.md b/docs/install/spark_cluster_mode.md index d2517bd2498..47f688cca96 100644 --- a/docs/install/spark_cluster_mode.md +++ b/docs/install/spark_cluster_mode.md @@ -1,7 +1,7 @@ --- layout: page title: "Apache Zeppelin on Spark cluster mode" -description: "" +description: "This document will guide you how you can build and configure the environment on 3 types of Spark cluster manager with Apache Zeppelin using docker scripts." group: install --- + + + fs.defaultFS + hdfs://0.0.0.0:9000 + + diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/hdfs-site.xml b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/hdfs-site.xml new file mode 100644 index 00000000000..b3f88af304c --- /dev/null +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/hdfs-site.xml @@ -0,0 +1,78 @@ + + + + dfs.replication + 1 + + + + dfs.data.dir + /data/hdfs + true + + + + dfs.permissions + false + + + + dfs.client.use.datanode.hostname + true + Whether clients should use datanode hostnames when + connecting to datanodes. + + + + + dfs.datanode.use.datanode.hostname + true + Whether datanodes should use datanode hostnames when + connecting to other datanodes for data transfer. + + + + + dfs.datanode.address + 0.0.0.0:50010 + + The address where the datanode server will listen to. + If the port is 0 then the server will start on a free port. + + + + + dfs.datanode.http.address + 0.0.0.0:50075 + + The datanode http server address and port. + If the port is 0 then the server will start on a free port. + + + + + dfs.datanode.ipc.address + 0.0.0.0:50020 + + The datanode ipc server address and port. + If the port is 0 then the server will start on a free port. + + + + + diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/mapred-site.xml b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/mapred-site.xml new file mode 100644 index 00000000000..f8280f74657 --- /dev/null +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/mapred-site.xml @@ -0,0 +1,22 @@ + + + + mapreduce.framework.name + yarn + + diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/yarn-site.xml b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/yarn-site.xml new file mode 100644 index 00000000000..89848165c77 --- /dev/null +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/hdfs_conf/yarn-site.xml @@ -0,0 +1,42 @@ + + + + yarn.resourcemanager.scheduler.address + 0.0.0.0:8030 + + + yarn.resourcemanager.address + 0.0.0.0:8032 + + + yarn.resourcemanager.webapp.address + 0.0.0.0:8088 + + + yarn.resourcemanager.resource-tracker.address + 0.0.0.0:8031 + + + yarn.resourcemanager.admin.address + 0.0.0.0:8033 + + + yarn.application.classpath + /usr/local/hadoop/etc/hadoop, /usr/local/hadoop/share/hadoop/common/*, /usr/local/hadoop/share/hadoop/common/lib/*, /usr/local/hadoop/share/hadoop/hdfs/*, /usr/local/hadoop/share/hadoop/hdfs/lib/*, /usr/local/hadoop/share/hadoop/mapreduce/*, /usr/local/hadoop/share/hadoop/mapreduce/lib/*, /usr/local/hadoop/share/hadoop/yarn/*, /usr/local/hadoop/share/hadoop/yarn/lib/*, /usr/local/hadoop/share/spark/* + + diff --git a/scripts/docker/spark-cluster-managers/spark_yarn_cluster/ssh_config b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/ssh_config new file mode 100644 index 00000000000..537a95f48e5 --- /dev/null +++ b/scripts/docker/spark-cluster-managers/spark_yarn_cluster/ssh_config @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +Host * + UserKnownHostsFile /dev/null + StrictHostKeyChecking no + LogLevel quiet \ No newline at end of file