Skip to content

Commit 98d99bc

Browse files
zhuguangbinzhuguangbin
committed
Merge pull request #3 from MediaV/v1.3.0-cdh4.6.0
merge from v1.3.0-cdh4.6.0 branch for cdh4.6.0 yarn api compatibility
2 parents 3e83913 + 52afda7 commit 98d99bc

File tree

23 files changed

+4461
-136
lines changed

23 files changed

+4461
-136
lines changed

‎README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,25 @@
1+
# MVAD Spark
2+
3+
MVAD Spark is based on Apache Spark version v1.3.0 (currently latest version).
4+
We rebuild it because Spark 1.3 or later version will not support yarn-alpha API,
5+
but we are still using CDH4.6.0 in production.
6+
7+
we rewrite two modules :
8+
9+
* yarn-cdh4.6.0 : spark on yarn main module
10+
* network/yarn-cdh4.6.0 : a YarnShuffleService used by [Dynamic Allocation](http://spark.apache.org/docs/latest/job-scheduling.html#dynamic-resource-allocation).
11+
12+
and we add two extra profile in pom.xml
13+
14+
* hadoop-cdh4.6.0
15+
* yarn-cdh4.6.0
16+
17+
we build our package using the following command:
18+
19+
./make-distribution.sh --tgz -Phadoop-cdh4.6.0 -Pyarn-cdh4.6.0 -Phive -Phive-0.12.0 -Phive-thriftserver -Pspark-ganglia-lgpl -Phadoop-provided -Phbase-provided
20+
21+
-------------------------------------------------------------------------
22+
123
# Apache Spark
224

325
Spark is a fast and general cluster computing system for Big Data. It provides

‎assembly/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,6 +159,16 @@
159159
</dependency>
160160
</dependencies>
161161
</profile>
162+
<profile>
163+
<id>yarn-cdh4.6.0</id>
164+
<dependencies>
165+
<dependency>
166+
<groupId>org.apache.spark</groupId>
167+
<artifactId>spark-yarn_${scala.binary.version}</artifactId>
168+
<version>${project.version}</version>
169+
</dependency>
170+
</dependencies>
171+
</profile>
162172
<profile>
163173
<id>hive</id>
164174
<dependencies>

‎core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -317,7 +317,7 @@ private[spark] class ExecutorAllocationManager(
317317
val newNumExecutorsPending =
318318
newTotalExecutors - executorIds.size + executorsPendingToRemove.size
319319
val delta = newNumExecutorsPending - numExecutorsPending
320-
numExecutorsPending = newNumExecutorsPending
320+
numExecutorsPending = if (newNumExecutorsPending < 0) 0 else newNumExecutorsPending
321321
delta
322322
}
323323

‎examples/pom.xml‎

Lines changed: 1 addition & 135 deletions
Original file line numberDiff line numberDiff line change
@@ -93,143 +93,9 @@
9393
</dependency>
9494
<dependency>
9595
<groupId>org.apache.hbase</groupId>
96-
<artifactId>hbase-testing-util</artifactId>
96+
<artifactId>hbase</artifactId>
9797
<version>${hbase.version}</version>
9898
<scope>${hbase.deps.scope}</scope>
99-
<exclusions>
100-
<exclusion>
101-
<!-- SPARK-4455 -->
102-
<groupId>org.apache.hbase</groupId>
103-
<artifactId>hbase-annotations</artifactId>
104-
</exclusion>
105-
<exclusion>
106-
<groupId>org.jruby</groupId>
107-
<artifactId>jruby-complete</artifactId>
108-
</exclusion>
109-
</exclusions>
110-
</dependency>
111-
<dependency>
112-
<groupId>org.apache.hbase</groupId>
113-
<artifactId>hbase-protocol</artifactId>
114-
<version>${hbase.version}</version>
115-
<scope>${hbase.deps.scope}</scope>
116-
</dependency>
117-
<dependency>
118-
<groupId>org.apache.hbase</groupId>
119-
<artifactId>hbase-common</artifactId>
120-
<version>${hbase.version}</version>
121-
<scope>${hbase.deps.scope}</scope>
122-
<exclusions>
123-
<exclusion>
124-
<!-- SPARK-4455 -->
125-
<groupId>org.apache.hbase</groupId>
126-
<artifactId>hbase-annotations</artifactId>
127-
</exclusion>
128-
</exclusions>
129-
</dependency>
130-
<dependency>
131-
<groupId>org.apache.hbase</groupId>
132-
<artifactId>hbase-client</artifactId>
133-
<version>${hbase.version}</version>
134-
<scope>${hbase.deps.scope}</scope>
135-
<exclusions>
136-
<exclusion>
137-
<!-- SPARK-4455 -->
138-
<groupId>org.apache.hbase</groupId>
139-
<artifactId>hbase-annotations</artifactId>
140-
</exclusion>
141-
<exclusion>
142-
<groupId>io.netty</groupId>
143-
<artifactId>netty</artifactId>
144-
</exclusion>
145-
</exclusions>
146-
</dependency>
147-
<dependency>
148-
<groupId>org.apache.hbase</groupId>
149-
<artifactId>hbase-server</artifactId>
150-
<version>${hbase.version}</version>
151-
<scope>${hbase.deps.scope}</scope>
152-
<exclusions>
153-
<exclusion>
154-
<!-- SPARK-4455 -->
155-
<groupId>org.apache.hbase</groupId>
156-
<artifactId>hbase-annotations</artifactId>
157-
</exclusion>
158-
<exclusion>
159-
<groupId>org.apache.hadoop</groupId>
160-
<artifactId>hadoop-core</artifactId>
161-
</exclusion>
162-
<exclusion>
163-
<groupId>org.apache.hadoop</groupId>
164-
<artifactId>hadoop-client</artifactId>
165-
</exclusion>
166-
<exclusion>
167-
<groupId>org.apache.hadoop</groupId>
168-
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
169-
</exclusion>
170-
<exclusion>
171-
<groupId>org.apache.hadoop</groupId>
172-
<artifactId>hadoop-mapreduce-client-core</artifactId>
173-
</exclusion>
174-
<exclusion>
175-
<groupId>org.apache.hadoop</groupId>
176-
<artifactId>hadoop-auth</artifactId>
177-
</exclusion>
178-
<exclusion>
179-
<groupId>org.apache.hadoop</groupId>
180-
<artifactId>hadoop-annotations</artifactId>
181-
</exclusion>
182-
<exclusion>
183-
<groupId>org.apache.hadoop</groupId>
184-
<artifactId>hadoop-hdfs</artifactId>
185-
</exclusion>
186-
<exclusion>
187-
<groupId>org.apache.hbase</groupId>
188-
<artifactId>hbase-hadoop1-compat</artifactId>
189-
</exclusion>
190-
<exclusion>
191-
<groupId>org.apache.commons</groupId>
192-
<artifactId>commons-math</artifactId>
193-
</exclusion>
194-
<exclusion>
195-
<groupId>com.sun.jersey</groupId>
196-
<artifactId>jersey-core</artifactId>
197-
</exclusion>
198-
<exclusion>
199-
<groupId>org.slf4j</groupId>
200-
<artifactId>slf4j-api</artifactId>
201-
</exclusion>
202-
<exclusion>
203-
<groupId>com.sun.jersey</groupId>
204-
<artifactId>jersey-server</artifactId>
205-
</exclusion>
206-
<exclusion>
207-
<groupId>com.sun.jersey</groupId>
208-
<artifactId>jersey-core</artifactId>
209-
</exclusion>
210-
<exclusion>
211-
<groupId>com.sun.jersey</groupId>
212-
<artifactId>jersey-json</artifactId>
213-
</exclusion>
214-
<exclusion>
215-
<!-- hbase uses v2.4, which is better, but ...-->
216-
<groupId>commons-io</groupId>
217-
<artifactId>commons-io</artifactId>
218-
</exclusion>
219-
</exclusions>
220-
</dependency>
221-
<dependency>
222-
<groupId>org.apache.hbase</groupId>
223-
<artifactId>hbase-hadoop-compat</artifactId>
224-
<version>${hbase.version}</version>
225-
<scope>${hbase.deps.scope}</scope>
226-
</dependency>
227-
<dependency>
228-
<groupId>org.apache.hbase</groupId>
229-
<artifactId>hbase-hadoop-compat</artifactId>
230-
<version>${hbase.version}</version>
231-
<type>test-jar</type>
232-
<scope>test</scope>
23399
</dependency>
234100
<dependency>
235101
<groupId>org.apache.commons</groupId>

‎network/yarn-cdh4.6.0/pom.xml‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<modelVersion>4.0.0</modelVersion>
22+
<parent>
23+
<groupId>org.apache.spark</groupId>
24+
<artifactId>spark-parent_2.10</artifactId>
25+
<version>1.3.0</version>
26+
<relativePath>../../pom.xml</relativePath>
27+
</parent>
28+
29+
<groupId>org.apache.spark</groupId>
30+
<artifactId>spark-network-yarn_2.10</artifactId>
31+
<packaging>jar</packaging>
32+
<name>Spark Project YARN Shuffle Service</name>
33+
<url>http://spark.apache.org/</url>
34+
<properties>
35+
<sbt.project.name>network-yarn</sbt.project.name>
36+
<!-- Make sure all Hadoop dependencies are provided to avoid repackaging. -->
37+
<hadoop.deps.scope>provided</hadoop.deps.scope>
38+
</properties>
39+
40+
<dependencies>
41+
<!-- Core dependencies -->
42+
<dependency>
43+
<groupId>org.apache.spark</groupId>
44+
<artifactId>spark-network-shuffle_${scala.binary.version}</artifactId>
45+
<version>${project.version}</version>
46+
</dependency>
47+
48+
<!-- Provided dependencies -->
49+
<dependency>
50+
<groupId>org.apache.hadoop</groupId>
51+
<artifactId>hadoop-client</artifactId>
52+
</dependency>
53+
<dependency>
54+
<groupId>org.apache.hadoop</groupId>
55+
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
56+
<version>2.0.0-cdh4.6.0</version>
57+
</dependency>
58+
</dependencies>
59+
60+
<build>
61+
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
62+
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
63+
<plugins>
64+
<plugin>
65+
<groupId>org.apache.maven.plugins</groupId>
66+
<artifactId>maven-shade-plugin</artifactId>
67+
<configuration>
68+
<shadedArtifactAttached>false</shadedArtifactAttached>
69+
<outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-${project.version}-yarn-shuffle.jar</outputFile>
70+
<artifactSet>
71+
<includes>
72+
<include>*:*</include>
73+
</includes>
74+
</artifactSet>
75+
<filters>
76+
<filter>
77+
<artifact>*:*</artifact>
78+
<excludes>
79+
<exclude>META-INF/*.SF</exclude>
80+
<exclude>META-INF/*.DSA</exclude>
81+
<exclude>META-INF/*.RSA</exclude>
82+
</excludes>
83+
</filter>
84+
</filters>
85+
</configuration>
86+
<executions>
87+
<execution>
88+
<phase>package</phase>
89+
<goals>
90+
<goal>shade</goal>
91+
</goals>
92+
</execution>
93+
</executions>
94+
</plugin>
95+
</plugins>
96+
</build>
97+
</project>

0 commit comments

Comments
 (0)