Skip to content

Commit 45ddd5b

Browse files
scwfscwf
committed
fix conflicts
2 parents 204bab1 + ef84dab commit 45ddd5b

File tree

861 files changed

+33515
-9124
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

861 files changed

+33515
-9124
lines changed

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*.ipr
66
*.iml
77
*.iws
8+
*.pyc
89
.idea/
910
.idea_modules/
1011
sbt/*.jar
@@ -49,7 +50,9 @@ dependency-reduced-pom.xml
4950
checkpoint
5051
derby.log
5152
dist/
52-
spark-*-bin.tar.gz
53+
dev/create-release/*txt
54+
dev/create-release/*new
55+
spark-*-bin-*.tgz
5356
unit-tests.log
5457
/lib/
5558
rat-results.txt

‎.rat-excludes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ SparkImports.scala
4444
SparkJLineCompletion.scala
4545
SparkJLineReader.scala
4646
SparkMemberHandlers.scala
47+
SparkReplReporter.scala
4748
sbt
4849
sbt-launch-lib.bash
4950
plugins.sbt

‎LICENSE‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -646,7 +646,8 @@ THE SOFTWARE.
646646

647647
========================================================================
648648
For Scala Interpreter classes (all .scala files in repl/src/main/scala
649-
except for Main.Scala, SparkHelper.scala and ExecutorClassLoader.scala):
649+
except for Main.Scala, SparkHelper.scala and ExecutorClassLoader.scala),
650+
and for SerializableMapWrapper in JavaUtils.scala:
650651
========================================================================
651652

652653
Copyright (c) 2002-2013 EPFL
@@ -754,7 +755,7 @@ SUCH DAMAGE.
754755

755756

756757
========================================================================
757-
For Timsort (core/src/main/java/org/apache/spark/util/collection/Sorter.java):
758+
For Timsort (core/src/main/java/org/apache/spark/util/collection/TimSort.java):
758759
========================================================================
759760
Copyright (C) 2008 The Android Open Source Project
760761

@@ -771,6 +772,25 @@ See the License for the specific language governing permissions and
771772
limitations under the License.
772773

773774

775+
========================================================================
776+
For LimitedInputStream
777+
(network/common/src/main/java/org/apache/spark/network/util/LimitedInputStream.java):
778+
========================================================================
779+
Copyright (C) 2007 The Guava Authors
780+
781+
Licensed under the Apache License, Version 2.0 (the "License");
782+
you may not use this file except in compliance with the License.
783+
You may obtain a copy of the License at
784+
785+
http://www.apache.org/licenses/LICENSE-2.0
786+
787+
Unless required by applicable law or agreed to in writing, software
788+
distributed under the License is distributed on an "AS IS" BASIS,
789+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
790+
See the License for the specific language governing permissions and
791+
limitations under the License.
792+
793+
774794
========================================================================
775795
BSD-style licenses
776796
========================================================================

‎README.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ and Spark Streaming for stream processing.
1313
## Online Documentation
1414

1515
You can find the latest Spark documentation, including a programming
16-
guide, on the [project web page](http://spark.apache.org/documentation.html).
16+
guide, on the [project web page](http://spark.apache.org/documentation.html)
17+
and [project wiki](https://cwiki.apache.org/confluence/display/SPARK).
1718
This README file only contains basic setup instructions.
1819

1920
## Building Spark

‎assembly/pom.xml‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
<parent>
2222
<groupId>org.apache.spark</groupId>
2323
<artifactId>spark-parent</artifactId>
24-
<version>1.2.0-SNAPSHOT</version>
24+
<version>1.3.0-SNAPSHOT</version>
2525
<relativePath>../pom.xml</relativePath>
2626
</parent>
2727

@@ -66,22 +66,22 @@
6666
</dependency>
6767
<dependency>
6868
<groupId>org.apache.spark</groupId>
69-
<artifactId>spark-repl_${scala.binary.version}</artifactId>
69+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
7070
<version>${project.version}</version>
7171
</dependency>
7272
<dependency>
7373
<groupId>org.apache.spark</groupId>
74-
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
74+
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
7575
<version>${project.version}</version>
7676
</dependency>
7777
<dependency>
7878
<groupId>org.apache.spark</groupId>
79-
<artifactId>spark-graphx_${scala.binary.version}</artifactId>
79+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
8080
<version>${project.version}</version>
8181
</dependency>
8282
<dependency>
8383
<groupId>org.apache.spark</groupId>
84-
<artifactId>spark-sql_${scala.binary.version}</artifactId>
84+
<artifactId>spark-repl_${scala.binary.version}</artifactId>
8585
<version>${project.version}</version>
8686
</dependency>
8787
</dependencies>
@@ -170,16 +170,6 @@
170170
</build>
171171

172172
<profiles>
173-
<profile>
174-
<id>yarn-alpha</id>
175-
<dependencies>
176-
<dependency>
177-
<groupId>org.apache.spark</groupId>
178-
<artifactId>spark-yarn-alpha_${scala.binary.version}</artifactId>
179-
<version>${project.version}</version>
180-
</dependency>
181-
</dependencies>
182-
</profile>
183173
<profile>
184174
<id>yarn</id>
185175
<dependencies>
@@ -198,6 +188,11 @@
198188
<artifactId>spark-hive_${scala.binary.version}</artifactId>
199189
<version>${project.version}</version>
200190
</dependency>
191+
</dependencies>
192+
</profile>
193+
<profile>
194+
<id>hive-thriftserver</id>
195+
<dependencies>
201196
<dependency>
202197
<groupId>org.apache.spark</groupId>
203198
<artifactId>spark-hive-thriftserver_${scala.binary.version}</artifactId>

‎bagel/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
<parent>
2222
<groupId>org.apache.spark</groupId>
2323
<artifactId>spark-parent</artifactId>
24-
<version>1.2.0-SNAPSHOT</version>
24+
<version>1.3.0-SNAPSHOT</version>
2525
<relativePath>../pom.xml</relativePath>
2626
</parent>
2727

‎bin/beeline.cmd‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
@echo off
2+
3+
rem
4+
rem Licensed to the Apache Software Foundation (ASF) under one or more
5+
rem contributor license agreements. See the NOTICE file distributed with
6+
rem this work for additional information regarding copyright ownership.
7+
rem The ASF licenses this file to You under the Apache License, Version 2.0
8+
rem (the "License"); you may not use this file except in compliance with
9+
rem the License. You may obtain a copy of the License at
10+
rem
11+
rem http://www.apache.org/licenses/LICENSE-2.0
12+
rem
13+
rem Unless required by applicable law or agreed to in writing, software
14+
rem distributed under the License is distributed on an "AS IS" BASIS,
15+
rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
rem See the License for the specific language governing permissions and
17+
rem limitations under the License.
18+
rem
19+
20+
set SPARK_HOME=%~dp0..
21+
cmd /V /E /C %SPARK_HOME%\bin\spark-class.cmd org.apache.hive.beeline.BeeLine %*

‎bin/compute-classpath.sh‎

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@
2020
# This script computes Spark's classpath and prints it to stdout; it's used by both the "run"
2121
# script and the ExecutorRunner in standalone cluster mode.
2222

23-
SCALA_VERSION=2.10
24-
2523
# Figure out where Spark is installed
2624
FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
2725

@@ -36,7 +34,7 @@ else
3634
CLASSPATH="$CLASSPATH:$FWDIR/conf"
3735
fi
3836

39-
ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SCALA_VERSION"
37+
ASSEMBLY_DIR="$FWDIR/assembly/target/scala-$SPARK_SCALA_VERSION"
4038

4139
if [ -n "$JAVA_HOME" ]; then
4240
JAR_CMD="$JAVA_HOME/bin/jar"
@@ -48,19 +46,19 @@ fi
4846
if [ -n "$SPARK_PREPEND_CLASSES" ]; then
4947
echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\
5048
"classes ahead of assembly." >&2
51-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SCALA_VERSION/classes"
49+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes"
5250
CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*"
53-
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SCALA_VERSION/classes"
54-
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SCALA_VERSION/classes"
55-
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SCALA_VERSION/classes"
56-
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SCALA_VERSION/classes"
57-
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SCALA_VERSION/classes"
58-
CLASSPATH="$CLASSPATH:$FWDIR/tools/target/scala-$SCALA_VERSION/classes"
59-
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes"
60-
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes"
61-
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes"
62-
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SCALA_VERSION/classes"
63-
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes"
51+
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes"
52+
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes"
53+
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes"
54+
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SPARK_SCALA_VERSION/classes"
55+
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SPARK_SCALA_VERSION/classes"
56+
CLASSPATH="$CLASSPATH:$FWDIR/tools/target/scala-$SPARK_SCALA_VERSION/classes"
57+
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SPARK_SCALA_VERSION/classes"
58+
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SPARK_SCALA_VERSION/classes"
59+
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes"
60+
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes"
61+
CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes"
6462
fi
6563

6664
# Use spark-assembly jar from either RELEASE or assembly directory
@@ -70,14 +68,14 @@ else
7068
assembly_folder="$ASSEMBLY_DIR"
7169
fi
7270

73-
num_jars="$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar" | wc -l)"
71+
num_jars="$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*\.jar$" | wc -l)"
7472
if [ "$num_jars" -eq "0" ]; then
7573
echo "Failed to find Spark assembly in $assembly_folder"
7674
echo "You need to build Spark before running this program."
7775
exit 1
7876
fi
7977
if [ "$num_jars" -gt "1" ]; then
80-
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar")
78+
jars_list=$(ls "$assembly_folder" | grep "spark-assembly.*hadoop.*.jar$")
8179
echo "Found multiple Spark assembly jars in $assembly_folder:"
8280
echo "$jars_list"
8381
echo "Please remove all but one jar."
@@ -110,7 +108,7 @@ fi
110108
# built with Hive, so first check if the datanucleus jars exist, and then ensure the current Spark
111109
# assembly is built for Hive, before actually populating the CLASSPATH with the jars.
112110
# Note that this check order is faster (by up to half a second) in the case where Hive is not used.
113-
datanucleus_jars="$(find "$datanucleus_dir" 2>/dev/null | grep "datanucleus-.*\\.jar")"
111+
datanucleus_jars="$(find "$datanucleus_dir" 2>/dev/null | grep "datanucleus-.*\\.jar$")"
114112
datanucleus_jars="$(echo "$datanucleus_jars" | tr "\n" : | sed s/:$//g)"
115113

116114
# SPARK-4261: make right version info for beeline, copy hive-beeline*.jar to "lib_managed/jars/".
@@ -131,15 +129,15 @@ fi
131129

132130
# Add test classes if we're running from SBT or Maven with SPARK_TESTING set to 1
133131
if [[ $SPARK_TESTING == 1 ]]; then
134-
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SCALA_VERSION/test-classes"
135-
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SCALA_VERSION/test-classes"
136-
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SCALA_VERSION/test-classes"
137-
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SCALA_VERSION/test-classes"
138-
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SCALA_VERSION/test-classes"
139-
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SCALA_VERSION/test-classes"
140-
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/test-classes"
141-
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/test-classes"
142-
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/test-classes"
132+
CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/test-classes"
133+
CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/test-classes"
134+
CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/test-classes"
135+
CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/test-classes"
136+
CLASSPATH="$CLASSPATH:$FWDIR/graphx/target/scala-$SPARK_SCALA_VERSION/test-classes"
137+
CLASSPATH="$CLASSPATH:$FWDIR/streaming/target/scala-$SPARK_SCALA_VERSION/test-classes"
138+
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SPARK_SCALA_VERSION/test-classes"
139+
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SPARK_SCALA_VERSION/test-classes"
140+
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/test-classes"
143141
fi
144142

145143
# Add hadoop conf dir if given -- otherwise FileSystem.*, etc fail !

‎bin/load-spark-env.sh‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,3 +36,23 @@ if [ -z "$SPARK_ENV_LOADED" ]; then
3636
set +a
3737
fi
3838
fi
39+
40+
# Setting SPARK_SCALA_VERSION if not already set.
41+
42+
if [ -z "$SPARK_SCALA_VERSION" ]; then
43+
44+
ASSEMBLY_DIR2="$FWDIR/assembly/target/scala-2.11"
45+
ASSEMBLY_DIR1="$FWDIR/assembly/target/scala-2.10"
46+
47+
if [[ -d "$ASSEMBLY_DIR2" && -d "$ASSEMBLY_DIR1" ]]; then
48+
echo -e "Presence of build for both scala versions(SCALA 2.10 and SCALA 2.11) detected." 1>&2
49+
echo -e 'Either clean one of them or, export SPARK_SCALA_VERSION=2.11 in spark-env.sh.' 1>&2
50+
exit 1
51+
fi
52+
53+
if [ -d "$ASSEMBLY_DIR2" ]; then
54+
export SPARK_SCALA_VERSION="2.11"
55+
else
56+
export SPARK_SCALA_VERSION="2.10"
57+
fi
58+
fi

‎bin/pyspark‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ export SPARK_HOME="$FWDIR"
2525

2626
source "$FWDIR/bin/utils.sh"
2727

28-
SCALA_VERSION=2.10
28+
source "$FWDIR"/bin/load-spark-env.sh
2929

3030
function usage() {
3131
echo "Usage: ./bin/pyspark [options]" 1>&2
@@ -40,16 +40,14 @@ fi
4040
# Exit if the user hasn't compiled Spark
4141
if [ ! -f "$FWDIR/RELEASE" ]; then
4242
# Exit if the user hasn't compiled Spark
43-
ls "$FWDIR"/assembly/target/scala-$SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null
43+
ls "$FWDIR"/assembly/target/scala-$SPARK_SCALA_VERSION/spark-assembly*hadoop*.jar >& /dev/null
4444
if [[ $? != 0 ]]; then
4545
echo "Failed to find Spark assembly in $FWDIR/assembly/target" 1>&2
4646
echo "You need to build Spark before running this program" 1>&2
4747
exit 1
4848
fi
4949
fi
5050

51-
. "$FWDIR"/bin/load-spark-env.sh
52-
5351
# In Spark <= 1.1, setting IPYTHON=1 would cause the driver to be launched using the `ipython`
5452
# executable, while the worker would still be launched using PYSPARK_PYTHON.
5553
#
@@ -134,7 +132,5 @@ if [[ "$1" =~ \.py$ ]]; then
134132
gatherSparkSubmitOpts "$@"
135133
exec "$FWDIR"/bin/spark-submit "${SUBMISSION_OPTS[@]}" "$primary" "${APPLICATION_OPTS[@]}"
136134
else
137-
# PySpark shell requires special handling downstream
138-
export PYSPARK_SHELL=1
139135
exec "$PYSPARK_DRIVER_PYTHON" $PYSPARK_DRIVER_PYTHON_OPTS
140136
fi

0 commit comments

Comments
 (0)