Skip to content

Commit 5f28ee4

Browse files
pwendellpwendell
committed
SPARK-1167: Remove metrics-ganglia from default build due to LGPL issues.
This patch removes Ganglia integration from the default build. It allows users willing to link against LGPL code to use Ganglia by adding build flags or linking against a new Spark artifact called spark-ganglia-lgpl. This brings Spark in line with the Apache policy on LGPL code enumerated here: https://www.apache.org/legal/3party.html#options-optional
1 parent e59a3b6 commit 5f28ee4

File tree

12 files changed

+190
-16
lines changed

12 files changed

+190
-16
lines changed

‎assembly/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,16 @@
158158
</dependency>
159159
</dependencies>
160160
</profile>
161+
<profile>
162+
<id>spark-ganglia-lgpl</id>
163+
<dependencies>
164+
<dependency>
165+
<groupId>org.apache.spark</groupId>
166+
<artifactId>spark-ganglia-lgpl_${scala.binary.version}</artifactId>
167+
<version>${project.version}</version>
168+
</dependency>
169+
</dependencies>
170+
</profile>
161171
<profile>
162172
<id>bigtop-dist</id>
163173
<!-- This profile uses the assembly plugin to create a special "dist" package for BigTop

‎core/pom.xml‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,10 +183,6 @@
183183
<groupId>com.codahale.metrics</groupId>
184184
<artifactId>metrics-json</artifactId>
185185
</dependency>
186-
<dependency>
187-
<groupId>com.codahale.metrics</groupId>
188-
<artifactId>metrics-ganglia</artifactId>
189-
</dependency>
190186
<dependency>
191187
<groupId>com.codahale.metrics</groupId>
192188
<artifactId>metrics-graphite</artifactId>

‎dev/audit-release/README.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
# Test Application Builds
2+
This directory includes test applications which are built when auditing releases. You can
3+
run them locally by setting appropriate environment variables.
4+
5+
```
6+
$ cd sbt_app_core
7+
$ SCALA_VERSION=2.10.3 \
8+
SPARK_VERSION=1.0.0-SNAPSHOT \
9+
SPARK_RELEASE_REPOSITORY=file:///home/patrick/.ivy2/local \
10+
sbt run
11+
```

‎dev/audit-release/sbt_app_core/src/main/scala/SparkApp.scala‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package main.scala
1919

20+
import scala.util.Try
21+
2022
import org.apache.spark.SparkContext
2123
import org.apache.spark.SparkContext._
2224

@@ -31,6 +33,17 @@ object SimpleApp {
3133
println("Failed to parse log files with Spark")
3234
System.exit(-1)
3335
}
34-
println("Test succeeded")
36+
37+
// Regression test for SPARK-1167: Remove metrics-ganglia from default build due to LGPL issue
38+
val foundConsole = Try(Class.forName("org.apache.spark.metrics.sink.ConsoleSink")).isSuccess
39+
val foundGanglia = Try(Class.forName("org.apache.spark.metrics.sink.GangliaSink")).isSuccess
40+
if (!foundConsole) {
41+
println("Console sink not loaded via spark-core")
42+
System.exit(-1)
43+
}
44+
if (foundGanglia) {
45+
println("Ganglia sink was loaded via spark-core")
46+
System.exit(-1)
47+
}
3548
}
3649
}

‎dev/audit-release/sbt_app_ganglia/build.sbt‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one or more
3+
// contributor license agreements. See the NOTICE file distributed with
4+
// this work for additional information regarding copyright ownership.
5+
// The ASF licenses this file to You under the Apache License, Version 2.0
6+
// (the "License"); you may not use this file except in compliance with
7+
// the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing, software
12+
// distributed under the License is distributed on an "AS IS" BASIS,
13+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
// See the License for the specific language governing permissions and
15+
// limitations under the License.
16+
//
17+
18+
name := "Ganglia Test"
19+
20+
version := "1.0"
21+
22+
scalaVersion := System.getenv.get("SCALA_VERSION")
23+
24+
libraryDependencies += "org.apache.spark" %% "spark-core" % System.getenv.get("SPARK_VERSION")
25+
26+
libraryDependencies += "org.apache.spark" %% "spark-ganglia-lgpl" % System.getenv.get("SPARK_VERSION")
27+
28+
resolvers ++= Seq(
29+
"Spark Release Repository" at System.getenv.get("SPARK_RELEASE_REPOSITORY"),
30+
"Akka Repository" at "http://repo.akka.io/releases/",
31+
"Spray Repository" at "http://repo.spray.cc/")

‎dev/audit-release/sbt_app_ganglia/src/main/scala/SparkApp.scala‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package main.scala
19+
20+
import scala.util.Try
21+
22+
import org.apache.spark.SparkContext
23+
import org.apache.spark.SparkContext._
24+
25+
object SimpleApp {
26+
def main(args: Array[String]) {
27+
// Regression test for SPARK-1167: Remove metrics-ganglia from default build due to LGPL issue
28+
val foundConsole = Try(Class.forName("org.apache.spark.metrics.sink.ConsoleSink")).isSuccess
29+
val foundGanglia = Try(Class.forName("org.apache.spark.metrics.sink.GangliaSink")).isSuccess
30+
if (!foundConsole) {
31+
println("Console sink not loaded via spark-core")
32+
System.exit(-1)
33+
}
34+
if (!foundGanglia) {
35+
println("Ganglia sink not loaded via spark-ganglia-lgpl")
36+
System.exit(-1)
37+
}
38+
}
39+
}

‎dev/create-release/create-release.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,14 +49,14 @@ mvn -DskipTests \
4949
-Darguments="-DskipTests=true -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -Dgpg.passphrase=${GPG_PASSPHRASE}" \
5050
-Dusername=$GIT_USERNAME -Dpassword=$GIT_PASSWORD \
5151
-Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
52-
-Pyarn \
52+
-Pyarn -Pspark-ganglia-lgpl \
5353
-Dtag=$GIT_TAG -DautoVersionSubmodules=true \
5454
--batch-mode release:prepare
5555

5656
mvn -DskipTests \
5757
-Darguments="-DskipTests=true -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -Dgpg.passphrase=${GPG_PASSPHRASE}" \
5858
-Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 \
59-
-Pyarn \
59+
-Pyarn -Pspark-ganglia-lgpl\
6060
release:perform
6161

6262
rm -rf spark

‎docs/monitoring.md‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,22 @@ Each instance can report to zero or more _sinks_. Sinks are contained in the
4848

4949
* `ConsoleSink`: Logs metrics information to the console.
5050
* `CSVSink`: Exports metrics data to CSV files at regular intervals.
51-
* `GangliaSink`: Sends metrics to a Ganglia node or multicast group.
5251
* `JmxSink`: Registers metrics for viewing in a JXM console.
5352
* `MetricsServlet`: Adds a servlet within the existing Spark UI to serve metrics data as JSON data.
5453
* `GraphiteSink`: Sends metrics to a Graphite node.
5554

55+
Spark also supports a Ganglia sink which is not included in the default build due to
56+
licensing restrictions:
57+
58+
* `GangliaSink`: Sends metrics to a Ganglia node or multicast group.
59+
60+
To install the `GangliaSink` you'll need to perform a custom build of Spark. _**Note that
61+
by embedding this library you will include [LGPL](http://www.gnu.org/copyleft/lesser.html)-licensed
62+
code in your Spark package**_. For sbt users, set the
63+
`SPARK_GANGLIA_LGPL` environment varaible before building. For Maven users, enable
64+
the `-Pspark-ganglia-lgpl` profile. For users linking applications against Spark, link
65+
include the `spark-ganglia-lgpl` artifact as a dependency.
66+
5667
The syntax of the metrics configuration file is defined in an example configuration file,
5768
`$SPARK_HOME/conf/metrics.properties.template`.
5869

‎extras/spark-ganglia-lgpl/pom.xml‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<modelVersion>4.0.0</modelVersion>
20+
<parent>
21+
<groupId>org.apache.spark</groupId>
22+
<artifactId>spark-parent</artifactId>
23+
<version>1.0.0-SNAPSHOT</version>
24+
<relativePath>../../pom.xml</relativePath>
25+
</parent>
26+
27+
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
28+
<groupId>org.apache.spark</groupId>
29+
<artifactId>spark-ganglia-lgpl_2.10</artifactId>
30+
<packaging>jar</packaging>
31+
<name>Spark Ganglia Integration</name>
32+
33+
<dependencies>
34+
<dependency>
35+
<groupId>org.apache.spark</groupId>
36+
<artifactId>spark-core_${scala.binary.version}</artifactId>
37+
<version>${project.version}</version>
38+
</dependency>
39+
40+
<dependency>
41+
<groupId>com.codahale.metrics</groupId>
42+
<artifactId>metrics-ganglia</artifactId>
43+
</dependency>
44+
</dependencies>
45+
</project>

‎core/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala‎ renamed to ‎extras/spark-ganglia-lgpl/src/main/scala/org/apache/spark/metrics/sink/GangliaSink.scala‎

File renamed without changes.

0 commit comments

Comments
 (0)