Skip to content

Commit 3d44f64

Browse files
HanumathRaomgorbov
HanumathRao
authored and
mgorbov
committed
[MSPARK-45] Move Spark-OJAI connector code to Spark github repo (apache#132)
* SPARK-45 Move Spark-OJAI connector code to Spark github repo * Fixing pom versions for maprdb spark connector. * Changes made to the connector code to be compatible with 5.2.* and 6.0 clients.
1 parent aaca3b0 commit 3d44f64

File tree

79 files changed

+15351
-1
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+15351
-1
lines changed

‎external/maprdb/build.gradle‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
import PomUtil
2+
3+
apply from: "${topDir}/javatest.gradle"
4+
apply plugin: "scala"
5+
apply plugin: "maven-publish"
6+
7+
defaultTasks "publishToMavenLocal"
8+
9+
group = "com.mapr.db"
10+
version = maprVersion
11+
12+
dependencies {
13+
compile project(path: ":src/java-utils", configuration: "main")
14+
compile project(path: ":src/fs/proto-java", configuration: "main")
15+
compile project(path: ":src/fs/client/fileclient/java/maprfs", configuration: "main")
16+
compile project(path: ":src/fs/client/fileclient/java/maprfs-jni", configuration: "main")
17+
compile project(path: ":src/fs/client/dbclient-parent/client", configuration: "main")
18+
compile project(path: ":src/fs/client/dbclient-parent/map-reduce", configuration: "main")
19+
compile(group: "org.ojai", name: "ojai", version: ojaiVersion) {
20+
exclude group: "org.apache.hadoop"
21+
}
22+
compile(group: "org.ojai", name: "ojai-scala", version: ojaiVersion) {
23+
exclude group: "org.apache.hadoop"
24+
}
25+
compile group: "org.apache.spark", name: "spark-core_${scalaCompatVersion}", version: sparkCoreVersion
26+
compile group: "org.apache.spark", name: "spark-streaming_${scalaCompatVersion}", version: sparkCoreVersion
27+
compile group: "org.apache.spark", name: "spark-sql_${scalaCompatVersion}", version: sparkCoreVersion
28+
compile group: "org.scala-lang", name: "scala-library", version: scalaVersion
29+
compile group: "com.fasterxml.jackson.module", name: "jackson-module-scala_${scalaCompatVersion}", version: jacksonVersion
30+
compile group: "org.specs2", name: "specs2-junit_${scalaCompatVersion}", version: specs2Version
31+
compile group: "org.scalatest", name: "scalatest_${scalaCompatVersion}", version: scalatestVersion
32+
33+
testCompile project(path: ":src/fs/client/dbclient-parent/client", configuration: "testjar")
34+
}
35+
36+
tasks.withType(Jar) {
37+
baseName "maprdb-spark"
38+
}
39+
40+
File originalPom = file("pom.xml")
41+
List<Node> originalNodes = null
42+
43+
task eatPom() {
44+
doFirst {
45+
originalNodes = PomUtil.parsePom(originalPom, maprVersion)
46+
}
47+
}
48+
49+
publishing {
50+
publications {
51+
mvnArtifact {
52+
artifactId "maprdb-spark"
53+
pom.withXml {
54+
originalNodes.each { asNode().append(it) }
55+
}
56+
}
57+
}
58+
}
59+
60+
model {
61+
tasks.generatePomFileForMvnArtifactPublication {
62+
dependsOn "eatPom"
63+
}
64+
}
65+

‎external/maprdb/pom.xml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
<?xml version="1.0"?>
2+
<!-- Copyright (c) 2015 MapR, Inc. -->
3+
<project
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
5+
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
6+
<modelVersion>4.0.0</modelVersion>
7+
8+
<parent>
9+
<groupId>org.apache.spark</groupId>
10+
<artifactId>spark-parent_2.11</artifactId>
11+
<version>2.1.0-mapr-SNAPSHOT</version>
12+
<relativePath>../../pom.xml</relativePath>
13+
</parent>
14+
15+
<groupId>com.mapr.db</groupId>
16+
<artifactId>maprdb-spark</artifactId>
17+
<packaging>jar</packaging>
18+
19+
<name>MapR-DB Spark Library</name>
20+
<url>http://www.mapr.com/</url>
21+
22+
<dependencies>
23+
<dependency>
24+
<groupId>com.mapr.db</groupId>
25+
<artifactId>maprdb</artifactId>
26+
<version>6.0.0-mapr-SNAPSHOT</version>
27+
</dependency>
28+
<dependency>
29+
<groupId>org.apache.spark</groupId>
30+
<artifactId>spark-core_${scala.binary.version}</artifactId>
31+
<version>${project.version}</version>
32+
<scope>provided</scope>
33+
</dependency>
34+
<dependency>
35+
<groupId>org.apache.spark</groupId>
36+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
37+
<version>${project.version}</version>
38+
<scope>provided</scope>
39+
</dependency>
40+
<dependency>
41+
<groupId>org.apache.spark</groupId>
42+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
43+
<version>${project.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.scala-lang</groupId>
48+
<artifactId>scala-library</artifactId>
49+
<scope>provided</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>com.fasterxml.jackson.module</groupId>
53+
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
54+
<version>${fasterxml.jackson.version}</version>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.specs2</groupId>
58+
<artifactId>specs2-core_${scala.binary.version}</artifactId>
59+
<version>2.4.17</version>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.specs2</groupId>
63+
<artifactId>specs2-junit_${scala.binary.version}</artifactId>
64+
<version>2.4.17</version>
65+
</dependency>
66+
<dependency>
67+
<groupId>org.scalatest</groupId>
68+
<artifactId>scalatest_${scala.binary.version}</artifactId>
69+
</dependency>
70+
<dependency>
71+
<groupId>com.mapr.db</groupId>
72+
<artifactId>maprdb-mapreduce</artifactId>
73+
<version>6.0.0-mapr-SNAPSHOT</version>
74+
</dependency>
75+
<dependency>
76+
<groupId>org.ojai</groupId>
77+
<artifactId>ojai</artifactId>
78+
<version>2.0-mapr-1707-beta</version>
79+
</dependency>
80+
<dependency>
81+
<groupId>org.ojai</groupId>
82+
<artifactId>ojai-scala</artifactId>
83+
<version>2.0-mapr-1707-beta</version>
84+
</dependency>
85+
<dependency>
86+
<groupId>com.mapr</groupId>
87+
<artifactId>mapr-java-utils</artifactId>
88+
<version>6.0.0-mapr-SNAPSHOT</version>
89+
<classifier>tests</classifier>
90+
<scope>test</scope>
91+
</dependency>
92+
</dependencies>
93+
94+
<build>
95+
<sourceDirectory>src/main/scala</sourceDirectory>
96+
<testSourceDirectory>src/test/scala</testSourceDirectory>
97+
<plugins>
98+
<plugin>
99+
<groupId>net.alchim31.maven</groupId>
100+
<artifactId>scala-maven-plugin</artifactId>
101+
<configuration>
102+
<recompileMode>incremental</recompileMode>
103+
<args>
104+
<arg>-explaintypes</arg>
105+
</args>
106+
</configuration>
107+
<executions>
108+
<execution>
109+
<id>scala-compile-first</id>
110+
<phase>process-resources</phase>
111+
<goals>
112+
<goal>add-source</goal>
113+
<goal>compile</goal>
114+
</goals>
115+
</execution>
116+
<execution>
117+
<id>scala-test-compile</id>
118+
<phase>process-test-resources</phase>
119+
<goals>
120+
<goal>add-source</goal>
121+
<goal>testCompile</goal>
122+
</goals>
123+
</execution>
124+
</executions>
125+
</plugin>
126+
<plugin>
127+
<groupId>org.apache.maven.plugins</groupId>
128+
<artifactId>maven-surefire-plugin</artifactId>
129+
<configuration>
130+
<includes>
131+
<include>**/*Tests*.*</include>
132+
</includes>
133+
</configuration>
134+
</plugin>
135+
<plugin>
136+
<groupId>org.apache.maven.plugins</groupId>
137+
<artifactId>maven-compiler-plugin</artifactId>
138+
<configuration>
139+
<source>1.8</source>
140+
<target>1.8</target>
141+
</configuration>
142+
</plugin>
143+
</plugins>
144+
</build>
145+
</project>

‎external/maprdb/pom.xml.5.0‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
<?xml version="1.0"?>
2+
<!-- Copyright (c) 2015 MapR, Inc. -->
3+
<project
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
5+
xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
6+
<modelVersion>4.0.0</modelVersion>
7+
8+
<parent>
9+
<groupId>org.apache.spark</groupId>
10+
<artifactId>spark-parent_2.11</artifactId>
11+
<version>2.1.0-mapr-SNAPSHOT</version>
12+
<relativePath>../../pom.xml</relativePath>
13+
</parent>
14+
15+
<groupId>com.mapr.db</groupId>
16+
<artifactId>maprdb-spark</artifactId>
17+
<packaging>jar</packaging>
18+
19+
<name>MapR-DB Spark Library</name>
20+
<url>http://www.mapr.com/</url>
21+
22+
<dependencies>
23+
<dependency>
24+
<groupId>com.mapr.db</groupId>
25+
<artifactId>maprdb</artifactId>
26+
<version>5.2.2-mapr</version>
27+
</dependency>
28+
<dependency>
29+
<groupId>org.apache.spark</groupId>
30+
<artifactId>spark-core_${scala.binary.version}</artifactId>
31+
<version>${project.version}</version>
32+
<scope>provided</scope>
33+
</dependency>
34+
<dependency>
35+
<groupId>org.apache.spark</groupId>
36+
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
37+
<version>${project.version}</version>
38+
<scope>provided</scope>
39+
</dependency>
40+
<dependency>
41+
<groupId>org.apache.spark</groupId>
42+
<artifactId>spark-sql_${scala.binary.version}</artifactId>
43+
<version>${project.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
<dependency>
47+
<groupId>org.scala-lang</groupId>
48+
<artifactId>scala-library</artifactId>
49+
<scope>provided</scope>
50+
</dependency>
51+
<dependency>
52+
<groupId>com.fasterxml.jackson.module</groupId>
53+
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
54+
<version>${fasterxml.jackson.version}</version>
55+
</dependency>
56+
<dependency>
57+
<groupId>org.specs2</groupId>
58+
<artifactId>specs2-core_${scala.binary.version}</artifactId>
59+
<version>2.4.17</version>
60+
</dependency>
61+
<dependency>
62+
<groupId>org.specs2</groupId>
63+
<artifactId>specs2-junit_${scala.binary.version}</artifactId>
64+
<version>2.4.17</version>
65+
</dependency>
66+
<dependency>
67+
<groupId>org.scalatest</groupId>
68+
<artifactId>scalatest_${scala.binary.version}</artifactId>
69+
</dependency>
70+
<dependency>
71+
<groupId>com.mapr.db</groupId>
72+
<artifactId>maprdb-mapreduce</artifactId>
73+
<version>5.2.2-mapr</version>
74+
</dependency>
75+
<dependency>
76+
<groupId>org.ojai</groupId>
77+
<artifactId>ojai</artifactId>
78+
<version>1.1-mapr-1703</version>
79+
</dependency>
80+
<dependency>
81+
<groupId>org.ojai</groupId>
82+
<artifactId>ojai-scala</artifactId>
83+
<version>1.1-mapr-1703</version>
84+
</dependency>
85+
<dependency>
86+
<groupId>com.mapr</groupId>
87+
<artifactId>mapr-java-utils</artifactId>
88+
<version>5.2.2-mapr</version>
89+
<classifier>tests</classifier>
90+
<scope>test</scope>
91+
</dependency>
92+
</dependencies>
93+
94+
<build>
95+
<sourceDirectory>src/main/scala</sourceDirectory>
96+
<testSourceDirectory>src/test/scala</testSourceDirectory>
97+
<plugins>
98+
<plugin>
99+
<groupId>net.alchim31.maven</groupId>
100+
<artifactId>scala-maven-plugin</artifactId>
101+
<configuration>
102+
<recompileMode>incremental</recompileMode>
103+
<args>
104+
<arg>-explaintypes</arg>
105+
</args>
106+
</configuration>
107+
<executions>
108+
<execution>
109+
<id>scala-compile-first</id>
110+
<phase>process-resources</phase>
111+
<goals>
112+
<goal>add-source</goal>
113+
<goal>compile</goal>
114+
</goals>
115+
</execution>
116+
<execution>
117+
<id>scala-test-compile</id>
118+
<phase>process-test-resources</phase>
119+
<goals>
120+
<goal>add-source</goal>
121+
<goal>testCompile</goal>
122+
</goals>
123+
</execution>
124+
</executions>
125+
</plugin>
126+
<plugin>
127+
<groupId>org.apache.maven.plugins</groupId>
128+
<artifactId>maven-surefire-plugin</artifactId>
129+
<configuration>
130+
<includes>
131+
<include>**/*Tests*.*</include>
132+
</includes>
133+
</configuration>
134+
</plugin>
135+
<plugin>
136+
<groupId>org.apache.maven.plugins</groupId>
137+
<artifactId>maven-compiler-plugin</artifactId>
138+
<configuration>
139+
<source>1.8</source>
140+
<target>1.8</target>
141+
</configuration>
142+
</plugin>
143+
</plugins>
144+
</build>
145+
</project>

0 commit comments

Comments
 (0)