Merge pull request #188 from kkonstantine/CC-397-Refactor-reusable-co…

…mponents-from-HDFS-connector-into-shared-library CC-397: Refactoring on top of storage-common
confluentinc · Aug 9, 2017 · 73b0107 · 73b0107
2 parents cd06434 + cff5bde
commit 73b0107
Show file tree

Hide file tree

Showing 78 changed files with 2,143 additions and 2,128 deletions.
diff --git a/docs/hdfs_connector.rst b/docs/hdfs_connector.rst
@@ -306,7 +306,7 @@ To work with secure HDFS and Hive metastore, you need to specify ``hdfs.authenti
   connect.hdfs.keytab=path to the connector keytab
   hdfs.namenode.principal=namenode principal
 
-You need to create the Kafka connect principals and keytab files via Kerboros and distribute the
+You need to create the Kafka connect principals and keytab files via Kerberos and distribute the
 keytab file to all hosts that running the connector and ensures that only the connector user
 has read access to the keytab file.
 

diff --git a/pom.xml b/pom.xml
@@ -84,39 +84,29 @@
             <version>${confluent.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-client</artifactId>
-            <version>${hadoop.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hive</groupId>
-            <artifactId>hive-cli</artifactId>
-            <version>${hive.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hive</groupId>
-            <artifactId>hive-common</artifactId>
-            <version>${hive.version}</version>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-connect-storage-common</artifactId>
+            <version>${confluent.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.avro</groupId>
-            <artifactId>avro-mapred</artifactId>
-            <version>${avro.version}</version>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-connect-storage-core</artifactId>
+            <version>${confluent.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-column</artifactId>
-            <version>${parquet.version}</version>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-connect-storage-format</artifactId>
+            <version>${confluent.version}</version>
         </dependency>
         <dependency>
-            <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-avro</artifactId>
-            <version>${parquet.version}</version>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-connect-storage-partitioner</artifactId>
+            <version>${confluent.version}</version>
         </dependency>
         <dependency>
-            <groupId>commons-io</groupId>
-            <artifactId>commons-io</artifactId>
-            <version>${commons-io.version}</version>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-connect-storage-wal</artifactId>
+            <version>${confluent.version}</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -207,6 +197,7 @@
                         <descriptor>src/assembly/development.xml</descriptor>
                         <descriptor>src/assembly/package.xml</descriptor>
                     </descriptors>
+                    <attach>false</attach>
                 </configuration>
                 <executions>
                     <execution>

diff --git a/src/assembly/development.xml b/src/assembly/development.xml
@@ -19,7 +19,6 @@
                 shipped with Confluent platform and other dependencies such as Hadoop and Avro.
                 This allows correctly setup of CLASSPATH in kafka-run-class.sh when running
                 kafka connect hdfs connector. -->
-                <exclude>org.apache.kafka:connect-api</exclude>
                 <exclude>org.mortbay.jetty:*</exclude>
                 <exclude>com.sun.jersey:*</exclude>
                 <exclude>org.eclipse.jetty.aggregate:jetty-all</exclude>

diff --git a/src/assembly/package.xml b/src/assembly/package.xml
@@ -38,11 +38,16 @@
                 shipped with Confluent platform and other dependencies such as Hadoop and Avro.
                 This allows correctly setup of CLASSPATH in kafka-run-class.sh when running
                 kafka connect hdfs connector. -->
-                <exclude>org.apache.kafka:connect-api</exclude>
                 <exclude>org.mortbay.jetty:*</exclude>
                 <exclude>com.sun.jersey:*</exclude>
                 <exclude>org.eclipse.jetty.aggregate:jetty-all</exclude>
                 <exclude>com.sun.jersey.contribs:jersey-guice</exclude>
+                <exclude>io.confluent:kafka-connect-storage-common</exclude>
+                <exclude>io.confluent:kafka-connect-storage-core</exclude>
+                <exclude>io.confluent:kafka-connect-storage-format</exclude>
+                <exclude>io.confluent:kafka-connect-storage-hive</exclude>
+                <exclude>io.confluent:kafka-connect-storage-partitioner</exclude>
+                <exclude>io.confluent:kafka-connect-storage-wal</exclude>
             </excludes>
         </dependencySet>
     </dependencySets>