diff --git a/hbase-connectors-assembly/src/main/resources/supplemental-models.xml b/hbase-connectors-assembly/src/main/resources/supplemental-models.xml index 3c0a2ad..93e8ccc 100644 --- a/hbase-connectors-assembly/src/main/resources/supplemental-models.xml +++ b/hbase-connectors-assembly/src/main/resources/supplemental-models.xml @@ -50,4 +50,22 @@ under the License. + + + org.bouncycastle + bcprov-jdk18on + 1.78.1 + + + + MIT License + http://www.opensource.org/licenses/mit-license.php + repo + + Copyright (c) 2000 - 2018 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org) + + + + + diff --git a/kafka/pom.xml b/kafka/pom.xml index 50b9e05..9a0d369 100644 --- a/kafka/pom.xml +++ b/kafka/pom.xml @@ -34,6 +34,9 @@ hbase-kafka-model hbase-kafka-proxy + + ${hbase-hadoop2.version} + diff --git a/pom.xml b/pom.xml index 8335e7e..3d99059 100644 --- a/pom.xml +++ b/pom.xml @@ -137,6 +137,7 @@ 1.4 3.5.0 + 2.6.2 2.6.2-hadoop3 1.6.0 0.5.0 diff --git a/spark/README.md b/spark/README.md index 7515fb8..d069eaf 100755 --- a/spark/README.md +++ b/spark/README.md @@ -27,8 +27,6 @@ To generate an artifact for a different [Spark version](https://mvnrepository.co $ mvn -Dspark.version=3.1.2 -Dscala.version=2.12.10 -Dhadoop-three.version=3.2.0 -Dscala.binary.version=2.12 -Dhbase.version=2.4.8 clean install ``` -Note: to build the connector with Spark 2.x, compile it with `-Dscala.binary.version=2.11` and use the profile `-Dhadoop.profile=2.0` - ## Configuration and Installation **Client-side** (Spark) configuration: - The HBase configuration file `hbase-site.xml` should be made available to Spark, it can be copied to `$SPARK_CONF_DIR` (default is $SPARK_HOME/conf`) diff --git a/spark/hbase-spark-it/pom.xml b/spark/hbase-spark-it/pom.xml index 31adbc7..48c3bf4 100644 --- a/spark/hbase-spark-it/pom.xml +++ b/spark/hbase-spark-it/pom.xml @@ -72,54 +72,17 @@ org.apache.hadoop hadoop-common - ${hadoop-three.version} - - - com.google.code.findbugs - jsr305 - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - org.apache.hadoop hadoop-hdfs - ${hadoop-three.version} test-jar test - - - com.google.code.findbugs - jsr305 - - - xerces - xercesImpl - - org.apache.hadoop hadoop-client - ${hadoop-three.version} test - - - com.google.code.findbugs - jsr305 - - - xerces - xercesImpl - - diff --git a/spark/hbase-spark/pom.xml b/spark/hbase-spark/pom.xml index 58fb5b0..d9facf3 100644 --- a/spark/hbase-spark/pom.xml +++ b/spark/hbase-spark/pom.xml @@ -178,6 +178,31 @@ spark-catalyst_${scala.binary.version} ${spark.version} + + org.apache.hadoop + hadoop-client + + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-common + test-jar + test + + + org.apache.hadoop + hadoop-hdfs + test-jar + test + + + org.apache.hadoop + hadoop-minikdc + test + @@ -279,149 +304,6 @@ true - - - hadoop-2.0 - - - hadoop.profile - 2.0 - - - - - org.apache.hadoop - hadoop-client - ${hadoop-two.version} - - - org.apache.hadoop - hadoop-common - ${hadoop-two.version} - - - com.google.code.findbugs - jsr305 - - - - - org.apache.hadoop - hadoop-common - ${hadoop-two.version} - test-jar - test - - - com.google.code.findbugs - jsr305 - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop-two.version} - test-jar - test - - - com.google.code.findbugs - jsr305 - - - xerces - xercesImpl - - - - - org.apache.hadoop - hadoop-minikdc - ${hadoop-two.version} - test - - - org.apache.directory.jdbm - apacheds-jdbm1 - - - - - - - - hadoop-3.0 - - - !hadoop.profile - - - - ${hadoop-three.version} - - - - org.apache.hadoop - hadoop-client - ${hadoop-three.version} - - - org.apache.hadoop - hadoop-common - ${hadoop-three.version} - - - com.google.code.findbugs - jsr305 - - - - - org.apache.hadoop - hadoop-common - ${hadoop-three.version} - test-jar - test - - - com.google.code.findbugs - jsr305 - - - log4j - log4j - - - org.slf4j - slf4j-log4j12 - - - - - org.apache.hadoop - hadoop-hdfs - ${hadoop-three.version} - test-jar - test - - - com.google.code.findbugs - jsr305 - - - - - org.apache.hadoop - hadoop-minikdc - ${hadoop-three.version} - test - - - diff --git a/spark/pom.xml b/spark/pom.xml index 573b904..68618a8 100644 --- a/spark/pom.xml +++ b/spark/pom.xml @@ -74,6 +74,332 @@ hbase-shaded-miscellaneous ${hbase-thirdparty.version} + + org.apache.hadoop + hadoop-common + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-common + ${hadoop-three.version} + test-jar + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-client + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-hdfs + ${hadoop-three.version} + test-jar + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-minikdc + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-yarn-server-resourcemanager + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-shuffle + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-app + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + + + + org.apache.hadoop + hadoop-mapreduce-client-hs + ${hadoop-three.version} + + + com.google.code.findbugs + jsr305 + + + log4j + log4j + + + org.slf4j + slf4j-log4j12 + + + ch.qos.reload4j + reload4j + + + org.slf4j + slf4j-reload4j + + +