From 4939b84adc90c2c0791ec2969515d1eb9a520bfe Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Sun, 12 Nov 2017 18:32:47 +1100 Subject: [PATCH 1/7] upgrade hadoop to 2.9.0-palantir.1-rc9 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99e1bd4d2ffd6..1ed445138dbe1 100644 --- a/pom.xml +++ b/pom.xml @@ -2952,7 +2952,7 @@ hadoop-palantir - 2.8.2-palantir.2 + 2.9.0-palantir.1-rc9 2.7.1 From 2b0ae33ef1c14efb3e95c974fb38f51bc7cbd807 Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Sun, 12 Nov 2017 18:48:38 +1100 Subject: [PATCH 2/7] run test-dependencies.sh --replace-manifest --- dev/deps/spark-deps-hadoop-palantir | 48 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index fee1fbb36ff63..83c4c2c738c3c 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -1,3 +1,4 @@ +HikariCP-java7-2.4.12.jar JavaEWAH-0.3.2.jar RoaringBitmap-0.6.43.jar ST4-4.0.4.jar @@ -21,9 +22,7 @@ automaton-1.11-8.jar avro-1.8.1.jar avro-ipc-1.8.1.jar avro-mapred-1.8.1-hadoop2.jar -aws-java-sdk-core-1.11.45.jar -aws-java-sdk-kms-1.11.45.jar -aws-java-sdk-s3-1.11.45.jar +aws-java-sdk-bundle-1.11.199.jar base64-2.3.8.jar bcpkix-jdk15on-1.54.jar bcprov-jdk15on-1.54.jar @@ -72,40 +71,41 @@ dropwizard-metrics-0.9.2.jar dropwizard-metrics-influxdb-1.1.8.jar dropwizard-util-0.9.2.jar dropwizard-validation-0.9.2.jar +ehcache-3.3.1.jar eigenbase-properties-1.1.5.jar fastutil-7.0.13.jar flatbuffers-1.2.0-3f79e055.jar generex-1.0.1.jar +geronimo-jcache_1.0_spec-1.0-alpha-1.jar gson-2.2.4.jar guava-14.0.1.jar guice-3.0.jar guice-servlet-3.0.jar -hadoop-annotations-2.8.2-palantir.2.jar -hadoop-auth-2.8.2-palantir.2.jar -hadoop-aws-2.8.2-palantir.2.jar -hadoop-client-2.8.2-palantir.2.jar -hadoop-common-2.8.2-palantir.2.jar -hadoop-hdfs-client-2.8.2-palantir.2.jar -hadoop-mapreduce-client-app-2.8.2-palantir.2.jar -hadoop-mapreduce-client-common-2.8.2-palantir.2.jar -hadoop-mapreduce-client-core-2.8.2-palantir.2.jar -hadoop-mapreduce-client-jobclient-2.8.2-palantir.2.jar -hadoop-mapreduce-client-shuffle-2.8.2-palantir.2.jar -hadoop-openstack-2.8.2-palantir.2.jar -hadoop-yarn-api-2.8.2-palantir.2.jar -hadoop-yarn-client-2.8.2-palantir.2.jar -hadoop-yarn-common-2.8.2-palantir.2.jar -hadoop-yarn-server-common-2.8.2-palantir.2.jar -hadoop-yarn-server-web-proxy-2.8.2-palantir.2.jar +hadoop-annotations-2.9.0-palantir.1-rc9.jar +hadoop-auth-2.9.0-palantir.1-rc9.jar +hadoop-aws-2.9.0-palantir.1-rc9.jar +hadoop-client-2.9.0-palantir.1-rc9.jar +hadoop-common-2.9.0-palantir.1-rc9.jar +hadoop-hdfs-client-2.9.0-palantir.1-rc9.jar +hadoop-mapreduce-client-app-2.9.0-palantir.1-rc9.jar +hadoop-mapreduce-client-common-2.9.0-palantir.1-rc9.jar +hadoop-mapreduce-client-core-2.9.0-palantir.1-rc9.jar +hadoop-mapreduce-client-jobclient-2.9.0-palantir.1-rc9.jar +hadoop-mapreduce-client-shuffle-2.9.0-palantir.1-rc9.jar +hadoop-openstack-2.9.0-palantir.1-rc9.jar +hadoop-yarn-api-2.9.0-palantir.1-rc9.jar +hadoop-yarn-client-2.9.0-palantir.1-rc9.jar +hadoop-yarn-common-2.9.0-palantir.1-rc9.jar +hadoop-yarn-server-common-2.9.0-palantir.1-rc9.jar +hadoop-yarn-server-web-proxy-2.9.0-palantir.1-rc9.jar hibernate-validator-5.2.2.Final.jar hk2-api-2.5.0-b32.jar hk2-locator-2.5.0-b32.jar hk2-utils-2.5.0-b32.jar hppc-0.7.1.jar -htrace-core4-4.0.1-incubating.jar +htrace-core4-4.1.0-incubating.jar httpclient-4.5.3.jar httpcore-4.4.6.jar -ion-java-1.0.1.jar ivy-2.4.0.jar jackson-annotations-2.6.7.jar jackson-core-2.6.7.jar @@ -152,7 +152,6 @@ jets3t-0.9.3.jar jetty-6.1.26.jar jetty-sslengine-6.1.26.jar jetty-util-6.1.26.jar -jmespath-java-1.0.jar joda-time-2.9.9.jar jodd-core-3.5.2.jar json-smart-1.1.1.jar @@ -183,6 +182,7 @@ metrics-json-3.2.5.jar metrics-jvm-3.2.5.jar mimepull-1.9.6.jar minlog-1.3.0.jar +mssql-jdbc-6.2.1.jre7.jar mx4j-3.0.2.jar netty-3.10.6.Final.jar netty-all-4.0.50.Final.jar @@ -224,10 +224,12 @@ spire-macros_2.11-0.13.0.jar spire_2.11-0.13.0.jar stax-api-1.0-2.jar stax-api-1.0.1.jar +stax2-api-3.1.4.jar stream-2.7.0.jar stringtemplate-3.2.1.jar univocity-parsers-2.5.4.jar validation-api-1.1.0.Final.jar +woodstox-core-5.0.3.jar xbean-asm5-shaded-4.4.jar xmlenc-0.52.jar xz-1.5.jar From ba712f1b276889119fe9d577b4db6b15eacb67e1 Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Sun, 12 Nov 2017 18:58:37 +1100 Subject: [PATCH 3/7] missed one --- dists/hadoop-palantir/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dists/hadoop-palantir/pom.xml b/dists/hadoop-palantir/pom.xml index 7d713f0a6811f..72f57317ca693 100644 --- a/dists/hadoop-palantir/pom.xml +++ b/dists/hadoop-palantir/pom.xml @@ -34,7 +34,7 @@ spark-dist-hadoop-palantir none none - 2.8.2-palantir.2 + 2.9.0-palantir.1-rc9 2.7.1 From c166e2d2eca7ed2de1e6c2716f2302f233ea4a47 Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Mon, 13 Nov 2017 22:53:06 +1100 Subject: [PATCH 4/7] no more rc for deps --- dev/deps/spark-deps-hadoop-palantir | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index 83c4c2c738c3c..f45c9ac12f316 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -81,23 +81,23 @@ gson-2.2.4.jar guava-14.0.1.jar guice-3.0.jar guice-servlet-3.0.jar -hadoop-annotations-2.9.0-palantir.1-rc9.jar -hadoop-auth-2.9.0-palantir.1-rc9.jar -hadoop-aws-2.9.0-palantir.1-rc9.jar -hadoop-client-2.9.0-palantir.1-rc9.jar -hadoop-common-2.9.0-palantir.1-rc9.jar -hadoop-hdfs-client-2.9.0-palantir.1-rc9.jar -hadoop-mapreduce-client-app-2.9.0-palantir.1-rc9.jar -hadoop-mapreduce-client-common-2.9.0-palantir.1-rc9.jar -hadoop-mapreduce-client-core-2.9.0-palantir.1-rc9.jar -hadoop-mapreduce-client-jobclient-2.9.0-palantir.1-rc9.jar -hadoop-mapreduce-client-shuffle-2.9.0-palantir.1-rc9.jar -hadoop-openstack-2.9.0-palantir.1-rc9.jar -hadoop-yarn-api-2.9.0-palantir.1-rc9.jar -hadoop-yarn-client-2.9.0-palantir.1-rc9.jar -hadoop-yarn-common-2.9.0-palantir.1-rc9.jar -hadoop-yarn-server-common-2.9.0-palantir.1-rc9.jar -hadoop-yarn-server-web-proxy-2.9.0-palantir.1-rc9.jar +hadoop-annotations-2.9.0-palantir.1.jar +hadoop-auth-2.9.0-palantir.1.jar +hadoop-aws-2.9.0-palantir.1.jar +hadoop-client-2.9.0-palantir.1.jar +hadoop-common-2.9.0-palantir.1.jar +hadoop-hdfs-client-2.9.0-palantir.1.jar +hadoop-mapreduce-client-app-2.9.0-palantir.1.jar +hadoop-mapreduce-client-common-2.9.0-palantir.1.jar +hadoop-mapreduce-client-core-2.9.0-palantir.1.jar +hadoop-mapreduce-client-jobclient-2.9.0-palantir.1.jar +hadoop-mapreduce-client-shuffle-2.9.0-palantir.1.jar +hadoop-openstack-2.9.0-palantir.1.jar +hadoop-yarn-api-2.9.0-palantir.1.jar +hadoop-yarn-client-2.9.0-palantir.1.jar +hadoop-yarn-common-2.9.0-palantir.1.jar +hadoop-yarn-server-common-2.9.0-palantir.1.jar +hadoop-yarn-server-web-proxy-2.9.0-palantir.1.jar hibernate-validator-5.2.2.Final.jar hk2-api-2.5.0-b32.jar hk2-locator-2.5.0-b32.jar From 1ae86305c62908d4fc5ac0a0f95014ba2156e4da Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Mon, 13 Nov 2017 22:55:00 +1100 Subject: [PATCH 5/7] and the poms --- dists/hadoop-palantir/pom.xml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dists/hadoop-palantir/pom.xml b/dists/hadoop-palantir/pom.xml index 72f57317ca693..d3207e5a1f3a9 100644 --- a/dists/hadoop-palantir/pom.xml +++ b/dists/hadoop-palantir/pom.xml @@ -34,7 +34,7 @@ spark-dist-hadoop-palantir none none - 2.9.0-palantir.1-rc9 + 2.9.0-palantir.1 2.7.1 diff --git a/pom.xml b/pom.xml index 1ed445138dbe1..4519bfc92f677 100644 --- a/pom.xml +++ b/pom.xml @@ -2952,7 +2952,7 @@ hadoop-palantir - 2.9.0-palantir.1-rc9 + 2.9.0-palantir.1 2.7.1 From 6e545cd832200609e6a8a740d8d1cd6606df234c Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Thu, 16 Nov 2017 16:27:46 +1100 Subject: [PATCH 6/7] fix the test --- .../test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala index cfbf56fb8c369..469ef00a57fb3 100644 --- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala @@ -932,6 +932,8 @@ class SparkSubmitSuite if (supportMockHttpFs) { hadoopConf.set("fs.http.impl", classOf[TestFileSystem].getCanonicalName) hadoopConf.set("fs.http.impl.disable.cache", "true") + } else { + hadoopConf.set("fs.http.impl", "invalidClassName") } val tmpDir = Utils.createTempDir() From 9cd6b244ca30cad3f53f98bca8fa2977f2da218a Mon Sep 17 00:00:00 2001 From: Steven Rand Date: Thu, 16 Nov 2017 16:31:51 +1100 Subject: [PATCH 7/7] bump to 2.9.0-palantir.2 --- dev/deps/spark-deps-hadoop-palantir | 34 ++++++++++++++--------------- dists/hadoop-palantir/pom.xml | 2 +- pom.xml | 2 +- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index f45c9ac12f316..dbab352ce6430 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -81,23 +81,23 @@ gson-2.2.4.jar guava-14.0.1.jar guice-3.0.jar guice-servlet-3.0.jar -hadoop-annotations-2.9.0-palantir.1.jar -hadoop-auth-2.9.0-palantir.1.jar -hadoop-aws-2.9.0-palantir.1.jar -hadoop-client-2.9.0-palantir.1.jar -hadoop-common-2.9.0-palantir.1.jar -hadoop-hdfs-client-2.9.0-palantir.1.jar -hadoop-mapreduce-client-app-2.9.0-palantir.1.jar -hadoop-mapreduce-client-common-2.9.0-palantir.1.jar -hadoop-mapreduce-client-core-2.9.0-palantir.1.jar -hadoop-mapreduce-client-jobclient-2.9.0-palantir.1.jar -hadoop-mapreduce-client-shuffle-2.9.0-palantir.1.jar -hadoop-openstack-2.9.0-palantir.1.jar -hadoop-yarn-api-2.9.0-palantir.1.jar -hadoop-yarn-client-2.9.0-palantir.1.jar -hadoop-yarn-common-2.9.0-palantir.1.jar -hadoop-yarn-server-common-2.9.0-palantir.1.jar -hadoop-yarn-server-web-proxy-2.9.0-palantir.1.jar +hadoop-annotations-2.9.0-palantir.2.jar +hadoop-auth-2.9.0-palantir.2.jar +hadoop-aws-2.9.0-palantir.2.jar +hadoop-client-2.9.0-palantir.2.jar +hadoop-common-2.9.0-palantir.2.jar +hadoop-hdfs-client-2.9.0-palantir.2.jar +hadoop-mapreduce-client-app-2.9.0-palantir.2.jar +hadoop-mapreduce-client-common-2.9.0-palantir.2.jar +hadoop-mapreduce-client-core-2.9.0-palantir.2.jar +hadoop-mapreduce-client-jobclient-2.9.0-palantir.2.jar +hadoop-mapreduce-client-shuffle-2.9.0-palantir.2.jar +hadoop-openstack-2.9.0-palantir.2.jar +hadoop-yarn-api-2.9.0-palantir.2.jar +hadoop-yarn-client-2.9.0-palantir.2.jar +hadoop-yarn-common-2.9.0-palantir.2.jar +hadoop-yarn-server-common-2.9.0-palantir.2.jar +hadoop-yarn-server-web-proxy-2.9.0-palantir.2.jar hibernate-validator-5.2.2.Final.jar hk2-api-2.5.0-b32.jar hk2-locator-2.5.0-b32.jar diff --git a/dists/hadoop-palantir/pom.xml b/dists/hadoop-palantir/pom.xml index d3207e5a1f3a9..30d32e66e7599 100644 --- a/dists/hadoop-palantir/pom.xml +++ b/dists/hadoop-palantir/pom.xml @@ -34,7 +34,7 @@ spark-dist-hadoop-palantir none none - 2.9.0-palantir.1 + 2.9.0-palantir.2 2.7.1 diff --git a/pom.xml b/pom.xml index 4519bfc92f677..09c6d8250171e 100644 --- a/pom.xml +++ b/pom.xml @@ -2952,7 +2952,7 @@ hadoop-palantir - 2.9.0-palantir.1 + 2.9.0-palantir.2 2.7.1