Skip to content

[SPARK-29957][TEST] Reset MiniKDC's default enctypes to fit jdk8/jdk11 #26594

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
AngersZhuuuu wants to merge 14 commits into from
Closed

[SPARK-29957][TEST] Reset MiniKDC's default enctypes to fit jdk8/jdk11 #26594

Changes from 11 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
e9ef3ba
Update pom.xml
AngersZhuuuu Nov 19, 2019
3ec2891
miniKdc -> minikdc
AngersZhuuuu Nov 24, 2019
2e282f2
Update KafkaTestUtils.scala
AngersZhuuuu Nov 24, 2019
a224fad
Update KafkaTestUtils.scala
AngersZhuuuu Nov 24, 2019
3456ff9
Follow comment
AngersZhuuuu Nov 26, 2019
dfdf589
Merge branch 'master' into minikdc-3.2.0
AngersZhuuuu Nov 26, 2019
7ab6192
Update KafkaTestUtils.scala
AngersZhuuuu Nov 28, 2019
476d60e
Update KafkaTestUtils.scala
AngersZhuuuu Nov 29, 2019
b782e99
Update KafkaTestUtils.scala
AngersZhuuuu Dec 4, 2019
10a0006
Update KafkaTestUtils.scala
AngersZhuuuu Dec 4, 2019
03f7fb1
Update KafkaTestUtils.scala
AngersZhuuuu Dec 4, 2019
e252dd0
Update KafkaTestUtils.scala
AngersZhuuuu Dec 5, 2019
4facbee
Update KafkaTestUtils.scala
AngersZhuuuu Dec 6, 2019
223533f
Update KafkaTestUtils.scala
AngersZhuuuu Dec 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion external/kafka-0-10-sql/src/test/scala/org/apache/spark/sql/kafka010/KafkaTestUtils.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,15 @@

package org.apache.spark.sql.kafka010

import java.io.{File, IOException}
import java.io.{File, IOException, PrintWriter}
import java.net.{InetAddress, InetSocketAddress}
import java.nio.charset.StandardCharsets
import java.util.{Collections, Properties, UUID}
import java.util.concurrent.TimeUnit
import javax.security.auth.login.Configuration

import scala.collection.JavaConverters._
import scala.io.Source
import scala.util.Random

import com.google.common.io.Files
Expand Down Expand Up @@ -136,9 +137,49 @@ class KafkaTestUtils(
kdcConf.setProperty(MiniKdc.DEBUG, "true")
kdc = new MiniKdc(kdcConf, kdcDir)
Comment thread
dongjoon-hyun marked this conversation as resolved.
kdc.start()
// TODO https://issues.apache.org/jira/browse/SPARK-30037
// Need to customize krb5.conf like Kafka
Comment thread
AngersZhuuuu marked this conversation as resolved.
Outdated
rewriteKrb5Conf()
kdcReady = true
}

/**
* In this method we rewrite krb5.conf to make kdc nad client use the same enctypes
*
Comment thread
AngersZhuuuu marked this conversation as resolved.
Outdated
*/
private def rewriteKrb5Conf(): Unit = {
val krb5Conf = Source.fromFile(kdc.getKrb5conf, "UTF-8").getLines()
Comment thread
dongjoon-hyun marked this conversation as resolved.
var rewritten = false
val addedConfig =
addedKrb5Config("default_tkt_enctypes", "aes128-cts-hmac-sha1-96") +
addedKrb5Config("default_tgs_enctypes", "aes128-cts-hmac-sha1-96")
val rewriteKrb5Conf = krb5Conf.map(s =>
if (s.contains("libdefaults")) {
rewritten = true
s + addedConfig
} else {
s
}).filter(!_.trim.startsWith("#")).mkString(System.lineSeparator())

val krb5confStr = if (!rewritten) {
"[libdefaults]" + addedConfig + System.lineSeparator() +
System.lineSeparator() + rewriteKrb5Conf
} else {
rewriteKrb5Conf
}

kdc.getKrb5conf.delete()
val writer = new PrintWriter(kdc.getKrb5conf)
// scalastyle:off println
writer.println(krb5confStr)
// scalastyle:on println
writer.close()

@dongjoon-hyun dongjoon-hyun Dec 5, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we use Files.write(content, file, StandardCharsets.UTF_8) instead of println?

@dongjoon-hyun dongjoon-hyun Dec 5, 2019

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more. 

-    val writer = new PrintWriter(kdc.getKrb5conf)
-    // scalastyle:off println
-    writer.println(krb5confStr)
-    // scalastyle:on println
-    writer.close()
+    Files.write(krb5confStr, kdc.getKrb5conf, StandardCharsets.UTF_8)

@AngersZhuuuu AngersZhuuuu Dec 6, 2019

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shall we use Files.write(content, file, StandardCharsets.UTF_8) instead of println?

Done thanks.

}

private def addedKrb5Config(key: String, value: String): String = {
System.lineSeparator() + s" $key=$value"
}

private def createKeytabsAndJaasConfigFile(): String = {
assert(kdcReady, "KDC should be set up beforehand")
val baseDir = Utils.createTempDir()
Expand Down Expand Up @@ -171,6 +212,7 @@ class KafkaTestUtils(
| useKeyTab=true
| storeKey=true
| useTicketCache=false
| refreshKrb5Config=true
Comment thread
AngersZhuuuu marked this conversation as resolved.
| keyTab="${zkServerKeytabFile.getAbsolutePath()}"
| principal="$zkServerUser@$realm";
|};
Expand All @@ -180,6 +222,7 @@ class KafkaTestUtils(
| useKeyTab=true
| storeKey=true
| useTicketCache=false
| refreshKrb5Config=true
| keyTab="${zkClientKeytabFile.getAbsolutePath()}"
| principal="$zkClientUser@$realm";
|};
Expand Down