From b7a3106b1d3024345bd3b83e85521dae5b7a0f7f Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Wed, 1 May 2019 18:11:26 +1000 Subject: [PATCH 1/7] Enable krb5kdc-fixture, kerberos tests mount urandom for kdc container Infra has fixed #10462 by installing `haveged` on CI workers. This commit enables the disabled fixture and tests, and mounts `/dev/urandom` for the container so there is enough entropy required for kdc. Closes #40624 --- test/fixtures/krb5kdc-fixture/build.gradle | 3 +-- test/fixtures/krb5kdc-fixture/docker-compose.yml | 2 ++ x-pack/qa/kerberos-tests/build.gradle | 9 ++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/fixtures/krb5kdc-fixture/build.gradle b/test/fixtures/krb5kdc-fixture/build.gradle index 9787c3527af71..9ad1097af5297 100644 --- a/test/fixtures/krb5kdc-fixture/build.gradle +++ b/test/fixtures/krb5kdc-fixture/build.gradle @@ -47,8 +47,7 @@ postProcessFixture { } } -// https://github.com/elastic/elasticsearch/issues/40624 -buildFixture.enabled = false +buildFixture.enabled = true project.ext.krb5Conf = { service -> file("$buildDir/shared/${service}/krb5.conf") } project.ext.krb5Keytabs = { service, fileName -> file("$buildDir/shared/${service}/keytabs/${fileName}") } diff --git a/test/fixtures/krb5kdc-fixture/docker-compose.yml b/test/fixtures/krb5kdc-fixture/docker-compose.yml index 4d018dd6c3e08..8417d29434580 100644 --- a/test/fixtures/krb5kdc-fixture/docker-compose.yml +++ b/test/fixtures/krb5kdc-fixture/docker-compose.yml @@ -8,6 +8,7 @@ services: command: "bash /fixture/src/main/resources/provision/peppa.sh" volumes: - ./build/shared/peppa:/fixture/build + - /dev/urandom:/dev/random ports: - "4444" - "88/udp" @@ -19,6 +20,7 @@ services: command: "bash /fixture/src/main/resources/provision/hdfs.sh" volumes: - ./build/shared/hdfs:/fixture/build + - /dev/urandom:/dev/random ports: - "4444" - "88/udp" diff --git a/x-pack/qa/kerberos-tests/build.gradle b/x-pack/qa/kerberos-tests/build.gradle index 88248f89b72c5..8d8f6559a6015 100644 --- a/x-pack/qa/kerberos-tests/build.gradle +++ b/x-pack/qa/kerberos-tests/build.gradle @@ -8,8 +8,7 @@ apply plugin: 'elasticsearch.test.fixtures' testFixtures.useFixture ":test:fixtures:krb5kdc-fixture" -// https://github.com/elastic/elasticsearch/issues/40624 -integTest.enabled = false +integTest.enabled = true dependencies { testCompile "org.elasticsearch.plugin:x-pack-core:${version}" @@ -56,9 +55,9 @@ integTestCluster { String realm = "BUILD.ELASTIC.CO" integTestRunner { Path peppaKeytab = Paths.get("${project.buildDir}", "generated-resources", "keytabs", "peppa.keytab") - systemProperty 'test.userkt', "peppa@${realm}" - systemProperty 'test.userkt.keytab', "${peppaKeytab}" - systemProperty 'test.userpwd', "george@${realm}" + nonInputProperties.systemProperty 'test.userkt', "peppa@${realm}" + nonInputProperties.systemProperty 'test.userkt.keytab', "${peppaKeytab}" + nonInputProperties.systemProperty 'test.userpwd', "george@${realm}" systemProperty 'test.userpwd.password', "dino" systemProperty 'tests.security.manager', 'true' jvmArgs([ From 56a4c960d96f30f6f3f3443b43d7eee4c4ac18a2 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Thu, 2 May 2019 12:28:17 +1000 Subject: [PATCH 2/7] check if marking nonInputProperties for lazy-evaluated properties works --- plugins/repository-hdfs/build.gradle | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 3d6dcd29d6d19..1c22dcabd5e1f 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -122,10 +122,10 @@ for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSec runner { if (integTestTaskName.contains("Secure")) { dependsOn secureHdfsFixture - systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" - systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" + nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" + nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" jvmArgs "-Djava.security.krb5.conf=${krb5conf}" - systemProperty ( + nonInputProperties.systemProperty ( "test.krb5.keytab.hdfs", project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab") ) From 449dd738ada3a24d2fadb71e785640ea1ba544ce Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Fri, 31 May 2019 13:59:47 +1000 Subject: [PATCH 3/7] set data node address to find suitable port as locally it failed to bind to default port --- test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java index 01315cdab01ca..cc24c2010173d 100644 --- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java +++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java @@ -94,6 +94,7 @@ public static void main(String[] args) throws Exception { cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true"); cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true"); cfg.set(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY, "true"); + cfg.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "localhost:0"); } UserGroupInformation.setConfiguration(cfg); From df7607065f3d9bc0defe94582d574f8f7f81a5b5 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Fri, 31 May 2019 16:22:46 +1000 Subject: [PATCH 4/7] for debug --- test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java index cc24c2010173d..78837e6870be1 100644 --- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java +++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java @@ -55,6 +55,7 @@ public class MiniHDFS { private static String PID_FILE_NAME = "pid"; public static void main(String[] args) throws Exception { + try { if (args.length != 1 && args.length != 3) { throw new IllegalArgumentException("Expected: MiniHDFS [ ], " + "got: " + Arrays.toString(args)); @@ -174,5 +175,8 @@ public static void main(String[] args) throws Exception { tmp = Files.createTempFile(baseDir, null, null); Files.write(tmp, portFileContent.getBytes(StandardCharsets.UTF_8)); Files.move(tmp, baseDir.resolve(PORT_FILE_NAME), StandardCopyOption.ATOMIC_MOVE); - } + } catch (Exception e) { + e.printStackTrace(); + } + } } From 7ce8832e8620c91cb1af3fb38a7d893d52972425 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Fri, 31 May 2019 17:59:51 +1000 Subject: [PATCH 5/7] check if hdfs fixture is ready --- plugins/repository-hdfs/build.gradle | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 1c22dcabd5e1f..ddc4591e5d2ca 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -85,7 +85,14 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', waitCondition = { fixture, ant -> // the hdfs.MiniHDFS fixture writes the ports file when // it's ready, so we can just wait for the file to exist - return fixture.portsFile.exists() + + if (fixture.portsFile.exists()) { + println "hdfs.MiniHDFS fixture " + fixtureName + " is ready!"; + return true; + } else { + println "hdfs.MiniHDFS fixture " + fixtureName + " is not yet ready, keep waiting!"; + return false; + } } final List miniHDFSArgs = [] From ccc205785d3ca2680c3354a33cc2002a6f272c03 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Wed, 12 Jun 2019 16:16:12 +1000 Subject: [PATCH 6/7] disable hdfs tests --- plugins/repository-hdfs/build.gradle | 36 ++++++++++--------- .../src/main/java/hdfs/MiniHDFS.java | 32 ++++++++--------- .../krb5kdc-fixture/docker-compose.yml | 1 + 3 files changed, 34 insertions(+), 35 deletions(-) diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 7de0b018e7082..713850c10de98 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -86,15 +86,8 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', waitCondition = { fixture, ant -> // the hdfs.MiniHDFS fixture writes the ports file when // it's ready, so we can just wait for the file to exist - - if (fixture.portsFile.exists()) { - println "hdfs.MiniHDFS fixture " + fixtureName + " is ready!"; - return true; - } else { - println "hdfs.MiniHDFS fixture " + fixtureName + " is not yet ready, keep waiting!"; - return false; - } - } + return fixture.portsFile.exists() + } final List miniHDFSArgs = [] @@ -123,20 +116,29 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', } } +Set disabledIntegTestTaskNames = ['integTestSecure', 'integTestSecureHa'] + for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) { task "${integTestTaskName}"(type: RestIntegTestTask) { description = "Runs rest tests against an elasticsearch cluster with HDFS." dependsOn(project.bundlePlugin) + + if (disabledIntegTestTaskNames.contains(integTestTaskName)) { + enabled = false; + } + runner { if (integTestTaskName.contains("Secure")) { - dependsOn secureHdfsFixture - nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" - nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" - jvmArgs "-Djava.security.krb5.conf=${krb5conf}" - nonInputProperties.systemProperty ( - "test.krb5.keytab.hdfs", - project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab") - ) + if (disabledIntegTestTaskNames.contains(integTestTaskName) == false) { + dependsOn secureHdfsFixture + nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" + nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" + jvmArgs "-Djava.security.krb5.conf=${krb5conf}" + nonInputProperties.systemProperty ( + "test.krb5.keytab.hdfs", + project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs","hdfs_hdfs.build.elastic.co.keytab") + ) + } } } } diff --git a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java index 78837e6870be1..b060d78b92b69 100644 --- a/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java +++ b/test/fixtures/hdfs-fixture/src/main/java/hdfs/MiniHDFS.java @@ -19,18 +19,6 @@ package hdfs; -import java.io.File; -import java.lang.management.ManagementFactory; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; @@ -45,6 +33,18 @@ import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil; import org.apache.hadoop.security.UserGroupInformation; +import java.io.File; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** * MiniHDFS test fixture. There is a CLI tool, but here we can * easily properly setup logging, avoid parsing JSON, etc. @@ -55,7 +55,6 @@ public class MiniHDFS { private static String PID_FILE_NAME = "pid"; public static void main(String[] args) throws Exception { - try { if (args.length != 1 && args.length != 3) { throw new IllegalArgumentException("Expected: MiniHDFS [ ], " + "got: " + Arrays.toString(args)); @@ -95,7 +94,6 @@ public static void main(String[] args) throws Exception { cfg.set(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, "true"); cfg.set(DFSConfigKeys.IGNORE_SECURE_PORTS_FOR_TESTING_KEY, "true"); cfg.set(DFSConfigKeys.DFS_ENCRYPT_DATA_TRANSFER_KEY, "true"); - cfg.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "localhost:0"); } UserGroupInformation.setConfiguration(cfg); @@ -175,8 +173,6 @@ public static void main(String[] args) throws Exception { tmp = Files.createTempFile(baseDir, null, null); Files.write(tmp, portFileContent.getBytes(StandardCharsets.UTF_8)); Files.move(tmp, baseDir.resolve(PORT_FILE_NAME), StandardCopyOption.ATOMIC_MOVE); - } catch (Exception e) { - e.printStackTrace(); - } - } + } + } diff --git a/test/fixtures/krb5kdc-fixture/docker-compose.yml b/test/fixtures/krb5kdc-fixture/docker-compose.yml index 8417d29434580..41518e07c1f6a 100644 --- a/test/fixtures/krb5kdc-fixture/docker-compose.yml +++ b/test/fixtures/krb5kdc-fixture/docker-compose.yml @@ -8,6 +8,7 @@ services: command: "bash /fixture/src/main/resources/provision/peppa.sh" volumes: - ./build/shared/peppa:/fixture/build + # containers have bad entropy so mount /dev/urandom. Less secure but this is a test fixture. - /dev/urandom:/dev/random ports: - "4444" From 527958e0f9d6f3918410eb1d571b3a25adff50b5 Mon Sep 17 00:00:00 2001 From: Yogesh Gaikwad Date: Wed, 12 Jun 2019 16:24:02 +1000 Subject: [PATCH 7/7] add comment --- test/fixtures/krb5kdc-fixture/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/fixtures/krb5kdc-fixture/docker-compose.yml b/test/fixtures/krb5kdc-fixture/docker-compose.yml index 41518e07c1f6a..8e7108d58e5c0 100644 --- a/test/fixtures/krb5kdc-fixture/docker-compose.yml +++ b/test/fixtures/krb5kdc-fixture/docker-compose.yml @@ -21,6 +21,7 @@ services: command: "bash /fixture/src/main/resources/provision/hdfs.sh" volumes: - ./build/shared/hdfs:/fixture/build + # containers have bad entropy so mount /dev/urandom. Less secure but this is a test fixture. - /dev/urandom:/dev/random ports: - "4444"