From 478b863eaa5be0963b006451897731a3903d13f1 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 18 Nov 2020 16:15:22 -0500 Subject: [PATCH 1/6] Implement the new read operation on HDFS Repositories --- .../repositories/hdfs/HdfsBlobContainer.java | 22 ++++++++- .../hdfs/HdfsBlobStoreContainerTests.java | 45 +++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index cf9328436b40f..17f9718d397c5 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -19,12 +19,14 @@ package org.elasticsearch.repositories.hdfs; import org.apache.hadoop.fs.CreateFlag; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSInputStream; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetadata; @@ -33,6 +35,7 @@ import org.elasticsearch.common.blobstore.fs.FsBlobContainer; import org.elasticsearch.common.blobstore.support.AbstractBlobContainer; import org.elasticsearch.common.blobstore.support.PlainBlobMetadata; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.repositories.hdfs.HdfsBlobStore.Operation; import java.io.FileNotFoundException; @@ -112,8 +115,23 @@ public InputStream readBlob(String blobName) throws IOException { } @Override - public InputStream readBlob(String blobName, long position, long length) { - throw new UnsupportedOperationException(); + public InputStream readBlob(String blobName, long position, long length) throws IOException { + // FSDataInputStream does buffering internally + // FSDataInputStream can open connections on read() or skip() so we wrap in + // HDFSPrivilegedInputSteam which will ensure that underlying methods will + // be called with the proper privileges. + try { + return store.execute(fileContext -> { + FSDataInputStream fsInput = fileContext.open(new Path(path, blobName), bufferSize); + // As long as no read operations have happened yet on the stream, seeking + // should direct the datanode to start on the appropriate block, at the + // appropriate target position. + fsInput.seek(position); + return new HDFSPrivilegedInputSteam(Streams.limitStream(fsInput, length), securityContext); + }); + } catch (FileNotFoundException fnfe) { + throw new NoSuchFileException("[" + blobName + "] blob not found"); + } } @Override diff --git a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java index 0c6e6f9044a2a..7948a5f715262 100644 --- a/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java +++ b/plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsBlobStoreContainerTests.java @@ -29,10 +29,14 @@ import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobPath; import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.Streams; import org.elasticsearch.test.ESTestCase; +import org.hamcrest.CoreMatchers; import javax.security.auth.Subject; +import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.net.URI; @@ -41,6 +45,7 @@ import java.security.PrivilegedAction; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.util.Arrays; import java.util.Collections; import static org.elasticsearch.repositories.blobstore.ESBlobStoreRepositoryIntegTestCase.randomBytes; @@ -133,4 +138,44 @@ public void testReadOnly() throws Exception { assertArrayEquals(readBlobFully(container, "foo", data.length), data); assertTrue(container.blobExists("foo")); } + + public void testReadRange() throws Exception { + FileContext fileContext = createTestContext(); + // Constructor will not create dir if read only + HdfsBlobStore hdfsBlobStore = new HdfsBlobStore(fileContext, "dir", 1024, true); + FileContext.Util util = fileContext.util(); + Path root = fileContext.makeQualified(new Path("dir")); + assertFalse(util.exists(root)); + BlobPath blobPath = BlobPath.cleanPath().add("path"); + + // blobContainer() will not create path if read only + hdfsBlobStore.blobContainer(blobPath); + Path hdfsPath = root; + for (String p : blobPath) { + hdfsPath = new Path(hdfsPath, p); + } + assertFalse(util.exists(hdfsPath)); + + // if not read only, directory will be created + hdfsBlobStore = new HdfsBlobStore(fileContext, "dir", 1024, false); + assertTrue(util.exists(root)); + BlobContainer container = hdfsBlobStore.blobContainer(blobPath); + assertTrue(util.exists(hdfsPath)); + + byte[] data = randomBytes(randomIntBetween(10, scaledRandomIntBetween(1024, 1 << 16))); + writeBlob(container, "foo", new BytesArray(data), randomBoolean()); + int pos = randomIntBetween(0, data.length / 2); + int len = randomIntBetween(pos, data.length) - pos; + assertArrayEquals(readBlobPartially(container, "foo", pos, len), Arrays.copyOfRange(data, pos, pos+len)); + assertTrue(container.blobExists("foo")); + } + + public static byte[] readBlobPartially(BlobContainer container, String name, int pos, int length) throws IOException { + byte[] data = new byte[length]; + try (InputStream inputStream = container.readBlob(name, pos, length)) { + assertThat(Streams.readFully(inputStream, data), CoreMatchers.equalTo(length)); + assertThat(inputStream.read(), CoreMatchers.equalTo(-1)); + } + return data; + } } From 81cf99ff5cf5e99aa1878c0dddfb89621f7ccd3f Mon Sep 17 00:00:00 2001 From: James Baiera Date: Wed, 9 Dec 2020 17:24:14 -0500 Subject: [PATCH 2/6] Get bare bones testing completed --- .../hdfs/HdfsSecurityContext.java | 4 +- .../searchable-snapshots/qa/hdfs/build.gradle | 284 ++++++++++++++++++ .../hdfs/HdfsSearchableSnapshotsIT.java | 38 +++ 3 files changed, 324 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle create mode 100644 x-pack/plugin/searchable-snapshots/qa/hdfs/src/test/java/org/elasticsearch/xpack/searchablesnapshots/hdfs/HdfsSearchableSnapshotsIT.java diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java index 4562174230c27..f605d8958fd88 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java @@ -50,7 +50,7 @@ class HdfsSecurityContext { static { // We can do FS ops with only a few elevated permissions: SIMPLE_AUTH_PERMISSIONS = new Permission[]{ - new SocketPermission("*", "connect"), + new SocketPermission("*", "connect,resolve"), // 1) hadoop dynamic proxy is messy with access rules new ReflectPermission("suppressAccessChecks"), // 2) allow hadoop to add credentials to our Subject @@ -61,7 +61,7 @@ class HdfsSecurityContext { // If Security is enabled, we need all the following elevated permissions: KERBEROS_AUTH_PERMISSIONS = new Permission[] { - new SocketPermission("*", "connect"), + new SocketPermission("*", "connect,resolve"), // 1) hadoop dynamic proxy is messy with access rules new ReflectPermission("suppressAccessChecks"), // 2) allow hadoop to add credentials to our Subject diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle new file mode 100644 index 0000000000000..19919446ace6c --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle @@ -0,0 +1,284 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.info.BuildParams +import org.elasticsearch.gradle.test.RestIntegTestTask + +import java.nio.file.Files +import java.nio.file.Paths + +import static org.elasticsearch.gradle.PropertyNormalization.IGNORE_VALUE + +apply plugin: 'elasticsearch.test.fixtures' +apply plugin: 'elasticsearch.standalone-rest-test' +apply plugin: 'elasticsearch.rest-test' +apply plugin: 'elasticsearch.rest-resources' + +final Project hdfsFixtureProject = project(':test:fixtures:hdfs-fixture') +final Project krbFixtureProject = project(':test:fixtures:krb5kdc-fixture') +final Project hdfsRepoPluginProject = project(':plugins:repository-hdfs') + +dependencies { + testImplementation project(path: xpackModule('searchable-snapshots'), configuration: 'testArtifacts') + testImplementation hdfsRepoPluginProject +} + +restResources { + restApi { + includeCore 'indices', 'search', 'bulk', 'snapshot', 'nodes', '_common' + includeXpack 'searchable_snapshots' + } +} + +testFixtures.useFixture(krbFixtureProject.path, 'hdfs-snapshot') + +configurations { + hdfsFixture +} + +dependencies { + hdfsFixture hdfsFixtureProject + // Set the keytab files in the classpath so that we can access them from test code without the security manager freaking out. + if (isEclipse == false) { + testRuntimeOnly files(krbFixtureProject.ext.krb5Keytabs("hdfs-snapshot", "hdfs_hdfs.build.elastic.co.keytab").parent) + } +} + +normalization { + runtimeClasspath { + // ignore generated keytab files for the purposes of build avoidance + ignore '*.keytab' + // ignore fixture ports file which is on the classpath primarily to pacify the security manager + ignore 'ports' + } +} + +String realm = "BUILD.ELASTIC.CO" +String krb5conf = krbFixtureProject.ext.krb5Conf("hdfs") + +// Create HDFS File System Testing Fixtures for HA/Secure combinations +// TODO: Do we need HA fixtures? Unlikely? +//for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) { +for (String fixtureName : ['hdfsFixture', 'secureHdfsFixture']) { + def tsk = project.tasks.register(fixtureName, org.elasticsearch.gradle.test.AntFixture) { + dependsOn project.configurations.hdfsFixture, krbFixtureProject.tasks.postProcessFixture + executable = "${BuildParams.runtimeJavaHome}/bin/java" + env 'CLASSPATH', "${-> project.configurations.hdfsFixture.asPath}" + maxWaitInSeconds 60 + onlyIf { BuildParams.inFipsJvm == false } + waitCondition = { fixture, ant -> + // the hdfs.MiniHDFS fixture writes the ports file when + // it's ready, so we can just wait for the file to exist + return fixture.portsFile.exists() + } + final List miniHDFSArgs = [] + + // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options + if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { + miniHDFSArgs.add("-Djava.security.krb5.conf=${krb5conf}") + } +// // If it's an HA fixture, set a nameservice to use in the JVM options +// if (fixtureName.equals('haHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { +// miniHDFSArgs.add("-Dha-nameservice=ha-hdfs") +// } + + // Common options + miniHDFSArgs.add('hdfs.MiniHDFS') + miniHDFSArgs.add(baseDir) + + // If it's a secure fixture, then set the principal name and keytab locations to use for auth. + if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { + miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}") + miniHDFSArgs.add( + project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") + ) + } + + args miniHDFSArgs.toArray() + } + + // TODO: The task configuration block has side effects that require it currently to be always executed. + // Otherwise tests start failing. Therefore we enforce the task creation for now. + tsk.get() +} + +Set disabledIntegTestTaskNames = [] + +testClusters.matching { it.name == "integTest" }.configureEach { + testDistribution = 'DEFAULT' + plugin(hdfsRepoPluginProject.path) + setting 'xpack.license.self_generated.type', 'trial' +} + +// FIXHERE Do we need HA testing for searchable snapshots? I imagine that it is not vitally important... +//for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) { +for (String integTestTaskName : ['integTestSecure']) { + task "${integTestTaskName}"(type: RestIntegTestTask) { + description = "Runs rest tests against an elasticsearch cluster with HDFS." + + if (disabledIntegTestTaskNames.contains(integTestTaskName)) { + enabled = false + } + +// if (integTestTaskName.contains("Secure")) { +// if (integTestTaskName.contains("Ha")) { +// dependsOn secureHaHdfsFixture +// } else { + dependsOn secureHdfsFixture +// } +// } + + // TODO: If secure + systemProperty 'test.hdfs.uri', 'hdfs://localhost:9998' + nonInputProperties.systemProperty 'test.hdfs.path', '/user/elasticsearch/test/searchable_snapshots/secure' + + onlyIf { BuildParams.inFipsJvm == false } +// if (integTestTaskName.contains("Ha")) { +// java.nio.file.Path portsFile +// File portsFileDir = file("${workingDir}/hdfsFixture") +// if (integTestTaskName.contains("Secure")) { +// portsFile = buildDir.toPath() +// .resolve("fixtures") +// .resolve("secureHaHdfsFixture") +// .resolve("ports") +// } else { +// portsFile = buildDir.toPath() +// .resolve("fixtures") +// .resolve("haHdfsFixture") +// .resolve("ports") +// } +// nonInputProperties.systemProperty "test.hdfs-fixture.ports", file("$portsFileDir/ports") +// classpath += files(portsFileDir) +// // Copy ports file to separate location which is placed on the test classpath +// doFirst { +// mkdir(portsFileDir) +// copy { +// from portsFile +// into portsFileDir +// } +// } +// } + +// if (integTestTaskName.contains("Secure")) { + if (disabledIntegTestTaskNames.contains(integTestTaskName) == false) { + nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" + nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" + nonInputProperties.systemProperty( + "test.krb5.keytab.hdfs", + project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") + ) + } +// } + + + } + + testClusters."${integTestTaskName}" { + testDistribution = 'DEFAULT' + plugin(hdfsRepoPluginProject.path) + setting 'xpack.license.self_generated.type', 'trial' + if (integTestTaskName.contains("Secure")) { + systemProperty "java.security.krb5.conf", krb5conf + extraConfigFile( + "repository-hdfs/krb5.keytab", + file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}"), IGNORE_VALUE + ) + } + } +} + +// Determine HDFS Fixture compatibility for the current build environment. +boolean fixtureSupported = false +if (Os.isFamily(Os.FAMILY_WINDOWS)) { + // hdfs fixture will not start without hadoop native libraries on windows + String nativePath = System.getenv("HADOOP_HOME") + if (nativePath != null) { + java.nio.file.Path path = Paths.get(nativePath); + if (Files.isDirectory(path) && + Files.exists(path.resolve("bin").resolve("winutils.exe")) && + Files.exists(path.resolve("bin").resolve("hadoop.dll")) && + Files.exists(path.resolve("bin").resolve("hdfs.dll"))) { + fixtureSupported = true + } else { + throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin"); + } + } +} else { + fixtureSupported = true +} + +boolean legalPath = rootProject.rootDir.toString().contains(" ") == false +if (legalPath == false) { + fixtureSupported = false +} + +// Disable integration test if Fips mode +integTest { + onlyIf { BuildParams.inFipsJvm == false } +} + +if (fixtureSupported) { + // Check depends on the HA test. Already depends on the standard test. +// project.check.dependsOn(integTestHa) + + // Both standard and HA tests depend on their respective HDFS fixtures + integTest.dependsOn hdfsFixture +// integTestHa.dependsOn haHdfsFixture + + // The normal test runner only runs the standard hdfs rest tests + integTest { + systemProperty 'test.hdfs.uri', 'hdfs://localhost:9999' + nonInputProperties.systemProperty 'test.hdfs.path', '/user/elasticsearch/test/searchable_snapshots/simple' + } + + // Only include the HA integration tests for the HA test task +// integTestHa { +// setIncludes(['**/Ha*TestSuiteIT.class']) +// } +} else { + if (legalPath) { + logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") + } else { + logger.warn("hdfsFixture unsupported since there are spaces in the path: '" + rootProject.rootDir.toString() + "'") + } + + // The normal integration test runner will just test that the plugin loads + // TODO: Not doing rest tests +// integTest { +// systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic' +// } + // HA fixture is unsupported. Don't run them. +// integTestHa.setEnabled(false) +} + +//check.dependsOn(integTestSecure, integTestSecureHa) +check.dependsOn(integTestSecure) + +// Run just the secure hdfs rest test suite. +// TODO: Need the rest suite? +//integTestSecure { +// systemProperty 'tests.rest.suite', 'secure_hdfs_repository' +//} +//// Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner. +//integTestSecure { +// exclude('**/Ha*TestSuiteIT.class') +//} +//// Only include the HA integration tests for the HA test task +//integTestSecureHa { +// setIncludes(['**/Ha*TestSuiteIT.class']) +//} + +//thirdPartyAudit { +// ignoreMissingClasses() +// ignoreViolations( +// // internal java api: sun.misc.Unsafe +// 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', +// 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', +// 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm', +// 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot', +// ) +//} diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/src/test/java/org/elasticsearch/xpack/searchablesnapshots/hdfs/HdfsSearchableSnapshotsIT.java b/x-pack/plugin/searchable-snapshots/qa/hdfs/src/test/java/org/elasticsearch/xpack/searchablesnapshots/hdfs/HdfsSearchableSnapshotsIT.java new file mode 100644 index 0000000000000..5f62fba2ed7a2 --- /dev/null +++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/src/test/java/org/elasticsearch/xpack/searchablesnapshots/hdfs/HdfsSearchableSnapshotsIT.java @@ -0,0 +1,38 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.searchablesnapshots.hdfs; + +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.xpack.searchablesnapshots.AbstractSearchableSnapshotsRestTestCase; + +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.not; + +public class HdfsSearchableSnapshotsIT extends AbstractSearchableSnapshotsRestTestCase { + @Override + protected String repositoryType() { + return "hdfs"; + } + + @Override + protected Settings repositorySettings() { + final String uri = System.getProperty("test.hdfs.uri"); + assertThat(uri, not(blankOrNullString())); + + final String path = System.getProperty("test.hdfs.path"); + assertThat(path, not(blankOrNullString())); + + // Optional based on type of test + final String principal = System.getProperty("test.krb5.principal.es"); + + Settings.Builder repositorySettings = Settings.builder().put("client", "searchable_snapshots").put("uri", uri).put("path", path); + if (principal != null) { + repositorySettings.put("security.principal", principal); + } + return repositorySettings.build(); + } +} From fa7937e268e7ac5cb59a5594dffe59c34ac5fad0 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 10 Dec 2020 12:50:26 -0500 Subject: [PATCH 3/6] clean up build file --- .../searchable-snapshots/qa/hdfs/build.gradle | 185 ++++-------------- 1 file changed, 37 insertions(+), 148 deletions(-) diff --git a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle index 19919446ace6c..8936c61a89f3f 100644 --- a/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle +++ b/x-pack/plugin/searchable-snapshots/qa/hdfs/build.gradle @@ -60,9 +60,7 @@ normalization { String realm = "BUILD.ELASTIC.CO" String krb5conf = krbFixtureProject.ext.krb5Conf("hdfs") -// Create HDFS File System Testing Fixtures for HA/Secure combinations -// TODO: Do we need HA fixtures? Unlikely? -//for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture', 'secureHaHdfsFixture']) { +// Create HDFS File System Testing Fixtures for (String fixtureName : ['hdfsFixture', 'secureHdfsFixture']) { def tsk = project.tasks.register(fixtureName, org.elasticsearch.gradle.test.AntFixture) { dependsOn project.configurations.hdfsFixture, krbFixtureProject.tasks.postProcessFixture @@ -78,24 +76,18 @@ for (String fixtureName : ['hdfsFixture', 'secureHdfsFixture']) { final List miniHDFSArgs = [] // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options - if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { + if (fixtureName.equals('secureHdfsFixture')) { miniHDFSArgs.add("-Djava.security.krb5.conf=${krb5conf}") } -// // If it's an HA fixture, set a nameservice to use in the JVM options -// if (fixtureName.equals('haHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { -// miniHDFSArgs.add("-Dha-nameservice=ha-hdfs") -// } // Common options miniHDFSArgs.add('hdfs.MiniHDFS') miniHDFSArgs.add(baseDir) // If it's a secure fixture, then set the principal name and keytab locations to use for auth. - if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) { + if (fixtureName.equals('secureHdfsFixture')) { miniHDFSArgs.add("hdfs/hdfs.build.elastic.co@${realm}") - miniHDFSArgs.add( - project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") - ) + miniHDFSArgs.add(project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab")) } args miniHDFSArgs.toArray() @@ -106,89 +98,40 @@ for (String fixtureName : ['hdfsFixture', 'secureHdfsFixture']) { tsk.get() } -Set disabledIntegTestTaskNames = [] +// Disable integration test if Fips mode +integTest { + description = "Runs rest tests against an elasticsearch cluster with HDFS." + systemProperty 'test.hdfs.uri', 'hdfs://localhost:9999' + nonInputProperties.systemProperty 'test.hdfs.path', '/user/elasticsearch/test/searchable_snapshots/simple' + onlyIf { BuildParams.inFipsJvm == false } +} -testClusters.matching { it.name == "integTest" }.configureEach { +task integTestSecure(type: RestIntegTestTask) { + description = "Runs rest tests against an elasticsearch cluster with Secured HDFS." + nonInputProperties.systemProperty 'test.hdfs.uri', 'hdfs://localhost:9998' + nonInputProperties.systemProperty 'test.hdfs.path', '/user/elasticsearch/test/searchable_snapshots/secure' + nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" + nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" + nonInputProperties.systemProperty( + "test.krb5.keytab.hdfs", + project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") + ) + onlyIf { BuildParams.inFipsJvm == false } +} +check.dependsOn(integTestSecure) + +testClusters.configureEach { testDistribution = 'DEFAULT' plugin(hdfsRepoPluginProject.path) setting 'xpack.license.self_generated.type', 'trial' } -// FIXHERE Do we need HA testing for searchable snapshots? I imagine that it is not vitally important... -//for (String integTestTaskName : ['integTestHa', 'integTestSecure', 'integTestSecureHa']) { -for (String integTestTaskName : ['integTestSecure']) { - task "${integTestTaskName}"(type: RestIntegTestTask) { - description = "Runs rest tests against an elasticsearch cluster with HDFS." - - if (disabledIntegTestTaskNames.contains(integTestTaskName)) { - enabled = false - } - -// if (integTestTaskName.contains("Secure")) { -// if (integTestTaskName.contains("Ha")) { -// dependsOn secureHaHdfsFixture -// } else { - dependsOn secureHdfsFixture -// } -// } - - // TODO: If secure - systemProperty 'test.hdfs.uri', 'hdfs://localhost:9998' - nonInputProperties.systemProperty 'test.hdfs.path', '/user/elasticsearch/test/searchable_snapshots/secure' - - onlyIf { BuildParams.inFipsJvm == false } -// if (integTestTaskName.contains("Ha")) { -// java.nio.file.Path portsFile -// File portsFileDir = file("${workingDir}/hdfsFixture") -// if (integTestTaskName.contains("Secure")) { -// portsFile = buildDir.toPath() -// .resolve("fixtures") -// .resolve("secureHaHdfsFixture") -// .resolve("ports") -// } else { -// portsFile = buildDir.toPath() -// .resolve("fixtures") -// .resolve("haHdfsFixture") -// .resolve("ports") -// } -// nonInputProperties.systemProperty "test.hdfs-fixture.ports", file("$portsFileDir/ports") -// classpath += files(portsFileDir) -// // Copy ports file to separate location which is placed on the test classpath -// doFirst { -// mkdir(portsFileDir) -// copy { -// from portsFile -// into portsFileDir -// } -// } -// } - -// if (integTestTaskName.contains("Secure")) { - if (disabledIntegTestTaskNames.contains(integTestTaskName) == false) { - nonInputProperties.systemProperty "test.krb5.principal.es", "elasticsearch@${realm}" - nonInputProperties.systemProperty "test.krb5.principal.hdfs", "hdfs/hdfs.build.elastic.co@${realm}" - nonInputProperties.systemProperty( - "test.krb5.keytab.hdfs", - project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "hdfs_hdfs.build.elastic.co.keytab") - ) - } -// } - - - } - - testClusters."${integTestTaskName}" { - testDistribution = 'DEFAULT' - plugin(hdfsRepoPluginProject.path) - setting 'xpack.license.self_generated.type', 'trial' - if (integTestTaskName.contains("Secure")) { - systemProperty "java.security.krb5.conf", krb5conf - extraConfigFile( - "repository-hdfs/krb5.keytab", - file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}"), IGNORE_VALUE - ) - } - } +testClusters.integTestSecure { + systemProperty "java.security.krb5.conf", krb5conf + extraConfigFile( + "repository-hdfs/krb5.keytab", + file("${project(':test:fixtures:krb5kdc-fixture').ext.krb5Keytabs("hdfs", "elasticsearch.keytab")}"), IGNORE_VALUE + ) } // Determine HDFS Fixture compatibility for the current build environment. @@ -197,14 +140,14 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { // hdfs fixture will not start without hadoop native libraries on windows String nativePath = System.getenv("HADOOP_HOME") if (nativePath != null) { - java.nio.file.Path path = Paths.get(nativePath); + java.nio.file.Path path = Paths.get(nativePath) if (Files.isDirectory(path) && Files.exists(path.resolve("bin").resolve("winutils.exe")) && Files.exists(path.resolve("bin").resolve("hadoop.dll")) && Files.exists(path.resolve("bin").resolve("hdfs.dll"))) { fixtureSupported = true } else { - throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin"); + throw new IllegalStateException("HADOOP_HOME: ${path} is invalid, does not contain hadoop native libraries in \$HADOOP_HOME/bin") } } } else { @@ -216,69 +159,15 @@ if (legalPath == false) { fixtureSupported = false } -// Disable integration test if Fips mode -integTest { - onlyIf { BuildParams.inFipsJvm == false } -} - if (fixtureSupported) { - // Check depends on the HA test. Already depends on the standard test. -// project.check.dependsOn(integTestHa) - - // Both standard and HA tests depend on their respective HDFS fixtures integTest.dependsOn hdfsFixture -// integTestHa.dependsOn haHdfsFixture - - // The normal test runner only runs the standard hdfs rest tests - integTest { - systemProperty 'test.hdfs.uri', 'hdfs://localhost:9999' - nonInputProperties.systemProperty 'test.hdfs.path', '/user/elasticsearch/test/searchable_snapshots/simple' - } - - // Only include the HA integration tests for the HA test task -// integTestHa { -// setIncludes(['**/Ha*TestSuiteIT.class']) -// } + integTestSecure.dependsOn secureHdfsFixture } else { + integTest.enabled = false + integTestSecure.enabled = false if (legalPath) { logger.warn("hdfsFixture unsupported, please set HADOOP_HOME and put HADOOP_HOME\\bin in PATH") } else { logger.warn("hdfsFixture unsupported since there are spaces in the path: '" + rootProject.rootDir.toString() + "'") } - - // The normal integration test runner will just test that the plugin loads - // TODO: Not doing rest tests -// integTest { -// systemProperty 'tests.rest.suite', 'hdfs_repository/10_basic' -// } - // HA fixture is unsupported. Don't run them. -// integTestHa.setEnabled(false) } - -//check.dependsOn(integTestSecure, integTestSecureHa) -check.dependsOn(integTestSecure) - -// Run just the secure hdfs rest test suite. -// TODO: Need the rest suite? -//integTestSecure { -// systemProperty 'tests.rest.suite', 'secure_hdfs_repository' -//} -//// Ignore HA integration Tests. They are included below as part of integTestSecureHa test runner. -//integTestSecure { -// exclude('**/Ha*TestSuiteIT.class') -//} -//// Only include the HA integration tests for the HA test task -//integTestSecureHa { -// setIncludes(['**/Ha*TestSuiteIT.class']) -//} - -//thirdPartyAudit { -// ignoreMissingClasses() -// ignoreViolations( -// // internal java api: sun.misc.Unsafe -// 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator', -// 'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1', -// 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm', -// 'org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm$Slot', -// ) -//} From fa3aa54f63e284dc9fcffefd200a7add16d20e36 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 10 Dec 2020 13:03:55 -0500 Subject: [PATCH 4/6] add resolve permission to plugin permissions --- .../src/main/plugin-metadata/plugin-security.policy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy index eb8df4d63ba97..4b84d0462413d 100644 --- a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy @@ -74,7 +74,7 @@ grant { permission javax.security.auth.kerberos.ServicePermission "*", "initiate"; // hdfs client opens socket connections for to access repository - permission java.net.SocketPermission "*", "connect"; + permission java.net.SocketPermission "*", "connect,resolve"; // client binds to the address returned from the host name of any principal set up as a service principal // org.apache.hadoop.ipc.Client.Connection.setupConnection From 6d86e2205970431860f3823828f5ba3df234897d Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 10 Dec 2020 13:23:01 -0500 Subject: [PATCH 5/6] Precommit, also swap stream limit with privileged stream --- .../org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java index 17f9718d397c5..4703cdc296dec 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsBlobContainer.java @@ -26,7 +26,6 @@ import org.apache.hadoop.fs.Options; import org.apache.hadoop.fs.Options.CreateOpts; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdfs.DFSInputStream; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.blobstore.BlobContainer; import org.elasticsearch.common.blobstore.BlobMetadata; @@ -127,7 +126,7 @@ public InputStream readBlob(String blobName, long position, long length) throws // should direct the datanode to start on the appropriate block, at the // appropriate target position. fsInput.seek(position); - return new HDFSPrivilegedInputSteam(Streams.limitStream(fsInput, length), securityContext); + return Streams.limitStream(new HDFSPrivilegedInputSteam(fsInput, securityContext), length); }); } catch (FileNotFoundException fnfe) { throw new NoSuchFileException("[" + blobName + "] blob not found"); From eaabbcfd0c243c90f2695c20c2436323fd375111 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Mon, 14 Dec 2020 10:48:34 -0500 Subject: [PATCH 6/6] Remove resolve permission --- .../elasticsearch/repositories/hdfs/HdfsSecurityContext.java | 4 ++-- .../src/main/plugin-metadata/plugin-security.policy | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java index f605d8958fd88..4562174230c27 100644 --- a/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java +++ b/plugins/repository-hdfs/src/main/java/org/elasticsearch/repositories/hdfs/HdfsSecurityContext.java @@ -50,7 +50,7 @@ class HdfsSecurityContext { static { // We can do FS ops with only a few elevated permissions: SIMPLE_AUTH_PERMISSIONS = new Permission[]{ - new SocketPermission("*", "connect,resolve"), + new SocketPermission("*", "connect"), // 1) hadoop dynamic proxy is messy with access rules new ReflectPermission("suppressAccessChecks"), // 2) allow hadoop to add credentials to our Subject @@ -61,7 +61,7 @@ class HdfsSecurityContext { // If Security is enabled, we need all the following elevated permissions: KERBEROS_AUTH_PERMISSIONS = new Permission[] { - new SocketPermission("*", "connect,resolve"), + new SocketPermission("*", "connect"), // 1) hadoop dynamic proxy is messy with access rules new ReflectPermission("suppressAccessChecks"), // 2) allow hadoop to add credentials to our Subject diff --git a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy index 4b84d0462413d..eb8df4d63ba97 100644 --- a/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy +++ b/plugins/repository-hdfs/src/main/plugin-metadata/plugin-security.policy @@ -74,7 +74,7 @@ grant { permission javax.security.auth.kerberos.ServicePermission "*", "initiate"; // hdfs client opens socket connections for to access repository - permission java.net.SocketPermission "*", "connect,resolve"; + permission java.net.SocketPermission "*", "connect"; // client binds to the address returned from the host name of any principal set up as a service principal // org.apache.hadoop.ipc.Client.Connection.setupConnection