diff --git a/presto-accumulo/pom.xml b/presto-accumulo/pom.xml index eb38a5ff4f9c..032b3c2b00ef 100644 --- a/presto-accumulo/pom.xml +++ b/presto-accumulo/pom.xml @@ -87,12 +87,6 @@ guice - - commons-io - commons-io - 2.4 - - commons-lang commons-lang @@ -271,75 +265,6 @@ test - - org.apache.accumulo - accumulo-minicluster - ${dep.accumulo.version} - test - - - org.apache.hadoop - hadoop-client - - - org.apache.hadoop - hadoop-minicluster - - - org.apache.htrace - htrace-core - - - com.beust - jcommander - - - commons-beanutils - commons-beanutils-core - - - org.codehaus.plexus - plexus-utils - - - org.codehaus.plexus - plexus-utils - - - org.apache.accumulo - accumulo-monitor - - - org.slf4j - slf4j-log4j12 - - - com.sun.xml.bind - jaxb-impl - - - commons-logging - commons-logging - - - log4j - log4j - - - jline - jline - - - org.apache.maven.scm - maven-scm-api - - - org.apache.maven.scm - maven-scm-provider-svnexe - - - - org.assertj assertj-core @@ -349,7 +274,12 @@ org.jetbrains annotations - 13.0 + test + + + + org.testcontainers + testcontainers test diff --git a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/AccumuloQueryRunner.java b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/AccumuloQueryRunner.java index 1f3dd6493311..bc337b317478 100644 --- a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/AccumuloQueryRunner.java +++ b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/AccumuloQueryRunner.java @@ -22,45 +22,26 @@ import io.prestosql.plugin.accumulo.conf.AccumuloConfig; import io.prestosql.plugin.accumulo.serializers.LexicoderRowSerializer; import io.prestosql.plugin.tpch.TpchPlugin; -import io.prestosql.spi.PrestoException; import io.prestosql.testing.DistributedQueryRunner; import io.prestosql.testing.QueryRunner; import io.prestosql.tpch.TpchTable; -import org.apache.accumulo.core.client.AccumuloException; -import org.apache.accumulo.core.client.AccumuloSecurityException; -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.ZooKeeperInstance; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.minicluster.MiniAccumuloCluster; -import org.apache.commons.io.FileUtils; import org.apache.hadoop.io.Text; import org.intellij.lang.annotations.Language; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; import java.util.Map; import static io.airlift.units.Duration.nanosSince; -import static io.prestosql.plugin.accumulo.AccumuloErrorCode.MINI_ACCUMULO; -import static io.prestosql.plugin.accumulo.AccumuloErrorCode.UNEXPECTED_ACCUMULO_ERROR; -import static io.prestosql.plugin.accumulo.MiniAccumuloConfigUtil.setConfigClassPath; import static io.prestosql.plugin.tpch.TpchMetadata.TINY_SCHEMA_NAME; import static io.prestosql.spi.type.BigintType.BIGINT; import static io.prestosql.testing.TestingSession.testSessionBuilder; import static java.lang.String.format; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.apache.accumulo.minicluster.MemoryUnit.MEGABYTE; public final class AccumuloQueryRunner { private static final Logger LOG = Logger.get(AccumuloQueryRunner.class); - private static final String MAC_PASSWORD = "secret"; - private static final String MAC_USER = "root"; private static boolean tpchLoaded; - private static Connector connector = getAccumuloConnector(); private AccumuloQueryRunner() {} @@ -74,13 +55,14 @@ public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map< queryRunner.installPlugin(new TpchPlugin()); queryRunner.createCatalog("tpch", "tpch"); + TestingAccumuloServer server = TestingAccumuloServer.getInstance(); queryRunner.installPlugin(new AccumuloPlugin()); Map accumuloProperties = ImmutableMap.builder() - .put(AccumuloConfig.INSTANCE, connector.getInstance().getInstanceName()) - .put(AccumuloConfig.ZOOKEEPERS, connector.getInstance().getZooKeepers()) - .put(AccumuloConfig.USERNAME, MAC_USER) - .put(AccumuloConfig.PASSWORD, MAC_PASSWORD) + .put(AccumuloConfig.INSTANCE, server.getInstanceName()) + .put(AccumuloConfig.ZOOKEEPERS, server.getZooKeepers()) + .put(AccumuloConfig.USERNAME, server.getUser()) + .put(AccumuloConfig.PASSWORD, server.getPassword()) .put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/presto-accumulo-test") .build(); @@ -88,7 +70,7 @@ public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map< if (!tpchLoaded) { copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables()); - connector.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L)))); + server.getConnector().tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L)))); tpchLoaded = true; } @@ -158,71 +140,6 @@ public static Session createSession() return testSessionBuilder().setCatalog("accumulo").setSchema("tpch").build(); } - /** - * Gets the AccumuloConnector singleton, starting the MiniAccumuloCluster on initialization. - * This singleton instance is required so all test cases access the same MiniAccumuloCluster. - * - * @return Accumulo connector - */ - public static Connector getAccumuloConnector() - { - if (connector != null) { - return connector; - } - - try { - MiniAccumuloCluster accumulo = createMiniAccumuloCluster(); - Instance instance = new ZooKeeperInstance(accumulo.getInstanceName(), accumulo.getZooKeepers()); - connector = instance.getConnector(MAC_USER, new PasswordToken(MAC_PASSWORD)); - LOG.info("Connection to MAC instance %s at %s established, user %s password %s", accumulo.getInstanceName(), accumulo.getZooKeepers(), MAC_USER, MAC_PASSWORD); - return connector; - } - catch (AccumuloException | AccumuloSecurityException | InterruptedException | IOException e) { - throw new PrestoException(UNEXPECTED_ACCUMULO_ERROR, "Failed to get connector to Accumulo", e); - } - } - - /** - * Creates and starts an instance of MiniAccumuloCluster, returning the new instance. - * - * @return New MiniAccumuloCluster - */ - private static MiniAccumuloCluster createMiniAccumuloCluster() - throws IOException, InterruptedException - { - // Create MAC directory - File macDir = Files.createTempDirectory("mac-").toFile(); - LOG.info("MAC is enabled, starting MiniAccumuloCluster at %s", macDir); - - // Start MAC and connect to it - MiniAccumuloCluster accumulo = new MiniAccumuloCluster(macDir, MAC_PASSWORD); - accumulo.getConfig().setDefaultMemory(512, MEGABYTE); - setConfigClassPath(accumulo.getConfig()); - accumulo.start(); - - // Add shutdown hook to stop MAC and cleanup temporary files - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - try { - LOG.info("Shutting down MAC"); - accumulo.stop(); - } - catch (IOException | InterruptedException e) { - Thread.currentThread().interrupt(); - throw new PrestoException(MINI_ACCUMULO, "Failed to shut down MAC instance", e); - } - - try { - LOG.info("Cleaning up MAC directory"); - FileUtils.forceDelete(macDir); - } - catch (IOException e) { - throw new PrestoException(MINI_ACCUMULO, "Failed to clean up MAC directory", e); - } - })); - - return accumulo; - } - public static void main(String[] args) throws Exception { diff --git a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/MiniAccumuloConfigUtil.java b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/MiniAccumuloConfigUtil.java deleted file mode 100644 index a501005b45db..000000000000 --- a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/MiniAccumuloConfigUtil.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.prestosql.plugin.accumulo; - -import com.google.common.base.Splitter; -import org.apache.accumulo.minicluster.MiniAccumuloConfig; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; - -import java.io.File; -import java.lang.reflect.Field; -import java.util.List; - -import static java.lang.management.ManagementFactory.getRuntimeMXBean; - -public final class MiniAccumuloConfigUtil -{ - private MiniAccumuloConfigUtil() {} - - /** - * MiniAccumuloClusterImpl will build the class path itself if not set, - * but the code fails on Java 9 due to assumptions about URLClassLoader. - */ - public static void setConfigClassPath(MiniAccumuloConfig config) - { - List items = Splitter.on(File.pathSeparatorChar) - .splitToList(getRuntimeMXBean().getClassPath()); - getConfigImpl(config).setClasspathItems(items.toArray(new String[0])); - } - - private static MiniAccumuloConfigImpl getConfigImpl(MiniAccumuloConfig config) - { - try { - Field field = MiniAccumuloConfig.class.getDeclaredField("impl"); - field.setAccessible(true); - return (MiniAccumuloConfigImpl) field.get(config); - } - catch (ReflectiveOperationException e) { - throw new AssertionError(e); - } - } -} diff --git a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestAccumuloClient.java b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestAccumuloClient.java index d180ce021622..8545d0359e4f 100644 --- a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestAccumuloClient.java +++ b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestAccumuloClient.java @@ -48,7 +48,7 @@ public TestAccumuloClient() .setUsername("root") .setPassword("secret"); - Connector connector = AccumuloQueryRunner.getAccumuloConnector(); + Connector connector = TestingAccumuloServer.getInstance().getConnector(); config.setZooKeepers(connector.getInstance().getZooKeepers()); zooKeeperMetadataManager = new ZooKeeperMetadataManager(config, new InternalTypeManager(createTestMetadataManager(), new TypeOperators())); client = new AccumuloClient(connector, config, zooKeeperMetadataManager, new AccumuloTableManager(connector), new IndexLookup(connector, new ColumnCardinalityCache(connector, config))); diff --git a/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestingAccumuloServer.java b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestingAccumuloServer.java new file mode 100644 index 000000000000..e49852b60f2a --- /dev/null +++ b/presto-accumulo/src/test/java/io/prestosql/plugin/accumulo/TestingAccumuloServer.java @@ -0,0 +1,88 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.prestosql.plugin.accumulo; + +import org.apache.accumulo.core.client.AccumuloException; +import org.apache.accumulo.core.client.AccumuloSecurityException; +import org.apache.accumulo.core.client.Connector; +import org.apache.accumulo.core.client.ZooKeeperInstance; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.testcontainers.containers.FixedHostPortGenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import java.time.Duration; + +import static java.lang.String.format; + +public class TestingAccumuloServer +{ + private static final int ACCUMULO_MASTER_PORT = 9999; + private static final int ACCUMULO_TSERVER_PORT = 9997; + private static final int ZOOKEEPER_PORT = 2181; + + private static final TestingAccumuloServer instance = new TestingAccumuloServer(); + + private final FixedHostPortGenericContainer accumuloContainer; + + public static TestingAccumuloServer getInstance() + { + return instance; + } + + private TestingAccumuloServer() + { + accumuloContainer = new FixedHostPortGenericContainer<>("prestodev/accumulo:35"); + accumuloContainer.withFixedExposedPort(ACCUMULO_MASTER_PORT, ACCUMULO_MASTER_PORT); + accumuloContainer.withFixedExposedPort(ACCUMULO_TSERVER_PORT, ACCUMULO_TSERVER_PORT); + accumuloContainer.withExposedPorts(ZOOKEEPER_PORT); + accumuloContainer.waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofMinutes(10))); + + // No need for an explicit stop since this server is a singleton + // and the container will be stopped by TestContainers on shutdown + // TODO Change this class to not be a singleton + // https://github.com/prestosql/presto/issues/5842 + accumuloContainer.start(); + } + + public String getInstanceName() + { + return "default"; + } + + public String getZooKeepers() + { + return format("%s:%s", accumuloContainer.getHost(), accumuloContainer.getMappedPort(ZOOKEEPER_PORT)); + } + + public String getUser() + { + return "root"; + } + + public String getPassword() + { + return "secret"; + } + + public Connector getConnector() + { + try { + ZooKeeperInstance instance = new ZooKeeperInstance(getInstanceName(), getZooKeepers()); + return instance.getConnector(getUser(), new PasswordToken(getPassword())); + } + catch (AccumuloException | AccumuloSecurityException e) { + throw new RuntimeException(e); + } + } +}