Skip to content

Use tracing for testing DatabaseMetaData ConnectorMetadata invocations #18849

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
findepi merged 5 commits into from
Aug 30, 2023
Merged

Use tracing for testing DatabaseMetaData ConnectorMetadata invocations #18849

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
97ae15b
Remove unused throws declarations
findepi Aug 29, 2023
5df9ece
Fix formatting
findepi Aug 29, 2023
98266b5
Move MultisetAssertions to trino-testing
findepi Aug 30, 2023
d20ddf3
Use tracing for testing DatabaseMetaData ConnectorMetadata invocations
findepi Aug 29, 2023
136ee7a
Remove deprecated counting method from CountingMockConnector
findepi Aug 29, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
307 changes: 183 additions & 124 deletions client/trino-jdbc/src/test/java/io/trino/jdbc/TestTrinoDatabaseMetaData.java
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion ...trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestDeltaLakeFileOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
import static io.trino.plugin.deltalake.TestDeltaLakeFileOperations.FileType.TRINO_EXTENDED_STATS_JSON;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS;
import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.TestingSession.testSessionBuilder;
import static java.lang.Math.toIntExact;
import static java.util.Collections.nCopies;
Expand Down
2 changes: 1 addition & 1 deletion ...in/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/TestTransactionLogAccess.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@
import static io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.TRANSACTION_LOG_DIRECTORY;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS;
import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.spi.type.TimeZoneKey.UTC_KEY;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.TestingConnectorSession.SESSION;
import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER;
import static java.lang.String.format;
Expand Down
2 changes: 1 addition & 1 deletion ...-delta-lake/src/test/java/io/trino/plugin/deltalake/transactionlog/TestTableSnapshot.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
import static io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointEntryIterator.EntryType.PROTOCOL;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS;
import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.TestingConnectorSession.SESSION;
import static io.trino.type.InternalTypeManager.TESTING_TYPE_MANAGER;
import static java.util.Collections.nCopies;
Expand Down
2 changes: 1 addition & 1 deletion ...no-hive/src/test/java/io/trino/plugin/hive/metastore/CountingAccessHiveMetastoreUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import io.trino.testing.QueryRunner;
import org.intellij.lang.annotations.Language;

import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;

public final class CountingAccessHiveMetastoreUtil
{
Expand Down
2 changes: 1 addition & 1 deletion ...ino-hive/src/test/java/io/trino/plugin/hive/s3/TestTrinoS3FileSystemAccessOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,8 @@
import static com.google.common.base.Preconditions.checkArgument;
import static io.trino.plugin.hive.HiveQueryRunner.TPCH_SCHEMA;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.DataProviders.toDataProvider;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.TestingNames.randomNameSuffix;
import static io.trino.testing.containers.Minio.MINIO_ACCESS_KEY;
import static io.trino.testing.containers.Minio.MINIO_SECRET_KEY;
Expand Down
2 changes: 1 addition & 1 deletion ...rino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergMetadataFileOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS;
import static io.trino.plugin.hive.metastore.file.TestingFileHiveMetastore.createTestingFileHiveMetastore;
import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG;
import static io.trino.plugin.iceberg.IcebergSessionProperties.COLLECT_EXTENDED_STATISTICS_ON_WRITE;
import static io.trino.plugin.iceberg.TestIcebergMetadataFileOperations.FileType.DATA;
Expand All @@ -54,6 +53,7 @@
import static io.trino.plugin.iceberg.TestIcebergMetadataFileOperations.FileType.SNAPSHOT;
import static io.trino.plugin.iceberg.TestIcebergMetadataFileOperations.FileType.STATS;
import static io.trino.plugin.iceberg.TestIcebergMetadataFileOperations.FileType.fromFilePath;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.TestingNames.randomNameSuffix;
import static io.trino.testing.TestingSession.testSessionBuilder;
import static java.lang.Math.min;
Expand Down
2 changes: 1 addition & 1 deletion ...est/java/io/trino/plugin/iceberg/catalog/glue/TestIcebergGlueCatalogAccessOperations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
import static io.trino.filesystem.TrackingFileSystemFactory.OperationType.INPUT_FILE_NEW_STREAM;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT;
import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_STATS;
import static io.trino.plugin.hive.util.MultisetAssertions.assertMultisetsEqual;
import static io.trino.plugin.iceberg.IcebergSessionProperties.COLLECT_EXTENDED_STATISTICS_ON_WRITE;
import static io.trino.plugin.iceberg.TableType.DATA;
import static io.trino.plugin.iceberg.TableType.FILES;
Expand All @@ -73,6 +72,7 @@
import static io.trino.plugin.iceberg.catalog.glue.GlueMetastoreMethod.UPDATE_TABLE;
import static io.trino.plugin.iceberg.catalog.glue.TestIcebergGlueCatalogAccessOperations.FileType.METADATA_JSON;
import static io.trino.plugin.iceberg.catalog.glue.TestIcebergGlueCatalogAccessOperations.FileType.fromFilePath;
import static io.trino.testing.MultisetAssertions.assertMultisetsEqual;
import static io.trino.testing.TestingNames.randomNameSuffix;
import static io.trino.testing.TestingSession.testSessionBuilder;
import static java.lang.annotation.ElementType.FIELD;
Expand Down
270 changes: 3 additions & 267 deletions testing/trino-testing/src/main/java/io/trino/testing/CountingMockConnector.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,10 @@
import io.trino.tracing.TracingConnectorMetadata;
import io.trino.util.AutoCloseableCloser;

import java.util.Objects;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.IntStream;
import java.util.stream.Stream;

import static com.google.common.base.MoreObjects.toStringHelper;
import static com.google.common.collect.ImmutableMultiset.toImmutableMultiset;
import static com.google.common.collect.ImmutableSet.toImmutableSet;
import static io.trino.connector.MockConnectorFactory.Builder.defaultGetColumns;
Expand Down Expand Up @@ -63,12 +58,6 @@ public class CountingMockConnector

private final AutoCloseableCloser closer = AutoCloseableCloser.create();

private final AtomicLong listSchemasCallsCounter = new AtomicLong();
private final AtomicLong listTablesCallsCounter = new AtomicLong();
private final AtomicLong getTableHandleCallsCounter = new AtomicLong();
private final AtomicLong getColumnsCallsCounter = new AtomicLong();
private final ListRoleGrantsCounter listRoleGrantCounter = new ListRoleGrantsCounter();

private final InMemorySpanExporter spanExporter;
private final SdkTracerProvider tracerProvider;

Expand Down Expand Up @@ -108,33 +97,6 @@ public Stream<SchemaTableName> getAllTables()
.map(tableName -> new SchemaTableName("test_schema2", tableName)));
}

/**
* @deprecated Use {@link #runTracing}.
*/
@Deprecated
public MetadataCallsCount runCounting(Runnable runnable)
{
synchronized (lock) {
listSchemasCallsCounter.set(0);
listTablesCallsCounter.set(0);
getTableHandleCallsCounter.set(0);
getColumnsCallsCounter.set(0);
listRoleGrantCounter.reset();

runnable.run();

return new MetadataCallsCount(
listSchemasCallsCounter.get(),
listTablesCallsCounter.get(),
getTableHandleCallsCounter.get(),
getColumnsCallsCounter.get(),
listRoleGrantCounter.listRowGrantsCallsCounter.get(),
listRoleGrantCounter.rolesPushedCounter.get(),
listRoleGrantCounter.granteesPushedCounter.get(),
listRoleGrantCounter.limitPushedCounter.get());
}
}

public Multiset<String> runTracing(Runnable runnable)
{
synchronized (lock) {
Expand Down Expand Up @@ -163,12 +125,8 @@ private ConnectorFactory getConnectorFactory()
{
MockConnectorFactory mockConnectorFactory = MockConnectorFactory.builder()
.withMetadataWrapper(connectorMetadata -> new TracingConnectorMetadata(tracerProvider.get("test"), "mock", connectorMetadata))
.withListSchemaNames(connectorSession -> {
listSchemasCallsCounter.incrementAndGet();
return ImmutableList.of("test_schema1", "test_schema2");
})
.withListSchemaNames(connectorSession -> ImmutableList.of("test_schema1", "test_schema2"))
.withListTables((connectorSession, schemaName) -> {
listTablesCallsCounter.incrementAndGet();
if (schemaName.equals("test_schema1")) {
return ImmutableList.copyOf(tablesTestSchema1);
}
Expand All @@ -178,7 +136,6 @@ private ConnectorFactory getConnectorFactory()
return ImmutableList.of();
})
.withGetTableHandle((connectorSession, schemaTableName) -> {
getTableHandleCallsCounter.incrementAndGet();
switch (schemaTableName.getSchemaName()) {
case "test_schema1" -> {
if (!tablesTestSchema1.contains(schemaTableName.getTableName())) {
Expand All @@ -196,231 +153,10 @@ private ConnectorFactory getConnectorFactory()
}
return defaultGetTableHandle().apply(connectorSession, schemaTableName);
})
.withGetColumns(schemaTableName -> {
getColumnsCallsCounter.incrementAndGet();
return defaultGetColumns().apply(schemaTableName);
})
.withListRoleGrants((connectorSession, roles, grantees, limit) -> {
listRoleGrantCounter.incrementListRoleGrants(roles, grantees, limit);
return roleGrants;
})
.withGetColumns(schemaTableName -> defaultGetColumns().apply(schemaTableName))
.withListRoleGrants((connectorSession, roles, grantees, limit) -> roleGrants)
.build();

return mockConnectorFactory;
}

@Deprecated
public static final class MetadataCallsCount
{
private final long listSchemasCount;
private final long listTablesCount;
private final long getTableHandleCount;
private final long getColumnsCount;
private final long listRoleGrantsCount;
private final long rolesPushedCount;
private final long granteesPushedCount;
private final long limitPushedCount;

public MetadataCallsCount()
{
this(0, 0, 0, 0, 0, 0, 0, 0);
}

public MetadataCallsCount(
long listSchemasCount,
long listTablesCount,
long getTableHandleCount,
long getColumnsCount,
long listRoleGrantsCount,
long rolesPushedCount,
long granteesPushedCount,
long limitPushedCount)
{
this.listSchemasCount = listSchemasCount;
this.listTablesCount = listTablesCount;
this.getTableHandleCount = getTableHandleCount;
this.getColumnsCount = getColumnsCount;
this.listRoleGrantsCount = listRoleGrantsCount;
this.rolesPushedCount = rolesPushedCount;
this.granteesPushedCount = granteesPushedCount;
this.limitPushedCount = limitPushedCount;
}

public MetadataCallsCount withListSchemasCount(long listSchemasCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withListTablesCount(long listTablesCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withGetTableHandleCount(long getTableHandleCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withGetColumnsCount(long getColumnsCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withListRoleGrantsCount(long listRoleGrantsCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withRolesPushedCount(long rolesPushedCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withGranteesPushedCount(long granteesPushedCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

public MetadataCallsCount withLimitPushedCount(long limitPushedCount)
{
return new MetadataCallsCount(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

@Override
public boolean equals(Object o)
{
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
MetadataCallsCount that = (MetadataCallsCount) o;
return listSchemasCount == that.listSchemasCount &&
listTablesCount == that.listTablesCount &&
getTableHandleCount == that.getTableHandleCount &&
getColumnsCount == that.getColumnsCount &&
listRoleGrantsCount == that.listRoleGrantsCount &&
rolesPushedCount == that.rolesPushedCount &&
granteesPushedCount == that.granteesPushedCount &&
limitPushedCount == that.limitPushedCount;
}

@Override
public int hashCode()
{
return Objects.hash(
listSchemasCount,
listTablesCount,
getTableHandleCount,
getColumnsCount,
listRoleGrantsCount,
rolesPushedCount,
granteesPushedCount,
limitPushedCount);
}

@Override
public String toString()
{
return toStringHelper(this)
.add("listSchemasCount", listSchemasCount)
.add("listTablesCount", listTablesCount)
.add("getTableHandleCount", getTableHandleCount)
.add("getColumnsCount", getColumnsCount)
.add("listRoleGrantsCount", listRoleGrantsCount)
.add("rolesPushedCount", rolesPushedCount)
.add("granteesPushedCount", granteesPushedCount)
.add("limitPushedCount", limitPushedCount)
.toString();
}
}

public static class ListRoleGrantsCounter
{
private final AtomicLong listRowGrantsCallsCounter = new AtomicLong();
private final AtomicLong rolesPushedCounter = new AtomicLong();
private final AtomicLong granteesPushedCounter = new AtomicLong();
private final AtomicLong limitPushedCounter = new AtomicLong();

public void reset()
{
listRowGrantsCallsCounter.set(0);
rolesPushedCounter.set(0);
granteesPushedCounter.set(0);
limitPushedCounter.set(0);
}

public void incrementListRoleGrants(Optional<Set<String>> roles, Optional<Set<String>> grantees, OptionalLong limit)
{
listRowGrantsCallsCounter.incrementAndGet();
roles.ifPresent(x -> rolesPushedCounter.incrementAndGet());
grantees.ifPresent(x -> granteesPushedCounter.incrementAndGet());
limit.ifPresent(x -> limitPushedCounter.incrementAndGet());
}
}
}
Loading