From 6651edf32373563c85d2a5eb5283ee595417e4e2 Mon Sep 17 00:00:00 2001 From: Marius Grama Date: Thu, 7 Jul 2022 14:16:18 +0200 Subject: [PATCH 1/2] Use specific JMX object name for the Delta Lake connector --- .../InternalDeltaLakeConnectorFactory.java | 2 + .../product/deltalake/TestDeltaLakeJmx.java | 51 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java diff --git a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/InternalDeltaLakeConnectorFactory.java b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/InternalDeltaLakeConnectorFactory.java index 979bc3e4dee7..11f529963bd2 100644 --- a/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/InternalDeltaLakeConnectorFactory.java +++ b/plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/InternalDeltaLakeConnectorFactory.java @@ -30,6 +30,7 @@ import io.trino.plugin.base.classloader.ClassLoaderSafeConnectorSplitManager; import io.trino.plugin.base.classloader.ClassLoaderSafeEventListener; import io.trino.plugin.base.classloader.ClassLoaderSafeNodePartitioningProvider; +import io.trino.plugin.base.jmx.ConnectorObjectNameGeneratorModule; import io.trino.plugin.base.jmx.MBeanServerModule; import io.trino.plugin.base.session.SessionPropertiesProvider; import io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule; @@ -77,6 +78,7 @@ public static Connector createConnector( Bootstrap app = new Bootstrap( new EventModule(), new MBeanModule(), + new ConnectorObjectNameGeneratorModule(catalogName, "io.trino.plugin.deltalake", "trino.plugin.deltalake"), new JsonModule(), new MBeanServerModule(), new HiveHdfsModule(), diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java new file mode 100644 index 000000000000..9da80ea94518 --- /dev/null +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java @@ -0,0 +1,51 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.tests.product.deltalake; + +import io.trino.tempto.ProductTest; +import org.testng.annotations.Test; + +import static io.trino.tempto.assertions.QueryAssert.Row.row; +import static io.trino.tempto.assertions.QueryAssert.assertThat; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_DATABRICKS; +import static io.trino.tests.product.TestGroups.DELTA_LAKE_OSS; +import static io.trino.tests.product.TestGroups.PROFILE_SPECIFIC_TESTS; +import static io.trino.tests.product.utils.QueryExecutors.onTrino; + +public class TestDeltaLakeJmx + extends ProductTest +{ + @Test(groups = {DELTA_LAKE_DATABRICKS, PROFILE_SPECIFIC_TESTS}) + public void testJmxTablesExposedByDeltaLakeConnectorBackedByGlueMetastore() + { + assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( + row("io.trino.plugin.hive.metastore.cache:name=delta,type=cachinghivemetastore"), + row("io.trino.plugin.hive.s3:name=delta,type=trinos3filesystem"), + row("io.trino.plugin.hive:catalog=delta,name=delta,type=fileformatdatasourcestats"), + row("io.trino.plugin.hive:name=delta,type=namenodestats"), + row("trino.plugin.deltalake.transactionlog:catalog=delta,name=delta,type=transactionlogaccess")); + } + + @Test(groups = {DELTA_LAKE_OSS, PROFILE_SPECIFIC_TESTS}) + public void testJmxTablesExposedByDeltaLakeConnectorBackedByThriftMetastore() + { + assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( + row("io.trino.plugin.hive.metastore.cache:name=delta,type=cachinghivemetastore"), + row("io.trino.plugin.hive.metastore.thrift:name=delta,type=thrifthivemetastore"), + row("io.trino.plugin.hive.s3:name=delta,type=trinos3filesystem"), + row("io.trino.plugin.hive:catalog=delta,name=delta,type=fileformatdatasourcestats"), + row("io.trino.plugin.hive:name=delta,type=namenodestats"), + row("trino.plugin.deltalake.transactionlog:catalog=delta,name=delta,type=transactionlogaccess")); + } +} From da21beae029ccd82b5cdbffb911b081699fe27b6 Mon Sep 17 00:00:00 2001 From: Marius Grama Date: Tue, 5 Jul 2022 16:30:37 +0200 Subject: [PATCH 2/2] Expose AWS Glue metastore stats via JMX The stats are exposed in JMX under the following object names: - hive: "trino.plugin.hive.metastore.glue:type=GlueHiveMetastore,name=hive" - iceberg: "io.trino.plugin.hive.metastore.glue:type=GlueMetastoreStats,name=iceberg" - delta: "io.trino.plugin.hive.metastore.glue:type=GlueHiveMetastore,name=delta" --- .../trino/plugin/hive/metastore/glue/GlueHiveMetastore.java | 4 ++++ .../plugin/iceberg/catalog/glue/IcebergGlueCatalogModule.java | 1 + .../io/trino/tests/product/deltalake/TestDeltaLakeJmx.java | 1 + 3 files changed, 6 insertions(+) diff --git a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java index c7734d53e361..9881e6fabb27 100644 --- a/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java +++ b/plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/glue/GlueHiveMetastore.java @@ -101,6 +101,8 @@ import io.trino.spi.statistics.ColumnStatisticType; import io.trino.spi.type.Type; import org.apache.hadoop.fs.Path; +import org.weakref.jmx.Flatten; +import org.weakref.jmx.Managed; import javax.annotation.Nullable; import javax.inject.Inject; @@ -230,6 +232,8 @@ else if (config.getPinGlueClientToCurrentRegion()) { return asyncGlueClientBuilder.build(); } + @Managed + @Flatten public GlueMetastoreStats getStats() { return stats; diff --git a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/IcebergGlueCatalogModule.java b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/IcebergGlueCatalogModule.java index eb22f5ccdd89..9d5d97cf8a39 100644 --- a/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/IcebergGlueCatalogModule.java +++ b/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/catalog/glue/IcebergGlueCatalogModule.java @@ -34,6 +34,7 @@ protected void setup(Binder binder) { configBinder(binder).bindConfig(GlueHiveMetastoreConfig.class); binder.bind(GlueMetastoreStats.class).in(Scopes.SINGLETON); + newExporter(binder).export(GlueMetastoreStats.class).withGeneratedName(); binder.bind(AWSCredentialsProvider.class).toProvider(GlueCredentialsProvider.class).in(Scopes.SINGLETON); binder.bind(IcebergTableOperationsProvider.class).to(GlueIcebergTableOperationsProvider.class).in(Scopes.SINGLETON); binder.bind(TrinoCatalogFactory.class).to(TrinoGlueCatalogFactory.class).in(Scopes.SINGLETON); diff --git a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java index 9da80ea94518..8c9542d490f7 100644 --- a/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java +++ b/testing/trino-product-tests/src/main/java/io/trino/tests/product/deltalake/TestDeltaLakeJmx.java @@ -31,6 +31,7 @@ public void testJmxTablesExposedByDeltaLakeConnectorBackedByGlueMetastore() { assertThat(onTrino().executeQuery("SHOW TABLES IN jmx.current LIKE '%name=delta%'")).containsOnly( row("io.trino.plugin.hive.metastore.cache:name=delta,type=cachinghivemetastore"), + row("io.trino.plugin.hive.metastore.glue:name=delta,type=gluehivemetastore"), row("io.trino.plugin.hive.s3:name=delta,type=trinos3filesystem"), row("io.trino.plugin.hive:catalog=delta,name=delta,type=fileformatdatasourcestats"), row("io.trino.plugin.hive:name=delta,type=namenodestats"),