Glue v2

lib/trino-cache/src/main/java/io/trino/cache/SafeCaches.java

@@ -70,4 +70,9 @@ private static <K, V> LoadingCache<K, V> buildUnsafeCache(CacheBuilder<? super K
     {
         return cacheBuilder.build(cacheLoader);
     }
+
+    public static <K, V> LoadingCache<K, V> emptyLoadingCache(CacheLoader<K, V> cacheLoader, boolean recordStats)
+    {
+        return new EmptyCache<>(cacheLoader, recordStats);
+    }
 }

plugin/trino-delta-lake/pom.xml

@@ -267,6 +267,12 @@
             <scope>runtime</scope>
         </dependency>
 
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>glue</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <dependency>
             <groupId>com.github.docker-java</groupId>
             <artifactId>docker-java-api</artifactId>

plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/DeltaLakeMetastoreModule.java

@@ -18,7 +18,7 @@
 import com.google.inject.Module;
 import io.airlift.configuration.AbstractConfigurationAwareModule;
 import io.trino.plugin.deltalake.metastore.file.DeltaLakeFileMetastoreModule;
-import io.trino.plugin.deltalake.metastore.glue.DeltaLakeGlueMetastoreModule;
+import io.trino.plugin.deltalake.metastore.glue.v1.DeltaLakeGlueV1MetastoreModule;
 import io.trino.plugin.deltalake.metastore.thrift.DeltaLakeThriftMetastoreModule;
 import io.trino.plugin.hive.HideDeltaLakeTables;
 import io.trino.plugin.hive.metastore.CachingHiveMetastoreModule;
@@ -35,7 +35,8 @@ protected void setup(Binder binder)
         binder.bind(Key.get(boolean.class, HideDeltaLakeTables.class)).toInstance(false);
         bindMetastoreModule("thrift", new DeltaLakeThriftMetastoreModule());
         bindMetastoreModule("file", new DeltaLakeFileMetastoreModule());
-        bindMetastoreModule("glue", new DeltaLakeGlueMetastoreModule());
+        bindMetastoreModule("glue", new io.trino.plugin.deltalake.metastore.glue.DeltaLakeGlueMetastoreModule());
+        bindMetastoreModule("glue-v1", new DeltaLakeGlueV1MetastoreModule());
 
         install(new CachingHiveMetastoreModule(false));
     }

plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/DeltaLakeGlueMetastoreModule.java

@@ -13,18 +13,19 @@
  */
 package io.trino.plugin.deltalake.metastore.glue;
 
-import com.amazonaws.services.glue.model.Table;
 import com.google.inject.Binder;
 import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
+import com.google.inject.Singleton;
+import com.google.inject.multibindings.ProvidesIntoOptional;
 import io.airlift.configuration.AbstractConfigurationAwareModule;
 import io.trino.plugin.deltalake.AllowDeltaLakeManagedTableRename;
-import io.trino.plugin.hive.metastore.glue.ForGlueHiveMetastore;
+import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore;
 import io.trino.plugin.hive.metastore.glue.GlueMetastoreModule;
 
-import java.util.function.Predicate;
+import java.util.EnumSet;
+import java.util.Set;
 
-import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
+import static com.google.inject.multibindings.ProvidesIntoOptional.Type.ACTUAL;
 import static io.airlift.configuration.ConfigBinder.configBinder;
 
 public class DeltaLakeGlueMetastoreModule
@@ -35,10 +36,17 @@ protected void setup(Binder binder)
     {
         configBinder(binder).bindConfig(DeltaLakeGlueMetastoreConfig.class);
 
-        newOptionalBinder(binder, Key.get(new TypeLiteral<Predicate<Table>>() {}, ForGlueHiveMetastore.class))
-                .setBinding().toProvider(DeltaLakeGlueMetastoreTableFilterProvider.class);
-
         install(new GlueMetastoreModule());
         binder.bind(Key.get(boolean.class, AllowDeltaLakeManagedTableRename.class)).toInstance(true);
     }
+
+    @ProvidesIntoOptional(ACTUAL)
+    @Singleton
+    public static Set<GlueHiveMetastore.TableKind> getTableKinds(DeltaLakeGlueMetastoreConfig config)
+    {
+        if (config.isHideNonDeltaLakeTables()) {
+            return EnumSet.of(GlueHiveMetastore.TableKind.DELTA);
+        }
+        return EnumSet.allOf(GlueHiveMetastore.TableKind.class);
+    }
 }

plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/DeltaLakeGlueMetastoreTableFilterProvider.java → plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueMetastoreTableFilterProvider.java

@@ -11,16 +11,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.trino.plugin.deltalake.metastore.glue;
+package io.trino.plugin.deltalake.metastore.glue.v1;
 
 import com.amazonaws.services.glue.model.Table;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
+import io.trino.plugin.deltalake.metastore.glue.DeltaLakeGlueMetastoreConfig;
 import io.trino.plugin.hive.util.HiveUtil;
 
 import java.util.function.Predicate;
 
-import static io.trino.plugin.hive.metastore.glue.converter.GlueToTrinoConverter.getTableParameters;
+import static io.trino.plugin.hive.metastore.glue.v1.converter.GlueToTrinoConverter.getTableParameters;
 
 public class DeltaLakeGlueMetastoreTableFilterProvider
         implements Provider<Predicate<Table>>

plugin/trino-delta-lake/src/main/java/io/trino/plugin/deltalake/metastore/glue/v1/DeltaLakeGlueV1MetastoreModule.java

@@ -0,0 +1,45 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package io.trino.plugin.deltalake.metastore.glue.v1;
+
+import com.amazonaws.services.glue.model.Table;
+import com.google.inject.Binder;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
+import io.airlift.configuration.AbstractConfigurationAwareModule;
+import io.trino.plugin.deltalake.AllowDeltaLakeManagedTableRename;
+import io.trino.plugin.deltalake.metastore.glue.DeltaLakeGlueMetastoreConfig;
+import io.trino.plugin.hive.metastore.glue.v1.ForGlueHiveMetastore;
+import io.trino.plugin.hive.metastore.glue.v1.GlueMetastoreModule;
+
+import java.util.function.Predicate;
+
+import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder;
+import static io.airlift.configuration.ConfigBinder.configBinder;
+
+public class DeltaLakeGlueV1MetastoreModule
+        extends AbstractConfigurationAwareModule
+{
+    @Override
+    protected void setup(Binder binder)
+    {
+        configBinder(binder).bindConfig(DeltaLakeGlueMetastoreConfig.class);
+
+        newOptionalBinder(binder, Key.get(new TypeLiteral<Predicate<Table>>() {}, ForGlueHiveMetastore.class))
+                .setBinding().toProvider(DeltaLakeGlueMetastoreTableFilterProvider.class);
+
+        install(new GlueMetastoreModule());
+        binder.bind(Key.get(boolean.class, AllowDeltaLakeManagedTableRename.class)).toInstance(true);
+    }
+}

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeCleanUpGlueMetastore.java

@@ -26,7 +26,7 @@
 import java.util.List;
 
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static io.trino.plugin.hive.metastore.glue.AwsSdkUtil.getPaginatedResults;
+import static io.trino.plugin.hive.metastore.glue.v1.AwsSdkUtil.getPaginatedResults;
 import static java.lang.System.currentTimeMillis;
 import static java.util.concurrent.TimeUnit.DAYS;
 

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeConcurrentModificationGlueMetastore.java

@@ -13,22 +13,24 @@
  */
 package io.trino.plugin.deltalake.metastore.glue;
 
-import com.amazonaws.services.glue.AWSGlueAsync;
-import com.amazonaws.services.glue.model.ConcurrentModificationException;
+import com.google.common.collect.ImmutableSet;
+import io.opentelemetry.api.OpenTelemetry;
 import io.trino.Session;
 import io.trino.plugin.deltalake.TestingDeltaLakePlugin;
 import io.trino.plugin.deltalake.metastore.TestingDeltaLakeMetastoreModule;
-import io.trino.plugin.hive.metastore.glue.DefaultGlueColumnStatisticsProviderFactory;
+import io.trino.plugin.hive.metastore.glue.GlueCache;
+import io.trino.plugin.hive.metastore.glue.GlueContext;
 import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore;
 import io.trino.plugin.hive.metastore.glue.GlueHiveMetastoreConfig;
-import io.trino.plugin.hive.metastore.glue.GlueMetastoreStats;
 import io.trino.spi.TrinoException;
 import io.trino.testing.AbstractTestQueryFramework;
 import io.trino.testing.DistributedQueryRunner;
 import io.trino.testing.QueryRunner;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
+import software.amazon.awssdk.services.glue.GlueClient;
+import software.amazon.awssdk.services.glue.model.ConcurrentModificationException;
 
 import java.io.IOException;
 import java.lang.reflect.InvocationTargetException;
@@ -39,10 +41,10 @@
 import static com.google.common.io.MoreFiles.deleteRecursively;
 import static com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE;
 import static com.google.common.reflect.Reflection.newProxy;
-import static com.google.common.util.concurrent.MoreExecutors.directExecutor;
 import static io.trino.plugin.hive.HiveErrorCode.HIVE_METASTORE_ERROR;
 import static io.trino.plugin.hive.HiveTestUtils.HDFS_FILE_SYSTEM_FACTORY;
-import static io.trino.plugin.hive.metastore.glue.TestingGlueHiveMetastore.createTestingAsyncGlueClient;
+import static io.trino.plugin.hive.metastore.glue.GlueHiveMetastore.TableKind.DELTA;
+import static io.trino.plugin.hive.metastore.glue.GlueMetastoreModule.createGlueClient;
 import static io.trino.testing.TestingNames.randomNameSuffix;
 import static io.trino.testing.TestingSession.testSessionBuilder;
 import static org.assertj.core.api.Assertions.assertThat;
@@ -70,18 +72,19 @@ protected QueryRunner createQueryRunner()
         QueryRunner queryRunner = DistributedQueryRunner.builder(deltaLakeSession).build();
 
         dataDirectory = queryRunner.getCoordinator().getBaseDataDir().resolve("data_delta_concurrent");
-        GlueMetastoreStats stats = new GlueMetastoreStats();
         GlueHiveMetastoreConfig glueConfig = new GlueHiveMetastoreConfig()
                 .setDefaultWarehouseDir(dataDirectory.toUri().toString());
 
-        AWSGlueAsync glueClient = createTestingAsyncGlueClient(glueConfig, stats);
-        AWSGlueAsync proxiedGlueClient = newProxy(AWSGlueAsync.class, (proxy, method, args) -> {
+        GlueClient glueClient = createGlueClient(new GlueHiveMetastoreConfig(), OpenTelemetry.noop());
+        GlueClient proxiedGlueClient = newProxy(GlueClient.class, (proxy, method, args) -> {
             Object result;
             try {
                 if (method.getName().equals("deleteTable") && failNextGlueDeleteTableCall.get()) {
                     // Simulate concurrent modifications on the table that is about to be dropped
                     failNextGlueDeleteTableCall.set(false);
-                    throw new TrinoException(HIVE_METASTORE_ERROR, new ConcurrentModificationException("Test-simulated metastore concurrent modification exception"));
+                    throw new TrinoException(HIVE_METASTORE_ERROR, ConcurrentModificationException.builder()
+                            .message("Test-simulated metastore concurrent modification exception")
+                            .build());
                 }
                 result = method.invoke(glueClient, args);
             }
@@ -92,13 +95,12 @@ protected QueryRunner createQueryRunner()
         });
 
         metastore = new GlueHiveMetastore(
+                proxiedGlueClient,
+                new GlueContext(glueConfig),
+                GlueCache.NOOP,
                 HDFS_FILE_SYSTEM_FACTORY,
                 glueConfig,
-                directExecutor(),
-                new DefaultGlueColumnStatisticsProviderFactory(directExecutor(), directExecutor()),
-                proxiedGlueClient,
-                stats,
-                table -> true);
+                ImmutableSet.of(DELTA));
 
         queryRunner.installPlugin(new TestingDeltaLakePlugin(dataDirectory, Optional.of(new TestingDeltaLakeMetastoreModule(metastore))));
         queryRunner.createCatalog(CATALOG_NAME, "delta_lake");

plugin/trino-delta-lake/src/test/java/io/trino/plugin/deltalake/metastore/glue/TestDeltaLakeGlueMetastore.java

@@ -34,9 +34,9 @@
 import io.trino.plugin.hive.metastore.Column;
 import io.trino.plugin.hive.metastore.Database;
 import io.trino.plugin.hive.metastore.HiveMetastore;
+import io.trino.plugin.hive.metastore.HiveMetastoreFactory;
 import io.trino.plugin.hive.metastore.PrincipalPrivileges;
 import io.trino.plugin.hive.metastore.Table;
-import io.trino.plugin.hive.metastore.glue.GlueHiveMetastore;
 import io.trino.spi.NodeManager;
 import io.trino.spi.PageIndexerFactory;
 import io.trino.spi.TrinoException;
@@ -137,7 +137,7 @@ public void setUp()
                 .initialize();
 
         lifeCycleManager = injector.getInstance(LifeCycleManager.class);
-        metastoreClient = injector.getInstance(GlueHiveMetastore.class);
+        metastoreClient = injector.getInstance(HiveMetastoreFactory.class).createMetastore(Optional.empty());
         metadataFactory = injector.getInstance(DeltaLakeMetadataFactory.class);
 
         session = TestingConnectorSession.builder()

plugin/trino-hive/pom.xml

@@ -132,6 +132,11 @@
             <artifactId>opentelemetry-aws-sdk-1.11</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>io.opentelemetry.instrumentation</groupId>
+            <artifactId>opentelemetry-aws-sdk-2.2</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>io.opentelemetry.semconv</groupId>
             <artifactId>opentelemetry-semconv</artifactId>
@@ -253,6 +258,57 @@
             <artifactId>jmxutils</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>apache-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-logging</groupId>
+                    <artifactId>commons-logging</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>auth</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>aws-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>glue</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>http-client-spi</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>regions</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>sdk-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>sts</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>utils</artifactId>
+        </dependency>
+
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-annotations</artifactId>

plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/Column.java

@@ -102,4 +102,14 @@ public int hashCode()
     {
         return Objects.hash(name, type, comment, properties);
     }
+
+    public Column withName(String newColumnName)
+    {
+        return new Column(newColumnName, type, comment, properties);
+    }
+
+    public Column withComment(Optional<String> comment)
+    {
+        return new Column(name, type, comment, properties);
+    }
 }

plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreFactory.java

@@ -21,6 +21,11 @@
 
 public interface HiveMetastoreFactory
 {
+    default boolean hasBuiltInCaching()
+    {
+        return false;
+    }
+
     boolean isImpersonationEnabled();
 
     /**

plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/HiveMetastoreModule.java

@@ -50,6 +50,7 @@ protected void setup(Binder binder)
             bindMetastoreModule("thrift", new ThriftMetastoreModule());
             bindMetastoreModule("file", new FileMetastoreModule());
             bindMetastoreModule("glue", new GlueMetastoreModule());
+            bindMetastoreModule("glue-v1", new io.trino.plugin.hive.metastore.glue.v1.GlueMetastoreModule());
         }
 
         install(new CachingHiveMetastoreModule(true));

plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/Partition.java

@@ -143,6 +143,13 @@ public static Builder builder(Partition partition)
         return new Builder(partition);
     }
 
+    public Partition withParameters(Map<String, String> parameters)
+    {
+        return builder(this)
+                .setParameters(parameters)
+                .build();
+    }
+
     public static class Builder
     {
         private final Storage.Builder storageBuilder;

plugin/trino-hive/src/main/java/io/trino/plugin/hive/metastore/Storage.java

@@ -174,6 +174,12 @@ public Builder setLocation(String location)
             return this;
         }
 
+        public Builder setBucketProperty(HiveBucketProperty bucketProperty)
+        {
+            this.bucketProperty = Optional.of(bucketProperty);
+            return this;
+        }
+
         public Builder setBucketProperty(Optional<HiveBucketProperty> bucketProperty)
         {
             this.bucketProperty = bucketProperty;