HIVE-29020: Iceberg: Validate HMS REST Catalog Client with OAuth2

Author: Dmitriy Fingerman

itests/hive-iceberg/pom.xml

@@ -46,6 +46,12 @@
       <classifier>tests</classifier>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>org.keycloak</groupId>
+      <artifactId>keycloak-admin-client</artifactId>
+      <version>${keycloak.version}</version>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.hive</groupId>
       <artifactId>hive-standalone-metastore-common</artifactId>

itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientIT.java renamed to itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITBase.java

@@ -23,7 +23,6 @@
 import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
-import org.apache.hadoop.hive.metastore.ServletSecurity;
 import org.apache.hadoop.hive.metastore.api.Database;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
 import org.apache.hadoop.hive.metastore.api.MetaException;
@@ -46,18 +45,15 @@
 import org.apache.iceberg.hive.CatalogUtils;
 import org.apache.iceberg.hive.HiveSchemaUtil;
 import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
-import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.TestInstance;
-import org.junit.jupiter.api.extension.RegisterExtension;
-
-import java.util.Collections;
-import java.util.Map;
 
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
@@ -67,38 +63,36 @@
   * The flow is as follows:
   * Hive ql wrapper --> HiveMetaStoreClient --> HiveRESTCatalogClient --> HMS RESTCatalog Server --> HMS
  */
-@TestInstance(TestInstance.Lifecycle.PER_CLASS)
-public class TestHiveRESTCatalogClientIT {
-
-  private static final String DB_NAME = "ice_db";
-  private static final String TABLE_NAME = "ice_tbl";
-  private static final String CATALOG_NAME = "ice01";
-  private static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+public abstract class TestHiveRESTCatalogClientITBase {
+
+  static final String DB_NAME = "ice_db";
+  static final String TABLE_NAME = "ice_tbl";
+  static final String CATALOG_NAME = "ice01";
+  static final String HIVE_ICEBERG_STORAGE_HANDLER = "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler";
+  static final String restCatalogPrefix = String.format("%s%s.", CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
+
+  HiveConf hiveConf;
+  Configuration conf;
+  Hive hive;
+  IMetaStoreClient msClient;
 
-  private Configuration conf;
-  private HiveConf hiveConf;
-  private Hive hive;
-
-  private IMetaStoreClient msClient;
+  abstract HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension();
 
-  @RegisterExtension
-  private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
-      HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
-          .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
-          .build();
+  public void setupConf() {
+    HiveRESTCatalogServerExtension restCatalogExtension = getHiveRESTCatalogServerExtension();
 
-  @BeforeAll
-  public void setup() throws Exception {
-    // Starting msClient with Iceberg REST Catalog client underneath
-    String restCatalogPrefix = String.format("%s%s.", CatalogUtils.CATALOG_CONFIG_PREFIX, CATALOG_NAME);
-
-    conf = REST_CATALOG_EXTENSION.getConf();
+    conf = restCatalogExtension.getConf();
 
     MetastoreConf.setVar(conf, MetastoreConf.ConfVars.METASTORE_CLIENT_IMPL,
         "org.apache.iceberg.hive.client.HiveRESTCatalogClient");
     conf.set(MetastoreConf.ConfVars.CATALOG_DEFAULT.getVarname(), CATALOG_NAME);
-    conf.set(restCatalogPrefix + "uri", REST_CATALOG_EXTENSION.getRestEndpoint());
+    conf.set(restCatalogPrefix + "uri", restCatalogExtension.getRestEndpoint());
     conf.set(restCatalogPrefix + "type", CatalogUtil.ICEBERG_CATALOG_TYPE_REST);
+  }
+
+  @BeforeEach
+  void setup() throws Exception {
+    setupConf();
 
     HiveMetaHookLoader hookLoader = tbl -> {
       HiveStorageHandler storageHandler;
@@ -109,18 +103,19 @@ public void setup() throws Exception {
       }
       return storageHandler == null ? null : storageHandler.getMetaHook();
     };
-    
+
     msClient = new HiveMetaStoreClient(conf, hookLoader);
     hiveConf = new HiveConf(conf, HiveConf.class);
     hive = Hive.get(hiveConf);
   }
 
-  @AfterAll public void tearDown() {
+  @AfterEach
+  public void tearDown() {
     if (msClient != null) {
       msClient.close();
     }
   }
-
+  
   @Test
   public void testIceberg() throws Exception {
 
@@ -142,7 +137,7 @@ public void testIceberg() throws Exception {
     // --- Get Databases ---
     List<String> dbs = msClient.getDatabases(CATALOG_NAME, "ice_*");
     Assertions.assertEquals(1, dbs.size());
-    Assertions.assertEquals(DB_NAME, dbs.get(0));
+    Assertions.assertEquals(DB_NAME, dbs.getFirst());
 
     // --- Get All Databases ---
     List<String> allDbs = msClient.getAllDatabases(CATALOG_NAME);
@@ -151,7 +146,7 @@ public void testIceberg() throws Exception {
     Assertions.assertTrue(allDbs.contains(DB_NAME));
 
     // --- Create Table ---
-    org.apache.hadoop.hive.metastore.api.Table tTable = createPartitionedTable(msClient,
+    Table tTable = createPartitionedTable(msClient,
         CATALOG_NAME, DB_NAME, TABLE_NAME, new java.util.HashMap<>());
     Assertions.assertNotNull(tTable);
     Assertions.assertEquals(HiveMetaHook.ICEBERG, tTable.getParameters().get(HiveMetaHook.TABLE_TYPE));
@@ -166,7 +161,7 @@ public void testIceberg() throws Exception {
     Assertions.assertTrue(msClient.tableExists(CATALOG_NAME, DB_NAME, TABLE_NAME));
 
     // --- Get Table ---
-    org.apache.hadoop.hive.metastore.api.Table table = msClient.getTable(CATALOG_NAME, DB_NAME, TABLE_NAME);
+    Table table = msClient.getTable(CATALOG_NAME, DB_NAME, TABLE_NAME);
     Assertions.assertEquals(DB_NAME, table.getDbName());
     Assertions.assertEquals(TABLE_NAME, table.getTableName());
     Assertions.assertEquals(HIVE_ICEBERG_STORAGE_HANDLER, table.getParameters().get("storage_handler"));

itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITNoAuth.java

@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive;
+
+import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestHiveRESTCatalogClientITNoAuth extends TestHiveRESTCatalogClientITBase {
+
+  @RegisterExtension
+  private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
+      HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.NONE)
+          .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
+          .build();
+
+  @Override
+  HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
+    return REST_CATALOG_EXTENSION;
+  }
+}

itests/hive-iceberg/src/test/java/org/apache/hive/TestHiveRESTCatalogClientITOauth2.java

@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hive;
+
+import org.apache.hadoop.hive.metastore.ServletSecurity;
+import org.apache.iceberg.rest.extension.HiveRESTCatalogServerExtension;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.extension.RegisterExtension;
+
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestHiveRESTCatalogClientITOauth2 extends TestHiveRESTCatalogClientITBase {
+
+  @RegisterExtension
+  private static final HiveRESTCatalogServerExtension REST_CATALOG_EXTENSION =
+      HiveRESTCatalogServerExtension.builder(ServletSecurity.AuthType.OAUTH2)
+          .addMetaStoreSchemaClassName(ITestsSchemaInfo.class)
+          .build();
+
+  public void setupConf() {
+    super.setupConf();
+    
+    // Oauth2 properties
+    conf.set(restCatalogPrefix + "rest.auth.type", "oauth2");
+    conf.set(restCatalogPrefix + "oauth2-server-uri", REST_CATALOG_EXTENSION.getOAuth2TokenEndpoint());
+    conf.set(restCatalogPrefix + "credential", REST_CATALOG_EXTENSION.getOAuth2ClientCredential());
+  }
+
+  @Override
+  HiveRESTCatalogServerExtension getHiveRESTCatalogServerExtension() {
+    return REST_CATALOG_EXTENSION;
+  }
+}

itests/qtest-iceberg/pom.xml

@@ -480,6 +480,37 @@
       <artifactId>testcontainers</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.keycloak</groupId>
+      <artifactId>keycloak-admin-client</artifactId>
+      <version>${keycloak.version}</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.annotation</groupId>
+      <artifactId>jakarta.annotation-api</artifactId>
+      <version>2.1.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.nimbusds</groupId>
+      <artifactId>oauth2-oidc-sdk</artifactId>
+      <version>${nimbus-oauth.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.xml.bind</groupId>
+      <artifactId>jakarta.xml.bind-api</artifactId>
+      <version>4.0.4</version>
+    </dependency>
+    <dependency>
+      <groupId>jakarta.activation</groupId>
+      <artifactId>jakarta.activation-api</artifactId>
+      <version>2.1.2</version>
+    </dependency>
+    <dependency>
+      <groupId>org.glassfish.jaxb</groupId>
+      <artifactId>jaxb-runtime</artifactId>
+      <version>${jaxb-runtime.version}</version>
+    </dependency>
   </dependencies>
   <build>
     <plugins>

itests/qtest-iceberg/src/test/java/org/apache/hadoop/hive/cli/HiveRESTCatalogServerExtension.java

@@ -24,43 +24,82 @@
 import org.apache.hadoop.hive.metastore.ServletSecurity.AuthType;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
 import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars;
+import org.apache.iceberg.rest.extension.OAuth2AuthorizationServer;
 import org.apache.iceberg.rest.extension.RESTCatalogServer;
 import org.junit.rules.ExternalResource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
 
 public class HiveRESTCatalogServerExtension extends ExternalResource {
   private final Configuration conf;
+  private final OAuth2AuthorizationServer authorizationServer;
   private final RESTCatalogServer restCatalogServer;
 
-  private HiveRESTCatalogServerExtension(AuthType authType, Class<? extends MetaStoreSchemaInfo> schemaInfoClass) {
+  static final String HMS_ID = "hive-metastore";
+  static final String HMS_SECRET = "hive-metastore-secret";
+  
+  private static final Logger LOG = LoggerFactory.getLogger(HiveRESTCatalogServerExtension.class);
+
+  private HiveRESTCatalogServerExtension(AuthType authType, Class<? extends MetaStoreSchemaInfo> schemaInfoClass,
+      Map<String, String> configurations) {
     this.conf = MetastoreConf.newMetastoreConf();
     MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, authType.name());
+    if (authType == AuthType.OAUTH2) {
+      authorizationServer = new OAuth2AuthorizationServer();
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH, "oauth2");
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_ID, HMS_ID);
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_CLIENT_SECRET, HMS_SECRET);
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_AUDIENCE, HMS_ID);
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_FIELD, "email");
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_PRINCIPAL_MAPPER_REGEX_PATTERN,
+          "(.*)@example.com");
+    } else {
+      authorizationServer = null;
+    }
+    configurations.forEach(conf::set);
     restCatalogServer = new RESTCatalogServer();
     if (schemaInfoClass != null) {
       restCatalogServer.setSchemaInfoClass(schemaInfoClass);
     }
   }
 
-  public Configuration getConf() {
-    return conf;
-  }
-
   @Override
   protected void before() throws Throwable {
+    if (authorizationServer != null) {
+      authorizationServer.start();
+      LOG.info("An authorization server {} started", authorizationServer.getIssuer());
+      MetastoreConf.setVar(conf, ConfVars.CATALOG_SERVLET_AUTH_OAUTH2_ISSUER, authorizationServer.getIssuer());
+    }
     restCatalogServer.start(conf);
   }
 
   @Override
   protected void after() {
+    if (authorizationServer != null) {
+      authorizationServer.stop();
+    }
     restCatalogServer.stop();
   }
 
   public String getRestEndpoint() {
     return restCatalogServer.getRestEndpoint();
   }
 
+  public String getOAuth2TokenEndpoint() {
+    return authorizationServer.getTokenEndpoint();
+  }
+
+  public String getOAuth2ClientCredential() {
+    return authorizationServer.getClientCredential();
+  }
+
   public static class Builder {
     private final AuthType authType;
     private Class<? extends MetaStoreSchemaInfo> metaStoreSchemaClass;
+    private final Map<String, String> configurations = new HashMap<>();
 
     private Builder(AuthType authType) {
       this.authType = authType;
@@ -72,7 +111,7 @@ public Builder addMetaStoreSchemaClassName(Class<? extends MetaStoreSchemaInfo>
     }
 
     public HiveRESTCatalogServerExtension build() {
-      return new HiveRESTCatalogServerExtension(authType, metaStoreSchemaClass);
+      return new HiveRESTCatalogServerExtension(authType, metaStoreSchemaClass, configurations);
     }
   }