diff --git a/build.gradle.kts b/build.gradle.kts index 8ad42871680..0a98b6779b1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -949,7 +949,8 @@ tasks { ":catalogs:catalog-lakehouse-hudi:copyLibAndConfig", ":catalogs:catalog-lakehouse-iceberg:copyLibAndConfig", ":catalogs:catalog-lakehouse-paimon:copyLibAndConfig", - ":catalogs:catalog-model:copyLibAndConfig" + ":catalogs:catalog-model:copyLibAndConfig", + ":catalogs:catalog-generic-lakehouse:copyLibAndConfig" ) } diff --git a/catalogs/catalog-generic-lakehouse/build.gradle.kts b/catalogs/catalog-generic-lakehouse/build.gradle.kts new file mode 100644 index 00000000000..c3ad842ac38 --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/build.gradle.kts @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +description = "catalog-generic-lakehouse" + +plugins { + `maven-publish` + id("java") + id("idea") +} + +dependencies { + implementation(project(":api")) { + exclude("*") + } + implementation(project(":catalogs:catalog-common")) + implementation(project(":common")) { + exclude("*") + } + implementation(project(":core")) { + exclude("*") + } + + implementation(libs.bundles.log4j) + implementation(libs.cglib) + implementation(libs.commons.collections4) + implementation(libs.commons.io) + implementation(libs.commons.lang3) + implementation(libs.guava) + + annotationProcessor(libs.lombok) + + compileOnly(libs.lombok) + + testImplementation(project(":catalogs:catalog-jdbc-common", "testArtifacts")) + testImplementation(project(":clients:client-java")) + testImplementation(project(":integration-test-common", "testArtifacts")) + testImplementation(project(":server")) + testImplementation(project(":server-common")) + + testImplementation(libs.junit.jupiter.api) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.mockito.core) + testImplementation(libs.mysql.driver) + testImplementation(libs.postgresql.driver) + testImplementation(libs.slf4j.api) + testImplementation(libs.testcontainers) + testImplementation(libs.testcontainers.mysql) + testImplementation(libs.testcontainers.postgresql) + + testRuntimeOnly(libs.junit.jupiter.engine) +} + +tasks { + val runtimeJars by registering(Copy::class) { + from(configurations.runtimeClasspath) + into("build/libs") + } + + jar { + finalizedBy("runtimeJars") + } + + val copyCatalogLibs by registering(Copy::class) { + dependsOn("jar", "runtimeJars") + from("build/libs") { + exclude("guava-*.jar") + exclude("log4j-*.jar") + exclude("slf4j-*.jar") + } + into("$rootDir/distribution/package/catalogs/generic-lakehouse/libs") + } + + val copyCatalogConfig by registering(Copy::class) { + from("src/main/resources") + into("$rootDir/distribution/package/catalogs/generic-lakehouse/conf") + + rename { original -> + if (original.endsWith(".template")) { + original.replace(".template", "") + } else { + original + } + } + + exclude { details -> + details.file.isDirectory() + } + + fileMode = 0b111101101 + } + + register("copyLibAndConfig", Copy::class) { + dependsOn(copyCatalogLibs, copyCatalogConfig) + } +} + +tasks.test { + val skipITs = project.hasProperty("skipITs") + if (skipITs) { + // Exclude integration tests + exclude("**/integration/test/**") + } else { + dependsOn(tasks.jar) + } +} + +tasks.getByName("generateMetadataFileForMavenJavaPublication") { + dependsOn("runtimeJars") +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalog.java b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalog.java new file mode 100644 index 00000000000..68072f55baa --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalog.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.lakehouse; + +import java.util.Map; +import org.apache.gravitino.connector.BaseCatalog; +import org.apache.gravitino.connector.CatalogOperations; +import org.apache.gravitino.connector.PropertiesMetadata; +import org.apache.gravitino.connector.capability.Capability; + +/** Implementation of a generic lakehouse catalog in Apache Gravitino. */ +public class GenericLakehouseCatalog extends BaseCatalog { + + static final GenericLakehouseCatalogPropertiesMetadata CATALOG_PROPERTIES_METADATA = + new GenericLakehouseCatalogPropertiesMetadata(); + + static final GenericLakehouseSchemaPropertiesMetadata SCHEMA_PROPERTIES_METADATA = + new GenericLakehouseSchemaPropertiesMetadata(); + + static final GenericLakehouseTablePropertiesMetadata TABLE_PROPERTIES_METADATA = + new GenericLakehouseTablePropertiesMetadata(); + + /** + * Returns the short name of the generic lakehouse catalog. + * + * @return The short name of the catalog. + */ + @Override + public String shortName() { + return "generic-lakehouse"; + } + + /** + * Creates a new instance of {@link GenericLakehouseCatalogOperations} with the provided + * configuration. + * + * @param config The configuration map for the generic catalog operations. + * @return A new instance of {@link GenericLakehouseCatalogOperations}. + */ + @Override + protected CatalogOperations newOps(Map config) { + return new GenericLakehouseCatalogOperations(); + } + + @Override + public Capability newCapability() { + return new GenericLakehouseCatalogCapability(); + } + + @Override + public PropertiesMetadata catalogPropertiesMetadata() throws UnsupportedOperationException { + return CATALOG_PROPERTIES_METADATA; + } + + @Override + public PropertiesMetadata schemaPropertiesMetadata() throws UnsupportedOperationException { + return SCHEMA_PROPERTIES_METADATA; + } + + @Override + public PropertiesMetadata tablePropertiesMetadata() throws UnsupportedOperationException { + return TABLE_PROPERTIES_METADATA; + } +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogCapability.java b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogCapability.java new file mode 100644 index 00000000000..08015f7fce1 --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogCapability.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.lakehouse; + +import org.apache.gravitino.connector.capability.Capability; +import org.apache.gravitino.connector.capability.CapabilityResult; + +public class GenericLakehouseCatalogCapability implements Capability { + + @Override + public CapabilityResult columnNotNull() { + throw new UnsupportedOperationException( + "Not implemented yet: GenericLakehouseCatalogCapability.columnNotNull"); + } + + @Override + public CapabilityResult columnDefaultValue() { + throw new UnsupportedOperationException( + "Not implemented yet: GenericLakehouseCatalogCapability.columnDefaultValue"); + } + + @Override + public CapabilityResult caseSensitiveOnName(Scope scope) { + switch (scope) { + case SCHEMA: + case TABLE: + case COLUMN: + throw new UnsupportedOperationException( + "Not implemented yet: GenericLakehouseCatalogCapability.caseSensitiveOnName"); + default: + return CapabilityResult.SUPPORTED; + } + } +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogOperations.java b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogOperations.java new file mode 100644 index 00000000000..64743488a01 --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogOperations.java @@ -0,0 +1,142 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.lakehouse; + +import java.util.Map; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.Namespace; +import org.apache.gravitino.connector.CatalogInfo; +import org.apache.gravitino.connector.CatalogOperations; +import org.apache.gravitino.connector.HasPropertyMetadata; +import org.apache.gravitino.connector.SupportsSchemas; +import org.apache.gravitino.exceptions.NoSuchCatalogException; +import org.apache.gravitino.exceptions.NoSuchSchemaException; +import org.apache.gravitino.exceptions.NoSuchTableException; +import org.apache.gravitino.exceptions.NonEmptySchemaException; +import org.apache.gravitino.exceptions.SchemaAlreadyExistsException; +import org.apache.gravitino.exceptions.TableAlreadyExistsException; +import org.apache.gravitino.rel.Column; +import org.apache.gravitino.rel.Table; +import org.apache.gravitino.rel.TableCatalog; +import org.apache.gravitino.rel.TableChange; +import org.apache.gravitino.rel.expressions.distributions.Distribution; +import org.apache.gravitino.rel.expressions.sorts.SortOrder; +import org.apache.gravitino.rel.expressions.transforms.Transform; +import org.apache.gravitino.rel.indexes.Index; + +/** Operations for interacting with a generic lakehouse catalog in Apache Gravitino. */ +public class GenericLakehouseCatalogOperations + implements CatalogOperations, SupportsSchemas, TableCatalog { + + /** + * Initializes the generic lakehouse catalog operations with the provided configuration. + * + * @param conf The configuration map for the generic catalog operations. + * @param info The catalog info associated with this operation instance. + * @param propertiesMetadata The properties metadata of generic lakehouse catalog. + * @throws RuntimeException if initialization fails. + */ + @Override + public void initialize( + Map conf, CatalogInfo info, HasPropertyMetadata propertiesMetadata) + throws RuntimeException { + // TODO: Implement initialization logic + } + + @Override + public void close() {} + + @Override + public void testConnection( + NameIdentifier catalogIdent, + org.apache.gravitino.Catalog.Type type, + String provider, + String comment, + Map properties) + throws Exception { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public org.apache.gravitino.NameIdentifier[] listSchemas(org.apache.gravitino.Namespace namespace) + throws NoSuchCatalogException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public org.apache.gravitino.Schema createSchema( + org.apache.gravitino.NameIdentifier ident, String comment, Map properties) + throws NoSuchCatalogException, SchemaAlreadyExistsException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public org.apache.gravitino.Schema loadSchema(org.apache.gravitino.NameIdentifier ident) + throws NoSuchSchemaException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public org.apache.gravitino.Schema alterSchema( + org.apache.gravitino.NameIdentifier ident, org.apache.gravitino.SchemaChange... changes) + throws NoSuchSchemaException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public boolean dropSchema(org.apache.gravitino.NameIdentifier ident, boolean cascade) + throws NonEmptySchemaException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public NameIdentifier[] listTables(Namespace namespace) throws NoSuchSchemaException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public Table loadTable(NameIdentifier ident) throws NoSuchTableException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public Table createTable( + NameIdentifier ident, + Column[] columns, + String comment, + Map properties, + Transform[] partitions, + Distribution distribution, + SortOrder[] sortOrders, + Index[] indexes) + throws NoSuchSchemaException, TableAlreadyExistsException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public Table alterTable(NameIdentifier ident, TableChange... changes) + throws NoSuchTableException, IllegalArgumentException { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public boolean dropTable(NameIdentifier ident) { + throw new UnsupportedOperationException("Not implemented yet."); + } +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogPropertiesMetadata.java b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogPropertiesMetadata.java new file mode 100644 index 00000000000..18543bd0a3c --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseCatalogPropertiesMetadata.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.catalog.lakehouse; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.connector.BaseCatalogPropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +public class GenericLakehouseCatalogPropertiesMetadata extends BaseCatalogPropertiesMetadata { + + private static final Map> GENERIC_LAKEHOUSE_CATALOG_PROPERTY_ENTRIES = + ImmutableMap.>builder().build(); + + @Override + protected Map> specificPropertyEntries() { + return GENERIC_LAKEHOUSE_CATALOG_PROPERTY_ENTRIES; + } +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseSchemaPropertiesMetadata.java b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseSchemaPropertiesMetadata.java new file mode 100644 index 00000000000..05da8443cd9 --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseSchemaPropertiesMetadata.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.lakehouse; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.connector.BasePropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +public class GenericLakehouseSchemaPropertiesMetadata extends BasePropertiesMetadata { + private static final Map> propertiesMetadata; + + static { + propertiesMetadata = ImmutableMap.of(); + } + + @Override + protected Map> specificPropertyEntries() { + return propertiesMetadata; + } +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseTablePropertiesMetadata.java b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseTablePropertiesMetadata.java new file mode 100644 index 00000000000..362b10dbe4a --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/java/org/apache/gravitino/catalog/lakehouse/GenericLakehouseTablePropertiesMetadata.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.catalog.lakehouse; + +import com.google.common.collect.ImmutableMap; +import java.util.Map; +import org.apache.gravitino.connector.BasePropertiesMetadata; +import org.apache.gravitino.connector.PropertyEntry; + +public class GenericLakehouseTablePropertiesMetadata extends BasePropertiesMetadata { + + private static final Map> propertiesMetadata; + + static { + propertiesMetadata = ImmutableMap.of(); + } + + @Override + protected Map> specificPropertyEntries() { + return propertiesMetadata; + } +} diff --git a/catalogs/catalog-generic-lakehouse/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider b/catalogs/catalog-generic-lakehouse/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider new file mode 100644 index 00000000000..927e28b4fd2 --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/resources/META-INF/services/org.apache.gravitino.CatalogProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.catalog.lakehouse.GenericLakehouseCatalog diff --git a/catalogs/catalog-generic-lakehouse/src/main/resources/generic-lakehouse.conf b/catalogs/catalog-generic-lakehouse/src/main/resources/generic-lakehouse.conf new file mode 100644 index 00000000000..f2a4c807f4a --- /dev/null +++ b/catalogs/catalog-generic-lakehouse/src/main/resources/generic-lakehouse.conf @@ -0,0 +1,17 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 21245ecf8bb..5355fe7bc5f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -51,6 +51,7 @@ include( "clients:client-python", "clients:cli" ) +include("catalogs:catalog-generic-lakehouse") if (gradle.startParameter.projectProperties["enableFuse"]?.toBoolean() == true) { include("clients:filesystem-fuse") } else {