Skip to content

Commit

Permalink
[#6462] fix(filesystem-hadoop3): Fix GVFS client can't find `HDFSFile…
Browse files Browse the repository at this point in the history
…SystemProvider` problem (#6463)

### What changes were proposed in this pull request?

In the current code base, we need to add `catalog-hadoop` to make GVFS
client works or class `HDFSFileSystemProvider` and
`LocalFileSystemProvider` can't be found.

### Why are the changes needed?

It's a bug.

Fix: #6462 

### Does this PR introduce _any_ user-facing change?

N/A

### How was this patch tested?

ITs, UTs and test locally.
  • Loading branch information
yuqi1129 authored Feb 18, 2025
1 parent 3dd35ad commit 07ae3f6
Show file tree
Hide file tree
Showing 10 changed files with 138 additions and 136 deletions.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
package org.apache.gravitino.catalog.hadoop;

import static org.apache.gravitino.catalog.hadoop.authentication.kerberos.KerberosConfig.KERBEROS_PROPERTY_ENTRIES;
import static org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_LOCAL_FS_PROVIDER;

import com.google.common.collect.ImmutableMap;
import java.util.Map;
Expand Down Expand Up @@ -56,9 +57,6 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada
static final String FILESYSTEM_CONNECTION_TIMEOUT_SECONDS = "filesystem-conn-timeout-secs";
static final int DEFAULT_GET_FILESYSTEM_TIMEOUT_SECONDS = 6;

public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local";
public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs";

private static final Map<String, PropertyEntry<?>> HADOOP_CATALOG_PROPERTY_ENTRIES =
ImmutableMap.<String, PropertyEntry<?>>builder()
.put(
Expand Down

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.gravitino.catalog.hadoop.fs;

public class Constants {

// Name of the built-in local file system provider
public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local";

// Name of the built-in HDFS file system provider
public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs";
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
*/
package org.apache.gravitino.catalog.hadoop.fs;

import static org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_HDFS_FS_PROVIDER;
import static org.apache.gravitino.catalog.hadoop.Constants.BUILTIN_LOCAL_FS_PROVIDER;
import static org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_HDFS_FS_PROVIDER;
import static org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_LOCAL_FS_PROVIDER;
import static org.apache.gravitino.catalog.hadoop.fs.FileSystemProvider.GRAVITINO_BYPASS;

import com.google.common.collect.Maps;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino.catalog.hadoop.fs;

import static org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_HDFS_FS_PROVIDER;

import java.io.IOException;
import java.util.Map;
import javax.annotation.Nonnull;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class HDFSFileSystemProvider implements FileSystemProvider {

@Override
public FileSystem getFileSystem(@Nonnull Path path, @Nonnull Map<String, String> config)
throws IOException {
Configuration configuration = new Configuration();
config.forEach(
(k, v) -> {
configuration.set(k.replace(GRAVITINO_BYPASS, ""), v);
});
return FileSystem.newInstance(path.toUri(), configuration);
}

@Override
public String scheme() {
return "hdfs";
}

@Override
public String name() {
return BUILTIN_HDFS_FS_PROVIDER;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.gravitino.catalog.hadoop.fs;

import static org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_HDFS_FS_PROVIDER;
import static org.apache.gravitino.catalog.hadoop.fs.Constants.BUILTIN_LOCAL_FS_PROVIDER;

import java.io.IOException;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;

public class LocalFileSystemProvider implements FileSystemProvider {

@Override
public FileSystem getFileSystem(Path path, Map<String, String> config) throws IOException {
Configuration configuration = new Configuration();
config.forEach(
(k, v) -> {
configuration.set(k.replace(BUILTIN_HDFS_FS_PROVIDER, ""), v);
});

return FileSystem.newInstance(path.toUri(), configuration);
}

@Override
public String scheme() {
return "file";
}

@Override
public String name() {
return BUILTIN_LOCAL_FS_PROVIDER;
}
}
2 changes: 2 additions & 0 deletions clients/filesystem-hadoop3-runtime/build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ tasks.withType<ShadowJar>(ShadowJar::class.java) {
relocate("com.github.benmanes.caffeine", "org.apache.gravitino.shaded.com.github.benmanes.caffeine")
// relocate common lang3 package
relocate("org.apache.commons.lang3", "org.apache.gravitino.shaded.org.apache.commons.lang3")

mergeServiceFiles()
}

tasks.jar {
Expand Down

0 comments on commit 07ae3f6

Please sign in to comment.