Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 0 additions & 3 deletions docs/src/main/sphinx/connector/hudi.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,6 @@ Additionally, following configuration properties can be set depending on the use
* - Property name
- Description
- Default
* - ``hudi.metadata-enabled``
- Fetch the list of file names and sizes from metadata rather than storage.
- ``false``
* - ``hudi.columns-to-hide``
- List of column names that are hidden from the query output.
It can be used to hide Hudi meta fields. By default, no fields are hidden.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,13 @@
@DefunctConfig({
"hudi.min-partition-batch-size",
"hudi.max-partition-batch-size",
"hudi.metadata-enabled",
})
public class HudiConfig
{
private static final Splitter COMMA_SPLITTER = Splitter.on(",").omitEmptyStrings().trimResults();

private List<String> columnsToHide = ImmutableList.of();
private boolean metadataEnabled;
private boolean shouldUseParquetColumnNames = true;
private boolean sizeBasedSplitWeightsEnabled = true;
private DataSize standardSplitWeightSize = DataSize.of(128, MEGABYTE);
Expand All @@ -67,19 +67,6 @@ public HudiConfig setColumnsToHide(String columnsToHide)
return this;
}

@Config("hudi.metadata-enabled")
@ConfigDescription("Fetch the list of file names and sizes from metadata rather than storage.")
public HudiConfig setMetadataEnabled(boolean metadataEnabled)
{
this.metadataEnabled = metadataEnabled;
return this;
}

public boolean isMetadataEnabled()
{
return this.metadataEnabled;
}

@Config("hudi.parquet.use-column-names")
@ConfigDescription("Access Parquet columns using names from the file. If disabled, then columns are accessed using index."
+ "Only applicable to Parquet file format.")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ public class HudiSessionProperties
implements SessionPropertiesProvider
{
private static final String COLUMNS_TO_HIDE = "columns_to_hide";
private static final String METADATA_ENABLED = "metadata_enabled";
private static final String USE_PARQUET_COLUMN_NAMES = "use_parquet_column_names";
private static final String SIZE_BASED_SPLIT_WEIGHTS_ENABLED = "size_based_split_weights_enabled";
private static final String STANDARD_SPLIT_WEIGHT_SIZE = "standard_split_weight_size";
Expand All @@ -65,11 +64,6 @@ public HudiSessionProperties(HudiConfig hudiConfig)
.map(name -> ((String) name).toLowerCase(ENGLISH))
.collect(toImmutableList()),
value -> value),
booleanProperty(
METADATA_ENABLED,
"For Hudi tables prefer to fetch the list of files from its metadata",
hudiConfig.isMetadataEnabled(),
false),
booleanProperty(
USE_PARQUET_COLUMN_NAMES,
"Access parquet columns using names from the file. If disabled, then columns are accessed using index.",
Expand Down Expand Up @@ -124,11 +118,6 @@ public static List<String> getColumnsToHide(ConnectorSession session)
return (List<String>) session.getProperty(COLUMNS_TO_HIDE, List.class);
}

public static boolean isHudiMetadataEnabled(ConnectorSession session)
{
return session.getProperty(METADATA_ENABLED, Boolean.class);
}

public static boolean shouldUseParquetColumnNames(ConnectorSession session)
{
return session.getProperty(USE_PARQUET_COLUMN_NAMES, Boolean.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ public void testDefaults()
{
assertRecordedDefaults(recordDefaults(HudiConfig.class)
.setColumnsToHide(null)
.setMetadataEnabled(false)
.setUseParquetColumnNames(true)
.setSizeBasedSplitWeightsEnabled(true)
.setStandardSplitWeightSize(DataSize.of(128, MEGABYTE))
Expand All @@ -48,7 +47,6 @@ public void testExplicitPropertyMappings()
{
Map<String, String> properties = ImmutableMap.<String, String>builder()
.put("hudi.columns-to-hide", "_hoodie_record_key")
.put("hudi.metadata-enabled", "true")
.put("hudi.parquet.use-column-names", "false")
.put("hudi.size-based-split-weights-enabled", "false")
.put("hudi.standard-split-weight-size", "64MB")
Expand All @@ -62,7 +60,6 @@ public void testExplicitPropertyMappings()

HudiConfig expected = new HudiConfig()
.setColumnsToHide("_hoodie_record_key")
.setMetadataEnabled(true)
.setUseParquetColumnNames(false)
.setSizeBasedSplitWeightsEnabled(false)
.setStandardSplitWeightSize(DataSize.of(64, MEGABYTE))
Expand Down

This file was deleted.