Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2094,13 +2094,13 @@ public void testCommitWritesRelativePaths(boolean populateMetaFields) throws Exc
HoodieCommitMetadata commitMetadata = HoodieCommitMetadata
.fromBytes(commitTimeline.getInstantDetails(commitInstant).get(), HoodieCommitMetadata.class);
String basePath = table.getMetaClient().getBasePath();
Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(basePath).values();
Collection<String> commitPathNames = commitMetadata.getFileIdAndFullPaths(new Path(basePath)).values();

// Read from commit file
try (FSDataInputStream inputStream = fs.open(testTable.getCommitFilePath(instantTime))) {
String everything = FileIOUtils.readAsUTFString(inputStream);
HoodieCommitMetadata metadata = HoodieCommitMetadata.fromJsonString(everything, HoodieCommitMetadata.class);
HashMap<String, String> paths = metadata.getFileIdAndFullPaths(basePath);
HashMap<String, String> paths = metadata.getFileIdAndFullPaths(new Path(basePath));
// Compare values in both to make sure they are equal.
for (String pathName : paths.values()) {
assertTrue(commitPathNames.contains(pathName));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ private static HashMap<String, String> getLatestFileIDsToFullPath(String basePat
for (HoodieInstant commit : commitsToReturn) {
HoodieCommitMetadata metadata =
HoodieCommitMetadata.fromBytes(commitTimeline.getInstantDetails(commit).get(), HoodieCommitMetadata.class);
fileIdToFullPath.putAll(metadata.getFileIdAndFullPaths(basePath));
fileIdToFullPath.putAll(metadata.getFileIdAndFullPaths(new Path(basePath)));
}
return fileIdToFullPath;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,15 @@ public static Schema createNullableSchema(Schema.Type avroType) {
return Schema.createUnion(Schema.create(Schema.Type.NULL), Schema.create(avroType));
}

/**
* Returns true in case when schema contains the field w/ provided name
*/
public static boolean containsFieldInSchema(Schema schema, String fieldName) {
try {
Schema.Field field = schema.getField(fieldName);
return field != null;
} catch (Exception e) {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -166,18 +166,6 @@ public static GenericRecord jsonBytesToAvro(byte[] bytes, Schema schema) throws
return reader.read(null, jsonDecoder);
}

/**
* True if the schema contains this name of field
*/
public static boolean containsFieldInSchema(Schema schema, String fieldName) {
try {
Field field = schema.getField(fieldName);
return field != null;
} catch (Exception e) {
return false;
}
}

public static boolean isMetadataField(String fieldName) {
return HoodieRecord.COMMIT_TIME_METADATA_FIELD.equals(fieldName)
|| HoodieRecord.COMMIT_SEQNO_METADATA_FIELD.equals(fieldName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ public WriteOperationType getOperationType() {
return this.operationType;
}

public HashMap<String, String> getFileIdAndFullPaths(String basePath) {
public HashMap<String, String> getFileIdAndFullPaths(Path basePath) {
HashMap<String, String> fullPaths = new HashMap<>();
for (Map.Entry<String, String> entry : getFileIdAndRelativePaths().entrySet()) {
String fullPath = entry.getValue() != null
Expand Down
Loading