linliu-code commented on code in PR #13300:
URL: https://github.com/apache/hudi/pull/13300#discussion_r2110538489
##########
hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroReaderContext.java:
##########
@@ -80,12 +85,29 @@ public ClosableIterator<IndexedRecord>
getFileRecordIterator(
long length,
Schema dataSchema,
Schema requiredSchema,
- HoodieStorage storage
- ) throws IOException {
+ HoodieStorage storage) throws IOException {
HoodieAvroFileReader reader = (HoodieAvroFileReader)
HoodieIOFactory.getIOFactory(storage)
.getReaderFactory(HoodieRecord.HoodieRecordType.AVRO).getFileReader(new
HoodieConfig(),
- filePath, HoodieFileFormat.PARQUET, Option.empty());
- return reader.getIndexedRecordIterator(dataSchema, requiredSchema);
+ filePath, baseFileFormat, Option.empty());
+ if (keyFilterOpt.isEmpty()) {
+ return reader.getIndexedRecordIterator(dataSchema, requiredSchema);
+ } else {
+ // Currently predicate is only supported for HFile reader.
+ if (!(reader instanceof HoodieAvroHFileReaderImplBase)) {
+ return reader.getIndexedRecordIterator(dataSchema, requiredSchema);
+ } else {
+ // For HFile reader, only two predicates are supported: IN and
StringStartsWithAny.
+ HoodieAvroHFileReaderImplBase hfileReader =
(HoodieAvroHFileReaderImplBase) reader;
+ List<Expression> children = keyFilterOpt.get().getChildren();
+ List<String> keysOrPrefixes = children.subList(1, children.size())
+ .stream().map(e -> (String)
e.eval(null)).collect(Collectors.toList());
+ if (keyFilterOpt.get().getOperator().equals(Expression.Operator.IN)) {
// With keys.
+ return hfileReader.getIndexedRecordsByKeysIterator(keysOrPrefixes,
requiredSchema);
+ } else { // With key prefixes.
Review Comment:
SG, will do.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
