linliu-code commented on code in PR #13300:
URL: https://github.com/apache/hudi/pull/13300#discussion_r2108747032
##########
hudi-hadoop-common/src/main/java/org/apache/hudi/common/util/HFileUtils.java:
##########
@@ -110,22 +115,61 @@ public Set<Pair<String, Long>>
filterRowKeys(HoodieStorage storage, StoragePath
@Override
public ClosableIterator<Pair<HoodieKey, Long>>
fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath) {
- throw new UnsupportedOperationException("HFileUtils does not support
fetchRecordKeysWithPositions");
+ return fetchRecordKeysWithPositions(storage, filePath, Option.empty(),
Option.empty());
}
@Override
public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage
storage, StoragePath filePath, Option<BaseKeyGenerator> keyGeneratorOpt,
Option<String> partitionPath) {
- throw new UnsupportedOperationException("HFileUtils does not support
getHoodieKeyIterator");
+ try {
+ Configuration conf = storage.getConf().unwrapCopyAs(Configuration.class);
+ conf.addResource(HadoopFSUtils.getFs(filePath.toString(),
conf).getConf());
+ HoodieNativeAvroHFileReader reader = new
HoodieNativeAvroHFileReader(storage, filePath, Option.empty());
+ ClosableIterator<String> keyIterator = reader.getRecordKeyIterator();
+ return new ClosableIterator<HoodieKey>() {
+ @Override
+ public void close() {
+ keyIterator.close();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return keyIterator.hasNext();
+ }
+
+ @Override
+ public HoodieKey next() {
+ String key = keyIterator.next();
+ if (partitionPath.isPresent()) {
+ return new HoodieKey(key, partitionPath.get());
+ } else {
+ return new HoodieKey(key, null);
+ }
+ }
+ };
+ } catch (IOException e) {
+ throw new HoodieIOException("Unable to read the HFile: ", e);
+ }
}
@Override
public ClosableIterator<HoodieKey> getHoodieKeyIterator(HoodieStorage
storage, StoragePath filePath) {
- throw new UnsupportedOperationException("HFileUtils does not support
getHoodieKeyIterator");
+ return getHoodieKeyIterator(storage, filePath, Option.empty(),
Option.empty());
}
@Override
public ClosableIterator<Pair<HoodieKey, Long>>
fetchRecordKeysWithPositions(HoodieStorage storage, StoragePath filePath,
Option<BaseKeyGenerator> keyGeneratorOpt, Option<String> partitionPath) {
- throw new UnsupportedOperationException("HFileUtils does not support
fetchRecordKeysWithPositions");
+ try {
+ if (filePath == null || !storage.exists(filePath)) {
+ return ClosableIterator.wrap(Collections.emptyIterator());
+ } else {
+ AtomicLong position = new AtomicLong(0);
Review Comment:
Removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
