This is an automated email from the ASF dual-hosted git repository.
zhangyue19921010 pushed a change to branch stream-binary-copy
in repository https://gitbox.apache.org/repos/asf/hudi.git
from 2ed9776f027 code review
add 39fab6d561c [HUDI-9466] Claim RFC 96: Introduce Unified Bucket Index
(#13362)
add 94e4d3763e7 [HUDI-9146][part3] Integrating FileGroup reader into Flink
CDC reader (#13357)
add 4284012d2b5 [HUDI-9389] Refactor table service schedule to lock
instant creation and plan generation (#13340)
add 2ddc4bdb4e4 [HUDI-8474] Introduce SparkRDDMetadatatWriteClient and
HoodieSparkMergeOnReadMetadataTable to support streaming writes to metadata
table (#13295)
add a23faa26125 [HUDI-941] Update restore/rollback/indexing planning and
instant generation (#13368)
add 6e767fed329 [HUDI-9473] Set memory config properly for FileGroup
reader in Flink reader (#13372)
add ed6f6928895 [Hudi-9475] Fix invalid log file name exception during CDC
query (#13373)
add 9dd5358c419 [HUDI-9365] Reduce overhead of Hive and AWS Glue sync
tools (#13249)
add 5ad9bcdda17 [HUDI-9235] Read MDT through FG reader (#13300)
add 133c2a5befc [MINOR] Optimizing metadata getter for metadata table
(#13298)
add 2206326502a [HUDI-8449] Fix deletion of record from FILES partition on
empty files list (#13346)
add 38778a298cc Revert "[HUDI-8449] Fix deletion of record from FILES
partition on empty file…" (#13379)
add 450a3c27ce4 [HUDI-9438] Fix conflict handling for compaction instants
for v8 tables (#13347)
add 59373c7af3a [HUDI-9479] Improve perf for getOrderingValue of timestamp
type for flink FileGroup reader (#13378)
add 09f9a16c262 [HUDI-9020] Remove HBase dependencies from Hudi (#12964)
add fca45562180 [HUDI-8480] Adding new write config for streaming writes
to metadata table and Enabling Non Blocking Concurrency Control with Metadata
(#13292)
add a4e7f8c6dd3 [HUDI-8284] Migrate HoodieFileSliceReader to
HoodieFileGroupReader in Java and Spark engines (#13361)
add 2db999a1a25 [MINOR] Renaming TransactionManager methods to begin/end x
StateChange (#13387)
add 7fda08485d1 Merge branch 'master' into stream-binary-copy
No new revisions were added by this update.
Summary of changes:
azure-pipelines-20230430.yml | 4 +-
.../hudi/aws/sync/AWSGlueCatalogSyncClient.java | 29 +-
.../org/apache/hudi/async/AsyncCleanerService.java | 4 +-
.../hudi/client/BaseHoodieTableServiceClient.java | 296 ++-
.../apache/hudi/client/BaseHoodieWriteClient.java | 123 +-
.../timeline/versioning/v1/TimelineArchiverV1.java | 4 +-
.../timeline/versioning/v2/TimelineArchiverV2.java | 4 +-
.../DirectMarkerTransactionManager.java | 2 +-
...urrentFileWritesConflictResolutionStrategy.java | 71 +-
.../client/transaction/TransactionManager.java | 40 +-
...or.java => CloseableConcatenatingIterator.java} | 23 +-
.../hudi/client/utils/ConcatenatingIterator.java | 8 +-
.../apache/hudi/client/utils/TransactionUtils.java | 2 +-
.../apache/hudi/config/HoodieHBaseIndexConfig.java | 554 -----
.../org/apache/hudi/config/HoodieIndexConfig.java | 30 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 159 +-
.../java/org/apache/hudi/index/HoodieIndex.java | 6 -
.../hbase/DefaultHBaseQPSResourceAllocator.java | 52 -
.../hbase/HBaseIndexQPSResourceAllocator.java | 51 -
.../metadata/HoodieBackedTableMetadataWriter.java | 27 +-
...ieBackedTableMetadataWriterTableVersionSix.java | 5 -
.../hudi/metadata/HoodieMetadataWriteUtils.java | 26 +-
.../java/org/apache/hudi/table/HoodieTable.java | 12 +-
.../org/apache/hudi/table/WorkloadProfile.java | 15 +-
.../hudi/table/action/BaseActionExecutor.java | 35 +-
.../table/action/clean/CleanActionExecutor.java | 49 +-
.../action/clean/CleanPlanActionExecutor.java | 54 +-
.../strategy/ClusteringExecutionStrategy.java | 115 +-
.../action/commit/BaseCommitActionExecutor.java | 6 +-
.../action/index/AbstractIndexingCatchupTask.java | 4 +-
.../table/action/index/RunIndexActionExecutor.java | 4 +-
.../action/index/ScheduleIndexActionExecutor.java | 7 -
.../action/restore/BaseRestoreActionExecutor.java | 4 +-
.../rollback/BaseRollbackActionExecutor.java | 4 +-
.../hudi/table/upgrade/UpgradeDowngrade.java | 2 +-
.../client/TestBaseHoodieTableServiceClient.java | 69 +-
.../hudi/client/TestBaseHoodieWriteClient.java | 4 +-
.../apache/hudi/client/WriteClientTestUtils.java | 7 +
.../TestConflictResolutionStrategyUtil.java | 23 +-
...urrentFileWritesConflictResolutionStrategy.java | 111 +-
...itesConflictResolutionStrategyWithMORTable.java | 90 -
.../client/transaction/TestTransactionManager.java | 54 +-
.../hudi/client/utils/TestTransactionUtils.java | 47 +
.../apache/hudi/config/TestHoodieWriteConfig.java | 27 +
.../metadata/TestHoodieMetadataWriteUtils.java | 56 +-
.../org/apache/hudi/table/TestBaseHoodieTable.java | 2 +-
.../action/clean/TestCleanPlanActionExecutor.java | 75 +-
.../hudi/utils/HoodieWriterClientTestHarness.java | 13 +-
hudi-client/hudi-flink-client/pom.xml | 30 -
.../hudi/client/HoodieFlinkTableServiceClient.java | 8 +-
.../apache/hudi/client/HoodieFlinkWriteClient.java | 2 +-
.../v2/FlinkFileGroupReaderBasedMergeHandle.java | 2 +-
.../FlinkHoodieBackedTableMetadataWriter.java | 2 +-
.../hudi/table/HoodieFlinkCopyOnWriteTable.java | 5 +-
.../org/apache/hudi/table/HoodieFlinkTable.java | 3 +
.../java/org/apache/hudi/util/FlinkClientUtil.java | 7 +-
.../java/org/apache/hudi/util/RowDataUtils.java | 2 +-
hudi-client/hudi-java-client/pom.xml | 7 +
.../run/strategy/JavaExecutionStrategy.java | 44 +-
.../hudi/table/HoodieJavaCopyOnWriteTable.java | 4 +-
.../org/apache/hudi/table/HoodieJavaTable.java | 3 +
.../commit/BaseJavaCommitActionExecutor.java | 4 +-
.../hudi/client/TestJavaHoodieBackedMetadata.java | 47 +-
.../TestHoodieJavaClientOnCopyOnWriteStorage.java | 5 +
.../read/TestHoodieFileGroupReaderOnJava.java | 6 +-
.../hudi/testutils/TestHoodieMetadataBase.java | 8 +-
hudi-client/hudi-spark-client/pom.xml | 35 +-
.../hudi/client/SparkRDDMetadataWriteClient.java | 96 +
.../MultipleSparkJobExecutionStrategy.java | 229 +-
...SparkJobConsistentHashingExecutionStrategy.java | 33 +-
.../strategy/SingleSparkJobExecutionStrategy.java | 61 +-
.../apache/hudi/index/SparkHoodieIndexFactory.java | 5 -
.../hbase/RebalancedSparkHoodieHBaseIndex.java | 38 -
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 723 -------
.../SparkHoodieBackedTableMetadataWriter.java | 3 +-
.../hudi/table/HoodieSparkCopyOnWriteTable.java | 6 +-
.../table/HoodieSparkMergeOnReadMetadataTable.java | 54 +
.../org/apache/hudi/table/HoodieSparkTable.java | 9 +-
.../commit/BaseSparkCommitActionExecutor.java | 31 +-
...arkMetadataTableUpsertCommitActionExecutor.java | 103 +
.../SparkMetadataTableUpsertPartitioner.java | 66 +
.../hudi/BaseSparkInternalRowReaderContext.java | 2 +-
.../apache/spark/sql/HoodieInternalRowUtils.scala | 3 +-
.../client/TestSparkRDDMetadataWriteClient.java | 418 ++++
.../client/functional/TestHoodieMetadataBase.java | 4 +-
.../apache/hudi/index/TestHoodieIndexConfigs.java | 11 +-
.../hudi/index/hbase/TestHBaseIndexUsage.java | 40 -
.../hbase/TestHBasePutBatchSizeCalculator.java | 61 -
.../index/hbase/TestHBaseQPSResourceAllocator.java | 96 -
.../index/hbase/TestSparkHoodieHBaseIndex.java | 905 --------
.../java/org/apache/hudi/table/TestCleaner.java | 95 +-
.../table/action/cluster/ClusteringTestUtils.java | 10 +-
...arkMetadataTableUpsertCommitActionExecutor.java | 108 +
.../TestSparkMetadataTableUpsertPartitioner.java | 94 +
.../table/action/commit/TestUpsertPartitioner.java | 7 +-
.../table/action/compact/CompactionTestBase.java | 5 +-
.../testutils/HoodieSparkClientTestHarness.java | 2 +-
hudi-common/pom.xml | 50 +-
.../apache/hudi/avro/HoodieAvroReaderContext.java | 43 +-
.../hudi/common/config/HoodieMetadataConfig.java | 31 +
.../common/engine/AvroReaderContextFactory.java | 3 +-
.../hudi/common/engine/HoodieReaderContext.java | 20 +-
.../java/org/apache/hudi/common/fs/FSUtils.java | 21 +-
.../hudi/common/table/cdc/HoodieCDCFileSplit.java | 3 +-
.../table/log/HoodieMergedLogRecordReader.java | 90 +-
.../common/table/read/HoodieFileGroupReader.java | 19 +-
.../table/view/AbstractTableFileSystemView.java | 3 +-
.../org/apache/hudi/common/util/HFileUtils.java | 38 +-
.../org/apache/hudi/expression/Predicates.java | 48 +
.../apache/hudi/io/storage}/HoodieHFileConfig.java | 3 +-
.../hudi/metadata/BaseFileRecordParsingUtils.java | 4 +-
.../hudi/metadata/HoodieBackedTableMetadata.java | 343 ++-
.../SecondaryIndexRecordGenerationUtils.java | 231 +--
hudi-common/src/main/resources/hbase-site.xml | 2185 --------------------
.../hudi/avro/TestHoodieAvroReaderContext.java | 11 +-
.../table/log/TestHoodieMergedLogRecordReader.java | 69 +
.../org/apache/hudi/expression/TestPredicates.java | 55 +
.../io/storage/TestHoodieReaderWriterUtils.java | 89 -
.../examples/quickstart/TestQuickstartData.java | 83 -
hudi-examples/hudi-examples-java/pom.xml | 4 +
hudi-examples/hudi-examples-k8s/pom.xml | 1 -
.../java/org/apache/hudi/sink/CleanFunction.java | 4 +-
.../hudi/sink/bootstrap/BootstrapOperator.java | 74 +-
.../hudi/sink/clustering/ClusteringOperator.java | 136 +-
.../sink/clustering/HoodieFlinkClusteringJob.java | 9 +-
.../apache/hudi/source/ExpressionEvaluators.java | 2 +-
.../table/format/FlinkRowDataReaderContext.java | 17 +-
.../org/apache/hudi/table/format/FormatUtils.java | 162 +-
.../hudi/table/format/cdc/CdcInputFormat.java | 127 +-
.../table/format/mor/MergeOnReadInputFormat.java | 336 +--
.../org/apache/hudi/util/FlinkWriteClients.java | 21 +-
...erter.java => RecordKeyToRowDataConverter.java} | 24 +-
.../java/org/apache/hudi/util/StreamerUtil.java | 21 -
.../hudi/sink/bucket/ITTestBucketStreamWrite.java | 6 +-
.../bucket/ITTestConsistentBucketStreamWrite.java | 5 +-
.../sink/cluster/ITTestHoodieFlinkClustering.java | 57 +-
.../org/apache/hudi/sink/utils/TestWriteBase.java | 10 +-
.../table/TestHoodieFileGroupReaderOnFlink.java | 37 +-
.../test/java/org/apache/hudi/utils/TestData.java | 134 +-
...r.java => TestRecordKeyToRowDataConverter.java} | 22 +-
hudi-hadoop-common/pom.xml | 6 +
.../bootstrap/index/HFileBootstrapIndex.java | 37 -
.../hfile/HBaseHFileBootstrapIndexReader.java | 283 ---
.../io/hadoop/HoodieAvroFileReaderFactory.java | 13 +-
.../io/hadoop/HoodieAvroFileWriterFactory.java | 1 +
.../hudi/io/hadoop/HoodieAvroHFileWriter.java | 5 +-
.../hudi/io/hadoop/HoodieHBaseAvroHFileReader.java | 674 ------
.../apache/hudi/io/hadoop/HoodieHFileUtils.java | 156 --
.../hudi/io/storage/HoodieHBaseKVComparator.java | 29 -
.../org/apache/hudi/common/fs/TestFSUtils.java | 32 +-
.../TestInLineFileSystemHFileInLiningBase.java | 181 --
.../TestInLineFileSystemWithHBaseHFileReader.java | 126 --
.../TestInLineFileSystemWithHFileReader.java | 136 +-
.../hudi/common/table/read/TestCustomMerger.java | 3 +-
.../common/table/read/TestEventTimeMerging.java | 4 +-
.../TestHoodieFileGroupReaderInflightCommit.java | 3 +-
.../table/read/TestOverwriteWithLatestMerger.java | 4 +-
.../hadoop/TestHoodieHBaseHFileReaderWriter.java | 145 --
.../io/hadoop/TestHoodieHFileReaderWriter.java | 490 ++++-
.../io/hadoop/TestHoodieHFileReaderWriterBase.java | 537 -----
hudi-hadoop-mr/pom.xml | 1 -
.../hudi/hadoop/HiveHoodieReaderContext.java | 5 +-
.../org/apache/hudi/integ/HoodieTestHiveBase.java | 3 +-
.../hudi-metaserver/hudi-metaserver-client/pom.xml | 9 +
hudi-spark-datasource/hudi-spark-common/pom.xml | 8 -
.../hudi/command/MergeIntoHoodieTableCommand.scala | 5 +-
.../procedures/PartitionBucketIndexManager.scala | 2 +-
.../command/procedures/RunCleanProcedure.scala | 41 +-
.../procedures/RunClusteringProcedure.scala | 8 +-
.../procedures/RunCompactionProcedure.scala | 8 +-
.../hudi/client/TestHoodieClientMultiWriter.java | 37 +-
.../TestMultiWriterWithPreferWriterIngestion.java | 36 +-
.../hudi/client/TestTableSchemaEvolution.java | 3 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 35 +-
.../TestHoodieClientOnMergeOnReadStorage.java | 41 +
.../TestMetadataUtilRLIandSIRecordGeneration.java | 12 +-
.../TestColStatsRecordWithMetadataRecord.java | 3 +-
.../TestDataSkippingWithMORColstats.java | 2 +-
.../hudi/functional/TestHoodieBackedMetadata.java | 68 +-
.../hudi/table/TestHoodieMergeOnReadTable.java | 3 +-
.../action/cluster/TestIncrementalClustering.java | 0
.../table/action/compact/TestAsyncCompaction.java | 3 +-
.../table/action/compact/TestHoodieCompactor.java | 13 +-
.../TestCopyOnWriteRollbackActionExecutor.java | 3 +-
.../TestMergeOnReadRollbackActionExecutor.java | 3 +-
...HoodieSparkMergeOnReadTableIncrementalRead.java | 3 +-
.../TestHoodieSparkMergeOnReadTableRollback.java | 13 +-
.../hudi/functional/TestSevenToEightUpgrade.scala | 3 +-
.../spark/hudi/benchmark/HoodieBenchmarks.scala | 2 +-
.../analysis/TestMergeIntoHoodieTableCommand.scala | 8 +-
.../sql/hudi/ddl/TestAlterTableDropPartition.scala | 12 +-
.../spark/sql/hudi/dml/TestInsertTable.scala | 3 +-
.../spark/sql/hudi/dml/TestUpdateTable.scala | 3 +-
.../hudi/procedure/TestClusteringProcedure.scala | 9 +-
.../procedure/TestShowFileStatusProcedure.scala | 12 +-
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 14 +-
hudi-spark-datasource/hudi-spark3.4.x/pom.xml | 6 +
hudi-spark-datasource/hudi-spark3.5.x/pom.xml | 6 +
.../java/org/apache/hudi/hive/HiveSyncTool.java | 35 +-
.../org/apache/hudi/hive/HoodieHiveSyncClient.java | 23 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 7 +-
.../apache/hudi/sync/common/HoodieSyncClient.java | 6 +-
hudi-tests-common/pom.xml | 6 +
hudi-utilities/pom.xml | 6 +
.../apache/hudi/utilities/HoodieClusteringJob.java | 4 -
.../org/apache/hudi/utilities/HoodieCompactor.java | 8 +-
packaging/hudi-aws-bundle/pom.xml | 1 -
packaging/hudi-datahub-sync-bundle/pom.xml | 1 -
packaging/hudi-flink-bundle/pom.xml | 4 -
packaging/hudi-gcp-bundle/pom.xml | 1 -
packaging/hudi-hadoop-mr-bundle/pom.xml | 1 -
packaging/hudi-hive-sync-bundle/pom.xml | 1 -
packaging/hudi-integ-test-bundle/pom.xml | 1 -
packaging/hudi-kafka-connect-bundle/pom.xml | 1 -
packaging/hudi-presto-bundle/pom.xml | 1 -
packaging/hudi-spark-bundle/pom.xml | 4 -
packaging/hudi-timeline-server-bundle/pom.xml | 1 -
packaging/hudi-trino-bundle/pom.xml | 1 -
packaging/hudi-utilities-bundle/pom.xml | 1 -
packaging/hudi-utilities-slim-bundle/pom.xml | 1 -
pom.xml | 115 +-
rfc/README.md | 1 +
222 files changed, 4007 insertions(+), 10137 deletions(-)
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/utils/{ClosableMergingIterator.java
=> CloseableConcatenatingIterator.java} (57%)
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieHBaseIndexConfig.java
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/hbase/DefaultHBaseQPSResourceAllocator.java
delete mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/index/hbase/HBaseIndexQPSResourceAllocator.java
delete mode 100644
hudi-client/hudi-client-common/src/test/java/org/apache/hudi/client/transaction/TestSimpleConcurrentFileWritesConflictResolutionStrategyWithMORTable.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/SparkRDDMetadataWriteClient.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/RebalancedSparkHoodieHBaseIndex.java
delete mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/index/hbase/SparkHoodieHBaseIndex.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/HoodieSparkMergeOnReadMetadataTable.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMetadataTableUpsertCommitActionExecutor.java
create mode 100644
hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/table/action/commit/SparkMetadataTableUpsertPartitioner.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/TestSparkRDDMetadataWriteClient.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseIndexUsage.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBasePutBatchSizeCalculator.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestHBaseQPSResourceAllocator.java
delete mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/index/hbase/TestSparkHoodieHBaseIndex.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestSparkMetadataTableUpsertCommitActionExecutor.java
create mode 100644
hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/table/action/commit/TestSparkMetadataTableUpsertPartitioner.java
rename {hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop =>
hudi-common/src/main/java/org/apache/hudi/io/storage}/HoodieHFileConfig.java
(98%)
delete mode 100644 hudi-common/src/main/resources/hbase-site.xml
create mode 100644
hudi-common/src/test/java/org/apache/hudi/common/table/log/TestHoodieMergedLogRecordReader.java
create mode 100644
hudi-common/src/test/java/org/apache/hudi/expression/TestPredicates.java
delete mode 100644
hudi-common/src/test/java/org/apache/hudi/io/storage/TestHoodieReaderWriterUtils.java
rename
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/{StringToRowDataConverter.java
=> RecordKeyToRowDataConverter.java} (81%)
rename
hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/{TestStringToRowDataConverter.java
=> TestRecordKeyToRowDataConverter.java} (84%)
delete mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/common/bootstrap/index/HFileBootstrapIndex.java
delete mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/common/bootstrap/index/hfile/HBaseHFileBootstrapIndexReader.java
delete mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHBaseAvroHFileReader.java
delete mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/io/hadoop/HoodieHFileUtils.java
delete mode 100644
hudi-hadoop-common/src/main/java/org/apache/hudi/io/storage/HoodieHBaseKVComparator.java
delete mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemHFileInLiningBase.java
delete mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/common/fs/inline/TestInLineFileSystemWithHBaseHFileReader.java
delete mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHBaseHFileReaderWriter.java
delete mode 100644
hudi-hadoop-common/src/test/java/org/apache/hudi/io/hadoop/TestHoodieHFileReaderWriterBase.java
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java
(98%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/table/action/cluster/TestIncrementalClustering.java
(100%)
rename {hudi-client/hudi-spark-client =>
hudi-spark-datasource/hudi-spark}/src/test/java/org/apache/hudi/table/action/rollback/TestCopyOnWriteRollbackActionExecutor.java
(99%)
