This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a change to branch release-feature-rfc46
in repository https://gitbox.apache.org/repos/asf/hudi.git
omit ec486855e5 [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
omit 34dd504721 [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
omit 2d6b966307 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
omit 798dfd9aa7 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
omit 4cb36befd5 [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
add 0f88efb278 [HUDI-4512][HUDI-4513] Fix bundle name for spark3 profile
(#6261)
add cdbd1561e0 [HUDI-4501] Throwing exception when restore is attempted
with hoodie.arhive.beyond.savepoint is enabled (#6239)
add 4ec060129f [HUDI-4516] fix Task not serializable error when run
HoodieCleaner after one failure (#6265)
add 2ac4872115 remove test resources (#6147)
add 8b35013658 [HUDI-4477] Adjust partition number of flink sink task
(#6218)
add e3dabbc3ad [HUDI-4298] Mor table reading for base and log files lost
sequence of events (#6286)
add 3042b2c7c5 [HUDI-4525] Fixing Spark 3.3 `AvroSerializer`
implementation (#6279)
add 7307ff3d5c [HUDI-4447] fix no partitioned path extractor error when
sync meta (#6263)
add 345dd81ec8 [HUDI-4520] Support qualified table 'db.table' in call
procedures (#6274)
add e22b6c12b0 [HUDI-4531] Wrong partition path for flink hive catalog
when the partition fields are not in the last (#6292)
add bab8254fb8 [HUDI-4487] support to create ro/rt table by spark sql
(#6262)
add b13cc76b5e [HUDI-4533] Fix RunCleanProcedure's
ArrayIndexOutOfBoundsException (#6293)
add 706108e5ea [HUDI-4536] ClusteringOperator causes the
NullPointerException when writing with BulkInsertWriterHelper in clustering
(#6298)
add 1761b897ca [HUDI-4385] Support online compaction in the flink batch
mode write (#6093)
add a75cc02273 [HUDI-4530] fix default payloadclass in mor is different
with cow (#6288)
add e03cd0a198 [HUDI-4545] Do not modify the current record directly for
OverwriteNonDefaultsWithLatestAvroPayload (#6306)
add fcdd4cf06c [HUDI-4544] support retain hour cleaning policy for flink
(#6300)
add ae7395df1e [HUDI-4547] Fix SortOperatorGen sort indices (#6309)
add b93f03ad06 [HUDI-4470] Remove spark dataPrefetch disabled prop in
DefaultSource
add 59527c3590 [HUDI-4540] Cover different table types in functional tests
of Spark structured streaming (#6317)
add 56ee5205e6 [HUDI-4514] optimize CTAS to adapt to saveAsTable api in
different modes (#6295)
add 4cc7d40924 [HUDI-4474] Fix inferring props for meta sync (#6310)
add a76f694902 [HUDI-4550] Fallback to listing based rollback for
completed instant (#6313)
add 767c196631 [HUDI-4303] Adding 4 to 5 upgrade handler to check for old
deprecated "default" partition value (#6248)
add 78eeb02beb [HUDI-4546] Optimize catalog cast logic in
HoodieSpark3Analysis (#6307)
add 98f01666c7 [HUDI-4534] Fixing upgrade to reload Metaclient for
deltastreamer writes (#6296)
add 95d7489bc5 [HUDI-4517] If no marker type file, fallback to timeline
based marker (#6266)
add 660177bce1 [HUDI-3669] Add a remote request retry mechanism for
'Remotehoodietablefiles… (#5884)
add 9796de9d4c [HUDI-4464] Clear warnings in Azure CI (#6210)
add 0fa2609815 [MINOR] Update PR description template (#6323)
add da60f2acdc [HUDI-4508] Repair the exception when reading optimized
query for mor in hive and presto/trino (#6254)
add 2be7c3502d [HUDI-4548] Unpack the column max/min to string instead of
Utf8 for Mor table (#6311)
add 61fc3c03a6 [HUDI-4447] fix SQL metasync when perform delete table
operation (#6180)
add 59f652a19c [HUDI-4424] Add new compactoin trigger stratgy:
NUM_COMMITS_AFTER_REQ… (#6144)
add 6b02877779 [MINOR] improve flink dummySink's parallelism (#6325)
add 8a1b2fe7de [HUDI-4568] Shade dropwizard metrics-core in
hudi-aws-bundle (#6327)
add f4b2782886 [HUDI-4572] Fix 'Not a valid schema field: ts' error in
HoodieFlinkCompactor if precombine field is not ts (#6331)
add 417cca94df [HUDI-4570] Fix hive sync path error due to reuse of
storage descriptors. (#6329)
add 75f6266594 [HUDI-4571] Fix partition extractor infer function when
partition field mismatch (#6333)
add 009259645f [HUDI-4570] Add test for updating multiple partitions in
hive sync (#6340)
add d2cca29546 [MINOR] Fix wrong key to determine sync sql cascade (#6339)
add 850d50b501 [HUDI-4581] Claim RFC-58 for data skipping integration with
query engines (#6346)
add 96892b31e1 [HUDI-4577] Adding test coverage for `DELETE FROM`, Spark
Quickstart guide (#6318)
add c75e2649d8 [HUDI-4556] Improve functional test coverage of column
stats index (#6319)
add fdd0c30d3d [HUDI-4558] lost 'hoodie.table.keygenerator.class' in
hoodie.properties (#6320)
add 676b3817bf [HUDI-4543] Support natural order when table schema
contains a field named 'ts' (#6246)
add 43d6710287 [HUDI-4569][RFC-58] Claim RFC-58 for adding a new feature
named 'Multiple event_time Fields Latest Verification in a Single Table' for
Hudi (#6328)
add f9eb91a9ef [HUDI-3503] Support more feature to call procedure
CleanCommand (#6353)
add 34f72fd0d0 [HUDI-4590] Add hudi-aws dependency to hudi-flink-bundle.
(#6356)
add bd2f999768 [MINOR] fix potential npe in spark writer (#6363)
add 71909f0f82 fix bug in cli show fsview all (#6314)
add 2ca1fdc1c8 [HUDI-4488] Improve S3EventsHoodieIncrSource efficiency
(#6228)
add 0d0a4152cf [HUDI-4611] Fix the duplicate creation of config in
HoodieFlinkStreamer (#6369)
add 553e280c6d [HUDI-3189] Fallback to full table scan with incremental
query when files are cleaned up or achived for MOR table (#6141)
add 997200f27f [MINOR] fix progress field calculate logic in
HoodieLogRecordReader (#6291)
add 2633e88a39 [HUDI-4608] Fix upgrade command in Hudi CLI (#6374)
add f5bb78bbef [HUDI-4609] Improve usability of upgrade/downgrade commands
in Hudi CLI (#6377)
add bad954c3b2 [HUDI-4574] Fixed timeline based marker thread safety issue
(#6383)
add 9bbd3dacd7 [HUDI-4621] Add validation that bucket index fields should
be subset of primary keys (#6396)
add 8c02e90a9b [HUDI-4354] Add --force-empty-sync flag to deltastreamer
(#6027)
add 642f87cc6b [HUDI-4601] Read error from MOR table after compaction with
timestamp partitioning (#6365)
add 042241fa2c [MINOR] Update DOAP with 0.12.0 Release (#6413)
add 1ad0e9560e [HUDI-4529] Tweak some default config options for flink
(#6287)
add 9055b2f65d [HUDI-4632] Remove the force active property for flink1.14
profile (#6415)
add efd9dab14c [HUDI-4551] Tweak the default parallelism of flink pipeline
to execution env parallelism (#6312)
add c855d8a31c [MINOR] Improve code style of CLI Command classes (#6427)
add ba6d7cb54b [HUDI-3625] Claim RFC-60 for Federated Storage Layer (#6440)
add a41386fda2 [HUDI-4616] Adding `PulsarSource` to `DeltaStreamer` to
support ingesting from Apache Pulsar (#6386)
add a6740d0767 [HUDI-3579] Add timeline commands in hudi-cli (#5139)
add 39b16c58c9 [HUDI-4638] Rename payload clazz and preCombine field
options for flink sql (#6434)
add b49417ff01 Revert "[HUDI-4632] Remove the force active property for
flink1.14 profile (#6415)" (#6449)
add 419f881167 [HUDI-4643] MergeInto syntax WHEN MATCHED is optional but
must be set (#6443)
add 1e95f14b6c [HUDI-4644] Change default flink profile to 1.15.x (#6445)
add 3adb571531 [HUDI-4678] Claim RFC-61 for Snapshot view management
(#6461)
add 4966978a55 [HUDI-4676] infer cleaner policy when write concurrency
mode is OCC (#6459)
add c677333f26 [HUDI-4683] Use enum class value for default value in flink
options (#6453)
add a9982d12c5 [HUDI-4584] Cleaning up Spark utilities (#6351)
add 1879efa45d [HUDI-4686] Flip option 'write.ignore.failed' to default
false (#6467)
add ca8a57a21d [HUDI-4515] Fix savepoints will be cleaned in keeping
latest versions policy (#6267)
add 16a80e6d41 [HUDI-4637] Release thread in RateLimiter doesn't been
terminated (#6433)
add 822c1397e0 [HUDI-4698] Rename the package
'org.apache.flink.table.data' to avoid conflicts with flink table core (#6481)
add 1e162bb73a HUDI-4687 add show_invalid_parquet procedure (#6480)
add e5584b3735 [HUDI-4584] Fixing `SQLConf` not being propagated to
executor (#6352)
add 5f92221655 [HUDI-4441] Log4j2 configuration fixes and removal of
log4j1 dependencies (#6170)
add c188852f49 [HUDI-4665] Flipping default for "ignore failed batch"
config in streaming sink to false (#6450)
add e46647341c [HUDI-4713] Fix flaky
ITTestHoodieDataSource#testAppendWrite (#6490)
add e90872b396 [HUDI-4696] Fix flaky TestHoodieCombineHiveInputFormat
(#6494)
add 11f85d1efb Revert "[HUDI-3669] Add a remote request retry mechanism
for 'Remotehoodietablefiles… (#5884)" (#6501)
add 797e7a67a9 [Stacked on 6386] Fixing `DebeziumSource` to properly
commit offsets; (#6416)
new 0126868d96 [HUDI-3378][HUDI-3379][HUDI-3381] Migrate usage of
HoodieRecordPayload and raw Avro payload to HoodieRecord (#5522)
new c0c3c5ea06 [MINOR] Fix type casting in TestHoodieHFileReaderWriter
new 9d4b3bf5b4 [HUDI-4292][RFC-46] Update doc to align with the Record
Merge API changes (#5927)
new 17d179ceeb [HUDI-4344] fix usage of HoodieDataBlock#getRecordIterator
(#6005)
new bb208e9861 [HUDI-3350][HUDI-3351] Support HoodieMerge API and Spark
engine-specific HoodieRecord (#5627)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ec486855e5)
\
N -- N -- N refs/heads/release-feature-rfc46 (bb208e9861)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/PULL_REQUEST_TEMPLATE.md | 48 +--
.github/workflows/bot.yml | 8 +-
azure-pipelines.yml | 30 +-
doap_HUDI.rdf | 5 +
docker/demo/config/hoodie-incr.properties | 5 +-
docker/demo/config/log4j.properties | 42 ---
docker/demo/config/log4j2.properties | 60 +++
docker/demo/setup_demo_container.sh | 2 +-
hudi-aws/pom.xml | 21 +-
.../src/test/resources/log4j-surefire.properties | 25 --
hudi-cli/pom.xml | 33 +-
.../org/apache/hudi/cli/HoodiePrintHelper.java | 73 +++-
.../apache/hudi/cli/HoodieTableHeaderFields.java | 15 +
.../src/main/java/org/apache/hudi/cli/Table.java | 53 ++-
.../main/java/org/apache/hudi/cli/TableHeader.java | 10 +
.../hudi/cli/commands/ArchivedCommitsCommand.java | 4 +-
.../apache/hudi/cli/commands/BootstrapCommand.java | 5 +-
.../apache/hudi/cli/commands/CleansCommand.java | 24 +-
.../hudi/cli/commands/ClusteringCommand.java | 2 +
.../apache/hudi/cli/commands/CommitsCommand.java | 117 +++---
.../hudi/cli/commands/CompactionCommand.java | 99 ++---
.../apache/hudi/cli/commands/ExportCommand.java | 13 +-
.../hudi/cli/commands/FileSystemViewCommand.java | 20 +-
.../cli/commands/HDFSParquetImportCommand.java | 6 +-
.../apache/hudi/cli/commands/MarkersCommand.java | 4 +-
.../apache/hudi/cli/commands/MetadataCommand.java | 2 +-
.../apache/hudi/cli/commands/RepairsCommand.java | 11 +-
.../apache/hudi/cli/commands/RollbacksCommand.java | 22 +-
.../hudi/cli/commands/SavepointsCommand.java | 7 +-
.../apache/hudi/cli/commands/SparkEnvCommand.java | 6 +-
.../org/apache/hudi/cli/commands/SparkMain.java | 42 ++-
.../org/apache/hudi/cli/commands/TableCommand.java | 27 +-
.../apache/hudi/cli/commands/TempViewCommand.java | 6 +-
.../apache/hudi/cli/commands/TimelineCommand.java | 410 +++++++++++++++++++++
.../cli/commands/UpgradeOrDowngradeCommand.java | 30 +-
.../org/apache/hudi/cli/commands/UtilsCommand.java | 1 +
.../cli/commands/TestArchivedCommitsCommand.java | 2 +-
.../hudi/cli/commands/TestCommitsCommand.java | 2 +-
.../hudi/cli/commands/TestCompactionCommand.java | 2 +-
.../cli/commands/TestFileSystemViewCommand.java | 15 +-
.../cli/commands/TestHoodieLogFileCommand.java | 6 +-
.../hudi/cli/commands/TestSavepointsCommand.java | 2 +-
.../cli/commands/TestUpgradeDowngradeCommand.java | 82 ++++-
.../test/resources/log4j-surefire-quiet.properties | 29 --
hudi-client/hudi-client-common/pom.xml | 20 +-
.../apache/hudi/client/BaseHoodieWriteClient.java | 12 +-
.../org/apache/hudi/config/HoodieCleanConfig.java | 10 +
.../org/apache/hudi/config/HoodieWriteConfig.java | 17 +
.../org/apache/hudi/io/HoodieAppendHandle.java | 6 +-
.../hudi/table/action/clean/CleanPlanner.java | 10 +-
.../action/compact/CompactionTriggerStrategy.java | 2 +
.../compact/ScheduleCompactionActionExecutor.java | 23 ++
.../restore/CopyOnWriteRestoreActionExecutor.java | 1 -
.../restore/MergeOnReadRestoreActionExecutor.java | 1 -
.../rollback/BaseRollbackActionExecutor.java | 10 +-
.../rollback/BaseRollbackPlanActionExecutor.java | 2 +-
.../CopyOnWriteRollbackActionExecutor.java | 3 +-
.../MergeOnReadRollbackActionExecutor.java | 3 +-
.../table/marker/MarkerBasedRollbackUtils.java | 40 +-
...andler.java => FiveToFourDowngradeHandler.java} | 15 +-
.../table/upgrade/FourToFiveUpgradeHandler.java | 74 ++++
.../hudi/table/upgrade/UpgradeDowngrade.java | 4 +
.../src/main/resources/log4j.properties | 23 --
.../hudi/callback/http/TestCallbackHttpClient.java | 14 +-
.../metrics/datadog/TestDatadogHttpClient.java | 17 +-
.../hudi/metrics/datadog/TestDatadogReporter.java | 3 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-client/hudi-flink-client/pom.xml | 14 +
.../hudi/table/action/commit/FlinkMergeHelper.java | 2 +-
.../src/main/resources/log4j-surefire.properties | 31 --
.../src/main/resources/log4j.properties | 23 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-client/hudi-java-client/pom.xml | 6 +
.../src/main/resources/log4j.properties | 23 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-client/hudi-spark-client/pom.xml | 30 ++
.../hudi/index/hbase/SparkHoodieHBaseIndex.java | 11 +-
.../apache/hudi/keygen/BuiltinKeyGenerator.java | 2 +-
.../table/upgrade/SparkUpgradeDowngradeHelper.java | 4 +-
.../src/main/resources/log4j.properties | 23 --
.../org/apache/hudi/AvroConversionUtils.scala | 9 +-
.../org/apache/hudi/HoodieConversionUtils.scala | 13 +
.../scala/org/apache/hudi/HoodieSparkUtils.scala | 226 +-----------
.../scala/org/apache/hudi/util/PathUtils.scala | 84 +++++
.../org/apache/hudi/util/SparkKeyGenUtils.scala | 60 +++
.../spark/sql/HoodieCatalystExpressionUtils.scala | 152 ++++++--
.../spark/sql/HoodieCatalystPlansUtils.scala | 5 -
.../spark/sql/execution/SQLConfInjectingRDD.scala | 61 +++
.../org/apache/spark/sql/hudi/SparkAdapter.scala | 2 +-
.../org/apache/hudi/client/TestClientRollback.java | 153 ++++++++
.../functional/TestHoodieBackedMetadata.java | 5 +-
.../TestHoodieClientOnCopyOnWriteStorage.java | 81 +++-
.../table/action/compact/TestInlineCompaction.java | 74 ++++
.../TestMergeOnReadRollbackActionExecutor.java | 19 +-
.../TestMarkerBasedRollbackStrategy.java | 22 ++
.../hudi/table/upgrade/TestUpgradeDowngrade.java | 92 ++++-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-common/pom.xml | 40 +-
.../java/org/apache/hudi/avro/HoodieAvroUtils.java | 31 +-
.../OverwriteNonDefaultsWithLatestAvroPayload.java | 8 +-
.../hudi/common/table/HoodieTableVersion.java | 6 +-
.../table/log/AbstractHoodieLogRecordReader.java | 2 +-
.../apache/hudi/common/util/CompactionUtils.java | 30 ++
.../hudi/common/util/DefaultSizeEstimator.java | 4 +-
.../common/util/HoodieRecordSizeEstimator.java | 4 +-
.../org/apache/hudi/common/util/MarkerUtils.java | 3 +-
.../hudi/common/util/PartitionPathEncodeUtils.java | 1 +
.../util/{SizeEstimator.java => ThreadUtils.java} | 26 +-
.../util/collection/ExternalSpillableMap.java | 2 +-
.../common/util/collection/RocksDBBasedMap.java | 4 +-
.../hudi/metadata/HoodieTableMetadataUtil.java | 2 +
.../org/apache/hudi/avro/TestHoodieAvroUtils.java | 8 +
...tOverwriteNonDefaultsWithLatestAvroPayload.java | 11 +-
.../view/TestPriorityBasedFileSystemView.java | 40 +-
.../testutils/minicluster/HdfsTestService.java | 12 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-examples/hudi-examples-flink/pom.xml | 14 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-examples/hudi-examples-java/pom.xml | 6 +
hudi-examples/hudi-examples-spark/pom.xml | 11 +
.../examples/quickstart/HoodieSparkQuickstart.java | 55 ++-
.../quickstart/TestHoodieSparkQuickstart.java | 30 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-flink-datasource/hudi-flink/pom.xml | 15 +-
.../apache/hudi/configuration/FlinkOptions.java | 117 +++---
.../hudi/configuration/OptionsInference.java | 65 ++++
.../hudi/sink/StreamWriteOperatorCoordinator.java | 5 +
.../hudi/sink/bulk/sort/SortOperatorGen.java | 5 +-
.../hudi/sink/clustering/ClusteringOperator.java | 1 +
.../hudi/sink/compact/CompactionPlanOperator.java | 9 +-
.../hudi/sink/compact/HoodieFlinkCompactor.java | 2 +
.../java/org/apache/hudi/sink/utils/Pipelines.java | 29 +-
.../apache/hudi/streamer/FlinkStreamerConfig.java | 13 +-
.../apache/hudi/streamer/HoodieFlinkStreamer.java | 11 +-
.../org/apache/hudi/table/HoodieTableFactory.java | 54 +--
.../org/apache/hudi/table/HoodieTableSink.java | 13 +-
.../org/apache/hudi/table/HoodieTableSource.java | 2 +
.../apache/hudi/table/catalog/HiveSchemaUtils.java | 31 +-
.../hudi/table/catalog/HoodieCatalogUtil.java | 20 +
.../hudi/table/catalog/HoodieHiveCatalog.java | 32 +-
.../hudi/table/catalog/TableOptionProperties.java | 8 +-
.../table/format/cow/CopyOnWriteInputFormat.java | 11 +-
.../table/format/mor/MergeOnReadInputFormat.java | 21 +-
.../java/org/apache/hudi/util/CompactionUtil.java | 14 +
.../java/org/apache/hudi/util/StreamerUtil.java | 28 +-
.../apache/hudi/sink/ITTestDataStreamWrite.java | 59 +--
.../sink/cluster/ITTestHoodieFlinkClustering.java | 4 +-
.../sink/compact/ITTestHoodieFlinkCompactor.java | 24 +-
.../apache/hudi/table/ITTestHoodieDataSource.java | 69 +++-
.../apache/hudi/table/TestHoodieTableFactory.java | 74 +++-
.../hudi/table/catalog/TestHoodieHiveCatalog.java | 28 +-
.../apache/hudi/table/format/TestInputFormat.java | 26 ++
.../org/apache/hudi/utils/TestStreamerUtil.java | 3 +
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-flink-datasource/hudi-flink1.13.x/pom.xml | 7 +-
.../table/data/ColumnarArrayData.java | 16 +-
.../table/data/ColumnarMapData.java | 4 +-
.../table/data/ColumnarRowData.java | 12 +-
.../table/data/vector/MapColumnVector.java | 3 +-
.../table/data/vector/RowColumnVector.java | 6 +-
.../table/data/vector/VectorizedColumnBatch.java | 14 +-
.../table/format/cow/ParquetSplitReaderUtil.java | 2 +-
.../table/format/cow/vector/HeapArrayVector.java | 3 +-
.../format/cow/vector/HeapMapColumnVector.java | 5 +-
.../format/cow/vector/HeapRowColumnVector.java | 7 +-
.../cow/vector/reader/ArrayColumnReader.java | 2 +-
.../reader/ParquetColumnarRowSplitReader.java | 4 +-
hudi-flink-datasource/hudi-flink1.14.x/pom.xml | 7 +-
hudi-flink-datasource/hudi-flink1.15.x/pom.xml | 7 +-
hudi-gcp/pom.xml | 21 +-
.../src/test/resources/log4j-surefire.properties | 29 --
hudi-hadoop-mr/pom.xml | 6 +
.../hadoop/HoodieCopyOnWriteTableInputFormat.java | 16 +
.../HoodieMergeOnReadTableInputFormat.java | 12 +
.../TestHoodieCombineHiveInputFormat.java | 11 +-
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-integ-test/pom.xml | 54 +--
.../org/apache/hudi/integ/ITTestHoodieDemo.java | 8 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-kafka-connect/pom.xml | 15 +-
.../src/main/resources/log4j.properties | 23 --
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-spark-datasource/hudi-spark-common/pom.xml | 6 +
.../scala/org/apache/hudi/DataSourceOptions.scala | 7 +-
.../main/scala/org/apache/hudi/DefaultSource.scala | 7 +-
.../scala/org/apache/hudi/HoodieBaseRelation.scala | 13 +-
.../scala/org/apache/hudi/HoodieCLIUtils.scala | 15 +
.../org/apache/hudi/HoodieSparkSqlWriter.scala | 9 +-
.../org/apache/hudi/HoodieStreamingSink.scala | 2 +-
.../scala/org/apache/hudi/HoodieWriterUtils.scala | 3 +-
.../org/apache/hudi/IncrementalRelation.scala | 122 +++---
.../hudi/MergeOnReadIncrementalRelation.scala | 68 +++-
.../apache/hudi/MergeOnReadSnapshotRelation.scala | 55 ---
.../sql/catalyst/catalog/HoodieCatalogTable.scala | 21 +-
.../apache/spark/sql/hudi/HoodieOptionConfig.scala | 3 +-
.../spark/sql/hudi/ProvidesHoodieConfig.scala | 5 +
.../hudi/command/CreateHoodieTableCommand.scala | 33 +-
.../sql/hudi/streaming/HoodieStreamSource.scala | 2 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-spark-datasource/hudi-spark/pom.xml | 35 +-
.../spark/sql/hudi/HoodieInternalRowUtils.scala | 5 +-
.../command/CreateHoodieTableAsSelectCommand.scala | 18 +-
.../hudi/command/MergeIntoHoodieTableCommand.scala | 6 +-
.../hudi/command/procedures/BaseProcedure.scala | 4 +-
.../procedures/CommitsCompareProcedure.scala | 3 +-
.../procedures/ExportInstantsProcedure.scala | 3 +-
.../hudi/command/procedures/HoodieProcedures.scala | 1 +
.../RollbackToInstantTimeProcedure.scala | 3 +-
.../command/procedures/RunCleanProcedure.scala | 28 +-
.../procedures/RunClusteringProcedure.scala | 13 +-
.../procedures/ShowArchivedCommitsProcedure.scala | 3 +-
.../procedures/ShowCommitFilesProcedure.scala | 3 +-
.../procedures/ShowCommitPartitionsProcedure.scala | 3 +-
.../procedures/ShowCommitWriteStatsProcedure.scala | 3 +-
.../command/procedures/ShowCommitsProcedure.scala | 3 +-
.../procedures/ShowInvalidParquetProcedure.scala | 83 +++++
.../src/test/java/HoodieJavaStreamingApp.java | 1 +
... => cow-updated2-column-stats-index-table.json} | 7 +-
... => mor-updated2-column-stats-index-table.json} | 7 +-
...-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json | 11 +
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
.../TestConvertFilterToCatalystExpression.scala | 19 +-
.../org/apache/hudi/TestDataSkippingUtils.scala | 7 +-
.../org/apache/hudi/TestHoodieSparkSqlWriter.scala | 34 +-
.../org/apache/hudi/TestHoodieSparkUtils.scala | 57 ---
.../apache/hudi/functional/TestCOWDataSource.scala | 153 ++++----
.../hudi/functional/TestColumnStatsIndex.scala | 217 ++++++-----
.../TestIncrementalReadWithFullTableScan.scala | 187 ++++++++++
.../hudi/functional/TestStructuredStreaming.scala | 164 +++++----
.../scala/org/apache/hudi/util/TestPathUtils.scala | 85 +++++
.../apache/spark/sql/hudi/TestCreateTable.scala | 134 ++++++-
.../spark/sql/hudi/TestDeleteFromTable.scala | 83 +++++
.../spark/sql/hudi/TestHoodieOptionConfig.scala | 4 +-
.../apache/spark/sql/hudi/TestMergeIntoTable.scala | 3 +-
.../spark/sql/hudi/TestMergeIntoTable2.scala | 94 ++++-
.../hudi/TestNestedSchemaPruningOptimization.scala | 1 -
.../procedure/HoodieSparkProcedureTestBase.scala} | 13 +-
.../hudi/procedure/TestBootstrapProcedure.scala | 3 +-
.../sql/hudi/procedure/TestCallProcedure.scala | 3 +-
.../sql/hudi/procedure/TestCleanProcedure.scala | 64 +++-
.../hudi/procedure/TestClusteringProcedure.scala | 3 +-
.../sql/hudi/procedure/TestCommitsProcedure.scala | 4 +-
.../hudi/procedure/TestCompactionProcedure.scala | 3 +-
.../hudi/procedure/TestCopyToTableProcedure.scala | 3 +-
.../procedure/TestExportInstantsProcedure.scala | 4 +-
.../sql/hudi/procedure/TestFsViewProcedure.scala | 4 +-
.../procedure/TestHdfsParquetImportProcedure.scala | 3 +-
.../procedure/TestHoodieLogFileProcedure.scala | 4 +-
.../sql/hudi/procedure/TestMetadataProcedure.scala | 4 +-
.../sql/hudi/procedure/TestRepairsProcedure.scala | 3 +-
.../hudi/procedure/TestSavepointsProcedure.scala | 4 +-
.../procedure/TestShowFsPathDetailProcedure.scala | 4 +-
...scala => TestShowInvalidParquetProcedure.scala} | 35 +-
.../sql/hudi/procedure/TestStatsProcedure.scala | 4 +-
.../TestUpgradeOrDowngradeProcedure.scala | 7 +-
hudi-spark-datasource/hudi-spark2/pom.xml | 6 +
.../sql/HoodieSpark2CatalystExpressionUtils.scala | 2 +-
.../spark/sql/HoodieSpark2CatalystPlanUtils.scala | 4 -
.../apache/spark/sql/adapter/Spark2Adapter.scala | 16 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-spark-datasource/hudi-spark3-common/pom.xml | 6 +
.../spark/sql/HoodieSpark3CatalystPlanUtils.scala | 4 -
.../spark/sql/adapter/BaseSpark3Adapter.scala | 10 +-
hudi-spark-datasource/hudi-spark3.1.x/pom.xml | 6 +
.../org/apache/spark/sql/avro/AvroSerializer.scala | 27 +-
.../sql/hudi/analysis/HoodieSpark3Analysis.scala | 11 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
hudi-spark-datasource/hudi-spark3.3.x/pom.xml | 6 +
.../apache/spark/sql/avro/AvroDeserializer.scala | 35 +-
.../org/apache/spark/sql/avro/AvroSerializer.scala | 121 ++++--
hudi-sync/hudi-adb-sync/pom.xml | 16 +-
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-sync/hudi-datahub-sync/pom.xml | 16 +-
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 29 --
hudi-sync/hudi-hive-sync/pom.xml | 14 +-
.../java/org/apache/hudi/hive/HiveSyncTool.java | 3 +-
.../org/apache/hudi/hive/ddl/HMSDDLExecutor.java | 5 +-
.../hudi/hive/ddl/QueryBasedDDLExecutor.java | 2 +-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 22 +-
.../replication/TestHiveSyncGlobalCommitTool.java | 3 +
.../apache/hudi/hive/testutils/HiveTestUtil.java | 3 +
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-sync/hudi-sync-common/pom.xml | 5 +-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 45 ++-
.../hudi/sync/common/TestHoodieSyncConfig.java | 62 +++-
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-tests-common/pom.xml | 71 ++++
.../resources/log4j2-surefire-quiet.properties | 24 +-
.../src/main/resources/log4j2-surefire.properties | 30 +-
hudi-timeline-service/pom.xml | 16 +-
.../timeline/service/handlers/MarkerHandler.java | 5 +-
.../marker/MarkerCreationDispatchingRunnable.java | 6 +-
.../service/handlers/marker/MarkerDirState.java | 2 +-
.../test/resources/log4j-surefire-quiet.properties | 29 --
.../src/test/resources/log4j-surefire.properties | 30 --
hudi-utilities/pom.xml | 33 +-
.../hudi/utilities/deltastreamer/DeltaSync.java | 20 +-
.../deltastreamer/HoodieDeltaStreamer.java | 9 +-
.../deltastreamer/SourceFormatAdapter.java | 17 +-
.../hudi/utilities/sources/PulsarSource.java | 297 +++++++++++++++
.../sources/S3EventsHoodieIncrSource.java | 65 ++--
.../utilities/sources/debezium/DebeziumSource.java | 8 +
.../utilities/sources/helpers/KafkaOffsetGen.java | 10 +-
.../functional/TestHoodieDeltaStreamer.java | 21 ++
.../src/test/resources/hive-site.xml | 21 +-
.../test/resources/log4j-surefire-quiet.properties | 30 --
.../src/test/resources/log4j-surefire.properties | 31 --
packaging/hudi-aws-bundle/pom.xml | 5 +
packaging/hudi-flink-bundle/pom.xml | 6 +-
packaging/hudi-kafka-connect-bundle/pom.xml | 1 -
packaging/hudi-timeline-server-bundle/pom.xml | 9 +-
packaging/hudi-utilities-bundle/pom.xml | 1 -
packaging/hudi-utilities-slim-bundle/pom.xml | 1 -
pom.xml | 282 +++++++++++---
rfc/README.md | 79 ++--
scripts/release/deploy_staging_jars.sh | 1 +
scripts/release/validate_staged_bundles.sh | 1 +
334 files changed, 5523 insertions(+), 3603 deletions(-)
delete mode 100644 docker/demo/config/log4j.properties
create mode 100644 docker/demo/config/log4j2.properties
delete mode 100644 hudi-aws/src/test/resources/log4j-surefire.properties
create mode 100644
hudi-cli/src/main/java/org/apache/hudi/cli/commands/TimelineCommand.java
delete mode 100644 hudi-cli/src/test/resources/log4j-surefire-quiet.properties
copy
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/{FourToThreeDowngradeHandler.java
=> FiveToFourDowngradeHandler.java} (66%)
create mode 100644
hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/FourToFiveUpgradeHandler.java
delete mode 100644
hudi-client/hudi-client-common/src/main/resources/log4j.properties
delete mode 100644
hudi-client/hudi-client-common/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-client/hudi-client-common/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-client/hudi-flink-client/src/main/resources/log4j-surefire.properties
delete mode 100644
hudi-client/hudi-flink-client/src/main/resources/log4j.properties
delete mode 100644
hudi-client/hudi-flink-client/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-client/hudi-java-client/src/main/resources/log4j.properties
delete mode 100644
hudi-client/hudi-java-client/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-client/hudi-spark-client/src/main/resources/log4j.properties
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/PathUtils.scala
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/hudi/util/SparkKeyGenUtils.scala
create mode 100644
hudi-client/hudi-spark-client/src/main/scala/org/apache/spark/sql/execution/SQLConfInjectingRDD.scala
delete mode 100644
hudi-client/hudi-spark-client/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-client/hudi-spark-client/src/test/resources/log4j-surefire.properties
copy hudi-common/src/main/java/org/apache/hudi/common/util/{SizeEstimator.java
=> ThreadUtils.java} (62%)
delete mode 100644
hudi-common/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644 hudi-common/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-examples/hudi-examples-flink/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-examples/hudi-examples-spark/src/test/resources/log4j-surefire.properties
create mode 100644
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/OptionsInference.java
delete mode 100644
hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-flink-datasource/hudi-flink/src/test/resources/log4j-surefire.properties
rename hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/{flink
=> hudi}/table/data/ColumnarArrayData.java (93%)
rename hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/{flink
=> hudi}/table/data/ColumnarMapData.java (94%)
rename hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/{flink
=> hudi}/table/data/ColumnarRowData.java (93%)
rename hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/{flink
=> hudi}/table/data/vector/MapColumnVector.java (90%)
rename hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/{flink
=> hudi}/table/data/vector/RowColumnVector.java (85%)
rename hudi-flink-datasource/hudi-flink1.13.x/src/main/java/org/apache/{flink
=> hudi}/table/data/vector/VectorizedColumnBatch.java (84%)
delete mode 100644 hudi-gcp/src/test/resources/log4j-surefire.properties
rename hudi-hadoop-mr/src/test/java/org/apache/hudi/hadoop/{functional =>
hive}/TestHoodieCombineHiveInputFormat.java (98%)
delete mode 100644
hudi-hadoop-mr/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644 hudi-hadoop-mr/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-integ-test/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644 hudi-integ-test/src/test/resources/log4j-surefire.properties
delete mode 100644 hudi-kafka-connect/src/main/resources/log4j.properties
delete mode 100644
hudi-kafka-connect/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-kafka-connect/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-spark-datasource/hudi-spark-common/src/test/resources/log4j-surefire.properties
create mode 100644
hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/procedures/ShowInvalidParquetProcedure.scala
copy
hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/{updated-column-stats-index-table.json
=> cow-updated2-column-stats-index-table.json} (61%)
copy
hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/{updated-column-stats-index-table.json
=> mor-updated2-column-stats-index-table.json} (61%)
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/resources/index/colstats/update-input-table-json/part-00000-7e680484-e7e1-48b6-8289-1a7c483b530b-c000.json
delete mode 100644
hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-spark-datasource/hudi-spark/src/test/resources/log4j-surefire.properties
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestIncrementalReadWithFullTableScan.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/util/TestPathUtils.scala
create mode 100644
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/TestDeleteFromTable.scala
copy
hudi-spark-datasource/hudi-spark/src/{main/scala/org/apache/spark/sql/hudi/command/procedures/ProcedureArgs.scala
=>
test/scala/org/apache/spark/sql/hudi/procedure/HoodieSparkProcedureTestBase.scala}
(75%)
copy
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/procedure/{TestShowFsPathDetailProcedure.scala
=> TestShowInvalidParquetProcedure.scala} (59%)
delete mode 100644
hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-spark-datasource/hudi-spark2/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-spark-datasource/hudi-spark3.2.x/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-sync/hudi-adb-sync/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-sync/hudi-datahub-sync/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-sync/hudi-hive-sync/src/test/resources/log4j-surefire.properties
delete mode 100644
hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-sync/hudi-sync-common/src/test/resources/log4j-surefire.properties
create mode 100644 hudi-tests-common/pom.xml
rename hudi-cli/src/test/resources/log4j-surefire.properties =>
hudi-tests-common/src/main/resources/log4j2-surefire-quiet.properties (64%)
rename hudi-gcp/src/test/resources/log4j-surefire-quiet.properties =>
hudi-tests-common/src/main/resources/log4j2-surefire.properties (58%)
delete mode 100644
hudi-timeline-service/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644
hudi-timeline-service/src/test/resources/log4j-surefire.properties
create mode 100644
hudi-utilities/src/main/java/org/apache/hudi/utilities/sources/PulsarSource.java
copy {hudi-flink-datasource/hudi-flink =>
hudi-utilities}/src/test/resources/hive-site.xml (73%)
delete mode 100644
hudi-utilities/src/test/resources/log4j-surefire-quiet.properties
delete mode 100644 hudi-utilities/src/test/resources/log4j-surefire.properties
