This is an automated email from the ASF dual-hosted git repository.
cambyzju pushed a change to branch auto-pick-51274-branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from e4548ae8b4d Merge branch 'branch-3.0' into auto-pick-51274-branch-3.0
add 4677e361337 branch-3.0: [chore](load) reduce log for data load #51172
(#51802)
add 97a8cd4db6d branch-3.0: [fix](memory-leak) skip libzip and libjvm
memory leak #51628 (#51631)
add 5970c82edc7 branch-3.0: [fix](filecache) fix
load_cache_info_into_memory crash #51684 (#51904)
add 945f776ac1d [cherry-pick](branch-30) fix split_by_regexp meet empty
string return error (#51293) (#51862)
add 66681cfe5cf branch-3.0: [feat](cloud) Support alter operation for
obj_info and s3 vault obj_info #51162 (#51685)
add 69912ecaafd branch-3.0: [fix](mow) remove delete bitmap when remove
unused rowsets (#50973) (#51871)
add 483440afa96 branch-3.0: [improve](auto partition) random choose dummy
partition location to enhance fault tolerance #51732 (#51885)
add f440b741fb5 branch-3.0: [fix](jdbc) make sure init the jdbc client
before using it (#51540)
add ec7314fd89a [feat](desc) add comment column in desc statement (#51047)
(#51916)
add abc190ae1c3 branch-3.0: [Fix](compile) Fix arm compile failure caused
by undefined symbol #51715 (#51926)
add 6f6335a2527 [ci](perf) adjust exsits meta and data path (#51912)
add 459632e0f4e [feat](jdbc) support custom function rules in catalog
properties (#51471) (#51918)
add 0ec6d154fe9 branch-3.0: [test](mv) Fix mv regression test is not
chosen stable by CBO #50690 (#51943)
add 3f733ceb791 branch-3.0: [improvement](cloud) enable s3 load from
volcano engine object storag… #51794 (#51909)
add d3148dced23 branch-3.0: [enhancement](cloud) monitor evict size of
file cache active gc #51197 (#51793)
add 75d29d8e104 branch-3.0: [fix](trash) Fix shadow variable causing
garbage scheduling time errors #51647 (#51687)
add a7dddf01183 branch-3.0: [fix](auto bucket)Set the estimated partition
size to 5G in non cloud #51258 (#51682)
add 542cc933588 branch-3.0: [chore](cloud) Add cloud meta mgr timeout
counter #51836 (#51882)
add b48e0c0d51e branch-3.0: [fix](alter) alter partition without
storage_policy property will also cancel storage_policy #51662 (#51779)
add 2251d1f89a3 branch-3.0: [fix](hudi catalog) Fix the Kerberos
authentication error when querying hudi table #51713 (#51902)
add c4884ad079c branch-3.0: [opt](storage) Add log and metric when
aws/azure sdk do retry operation #51485 (#51686)
add cec412a853d branch-3.0: [opt](cache) Reset initial capacity of all
caches after Cgroup memory limit changes (#51698)
add e757432bfd4 [bugfix](memleak) fix memory leak for tabletschema and
result cache (#51931) (#51952)
add 7df2b14a12a [cherry-pick](branch-30) support regexp_replace function
with ignore flag options (#50245) (#51917)
add 2d1de5558df [bugfix](memleak) fix memleak in arrow input stream
(#51929) (#51992) (#52039)
add d5423714f03 Merge branch 'branch-3.0' into auto-pick-51274-branch-3.0
No new revisions were added by this update.
Summary of changes:
be/src/cloud/cloud_cumulative_compaction.cpp | 4 +
be/src/cloud/cloud_delete_bitmap_action.cpp | 182 ---------
be/src/cloud/cloud_meta_mgr.cpp | 9 +-
be/src/cloud/cloud_tablet.cpp | 10 +-
be/src/common/config.cpp | 13 +-
be/src/common/config.h | 7 +
be/src/exprs/string_functions.cpp | 16 +-
be/src/exprs/string_functions.h | 3 +-
be/src/http/action/delete_bitmap_action.cpp | 213 ++++++++++
.../action/delete_bitmap_action.h} | 13 +-
be/src/io/cache/block_file_cache.cpp | 6 +
be/src/io/cache/block_file_cache.h | 8 +-
be/src/olap/compaction.cpp | 2 +
be/src/olap/lru_cache.cpp | 4 +
be/src/olap/olap_server.cpp | 4 +-
be/src/olap/storage_engine.cpp | 35 +-
be/src/olap/tablet.cpp | 2 +
be/src/olap/tablet_meta.cpp | 26 +-
be/src/olap/tablet_meta.h | 1 +
be/src/olap/tablet_schema.cpp | 21 +-
be/src/olap/tablet_schema.h | 3 -
be/src/runtime/cache/result_cache.h | 6 +-
be/src/runtime/exec_env_init.cpp | 1 +
be/src/runtime/memory/cache_manager.cpp | 7 +
be/src/runtime/memory/cache_manager.h | 2 +
be/src/runtime/memory/cache_policy.h | 1 +
be/src/runtime/memory/lru_cache_policy.h | 25 +-
be/src/runtime/tablets_channel.cpp | 9 +-
be/src/service/http_service.cpp | 21 +-
be/src/util/mem_info.cpp | 8 +-
be/src/util/s3_util.cpp | 36 +-
be/src/vec/common/arithmetic_overflow.h | 42 +-
.../exec/format/arrow/arrow_pip_input_stream.cpp | 2 +-
.../vec/exec/format/arrow/arrow_pip_input_stream.h | 3 +-
be/src/vec/exec/scan/new_olap_scanner.cpp | 2 +
be/src/vec/functions/function_regexp.cpp | 176 ++++++--
be/src/vec/functions/function_split_by_regexp.cpp | 3 -
be/src/vec/sink/writer/vtablet_writer.cpp | 4 +-
be/test/io/fs/azure_obj_storage_client_test.cpp | 18 +-
be/test/olap/lru_cache_test.cpp | 57 +++
cloud/src/common/config.h | 12 +-
cloud/src/meta-service/meta_service_http.cpp | 8 +-
cloud/src/meta-service/meta_service_resource.cpp | 131 ++++--
cloud/src/recycler/s3_accessor.cpp | 31 +-
cloud/test/meta_service_test.cpp | 301 ++++++++++++++
common/cpp/aws_logger.h | 2 +-
common/cpp/obj_retry_strategy.cpp | 38 +-
common/cpp/obj_retry_strategy.h | 8 +-
conf/be.conf | 9 +-
conf/lsan_suppr.conf | 3 +
.../main/java/org/apache/doris/common/Config.java | 8 +-
.../main/java/org/apache/doris/alter/Alter.java | 3 +-
.../org/apache/doris/analysis/DescribeStmt.java | 5 +
.../org/apache/doris/catalog/JdbcResource.java | 1 +
.../java/org/apache/doris/catalog/JdbcTable.java | 20 +
.../org/apache/doris/catalog/S3StorageVault.java | 6 +-
.../org/apache/doris/catalog/StorageVault.java | 6 +-
.../apache/doris/cloud/rpc/MetaServiceClient.java | 4 +
.../apache/doris/cloud/rpc/MetaServiceProxy.java | 4 +
.../org/apache/doris/cloud/storage/RemoteBase.java | 2 +
.../storage/TosRemote.java} | 32 +-
.../transaction/CloudGlobalTransactionMgr.java | 10 +-
.../doris/common/proc/IndexSchemaProcNode.java | 49 ++-
.../common/proc/RemoteIndexSchemaProcDir.java | 2 +-
.../common/proc/RemoteIndexSchemaProcNode.java | 2 +-
.../apache/doris/common/util/PropertyAnalyzer.java | 7 +
.../apache/doris/datasource/ExternalCatalog.java | 68 ++--
.../apache/doris/datasource/ExternalDatabase.java | 72 ++--
.../doris/datasource/ExternalFunctionRules.java | 287 +++++++++++++
.../doris/datasource/hudi/source/HudiScanNode.java | 9 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 12 +
.../doris/datasource/jdbc/JdbcExternalTable.java | 1 +
.../jdbc/source/JdbcFunctionPushDownRule.java | 84 ++--
.../doris/datasource/jdbc/source/JdbcScanNode.java | 10 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 6 +-
.../functions/scalar/RegexpReplace.java | 45 ++-
.../functions/scalar/RegexpReplaceOne.java | 45 ++-
.../org/apache/doris/planner/OlapTableSink.java | 15 +-
.../java/org/apache/doris/qe/SessionVariable.java | 9 +
.../ExternalFunctionPushDownRulesTest.java | 431 ++++++++++++++++++++
.../ExternalFunctionRewriteRulesTest.java | 443 +++++++++++++++++++++
...bleNameComparedLowercaseMetaCacheFalseTest.java | 1 -
gensrc/proto/cloud.proto | 1 +
.../test_mow_delete_unused_rowset_dm_docker.out} | Bin 204 -> 219 bytes
.../test_schema_change_add_key_column.csv.gz | Bin 0 -> 72233 bytes
.../test_schema_change_add_key_column1.csv.gz | Bin 0 -> 177558 bytes
.../test_external_and_internal_describe.out | Bin 0 -> 2972 bytes
.../mv_p0/test_dup_mv_plus/test_dup_mv_plus.out | Bin 436 -> 531 bytes
.../testAggQueryOnAggMV10.out | Bin 264 -> 349 bytes
.../testAggQueryOnAggMV11.out | Bin 229 -> 292 bytes
.../testAggQueryOnAggMV2/testAggQueryOnAggMV2.out | Bin 235 -> 300 bytes
.../testAggQueryOnAggMV3/testAggQueryOnAggMV3.out | Bin 241 -> 327 bytes
.../testAggQuqeryOnAggMV5.out | Bin 217 -> 301 bytes
.../testAggQuqeryOnAggMV7.out | Bin 217 -> 301 bytes
.../testAggregateMVCalcAggFunctionQuery.out | Bin 217 -> 301 bytes
.../testBitmapUnionInQuery.out | Bin 210 -> 261 bytes
.../testCountDistinctToBitmap.out | Bin 281 -> 332 bytes
.../testIncorrectMVRewriteInSubquery.out | Bin 189 -> 240 bytes
.../testIncorrectRewriteCountDistinct.out | Bin 192 -> 243 bytes
.../testJoinOnLeftProjectToJoin.out | Bin 129 -> 130 bytes
.../data/mv_p0/ut/testNDVToHll/testNDVToHll.out | Bin 218 -> 269 bytes
.../testOrderByQueryOnProjectView.out | Bin 225 -> 317 bytes
.../ut/testProjectionMV1/testProjectionMV1.out | Bin 260 -> 335 bytes
.../ut/testProjectionMV2/testProjectionMV2.out | Bin 224 -> 295 bytes
.../ut/testProjectionMV3/testProjectionMV3.out | Bin 227 -> 302 bytes
.../ut/testProjectionMV4/testProjectionMV4.out | Bin 220 -> 287 bytes
.../testSingleMVMultiUsage.out | Bin 217 -> 301 bytes
.../data/mv_p0/ut/testSubQuery/testSubQuery.out | Bin 241 -> 349 bytes
.../ut/testUnionDistinct/testUnionDistinct.out | Bin 225 -> 309 bytes
.../string_functions/test_split_by_regexp.out | Bin 987 -> 1387 bytes
.../pipeline/{p0 => nonConcurrent}/conf/be.conf | 10 +-
.../pipeline/{p0 => nonConcurrent}/conf/fe.conf | 6 +-
.../conf/regression-conf.groovy | 40 +-
regression-test/pipeline/performance/deploy.sh | 4 -
regression-test/pipeline/performance/prepare.sh | 4 -
.../pipeline/performance/run-clickbench.sh | 4 -
regression-test/pipeline/performance/run-load.sh | 4 -
...groovy => test_alter_s3_vault_with_role.groovy} | 55 ++-
.../node_mgr/test_ms_alter_obj_info.groovy | 206 ++++++++++
.../node_mgr/test_ms_alter_s3_vault.groovy | 237 +++++++++++
.../test_show_storage_policy_using.groovy | 13 +-
.../test_mow_compact_multi_segments.groovy | 253 ++++++++++++
.../test_mow_delete_unused_rowset_dm_docker.groovy | 231 +++++++++++
.../test_mow_stale_rowset_delete_bitmap.groovy | 249 ++++++++++++
.../jdbc/test_clickhouse_jdbc_catalog.groovy | 71 +++-
.../jdbc/test_jdbc_query_mysql.groovy | 134 ++++++-
.../jdbc/test_mysql_jdbc_catalog.groovy | 76 +++-
.../jdbc/test_oracle_jdbc_catalog.groovy | 72 +++-
.../test_external_and_internal_describe.groovy | 163 ++++++++
.../mv_p0/test_dup_mv_plus/test_dup_mv_plus.groovy | 6 +-
.../testAggQueryOnAggMV10.groovy | 6 +-
.../testAggQueryOnAggMV11.groovy | 6 +-
.../testAggQueryOnAggMV2.groovy | 5 +-
.../testAggQueryOnAggMV3.groovy | 6 +-
.../testAggQuqeryOnAggMV5.groovy | 6 +-
.../testAggQuqeryOnAggMV7.groovy | 7 +-
.../testAggTableCountDistinctInBitmapType.groovy | 5 +-
.../testAggregateMVCalcAggFunctionQuery.groovy | 6 +-
.../testBitmapUnionInQuery.groovy | 5 +-
.../testCountDistinctToBitmap.groovy | 5 +-
.../testIncorrectMVRewriteInSubquery.groovy | 5 +-
.../testIncorrectRewriteCountDistinct.groovy | 5 +-
.../testJoinOnLeftProjectToJoin.groovy | 10 +-
.../mv_p0/ut/testNDVToHll/testNDVToHll.groovy | 5 +-
.../testOrderByQueryOnProjectView.groovy | 6 +-
.../ut/testProjectionMV1/testProjectionMV1.groovy | 5 +-
.../ut/testProjectionMV2/testProjectionMV2.groovy | 5 +-
.../ut/testProjectionMV3/testProjectionMV3.groovy | 5 +-
.../ut/testProjectionMV4/testProjectionMV4.groovy | 5 +-
.../testSingleMVMultiUsage.groovy | 6 +-
.../mv_p0/ut/testSubQuery/testSubQuery.groovy | 6 +-
.../ut/testUnionDistinct/testUnionDistinct.groovy | 6 +-
.../mv/union_rewrite/usercase_union_rewrite.groovy | 2 +-
.../string_functions/test_split_by_regexp.groovy | 2 +
154 files changed, 4585 insertions(+), 631 deletions(-)
delete mode 100644 be/src/cloud/cloud_delete_bitmap_action.cpp
create mode 100644 be/src/http/action/delete_bitmap_action.cpp
rename be/src/{cloud/cloud_delete_bitmap_action.h =>
http/action/delete_bitmap_action.h} (78%)
copy
fe/fe-core/src/main/java/org/apache/doris/{load/sync/SyncDataConsumer.java =>
cloud/storage/TosRemote.java} (60%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java
copy
regression-test/data/{query_p0/sql_functions/conditional_functions/test_assert_true.out
=> compaction/test_mow_delete_unused_rowset_dm_docker.out} (61%)
create mode 100644
regression-test/data/compaction/test_schema_change_add_key_column.csv.gz
create mode 100644
regression-test/data/compaction/test_schema_change_add_key_column1.csv.gz
create mode 100644
regression-test/data/external_table_p0/test_external_and_internal_describe.out
copy regression-test/pipeline/{p0 => nonConcurrent}/conf/be.conf (93%)
copy regression-test/pipeline/{p0 => nonConcurrent}/conf/fe.conf (92%)
copy regression-test/pipeline/{p0 =>
nonConcurrent}/conf/regression-conf.groovy (81%)
copy regression-test/suites/aws_iam_role_p0/{test_s3_vault_with_role.groovy =>
test_alter_s3_vault_with_role.groovy} (58%)
create mode 100644
regression-test/suites/cloud_p0/node_mgr/test_ms_alter_obj_info.groovy
create mode 100644
regression-test/suites/cloud_p0/node_mgr/test_ms_alter_s3_vault.groovy
create mode 100644
regression-test/suites/compaction/test_mow_compact_multi_segments.groovy
create mode 100644
regression-test/suites/compaction/test_mow_delete_unused_rowset_dm_docker.groovy
create mode 100644
regression-test/suites/compaction/test_mow_stale_rowset_delete_bitmap.groovy
create mode 100644
regression-test/suites/external_table_p0/test_external_and_internal_describe.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
