This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new dc732fe33f3 [bugfix](iceberg)clear block for partition values for 2.1
(#39569) (#39729)
dc732fe33f3 is described below
commit dc732fe33f3d420c785666c87a62a85c5cc4a3fa
Author: wuwenchi <wuwenchi...@hotmail.com>
AuthorDate: Thu Aug 22 22:43:02 2024 +0800
[bugfix](iceberg)clear block for partition values for 2.1 (#39569) (#39729)
## Proposed changes
bp: #39569
clear block, or we will get wrong partition values.
---
.../sink/writer/iceberg/viceberg_table_writer.cpp | 1 +
...test_iceberg_overwrite_with_wrong_partition.out | 23 ++++++
...t_iceberg_overwrite_with_wrong_partition.groovy | 84 ++++++++++++++++++++++
3 files changed, 108 insertions(+)
diff --git a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
index 4b705b0e51b..1572e7d96f2 100644
--- a/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
+++ b/be/src/vec/sink/writer/iceberg/viceberg_table_writer.cpp
@@ -161,6 +161,7 @@ Status VIcebergTableWriter::write(vectorized::Block& block)
{
{
SCOPED_RAW_TIMER(&_partition_writers_dispatch_ns);
+ _transformed_block.clear();
_transformed_block.reserve(_iceberg_partition_columns.size());
for (auto& iceberg_partition_columns : _iceberg_partition_columns) {
_transformed_block.insert(iceberg_partition_columns.partition_column_transform().apply(
diff --git
a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_overwrite_with_wrong_partition.out
b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_overwrite_with_wrong_partition.out
new file mode 100644
index 00000000000..b17bf8063c7
--- /dev/null
+++
b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_overwrite_with_wrong_partition.out
@@ -0,0 +1,23 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !qt01 --
+2450841 2450841
+2450841 2450841
+2450842 2450842
+2450842 2450842
+2450843 2450843
+2450843 2450843
+2450844 2450844
+2450844 2450844
+2450845 2450845
+2450845 2450845
+2450846 2450846
+2450846 2450846
+2450847 2450847
+2450847 2450847
+2450848 2450848
+2450848 2450848
+2450849 2450849
+2450849 2450849
+2450850 2450850
+2450850 2450850
+
diff --git
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_overwrite_with_wrong_partition.groovy
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_overwrite_with_wrong_partition.groovy
new file mode 100644
index 00000000000..760611ab3b4
--- /dev/null
+++
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_overwrite_with_wrong_partition.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_overwrite_with_wrong_partition",
"p0,external,doris,external_docker,external_docker_doris") {
+ String enabled = context.config.otherConfigs.get("enableIcebergTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable iceberg test.")
+ return
+ }
+
+ String tb1 = "tb_dst";
+ String tb2 = "tb_src";
+
+ try {
+ String rest_port =
context.config.otherConfigs.get("iceberg_rest_uri_port")
+ String minio_port =
context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+ String catalog_name = "test_iceberg_overwrite_with_wrong_partition"
+
+ sql """drop catalog if exists ${catalog_name}"""
+ sql """CREATE CATALOG ${catalog_name} PROPERTIES (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='rest',
+ 'uri' = 'http://${externalEnvIp}:${rest_port}',
+ "s3.access_key" = "admin",
+ "s3.secret_key" = "password",
+ "s3.endpoint" = "http://${externalEnvIp}:${minio_port}";,
+ "s3.region" = "us-east-1"
+ );"""
+
+ sql """ switch ${catalog_name} """
+ sql """ use multi_catalog """
+
+ sql """ drop table if exists ${tb1} """
+ sql """ drop table if exists ${tb2} """
+
+ sql """
+ create table ${tb1} (
+ id bigint,
+ id2 bigint
+ ) PARTITION BY LIST(id2)() ;
+ """
+ sql """
+ create table ${tb2} (
+ id bigint,
+ id2 bigint
+ );
+ """
+
+ sql """ insert into ${tb2} values (2450841,2450841),
(2450842,2450842); """
+ sql """ insert into ${tb2} values (2450843,2450843),
(2450844,2450844); """
+ sql """ insert into ${tb2} values (2450845,2450845),
(2450846,2450846); """
+ sql """ insert into ${tb2} values (2450847,2450847),
(2450848,2450848); """
+ sql """ insert into ${tb2} values (2450849,2450849),
(2450850,2450850); """
+ sql """ insert into ${tb2} values (2450841,2450841),
(2450842,2450842); """
+ sql """ insert into ${tb2} values (2450843,2450843),
(2450844,2450844); """
+ sql """ insert into ${tb2} values (2450845,2450845),
(2450846,2450846); """
+ sql """ insert into ${tb2} values (2450847,2450847),
(2450848,2450848); """
+ sql """ insert into ${tb2} values (2450849,2450849),
(2450850,2450850); """
+
+ sql """ insert overwrite table ${tb1} (id, id2) select id, id2 from
${tb2} where id2 >= 2450841 AND id2 < 2450851; """
+
+ order_qt_qt01 """ select * from ${tb1} """
+
+ } finally {
+ sql """ drop table if exists ${tb1} """
+ sql """ drop table if exists ${tb2} """
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
