This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9f01c9bef7b9022d84058cfbeb03cc4f0da33c36 Author: Gabor Kaszab <[email protected]> AuthorDate: Wed Jan 3 15:16:48 2024 +0100 IMPALA-12673: Table migration fails if partition contains '/' Due to Iceberg #7612 migrating a table to Iceberg resulted in incorrect data and stats if some of the string partition fields contained '/' character. As a result we deliberately rejected migrating such tables. Now that Impala uses an Iceberg version that has the fix we can allow migrating such tables too. Change-Id: I05b4ca44c7edb81cee6747f83a5bd82c5a4b5c44 Reviewed-on: http://gerrit.cloudera.org:8080/20845 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../apache/impala/analysis/ConvertTableToIcebergStmt.java | 14 -------------- .../iceberg-migrate-from-external-hdfs-tables.test | 15 ++++++++------- 2 files changed, 8 insertions(+), 21 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java b/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java index e5f00ea6b..b8b0ee95f 100644 --- a/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java +++ b/fe/src/main/java/org/apache/impala/analysis/ConvertTableToIcebergStmt.java @@ -120,20 +120,6 @@ public class ConvertTableToIcebergStmt extends StatementBase { "location may change"); } - // TODO: this is a temporary check until https://github.com/apache/iceberg/issues/7612 - // is fixed. - for (PrunablePartition partition : ((FeFsTable) table).getPartitions()) { - for (LiteralExpr partitionExpr : partition.getPartitionValues()) { - if (!partitionExpr.getType().isStringType()) continue; - String partitionValue = partitionExpr.getStringValue(); - if (partitionValue == null) continue; - if (partitionValue.contains("/")) { - throw new AnalysisException ("Can't migrate table with '/' in the partition " + - "values until Iceberg #7612 is fixed. '" + partitionValue + "'"); - } - } - } - createSubQueryStrings((FeFsTable) table); } diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test index a2918744f..9d3fde193 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-migrate-from-external-hdfs-tables.test @@ -28,8 +28,7 @@ insert into alltypes partition (year, month, date_col, date_string_col) year, month, cast(date_string_col as date format 'MM/DD/YY') as date_col, - # removing '/' until Iceberg issue #7612 is fixed - replace(date_string_col, "/", "") + date_string_col from functional.alltypes t; insert into alltypes partition (year, month, date_col, date_string_col) values (10000, true, 1, 2, 3, 4, 5.1, 6.2, "str", "2023-05-01 01:02:03", 2023, 5, @@ -319,6 +318,7 @@ create table special_chars (i int) partitioned by (s string) stored as parquet; insert into special_chars partition (s='11 22-33&44%55"') values (1); insert into special_chars partition (s='aa - bb') values (2); insert into special_chars partition (s=null) values (3); +insert into special_chars partition (s='11/22/33') values (4); alter table special_chars convert to iceberg; ---- RESULTS 'Table has been migrated.' @@ -329,6 +329,7 @@ select * from special_chars; 1,'11 22-33&44%55"' 2,'aa - bb' 3,'NULL' +4,'11/22/33' ---- TYPES int, string ==== @@ -347,9 +348,9 @@ select * from special_chars where s is null; int, string ==== ---- QUERY -create table special_chars_with_slash (i int) partitioned by (s1 string, s2 string) stored as parquet; -insert into special_chars_with_slash partition (s1='abcde', s2='11/22/33') values (1); -alter table special_chars_with_slash convert to iceberg; ----- CATCH -AnalysisException: Can't migrate table with '/' in the partition values until Iceberg #7612 is fixed. '11/22/33' +select * from special_chars where s='11/22/33'; +---- RESULTS +4,'11/22/33' +---- TYPES +INT,STRING ====
