This is an automated email from the ASF dual-hosted git repository. stigahuang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 0549d9562bce876d93d9bfe188e828fa29c056f6 Author: noemi <[email protected]> AuthorDate: Mon Nov 7 15:26:42 2022 +0100 IMPALA-11620: Enable setting 'write.format.default' Enable setting 'write.format.default' to a different file format than what the table already contains. Before IMPALA-10610 Iceberg tables with mixed-format data files were not supported. We used 'write.format.default' to determine the file format of the table, which was only a temporary workaround. Because of this we did not allow changing this table property if the table already contained different table formats. E.g. we did not allow modifying 'write.format.default' to PARQUET if the table already contained ORC files, because it would have made the table unreadable for Impala. Since IMPALA-10610 'write.format.default' is not used to determine the Iceberg table's format anymore, so we can allow changing it. This table property change is not synchronized between HMS and Iceberg metadata files in case of true external Hive Catalog tables. See IMPALA-11710. Testing: - E2E test in iceberg-alter.test Change-Id: I22d0a8a18fce99015fcfe1fd15cb4a4d4c2deaec Reviewed-on: http://gerrit.cloudera.org:8080/19221 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../analysis/AlterTableSetTblProperties.java | 22 ---------------------- .../queries/QueryTest/iceberg-alter.test | 6 +++--- 2 files changed, 3 insertions(+), 25 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java index 411eca20e..85ed35108 100644 --- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java +++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java @@ -179,28 +179,6 @@ public class AlterTableSetTblProperties extends AlterTableSetStmt { if (IcebergUtil.getIcebergFileFormat(fileformat) == null) { throw new AnalysisException("Invalid fileformat for Iceberg table: " + fileformat); } - try { - FeIcebergTable iceTable = (FeIcebergTable)getTargetTable(); - List<DataFile> dataFiles = IcebergUtil.getIcebergFiles(iceTable, - new ArrayList<>(), /*timeTravelSpec=*/null).first; - if (dataFiles.isEmpty()) return; - DataFile firstFile = dataFiles.get(0); - String errorMsg = "Attempt to set Iceberg data file format to %s, but found data " + - "file %s with file format %s."; - if (!firstFile.format().name().equalsIgnoreCase(fileformat)) { - throw new AnalysisException(String.format(errorMsg, fileformat, firstFile.path(), - firstFile.format().name())); - } - //TODO(IMPALA-10610): Iceberg tables with mixed file formats are not readable. - for (DataFile df : dataFiles) { - if (df.format() != firstFile.format()) { - throw new AnalysisException(String.format(errorMsg, fileformat, df.path(), - df.format().name())); - } - } - } catch (TableLoadingException e) { - throw new AnalysisException(e); - } } private void icebergParquetCompressionCodecCheck() throws AnalysisException { diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test index df2e7adb9..b653199d0 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test +++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-alter.test @@ -174,13 +174,13 @@ INT ==== ---- QUERY ALTER TABLE iceberg_changing_fileformats set TBLPROPERTIES('write.format.default'='ORC'); ----- CATCH -Attempt to set Iceberg data file format to ORC +---- RESULTS: VERIFY_IS_SUBSET +'Updated table.' ==== ---- QUERY DESCRIBE FORMATTED iceberg_changing_fileformats; ---- RESULTS: VERIFY_IS_SUBSET -'','write.format.default','parquet ' +'','write.format.default','ORC ' ---- TYPES string, string, string ====
