This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 490e0ce699a [fix](paimon) Preserve Decimal scale during JNI schema
evolution (#68458)
490e0ce699a is described below
commit 490e0ce699a759fe220e313936af097ab9baa654
Author: Oliveira <[email protected]>
AuthorDate: Mon Sep 28 10:51:57 2026 +0800
[fix](paimon) Preserve Decimal scale during JNI schema evolution (#68458)
### What problem does this PR solve?
Issue Number: DORIS-29050
Related PR: None
Problem Summary: When a Paimon Decimal column evolves from DECIMAL(5,2)
to DECIMAL(6,3), the forced JNI scanner uses the destination Doris type
to decode the stored unscaled value. An old value such as 1.20 is
therefore interpreted as 0.120. The scanner now uses the precision and
scale from the physical Paimon DecimalType, so values written before and
after the schema evolution are decoded correctly.
### Release note
Fix incorrect Decimal values when reading Paimon files after Decimal
scale schema evolution with the forced JNI scanner.
### Check List (For Author)
- Test
- [x] Regression test: `test_paimon_decimal_scale_evolution` with
`force_jni_scanner=true`
- [x] Unit Test: `PaimonColumnValueTest`
- [ ] Manual test
- [ ] No need to test or manual test
- Behavior changed:
- [x] Yes. Paimon Decimal values are decoded using the physical schema
precision and scale.
- [ ] No
- Does this need documentation?
- [x] No
- [ ] Yes. <!-- Add document PR link here. -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label
---
.../org/apache/doris/paimon/PaimonColumnValue.java | 8 +-
.../apache/doris/paimon/PaimonColumnValueTest.java | 36 +++++++++
.../paimon/test_paimon_decimal_scale_evolution.out | 9 +++
.../test_paimon_decimal_scale_evolution.groovy | 87 ++++++++++++++++++++++
4 files changed, 139 insertions(+), 1 deletion(-)
diff --git
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
index 398f00a9552..f7e800f6a44 100644
---
a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
+++
b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java
@@ -27,6 +27,7 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.Timestamp;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.LocalZonedTimestampType;
import org.apache.paimon.types.MapType;
import org.apache.paimon.types.RowType;
@@ -147,7 +148,12 @@ public class PaimonColumnValue implements ColumnValue {
@Override
public BigDecimal getDecimal() {
- return record.getDecimal(idx, dorisType.getPrecision(),
dorisType.getScale()).toBigDecimal();
+ // Paimon decodes compact decimals from the physical unscaled value
using the
+ // precision and scale supplied here. During schema evolution those
values belong to
+ // the record's Paimon type, while dorisType is the destination type.
Passing the
+ // destination scale interprets an old value such as 1.20 as 0.120.
+ DecimalType decimalType = (DecimalType) dataType;
+ return record.getDecimal(idx, decimalType.getPrecision(),
decimalType.getScale()).toBigDecimal();
}
@Override
diff --git
a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java
b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java
index 9b5430f540a..43d84a7d6ba 100644
---
a/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java
+++
b/fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonColumnValueTest.java
@@ -21,6 +21,8 @@ import org.apache.doris.jni.spi.vec.ColumnType;
import org.apache.doris.jni.spi.vec.ColumnValue;
import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.DataGetters;
+import org.apache.paimon.data.Decimal;
import org.apache.paimon.data.GenericArray;
import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
@@ -30,6 +32,7 @@ import
org.apache.paimon.data.serializer.InternalRowSerializer;
import org.apache.paimon.types.ArrayType;
import org.apache.paimon.types.BigIntType;
import org.apache.paimon.types.DataType;
+import org.apache.paimon.types.DecimalType;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.LocalZonedTimestampType;
import org.apache.paimon.types.MapType;
@@ -40,6 +43,8 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Field;
+import java.lang.reflect.Proxy;
+import java.math.BigDecimal;
import java.time.Instant;
import java.time.LocalDateTime;
import java.util.ArrayList;
@@ -240,6 +245,37 @@ public class PaimonColumnValueTest {
localZonedValue.getDateTime());
}
+ @Test
+ public void testDecimalUsesPaimonScaleWhenSchemaEvolves() {
+ DataGetters record = (DataGetters) Proxy.newProxyInstance(
+ DataGetters.class.getClassLoader(), new Class<?>[]
{DataGetters.class},
+ (proxy, method, args) -> {
+ if (method.getName().equals("getDecimal")) {
+ long unscaled = (int) args[2] == 2 ? 120 : 1234;
+ return Decimal.fromUnscaledLong(unscaled, (int)
args[1], (int) args[2]);
+ }
+ if (method.getReturnType() == boolean.class) {
+ return false;
+ }
+ if (method.getReturnType() == byte.class ||
method.getReturnType() == short.class
+ || method.getReturnType() == int.class ||
method.getReturnType() == long.class) {
+ return 0;
+ }
+ if (method.getReturnType() == float.class ||
method.getReturnType() == double.class) {
+ return 0.0;
+ }
+ return null;
+ });
+ PaimonColumnValue value = new PaimonColumnValue(
+ record, 0, ColumnType.parseType("d", "decimal(6,3)"), new
DecimalType(5, 2), "UTC");
+
+ Assertions.assertEquals(new BigDecimal("1.20"), value.getDecimal());
+
+ PaimonColumnValue currentSchemaValue = new PaimonColumnValue(
+ record, 0, ColumnType.parseType("d", "decimal(6,3)"), new
DecimalType(6, 3), "UTC");
+ Assertions.assertEquals(new BigDecimal("1.234"),
currentSchemaValue.getDecimal());
+ }
+
private InternalRow nestedArrayRow(int outerSize, int innerSize, int
populatedIndex, int nullIndex) {
Object[] outerValues = new Object[outerSize];
for (int i = 0; i < outerSize; i++) {
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_decimal_scale_evolution.out
b/regression-test/data/external_table_p0/paimon/test_paimon_decimal_scale_evolution.out
new file mode 100644
index 00000000000..a572f72e286
--- /dev/null
+++
b/regression-test/data/external_table_p0/paimon/test_paimon_decimal_scale_evolution.out
@@ -0,0 +1,9 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !decimal_scale_evolution_v1 --
+1 1.200
+
+-- !decimal_scale_evolution_v2 --
+1 1.200
+
+-- !decimal_scale_evolution_jni --
+1 1.200
diff --git
a/regression-test/suites/external_table_p0/paimon/test_paimon_decimal_scale_evolution.groovy
b/regression-test/suites/external_table_p0/paimon/test_paimon_decimal_scale_evolution.groovy
new file mode 100644
index 00000000000..553f32bee01
--- /dev/null
+++
b/regression-test/suites/external_table_p0/paimon/test_paimon_decimal_scale_evolution.groovy
@@ -0,0 +1,87 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_paimon_decimal_scale_evolution", "p0,external,paimon") {
+ String enabled = context.config.otherConfigs.get("enablePaimonTest")
+ if (enabled == null || !enabled.equalsIgnoreCase("true")) {
+ logger.info("disable paimon test")
+ return
+ }
+
+ String minioPort =
context.config.otherConfigs.get("spark_paimon_minio_port")
+ ?: context.config.otherConfigs.get("iceberg_minio_port")
+ String externalEnvIp = context.config.otherConfigs.get("externalEnvIp")
+
+ sql "drop catalog if exists test_paimon_decimal_scale_evolution"
+ sql """
+ create catalog test_paimon_decimal_scale_evolution properties (
+ 'type'='paimon',
+ 'warehouse'='s3://warehouse/wh',
+ 's3.endpoint'='http://${externalEnvIp}:${minioPort}',
+ 's3.access_key'='admin',
+ 's3.secret_key'='password',
+ 's3.path.style.access'='true',
+ 'meta.cache.paimon.table.ttl-second'='0'
+ )
+ """
+
+ try {
+ spark_paimon_multi """
+ create database if not exists
paimon.paimon_decimal_scale_evolution_db;
+ drop table if exists
paimon.paimon_decimal_scale_evolution_db.decimal_scale_evolution;
+ create table
paimon.paimon_decimal_scale_evolution_db.decimal_scale_evolution (
+ id int, amount decimal(5,2)
+ ) using paimon tblproperties ('file.format'='parquet');
+ insert into
paimon.paimon_decimal_scale_evolution_db.decimal_scale_evolution
+ values (1, cast(1.20 as decimal(5,2)));
+ """
+
+ sql "select amount from
test_paimon_decimal_scale_evolution.paimon_decimal_scale_evolution_db.decimal_scale_evolution
limit 1"
+
+ spark_paimon_multi """
+ alter table
paimon.paimon_decimal_scale_evolution_db.decimal_scale_evolution
+ alter column amount type decimal(6,3);
+ """
+
+ sql "refresh catalog test_paimon_decimal_scale_evolution"
+ sql "set force_jni_scanner=false"
+ sql "set enable_file_scanner_v2=false"
+ order_qt_decimal_scale_evolution_v1 """
+ select id, amount
+ from
test_paimon_decimal_scale_evolution.paimon_decimal_scale_evolution_db.decimal_scale_evolution
+ order by id
+ """
+
+ sql "set enable_file_scanner_v2=true"
+ order_qt_decimal_scale_evolution_v2 """
+ select id, amount
+ from
test_paimon_decimal_scale_evolution.paimon_decimal_scale_evolution_db.decimal_scale_evolution
+ order by id
+ """
+
+ sql "set force_jni_scanner=true"
+ order_qt_decimal_scale_evolution_jni """
+ select id, amount
+ from
test_paimon_decimal_scale_evolution.paimon_decimal_scale_evolution_db.decimal_scale_evolution
+ order by id
+ """
+ } finally {
+ sql "set force_jni_scanner=false"
+ sql "set enable_file_scanner_v2=false"
+ sql "drop catalog if exists test_paimon_decimal_scale_evolution"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
