This is an automated email from the ASF dual-hosted git repository.

hello-stephen pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-4.2 by this push:
     new 97c99035a75 branch-4.2: [fix](cast) Declare Variant casts to 
non-Variant targets nullable (#70359)
97c99035a75 is described below

commit 97c99035a75f5b3b6bfc604bc7bd982601bb646e
Author: lihangyu <[email protected]>
AuthorDate: Tue Nov 17 15:15:26 2026 +0800

    branch-4.2: [fix](cast) Declare Variant casts to non-Variant targets 
nullable (#70359)
    
    ### What problem does this PR solve?
    
    Cherry-pick of #70365 (BE scan fix) to `branch-4.2`, together with the
    FE cast-nullability fix.
    
    Related PR: #70365
    
    This PR fixes two related problems for Variant casts on `branch-4.2`
    (the default legacy Variant
    representation):
    
    1. **FE type mismatch** (`declared=String, actual=Nullable(String)`):
    `Cast.castNullable` only
    declared a Variant -> non-Variant cast nullable when the Variant type
    used the V2 execution
    representation (`isExecutionV2()`). For the legacy representation it
    returned false, so the
    cast was serialized as non-nullable while the legacy kernel legitimately
    returns NULL for
    Variant nulls or non-convertible values. Direct casts and CTAS then
    failed with
       `Expr (...) return column type mismatch`.
    2. **Wrong results for null tests over Variant casts**: after fixing
    (1), `CAST(v AS INT) IS NULL`
    survives to the scan, where `_normalize_is_null_predicate` rewrites it
    into `v IS NULL` and
    drops rows whose Variant value is not convertible to INT (`{"a":1}`,
    `[1,2]`, ...). This is the
       same bug fixed on master in #70365 and included here as a backport.
    
    ### Fix
    
    - FE: reuse the existing `Cast.castNullable` rule to declare Variant ->
    non-Variant casts nullable
      for both representations. Identity casts keep the input nullability.
    - BE: keep the null test in the expression tree when the predicate's
    argument is a cast over a
      Variant source (backport of #70365).
    
    ### Tests
    
    - `CastTest#testCastFromVariant` updated (fails before, passes after).
    - New regression `variant_p0/test_variant_cast_isnull_pushdown` with a
    golden generated by the
    regression runner: `CAST(v AS INT)` values plus `IS NULL`, `IS NOT NULL`
    and `<=> NULL`
      filters over a `VARIANT NOT NULL` column.
    - Red/green verified end to end on a 4.2 FE+BE build:
    - before the BE fix: `WHERE CAST(v AS INT) IS NULL` returns no rows
    while the projection
        returns `0,1,1,1`;
    - after: the filter and `<=> NULL` return `2,3,4`, `IS NOT NULL` returns
    `1`;
    - CTAS `create table ... as select ... parse_to_variant(cast(v as
    string)) ...` succeeds after
        the FE fix (previously failed with the type mismatch).
    
    ### Release note
    
    Fix `return column type mismatch` when casting a NOT NULL Variant
    column, and fix wrong (empty)
    results of `CAST(variant_col AS T) IS NULL` / `<=> NULL` filters.
    
    ### Check List (For Author)
    
    - Test: Unit Test + Regression test + Manual end-to-end (4.2 FE+BE)
    - Behavior changed: Yes (casts are declared nullable and null filters
    return the correct rows)
    - Does this need documentation: No
---
 be/src/exec/operator/scan_operator.cpp             |  9 +++++
 .../doris/nereids/trees/expressions/Cast.java      | 12 +++----
 .../doris/nereids/trees/expressions/CastTest.java  | 12 ++++---
 .../test_variant_cast_isnull_pushdown.out          | 17 +++++++++
 .../test_variant_cast_isnull_pushdown.groovy       | 42 ++++++++++++++++++++++
 5 files changed, 81 insertions(+), 11 deletions(-)

diff --git a/be/src/exec/operator/scan_operator.cpp 
b/be/src/exec/operator/scan_operator.cpp
index 9152fe71adf..9c3424ef43d 100644
--- a/be/src/exec/operator/scan_operator.cpp
+++ b/be/src/exec/operator/scan_operator.cpp
@@ -946,6 +946,15 @@ Status ScanLocalStateBase::_normalize_is_null_predicate(
         return Status::OK();
     }
 
+    // A Variant cast can introduce NULLs for non-NULL stored values, so 
`CAST(v AS T) IS NULL`
+    // is not equivalent to `v IS NULL`. Keep the null test in the expression 
tree instead of
+    // pushing a NullPredicate down on the source Variant column.
+    if (root->children()[0]->node_type() == TExprNodeType::CAST_EXPR &&
+        
VExpr::expr_without_cast(root->children()[0])->data_type()->get_primitive_type()
 ==
+                PrimitiveType::TYPE_VARIANT) {
+        return Status::OK();
+    }
+
     auto fn_call = assert_cast<VectorizedFnCall*>(root.get());
     if (fn_call->fn().name.function_name == "is_null_pred") {
         pred = NullPredicate::create_shared(
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
index fd9d7138134..e78cb24f521 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java
@@ -36,7 +36,6 @@ import org.apache.doris.nereids.types.StructField;
 import org.apache.doris.nereids.types.StructType;
 import org.apache.doris.nereids.types.TimeStampNsType;
 import org.apache.doris.nereids.types.TinyIntType;
-import org.apache.doris.nereids.types.VariantType;
 
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
@@ -247,12 +246,11 @@ public class Cast extends Expression implements 
UnaryExpression, Monotonic {
         } else if (childDataType.isJsonType() && !targetType.isJsonType()) {
             // Json to other type is always nullable
             return true;
-        } else if (childDataType instanceof VariantType
-                && ((VariantType) childDataType).isExecutionV2() && 
!targetType.isVariantType()) {
-            // Variant V2 values can have a shape that is incompatible with 
the target type.
-            return true;
-        } else if (childDataType.isVariantType() && targetType.isJsonType()) {
-            // Legacy Variant to Json is always nullable.
+        } else if (childDataType.isVariantType() && 
!targetType.isVariantType()) {
+            // A Variant value can have a shape that is incompatible with the 
target type, and the
+            // legacy kernel may emit NULL for a non-NULL Variant input (it 
serializes through the
+            // nullable root/WRAP path), so the result is nullable for both 
execution
+            // representations. Keep this in sync with the BE columns the cast 
kernels produce.
             return true;
         }
         return false;
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
index 87fa64ee03a..da08ffa97ec 100644
--- 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/CastTest.java
@@ -679,11 +679,15 @@ public class CastTest {
         boolean originalEnableVariantV2 = Config.enable_variant_v2;
         try {
             Config.enable_variant_v2 = false;
-            Assertions.assertFalse(castNullable(false, VariantType.INSTANCE, 
BooleanType.INSTANCE));
-            Assertions.assertFalse(castNullable(false, VariantType.INSTANCE, 
IntegerType.INSTANCE));
-            Assertions.assertFalse(castNullable(false, VariantType.INSTANCE, 
StringType.INSTANCE));
+            // The legacy kernel serializes a Variant through the nullable 
root/WRAP path, so it can
+            // emit NULL for a non-NULL Variant input as well. The declared 
type must stay nullable
+            // for every non-Variant target, otherwise BE returns a nullable 
column against a
+            // non-nullable slot (VExpr rejects it with "return column type 
mismatch").
+            Assertions.assertTrue(castNullable(false, VariantType.INSTANCE, 
BooleanType.INSTANCE));
+            Assertions.assertTrue(castNullable(false, VariantType.INSTANCE, 
IntegerType.INSTANCE));
+            Assertions.assertTrue(castNullable(false, VariantType.INSTANCE, 
StringType.INSTANCE));
             Assertions.assertTrue(castNullable(false, VariantType.INSTANCE, 
JsonType.INSTANCE));
-            Assertions.assertFalse(castNullable(false, VariantType.INSTANCE, 
ArrayType.of(IntegerType.INSTANCE)));
+            Assertions.assertTrue(castNullable(false, VariantType.INSTANCE, 
ArrayType.of(IntegerType.INSTANCE)));
             Assertions.assertFalse(castNullable(false, VariantType.INSTANCE, 
VariantType.INSTANCE));
             Assertions.assertTrue(castNullable(true, VariantType.INSTANCE, 
VariantType.INSTANCE));
 
diff --git 
a/regression-test/data/variant_p0/test_variant_cast_isnull_pushdown.out 
b/regression-test/data/variant_p0/test_variant_cast_isnull_pushdown.out
new file mode 100644
index 00000000000..0ca199f6de2
--- /dev/null
+++ b/regression-test/data/variant_p0/test_variant_cast_isnull_pushdown.out
@@ -0,0 +1,17 @@
+-- This file is automatically generated. You should know what you did if you 
want to edit this
+-- !cast_int --
+1      1
+2      \N
+3      \N
+
+-- !is_null --
+2
+3
+
+-- !is_not_null --
+1
+
+-- !null_safe --
+2
+3
+
diff --git 
a/regression-test/suites/variant_p0/test_variant_cast_isnull_pushdown.groovy 
b/regression-test/suites/variant_p0/test_variant_cast_isnull_pushdown.groovy
new file mode 100644
index 00000000000..20287c01ac8
--- /dev/null
+++ b/regression-test/suites/variant_p0/test_variant_cast_isnull_pushdown.groovy
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_variant_cast_isnull_pushdown", "p0") {
+    // A Variant cast can turn a non-NULL Variant value into NULL, so `CAST(v 
AS INT) IS NULL`
+    // is not equivalent to `v IS NULL`. The scan node must keep this null 
test as an expression
+    // instead of pushing a null predicate down on the source Variant column.
+    sql "DROP TABLE IF EXISTS variant_cast_isnull_pushdown"
+    sql """
+        CREATE TABLE variant_cast_isnull_pushdown (
+            id INT NOT NULL,
+            v VARIANT NOT NULL
+        ) DUPLICATE KEY(id)
+        DISTRIBUTED BY HASH(id) BUCKETS 1
+        PROPERTIES("replication_num" = "1")
+    """
+    sql """
+        INSERT INTO variant_cast_isnull_pushdown VALUES
+            (1, '1'),
+            (2, '"x"'),
+            (3, '[1,2]')
+    """
+
+    order_qt_cast_int "SELECT id, CAST(v AS INT) FROM 
variant_cast_isnull_pushdown ORDER BY id"
+    order_qt_is_null "SELECT id FROM variant_cast_isnull_pushdown WHERE CAST(v 
AS INT) IS NULL ORDER BY id"
+    order_qt_is_not_null "SELECT id FROM variant_cast_isnull_pushdown WHERE 
CAST(v AS INT) IS NOT NULL ORDER BY id"
+    order_qt_null_safe "SELECT id FROM variant_cast_isnull_pushdown WHERE 
CAST(v AS INT) <=> NULL ORDER BY id"
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to