This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 639c9a039b9 [cherry-pick](branch-30)add some check for udf when result
is null (#51084) (#51198)
639c9a039b9 is described below
commit 639c9a039b936bc7d6826d0ff15d6c3d5e1164f2
Author: zhangstar333 <[email protected]>
AuthorDate: Sat May 24 10:38:20 2025 +0800
[cherry-pick](branch-30)add some check for udf when result is null (#51084)
(#51198)
Problem Summary:
cherry-pick from master (#51084)
---
.../apache/doris/common/jni/vec/VectorColumn.java | 44 ++++++++++++++++++++++
.../main/java/org/apache/doris/udf/StringTest.java | 3 ++
.../suites/javaudf_p0/test_javaudf_array.groovy | 12 +++++-
.../suites/javaudf_p0/test_javaudf_int.groovy | 26 +++++++++++++
.../suites/javaudf_p0/test_javaudf_string.groovy | 13 +++++++
5 files changed, 97 insertions(+), 1 deletion(-)
diff --git
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
index 1542174cc1c..b851afeebb5 100644
---
a/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
+++
b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
@@ -310,6 +310,16 @@ public class VectorColumn {
}
}
+ public void checkNullable(Object[] batch, int rows) {
+ for (int i = 0; i < rows; ++i) {
+ if (batch[i] == null) {
+ throw new RuntimeException(
+ "the result of " + i + " row is null, but the return
type is not nullable, please check "
+ + "the always_nullable property in create
function statement, it's should be true");
+ }
+ }
+ }
+
public final boolean isNullAt(int rowId) {
if (numNulls == 0 || nullMap == 0) {
return false;
@@ -410,6 +420,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = (byte) (batch[i] ? 1 : 0);
}
@@ -467,6 +478,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
@@ -524,6 +536,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
@@ -581,6 +594,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
@@ -638,6 +652,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
@@ -695,6 +710,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
@@ -752,6 +768,7 @@ public class VectorColumn {
}
OffHeap.UNSAFE.copyMemory(batchNulls, OffHeap.BYTE_ARRAY_OFFSET,
null, nullMap + appendIndex, rows);
} else {
+ checkNullable(batch, rows);
for (int i = 0; i < rows; ++i) {
batchData[i] = batch[i];
}
@@ -793,6 +810,9 @@ public class VectorColumn {
}
public void appendBigInteger(BigInteger[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
for (BigInteger v : batch) {
if (v == null) {
@@ -839,6 +859,9 @@ public class VectorColumn {
}
public void appendDecimal(BigDecimal[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
for (BigDecimal v : batch) {
if (v == null) {
@@ -885,6 +908,9 @@ public class VectorColumn {
}
public void appendDate(LocalDate[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
for (LocalDate v : batch) {
if (v == null) {
@@ -951,6 +977,9 @@ public class VectorColumn {
}
public void appendDateTime(LocalDateTime[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
for (LocalDateTime v : batch) {
if (v == null) {
@@ -1052,6 +1081,9 @@ public class VectorColumn {
}
public void appendStringAndOffset(String[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
for (String v : batch) {
byte[] bytes;
@@ -1068,6 +1100,9 @@ public class VectorColumn {
}
public void appendBinaryAndOffset(byte[][] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
for (byte[] v : batch) {
byte[] bytes = v;
@@ -1121,6 +1156,9 @@ public class VectorColumn {
}
public void appendArray(List<Object>[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
int offset = childColumns[0].appendIndex;
for (List<Object> v : batch) {
@@ -1181,6 +1219,9 @@ public class VectorColumn {
}
public void appendMap(Map<Object, Object>[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
int offset = childColumns[0].appendIndex;
for (Map<Object, Object> v : batch) {
@@ -1247,6 +1288,9 @@ public class VectorColumn {
}
public void appendStruct(Map<String, Object>[] batch, boolean isNullable) {
+ if (!isNullable) {
+ checkNullable(batch, batch.length);
+ }
reserve(appendIndex + batch.length);
Object[][] columnData = new Object[childColumns.length][];
Preconditions.checkArgument(this.getColumnType().getChildNames().size() ==
childColumns.length);
diff --git
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java
index cc1a6a2bca7..822c484c706 100644
---
a/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java
+++
b/regression-test/java-udf-src/src/main/java/org/apache/doris/udf/StringTest.java
@@ -22,6 +22,9 @@ import org.apache.hadoop.hive.ql.exec.UDF;
public class StringTest extends UDF {
public String evaluate(String field, Integer a, Integer b) {
+ if (field == null || a == null || b == null) {
+ return null;
+ }
return field.substring(0, a) + StringUtils.repeat("*", field.length()
- a -b) + field.substring(field.length()-b);
}
}
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_array.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_array.groovy
index 0d782c036b4..4cae4e872f9 100644
--- a/regression-test/suites/javaudf_p0/test_javaudf_array.groovy
+++ b/regression-test/suites/javaudf_p0/test_javaudf_array.groovy
@@ -124,9 +124,19 @@ suite("test_javaudf_array") {
"type"="JAVA_UDF"
); """
qt_select_14 """ SELECT java_udf_array_list_test(array(string_col)),
string_col, tinyint_col as result FROM ${tableName} ORDER BY result; """
-
+ sql """ CREATE FUNCTION
java_udf_array_list_test_not_nullable(array<string>) RETURNS array<string>
PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.ArrayListTest",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+ test {
+ sql """ SELECT java_udf_array_list_test_not_nullable(NULL); """
+ exception "but the return type is not nullable"
+ }
} finally {
try_sql("DROP FUNCTION IF EXISTS java_udf_array_int_test(array<int>);")
+ try_sql("DROP FUNCTION IF EXISTS
java_udf_array_list_test_not_nullable(array<string>);")
try_sql("DROP FUNCTION IF EXISTS
java_udf_array_return_int_test(array<int>);")
try_sql("DROP FUNCTION IF EXISTS
java_udf_array_return_string_test(array<string>);")
try_sql("DROP FUNCTION IF EXISTS
java_udf_array_string_test(array<string>);")
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_int.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_int.groovy
index cb9b87b7bf0..7433866a9f2 100644
--- a/regression-test/suites/javaudf_p0/test_javaudf_int.groovy
+++ b/regression-test/suites/javaudf_p0/test_javaudf_int.groovy
@@ -123,6 +123,30 @@ suite("test_javaudf_int") {
qt_select_global_3 """ SELECT java_udf_int_test_global(3) result FROM
${tableName} ORDER BY result; """
qt_select_global_4 """ SELECT abs(java_udf_int_test_global(3)) result
FROM ${tableName} ORDER BY result; """
+ sql """ CREATE FUNCTION java_udf_int_test_not_nullable(int) RETURNS
int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.IntTest",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ test {
+ sql """ SELECT java_udf_int_test_not_nullable(NULL); """
+ exception "but the return type is not nullable"
+ }
+
+ sql """ CREATE FUNCTION java_udf_largeint_test_not_nullable(largeint)
RETURNS largeint PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.LargeintTest",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ test {
+ sql """ SELECT java_udf_largeint_test_not_nullable(NULL); """
+ exception "but the return type is not nullable"
+ }
+
} finally {
try_sql("DROP GLOBAL FUNCTION IF EXISTS
java_udf_int_test_global(int);")
try_sql("DROP FUNCTION IF EXISTS java_udf_tinyint_test(tinyint);")
@@ -130,6 +154,8 @@ suite("test_javaudf_int") {
try_sql("DROP FUNCTION IF EXISTS java_udf_bigint_test(bigint);")
try_sql("DROP FUNCTION IF EXISTS java_udf_largeint_test(largeint);")
try_sql("DROP FUNCTION IF EXISTS java_udf_int_test(int);")
+ try_sql("DROP FUNCTION IF EXISTS java_udf_int_test_not_nullable(int);")
+ try_sql("DROP FUNCTION IF EXISTS
java_udf_largeint_test_not_nullable(largeint);")
try_sql("DROP TABLE IF EXISTS ${tableName}")
}
}
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
index e6484a1fde1..f309b88feb8 100644
--- a/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
+++ b/regression-test/suites/javaudf_p0/test_javaudf_string.groovy
@@ -98,8 +98,21 @@ suite("test_javaudf_string") {
}
sql """ insert into tbl1 select random()%10000 * 10000, "5" from
tbl1;"""
qt_select_5 """ select count(0) from (select k1, max(k2) as k2 from
tbl1 group by k1)v where java_udf_string_test(k2, 0, 1) = "asd" """;
+
+ sql """ CREATE FUNCTION java_udf_string_test_not_nullabel(string, int,
int) RETURNS string PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.StringTest",
+ "always_nullable"="false",
+ "type"="JAVA_UDF"
+ ); """
+
+ test {
+ sql """ SELECT java_udf_string_test_not_nullabel(NULL,NULL,NULL);
"""
+ exception "but the return type is not nullable"
+ }
} finally {
try_sql("DROP FUNCTION IF EXISTS java_udf_string_test(string, int,
int);")
+ try_sql("DROP FUNCTION IF EXISTS
java_udf_string_test_not_nullabel(string, int, int);")
try_sql("DROP TABLE IF EXISTS ${tableName}")
try_sql("DROP TABLE IF EXISTS tbl1")
try_sql("DROP TABLE IF EXISTS test_javaudf_string_2")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
