This is an automated email from the ASF dual-hosted git repository. lide pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new 59d8403c239 [branch-2.0](prepared statement) Handle unsigned numeric type in prepared statement in legacy planner (#36376) 59d8403c239 is described below commit 59d8403c239eddd6b55d3b2809ffe7e9f37047c8 Author: xy720 <22125576+xy...@users.noreply.github.com> AuthorDate: Tue Jun 18 19:05:21 2024 +0800 [branch-2.0](prepared statement) Handle unsigned numeric type in prepared statement in legacy planner (#36376) --- .../org/apache/doris/analysis/DateLiteral.java | 2 +- .../org/apache/doris/analysis/DecimalLiteral.java | 2 +- .../org/apache/doris/analysis/FloatLiteral.java | 2 +- .../java/org/apache/doris/analysis/IntLiteral.java | 9 +- .../org/apache/doris/analysis/LargeIntLiteral.java | 5 ++ .../org/apache/doris/analysis/LiteralExpr.java | 97 +++++++++++----------- .../org/apache/doris/analysis/PlaceHolderExpr.java | 25 +++++- .../org/apache/doris/analysis/PrepareStmt.java | 1 + .../org/apache/doris/analysis/StringLiteral.java | 2 +- .../apache/doris/common/util/ByteBufferUtil.java | 34 ++++++++ .../java/org/apache/doris/planner/ScanNode.java | 4 +- .../java/org/apache/doris/qe/ConnectProcessor.java | 3 +- .../doris/rewrite/RewriteInPredicateRule.java | 7 +- regression-test/data/insert_p0/prepare_insert.out | 1 + .../suites/insert_p0/prepare_insert.groovy | 17 +++- .../suites/prepared_stmt_p0/prepared_stmt.groovy | 2 +- 16 files changed, 149 insertions(+), 64 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 9baa583f115..a4908e37b2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -1893,7 +1893,7 @@ public class DateLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { int len = getParmLen(data); if (type.getPrimitiveType() == PrimitiveType.DATE) { if (len >= 4) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java index 3a3a8f8f66a..f568cadd1d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DecimalLiteral.java @@ -408,7 +408,7 @@ public class DecimalLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { int len = getParmLen(data); BigDecimal v = null; try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java index bbcfafb2890..55925821bda 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FloatLiteral.java @@ -271,7 +271,7 @@ public class FloatLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { if (type.getPrimitiveType() == PrimitiveType.FLOAT) { value = data.getFloat(); return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java index 88fe549a047..dfefae8f7db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IntLiteral.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.NotImplementedException; +import org.apache.doris.common.util.ByteBufferUtil; import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TExprNode; import org.apache.doris.thrift.TExprNodeType; @@ -364,19 +365,19 @@ public class IntLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { switch (type.getPrimitiveType()) { case TINYINT: value = data.get(); break; case SMALLINT: - value = data.getChar(); + value = !isUnsigned ? data.getChar() : ByteBufferUtil.getUnsignedByte(data); break; case INT: - value = data.getInt(); + value = !isUnsigned ? data.getInt() : ByteBufferUtil.getUnsignedShort(data); break; case BIGINT: - value = data.getLong(); + value = !isUnsigned ? data.getLong() : ByteBufferUtil.getUnsignedInt(data); break; default: Preconditions.checkState(false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java index 9d191797048..c13bc9171c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LargeIntLiteral.java @@ -261,4 +261,9 @@ public class LargeIntLiteral extends LiteralExpr { public int hashCode() { return 31 * super.hashCode() + Objects.hashCode(value); } + + @Override + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { + value = new BigInteger(Long.toUnsignedString(data.getLong())); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java index 3d8c7f4f06d..6ee4bd9a417 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java @@ -320,54 +320,57 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr return getStringValue(); } - public static LiteralExpr getLiteralByMysqlType(int mysqlType) throws AnalysisException { - switch (mysqlType) { - // MYSQL_TYPE_TINY - case 1: - return LiteralExpr.create("0", Type.TINYINT); - // MYSQL_TYPE_SHORT - case 2: - return LiteralExpr.create("0", Type.SMALLINT); - // MYSQL_TYPE_LONG - case 3: - return LiteralExpr.create("0", Type.INT); - // MYSQL_TYPE_LONGLONG - case 8: - return LiteralExpr.create("0", Type.BIGINT); - // MYSQL_TYPE_FLOAT - case 4: - return LiteralExpr.create("0", Type.FLOAT); - // MYSQL_TYPE_DOUBLE - case 5: - return LiteralExpr.create("0", Type.DOUBLE); - // MYSQL_TYPE_DECIMAL - case 0: - // MYSQL_TYPE_NEWDECIMAL - case 246: - return LiteralExpr.create("0", Type.DECIMAL32); - // MYSQL_TYPE_TIME - case 11: - return LiteralExpr.create("", Type.TIME); - // MYSQL_TYPE_DATE - case 10: - return LiteralExpr.create("1970-01-01", Type.DATE); - // MYSQL_TYPE_DATETIME - case 12: - // MYSQL_TYPE_TIMESTAMP - case 7: - // MYSQL_TYPE_TIMESTAMP2 - case 17: - return LiteralExpr.create("1970-01-01 00:00:00", Type.DATETIME); - // MYSQL_TYPE_STRING - case 254: - case 253: - return LiteralExpr.create("", Type.STRING); - // MYSQL_TYPE_VARCHAR - case 15: - return LiteralExpr.create("", Type.VARCHAR); + public static LiteralExpr getLiteralByMysqlType(int mysqlType, boolean isUnsigned) throws AnalysisException { + LiteralExpr literalExpr = null; + + // If this is an unsigned numeric type, we convert it by using larger data types. For example, we can use + // small int to represent unsigned tiny int (0-255), big int to represent unsigned ints (0-2 ^ 32-1), + // and so on. + switch (mysqlType & 0xFF) { + case 1: // MYSQL_TYPE_TINY + literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.TINYINT : Type.SMALLINT); + break; + case 2: // MYSQL_TYPE_SHORT + literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.SMALLINT : Type.INT); + break; + case 3: // MYSQL_TYPE_LONG + literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.INT : Type.BIGINT); + break; + case 8: // MYSQL_TYPE_LONGLONG + literalExpr = LiteralExpr.create("0", !isUnsigned ? Type.BIGINT : Type.LARGEINT); + break; + case 4: // MYSQL_TYPE_FLOAT + literalExpr = LiteralExpr.create("0", Type.FLOAT); + break; + case 5: // MYSQL_TYPE_DOUBLE + literalExpr = LiteralExpr.create("0", Type.DOUBLE); + break; + case 0: // MYSQL_TYPE_DECIMAL + case 246: // MYSQL_TYPE_NEWDECIMAL + literalExpr = LiteralExpr.create("0", Type.DECIMAL32); + break; + case 11: // MYSQL_TYPE_TIME + literalExpr = LiteralExpr.create("", Type.TIME); + break; + case 10: // MYSQL_TYPE_DATE + literalExpr = LiteralExpr.create("1970-01-01", Type.DATE); + break; + case 12: // MYSQL_TYPE_DATETIME + case 7: // MYSQL_TYPE_TIMESTAMP + case 17: // MYSQL_TYPE_TIMESTAMP2 + literalExpr = LiteralExpr.create("1970-01-01 00:00:00", Type.DATETIME); + break; + case 254: // MYSQL_TYPE_STRING + case 253: // MYSQL_TYPE_VAR_STRING + literalExpr = LiteralExpr.create("", Type.STRING); + break; + case 15: // MYSQL_TYPE_VARCHAR + literalExpr = LiteralExpr.create("", Type.VARCHAR); + break; default: - return null; + throw new AnalysisException("Unsupported MySQL type: " + mysqlType); } + return literalExpr; } // Port from mysql get_param_length @@ -434,7 +437,7 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr // Parse from binary data, the format follows mysql binary protocal // see https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html. // Return next offset - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { Preconditions.checkState(false, "should implement this in derived class. " + this.type.toSql()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java index 55cf1f1bba0..0114a3252aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PlaceHolderExpr.java @@ -48,6 +48,7 @@ public class PlaceHolderExpr extends LiteralExpr { protected PlaceHolderExpr(LiteralExpr literal) { this.lExpr = literal; + this.type = literal.getType(); } protected PlaceHolderExpr(PlaceHolderExpr other) { @@ -59,9 +60,23 @@ public class PlaceHolderExpr extends LiteralExpr { this.type = literal.getType(); } + public LiteralExpr getLiteral() { + return lExpr; + } + + @Override + protected void analysisDone() { + if (lExpr != null && !lExpr.isAnalyzed) { + lExpr.analysisDone(); + } + if (!isAnalyzed) { + super.analysisDone(); + } + } + public LiteralExpr createLiteralFromType() throws AnalysisException { Preconditions.checkState(mysqlTypeCode > 0); - return LiteralExpr.getLiteralByMysqlType(mysqlTypeCode); + return LiteralExpr.getLiteralByMysqlType(mysqlTypeCode, isUnsigned()); } public static PlaceHolderExpr create(String value, Type type) throws AnalysisException { @@ -88,6 +103,10 @@ public class PlaceHolderExpr extends LiteralExpr { return lExpr.isMinValue(); } + public boolean isUnsigned() { + return (mysqlTypeCode & 0x8000) != 0; + } + @Override public int compareLiteral(LiteralExpr expr) { return lExpr.compareLiteral(expr); @@ -175,7 +194,7 @@ public class PlaceHolderExpr extends LiteralExpr { return "\"" + getStringValue() + "\""; } - public void setupParamFromBinary(ByteBuffer data) { - lExpr.setupParamFromBinary(data); + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { + lExpr.setupParamFromBinary(data, isUnsigned); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java index bc78d8dde41..9ac1ce37fbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java @@ -268,6 +268,7 @@ public class PrepareStmt extends StatementBase { } for (int i = 0; i < values.size(); ++i) { inner.getPlaceHolders().get(i).setLiteral(values.get(i)); + inner.getPlaceHolders().get(i).analysisDone(); } if (!values.isEmpty()) { LOG.debug("assign values {}", values.get(0).toSql()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java index 57bc67fdc3e..0e0dbc81c47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StringLiteral.java @@ -300,7 +300,7 @@ public class StringLiteral extends LiteralExpr { } @Override - public void setupParamFromBinary(ByteBuffer data) { + public void setupParamFromBinary(ByteBuffer data, boolean isUnsigned) { int strLen = getParmLen(data); if (strLen > data.remaining()) { strLen = data.remaining(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java new file mode 100644 index 00000000000..4ec8f01149a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ByteBufferUtil.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common.util; + +import java.nio.ByteBuffer; + +public class ByteBufferUtil { + public static short getUnsignedByte(ByteBuffer buffer) { + return (short) (buffer.get() & 0xFF); + } + + public static int getUnsignedShort(ByteBuffer buffer) { + return buffer.getShort() & 0xFFFF; + } + + public static long getUnsignedInt(ByteBuffer buffer) { + return buffer.getInt() & 0xFFFFFFFFL; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 1c5ee53e34b..2f309d9e498 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -29,6 +29,7 @@ import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.IsNullPredicate; import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.analysis.NullLiteral; +import org.apache.doris.analysis.PlaceHolderExpr; import org.apache.doris.analysis.PredicateUtils; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; @@ -386,7 +387,8 @@ public abstract class ScanNode extends PlanNode { if (null == partitionColumnFilter) { partitionColumnFilter = new PartitionColumnFilter(); } - LiteralExpr literal = (LiteralExpr) slotBinding; + LiteralExpr literal = slotBinding instanceof PlaceHolderExpr + ? ((PlaceHolderExpr) slotBinding).getLiteral() : (LiteralExpr) slotBinding; BinaryPredicate.Operator op = binPredicate.getOp(); if (!binPredicate.slotIsLeft()) { op = op.commutative(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java index f83c9c3c53a..7c280566150 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java @@ -248,7 +248,8 @@ public class ConnectProcessor { continue; } LiteralExpr l = prepareCtx.stmt.placeholders().get(i).createLiteralFromType(); - l.setupParamFromBinary(packetBuf); + boolean isUnsigned = prepareCtx.stmt.placeholders().get(i).isUnsigned(); + l.setupParamFromBinary(packetBuf, isUnsigned); realValueExprs.add(l); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteInPredicateRule.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteInPredicateRule.java index c04ff432961..3541690d002 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteInPredicateRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/RewriteInPredicateRule.java @@ -23,6 +23,7 @@ import org.apache.doris.analysis.CastExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.InPredicate; import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.PlaceHolderExpr; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.Subquery; import org.apache.doris.catalog.Type; @@ -114,7 +115,11 @@ public class RewriteInPredicateRule implements ExprRewriteRule { // For example, 2.1 is converted to 2; // 3. childExpr is precisely converted to column type. For example, 2.0 is converted to 2. // In cases 1 and 2 above, childExpr should be discarded. - LiteralExpr newExpr = (LiteralExpr) childExpr.castTo(columnType); + Expr tmpExpr = childExpr.castTo(columnType); + if (tmpExpr instanceof CastExpr && tmpExpr.getChild(0) instanceof PlaceHolderExpr) { + tmpExpr = ((PlaceHolderExpr) tmpExpr.getChild(0)).getLiteral().castTo(columnType); + } + LiteralExpr newExpr = (LiteralExpr) tmpExpr; if (childExpr.compareLiteral(newExpr) == 0) { isCast = true; newInList.add(newExpr); diff --git a/regression-test/data/insert_p0/prepare_insert.out b/regression-test/data/insert_p0/prepare_insert.out index 743871f56a8..351107420ac 100644 --- a/regression-test/data/insert_p0/prepare_insert.out +++ b/regression-test/data/insert_p0/prepare_insert.out @@ -1,5 +1,6 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !sql -- +\N \N \N 1 a 90 2 ab 91 3 abc 92 diff --git a/regression-test/suites/insert_p0/prepare_insert.groovy b/regression-test/suites/insert_p0/prepare_insert.groovy index 300e245c88d..9972d359774 100644 --- a/regression-test/suites/insert_p0/prepare_insert.groovy +++ b/regression-test/suites/insert_p0/prepare_insert.groovy @@ -29,7 +29,7 @@ suite("prepare_insert") { sql """ DROP TABLE IF EXISTS ${tableName} """ sql """ CREATE TABLE ${tableName} ( - `id` int(11) NOT NULL, + `id` int(11) NULL, `name` varchar(50) NULL, `score` int(11) NULL DEFAULT "-1" ) ENGINE=OLAP @@ -87,6 +87,19 @@ suite("prepare_insert") { stmt.close() } + // insert with null + result1 = connect(user = user, password = password, url = url) { + def stmt = prepareStatement "insert into ${tableName} values(?, ?, ?)" + assertEquals(com.mysql.cj.jdbc.ServerPreparedStatement, stmt.class) + stmt.setNull(1, java.sql.Types.INTEGER) + stmt.setNull(2, java.sql.Types.VARCHAR) + stmt.setNull(3, java.sql.Types.INTEGER) + def result = stmt.execute() + logger.info("result: ${result}") + + stmt.close() + } + // insert with label def label = "insert_" + System.currentTimeMillis() result1 = connect(user = user, password = password, url = url) { @@ -135,4 +148,4 @@ suite("prepare_insert") { } qt_sql """ select * from ${tableName} order by id, name, score """ -} \ No newline at end of file +} diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index 7a746f41c97..8274eaa2f5f 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -72,7 +72,7 @@ suite("test_prepared_stmt", "nonConcurrent") { } else { stmt.setString(9, k9) } - exec stmt + stmt.execute() } sql """DROP TABLE IF EXISTS ${tableName} """ sql """ --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org