This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 99fb830023 [feature] datetime column type support auto-initialized
with default … (#9972)
99fb830023 is described below
commit 99fb830023aa38696890ef512a0535b4673a112e
Author: BePPPower <43782773+bepppo...@users.noreply.github.com>
AuthorDate: Thu Jun 9 00:28:03 2022 +0800
[feature] datetime column type support auto-initialized with default …
(#9972)
---
fe/fe-core/src/main/cup/sql_parser.cup | 8 ++-
.../java/org/apache/doris/analysis/ColumnDef.java | 39 +++++++++++++--
.../org/apache/doris/analysis/DataDescription.java | 4 +-
.../apache/doris/analysis/DefaultValueExprDef.java | 58 ++++++++++++++++++++++
.../apache/doris/analysis/FunctionCallExpr.java | 17 +++++++
.../java/org/apache/doris/analysis/InsertStmt.java | 10 +++-
.../main/java/org/apache/doris/catalog/Column.java | 19 +++++--
.../src/main/java/org/apache/doris/load/Load.java | 12 ++++-
.../org/apache/doris/planner/LoadScanNode.java | 6 ++-
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../data/correctness/test_current_timestamp.out | 7 +++
.../test_current_timestamp_streamload.csv | 4 ++
.../correctness/test_current_timestamp.groovy | 55 ++++++++++++++++++++
13 files changed, 225 insertions(+), 15 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 4affd1e466..2bdb818ed8 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -241,7 +241,7 @@ parser code {:
// Total keywords of doris
terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL,
KW_ALTER, KW_AND, KW_ANTI, KW_APPEND, KW_AS, KW_ASC, KW_AUTHORS, KW_ARRAY,
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG,
KW_BITMAP, KW_BITMAP_UNION, KW_QUANTILE_STATE, KW_QUANTILE_UNION, KW_BLOB,
KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY, KW_BUILTIN,
- KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK,
KW_CLUSTER, KW_CLUSTERS, KW_CLEAN,
+ KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK,
KW_CLUSTER, KW_CLUSTERS, KW_CLEAN, KW_CURRENT_TIMESTAMP,
KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT,
KW_COMMITTED, KW_COMPACT,
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT,
KW_COUNT, KW_CREATE, KW_CREATION, KW_CROSS, KW_CUBE, KW_CURRENT,
KW_CURRENT_USER,
KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DAY,
KW_DECIMAL, KW_DECOMMISSION, KW_DEFAULT, KW_DESC, KW_DESCRIBE,
@@ -2443,6 +2443,10 @@ opt_default_value ::=
{:
RESULT = ColumnDef.DefaultValue.NULL_DEFAULT_VALUE;
:}
+ | KW_DEFAULT KW_CURRENT_TIMESTAMP
+ {:
+ RESULT = ColumnDef.DefaultValue.CURRENT_TIMESTAMP_DEFAULT_VALUE;
+ :}
;
opt_is_key ::=
@@ -5808,6 +5812,8 @@ keyword ::=
{: RESULT = id; :}
| KW_POLICY:id
{: RESULT = id; :}
+ | KW_CURRENT_TIMESTAMP:id
+ {: RESULT = id; :}
;
// Identifier that contain keyword
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index d3831f2de6..cffa8df492 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -61,12 +61,31 @@ public class ColumnDef {
public static class DefaultValue {
public boolean isSet;
public String value;
+ // used for column which defaultValue is an expression.
+ public DefaultValueExprDef defaultValueExprDef;
public DefaultValue(boolean isSet, String value) {
this.isSet = isSet;
this.value = value;
+ this.defaultValueExprDef = null;
}
+ /**
+ * used for column which defaultValue is an expression.
+ * @param isSet is Set DefaultValue
+ * @param value default value
+ * @param exprName default value expression
+ */
+ public DefaultValue(boolean isSet, String value, String exprName) {
+ this.isSet = isSet;
+ this.value = value;
+ this.defaultValueExprDef = new DefaultValueExprDef(exprName);
+ }
+
+ // default "CURRENT_TIMESTAMP", only for DATETIME type
+ public static String CURRENT_TIMESTAMP = "CURRENT_TIMESTAMP";
+ public static String NOW = "now";
+ public static DefaultValue CURRENT_TIMESTAMP_DEFAULT_VALUE = new
DefaultValue(true, CURRENT_TIMESTAMP, NOW);
// no default value
public static DefaultValue NOT_SET = new DefaultValue(false, null);
// default null
@@ -279,11 +298,13 @@ public class ColumnDef {
}
if (defaultValue.isSet && defaultValue.value != null) {
- validateDefaultValue(type, defaultValue.value);
+ validateDefaultValue(type, defaultValue.value,
defaultValue.defaultValueExprDef);
}
}
- public static void validateDefaultValue(Type type, String defaultValue)
throws AnalysisException {
+ @SuppressWarnings("checkstyle:Indentation")
+ public static void validateDefaultValue(Type type, String defaultValue,
DefaultValueExprDef defaultValueExprDef)
+ throws AnalysisException {
Preconditions.checkNotNull(defaultValue);
Preconditions.checkArgument(type.isScalarType());
ScalarType scalarType = (ScalarType) type;
@@ -315,9 +336,19 @@ public class ColumnDef {
decimalLiteral.checkPrecisionAndScale(scalarType.getScalarPrecision(),
scalarType.getScalarScale());
break;
case DATE:
- case DATETIME:
new DateLiteral(defaultValue, type);
break;
+ case DATETIME:
+ if (defaultValueExprDef == null) {
+ new DateLiteral(defaultValue, type);
+ } else {
+ if
(defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) {
+ break;
+ } else {
+ throw new AnalysisException("date literal [" +
defaultValue + "] is invalid");
+ }
+ }
+ break;
case CHAR:
case VARCHAR:
case HLL:
@@ -368,7 +399,7 @@ public class ColumnDef {
public Column toColumn() {
return new Column(name, typeDef.getType(), isKey, aggregateType,
isAllowNull, defaultValue.value, comment,
- visible);
+ visible, defaultValue.defaultValueExprDef);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
index 34a2c23dcd..bda93a8651 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DataDescription.java
@@ -328,7 +328,7 @@ public class DataDescription {
}
if (args.get(0) != null) {
- ColumnDef.validateDefaultValue(column.getOriginType(),
args.get(0));
+ ColumnDef.validateDefaultValue(column.getOriginType(),
args.get(0), column.getDefaultValueExprDef());
}
}
@@ -363,7 +363,7 @@ public class DataDescription {
}
if (replaceValue != null) {
- ColumnDef.validateDefaultValue(column.getOriginType(),
replaceValue);
+ ColumnDef.validateDefaultValue(column.getOriginType(),
replaceValue, column.getDefaultValueExprDef());
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
new file mode 100644
index 0000000000..af81840dae
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/DefaultValueExprDef.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+// This file is copied from
+//
https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/ColumnDef.java
+// and modified by Doris
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.common.AnalysisException;
+
+import com.google.gson.annotations.SerializedName;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+
+/**
+ * This def used for column which defaultValue is an expression.
+ */
+public class DefaultValueExprDef {
+ private static final Logger LOG =
LogManager.getLogger(DefaultValueExprDef.class);
+ @SerializedName("exprName")
+ private String exprName;
+
+ public DefaultValueExprDef(String exprName) {
+ this.exprName = exprName;
+ }
+
+ /**
+ * generate a FunctionCallExpr
+ * @return FunctionCallExpr of exprName
+ */
+ public FunctionCallExpr getExpr() {
+ FunctionCallExpr expr = new FunctionCallExpr(exprName, new
FunctionParams(null));
+ try {
+ expr.analyzeImplForDefaultValue();
+ } catch (AnalysisException e) {
+ LOG.warn("analyzeImplForDefaultValue fail: {}", e);
+ }
+ return expr;
+ }
+
+ public String getExprName() {
+ return exprName;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 3c2451630b..4b6bfe2cb4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -778,6 +778,23 @@ public class FunctionCallExpr extends Expr {
fnName, fnParams.isStar() ? "*" : Joiner.on(",
").join(argTypesSql));
}
+ /**
+ * This analyzeImp used for DefaultValueExprDef
+ * to generate a builtinFunction.
+ * @throws AnalysisException
+ */
+ public void analyzeImplForDefaultValue() throws AnalysisException {
+ fn = getBuiltinFunction(null, fnName.getFunction(), new Type[0],
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ type = fn.getReturnType();
+ for (int i = 0; i < children.size(); ++i) {
+ if (getChild(i).getType().isNull()) {
+ uncheckedCastChild(Type.BOOLEAN, i);
+ }
+ }
+ return;
+ }
+
@Override
public void analyzeImpl(Analyzer analyzer) throws AnalysisException {
if (isMergeAggFn) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
index ff0fbd71d3..7dd92af01c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InsertStmt.java
@@ -687,9 +687,15 @@ public class InsertStmt extends DdlStmt {
*/
Preconditions.checkState(col.isAllowNull());
resultExprs.add(NullLiteral.create(col.getType()));
+
} else {
- StringLiteral defaultValueExpr = new
StringLiteral(col.getDefaultValue());
-
resultExprs.add(defaultValueExpr.checkTypeCompatibility(col.getType()));
+ if (col.getDefaultValueExprDef() != null) {
+ resultExprs.add(col.getDefaultValueExpr());
+ } else {
+ StringLiteral defaultValueExpr;
+ defaultValueExpr = new
StringLiteral(col.getDefaultValue());
+
resultExprs.add(defaultValueExpr.checkTypeCompatibility(col.getType()));
+ }
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 81eb14af0b..28e7a24303 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -18,6 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.alter.SchemaChangeHandler;
+import org.apache.doris.analysis.DefaultValueExprDef;
import org.apache.doris.analysis.Expr;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
@@ -87,6 +88,8 @@ public class Column implements Writable {
private Expr defineExpr; // use to define column in materialize view
@SerializedName(value = "visible")
private boolean visible;
+ @SerializedName(value = "defaultValueExprDef")
+ private DefaultValueExprDef defaultValueExprDef; // used for default value
public Column() {
this.name = "";
@@ -95,6 +98,7 @@ public class Column implements Writable {
this.isKey = false;
this.stats = new ColumnStats();
this.visible = true;
+ this.defineExpr = null;
this.children = new ArrayList<>(Type.MAX_NESTING_DEPTH);
}
@@ -117,10 +121,10 @@ public class Column implements Writable {
public Column(String name, Type type, boolean isKey, AggregateType
aggregateType, boolean isAllowNull,
String defaultValue, String comment) {
- this(name, type, isKey, aggregateType, isAllowNull, defaultValue,
comment, true);
+ this(name, type, isKey, aggregateType, isAllowNull, defaultValue,
comment, true, null);
}
public Column(String name, Type type, boolean isKey, AggregateType
aggregateType, boolean isAllowNull,
- String defaultValue, String comment, boolean visible) {
+ String defaultValue, String comment, boolean visible,
DefaultValueExprDef defaultValueExprDef) {
this.name = name;
if (this.name == null) {
this.name = "";
@@ -136,6 +140,7 @@ public class Column implements Writable {
this.isKey = isKey;
this.isAllowNull = isAllowNull;
this.defaultValue = defaultValue;
+ this.defaultValueExprDef = defaultValueExprDef;
this.comment = comment;
this.stats = new ColumnStats();
this.visible = visible;
@@ -151,6 +156,7 @@ public class Column implements Writable {
this.isKey = column.isKey();
this.isAllowNull = column.isAllowNull();
this.defaultValue = column.getDefaultValue();
+ this.defaultValueExprDef = column.defaultValueExprDef;
this.comment = column.getComment();
this.stats = column.getStats();
this.visible = column.visible;
@@ -297,11 +303,15 @@ public class Column implements Writable {
if (getDataType() == PrimitiveType.VARCHAR) {
return defaultValueLiteral;
}
+ if (defaultValueExprDef != null) {
+ return defaultValueExprDef.getExpr();
+ }
Expr result = defaultValueLiteral.castTo(getType());
result.checkValueValid();
return result;
}
+
public void setStats(ColumnStats stats) {
this.stats = stats;
}
@@ -497,6 +507,10 @@ public class Column implements Writable {
defineExpr = expr;
}
+ public DefaultValueExprDef getDefaultValueExprDef() {
+ return defaultValueExprDef;
+ }
+
public SlotRef getRefColumn() {
List<Expr> slots = new ArrayList<>();
if (defineExpr == null) {
@@ -642,7 +656,6 @@ public class Column implements Writable {
}
public static Column read(DataInput in) throws IOException {
-
String json = Text.readString(in);
return GsonUtils.GSON.fromJson(json, Column.class);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index 5a91bee9cb..91eee1c6c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -1351,7 +1351,11 @@ public class Load {
exprs.add(funcExpr.getChild(1));
} else {
if (column.getDefaultValue() != null) {
- exprs.add(new
StringLiteral(column.getDefaultValue()));
+ if (column.getDefaultValueExprDef() != null) {
+ exprs.add(column.getDefaultValueExpr());
+ } else {
+ exprs.add(new
StringLiteral(column.getDefaultValue()));
+ }
} else {
if (column.isAllowNull()) {
exprs.add(NullLiteral.create(Type.VARCHAR));
@@ -1370,7 +1374,11 @@ public class Load {
innerIfExprs.add(funcExpr.getChild(1));
} else {
if (column.getDefaultValue() != null) {
- innerIfExprs.add(new
StringLiteral(column.getDefaultValue()));
+ if (column.getDefaultValueExprDef() != null) {
+ innerIfExprs.add(column.getDefaultValueExpr());
+ } else {
+ innerIfExprs.add(new
StringLiteral(column.getDefaultValue()));
+ }
} else {
if (column.isAllowNull()) {
innerIfExprs.add(NullLiteral.create(Type.VARCHAR));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
index 9f063a47ed..980876c630 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/LoadScanNode.java
@@ -159,7 +159,11 @@ public abstract class LoadScanNode extends ScanNode {
} else {
Column column = destSlotDesc.getColumn();
if (column.getDefaultValue() != null) {
- expr = new
StringLiteral(destSlotDesc.getColumn().getDefaultValue());
+ if (column.getDefaultValueExprDef() != null) {
+ expr = column.getDefaultValueExpr();
+ } else {
+ expr = new
StringLiteral(destSlotDesc.getColumn().getDefaultValue());
+ }
} else {
if (column.isAllowNull()) {
expr = NullLiteral.create(column.getType());
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex
b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 8f3dd68a35..fceb550129 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -431,6 +431,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("write", new Integer(SqlParserSymbols.KW_WRITE));
keywordMap.put("year", new Integer(SqlParserSymbols.KW_YEAR));
keywordMap.put("||", new Integer(SqlParserSymbols.KW_PIPE));
+ keywordMap.put("current_timestamp", new
Integer(SqlParserSymbols.KW_CURRENT_TIMESTAMP));
}
// map from token id to token description
diff --git a/regression-test/data/correctness/test_current_timestamp.out
b/regression-test/data/correctness/test_current_timestamp.out
new file mode 100644
index 0000000000..dca4f4dd5a
--- /dev/null
+++ b/regression-test/data/correctness/test_current_timestamp.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !insert_into --
+4
+
+-- !stream_load --
+4
+
diff --git
a/regression-test/data/correctness/test_current_timestamp_streamload.csv
b/regression-test/data/correctness/test_current_timestamp_streamload.csv
new file mode 100644
index 0000000000..55f24a1c84
--- /dev/null
+++ b/regression-test/data/correctness/test_current_timestamp_streamload.csv
@@ -0,0 +1,4 @@
+5,ee
+6,ff
+7,gg
+8,hh
\ No newline at end of file
diff --git a/regression-test/suites/correctness/test_current_timestamp.groovy
b/regression-test/suites/correctness/test_current_timestamp.groovy
new file mode 100644
index 0000000000..3f84c86fd5
--- /dev/null
+++ b/regression-test/suites/correctness/test_current_timestamp.groovy
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+ // or more contributor license agreements. See the NOTICE file
+ // distributed with this work for additional information
+ // regarding copyright ownership. The ASF licenses this file
+ // to you under the Apache License, Version 2.0 (the
+ // "License"); you may not use this file except in compliance
+ // with the License. You may obtain a copy of the License at
+ //
+ // http://www.apache.org/licenses/LICENSE-2.0
+ //
+ // Unless required by applicable law or agreed to in writing,
+ // software distributed under the License is distributed on an
+ // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ // KIND, either express or implied. See the License for the
+ // specific language governing permissions and limitations
+ // under the License.
+
+suite("test_current_timestamp", "correctness") {
+ def tableName = "test_current_timestamp"
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName}
+ (
+ id TINYINT,
+ name CHAR(10) NOT NULL DEFAULT "zs",
+ dt_0 DATETIME,
+ dt_1 DATETIME DEFAULT CURRENT_TIMESTAMP
+ )
+ COMMENT "test current_timestamp table"
+ DISTRIBUTED BY HASH(id)
+ PROPERTIES("replication_num" = "1");
+ """
+
+ // test insert into.
+ sql " insert into ${tableName} (id,name,dt_0) values
(1,'aa',current_timestamp()); "
+ sql " insert into ${tableName} (id,name,dt_0) values
(2,'bb',current_timestamp()); "
+ sql " insert into ${tableName} (id,name,dt_0) values
(3,'cc',current_timestamp()); "
+ sql " insert into ${tableName} (id,name,dt_0) values
(4,'dd',current_timestamp()); "
+
+ qt_insert_into """ select count(*) from ${tableName} where to_date(dt_0) =
to_date(dt_1); """
+
+ // test stream load.
+ streamLoad {
+ table "${tableName}"
+
+ set 'column_separator', ','
+ set 'columns', 'id, name, dt_0 = current_timestamp()'
+
+ file 'test_current_timestamp_streamload.csv'
+
+ time 10000 // limit inflight 10s
+ }
+ qt_stream_load """ select count(*) from ${tableName} where id > 4 and
to_date(dt_0) = to_date(dt_1); """
+ }
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
