This is an automated email from the ASF dual-hosted git repository.
rongr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 59599872fb [multistage]: Added support for Json Data Type. (#10765)
59599872fb is described below
commit 59599872fbdc39af364996d9212d8a6066e56187
Author: Abhishek Sharma <[email protected]>
AuthorDate: Wed May 17 11:43:26 2023 -0400
[multistage]: Added support for Json Data Type. (#10765)
* Added support for Json data type in multistage.
* Added more queries for testing as per the PR comment.
---
.../RegexpLikePredicateEvaluatorFactory.java | 3 +-
.../org/apache/pinot/query/type/TypeFactory.java | 3 +-
.../apache/pinot/query/type/TypeFactoryTest.java | 2 +
.../src/test/resources/queries/JsonType.json | 64 ++++++++++++++++++++++
4 files changed, 69 insertions(+), 3 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
index a3af9de194..82477c3560 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/filter/predicate/RegexpLikePredicateEvaluatorFactory.java
@@ -44,7 +44,8 @@ public class RegexpLikePredicateEvaluatorFactory {
*/
public static BaseDictionaryBasedPredicateEvaluator
newDictionaryBasedEvaluator(
RegexpLikePredicate regexpLikePredicate, Dictionary dictionary, DataType
dataType) {
- Preconditions.checkArgument(dataType == DataType.STRING, "Unsupported data
type: " + dataType);
+ boolean condition = (dataType == DataType.STRING || dataType ==
DataType.JSON);
+ Preconditions.checkArgument(condition, "Unsupported data type: " +
dataType);
return new
DictionaryBasedRegexpLikePredicateEvaluator(regexpLikePredicate, dictionary);
}
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
index 21183ce7da..d5e21ede1e 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/type/TypeFactory.java
@@ -81,8 +81,7 @@ public class TypeFactory extends JavaTypeFactoryImpl {
return fieldSpec.isSingleValueField() ?
createSqlType(SqlTypeName.DECIMAL)
: createArrayType(createSqlType(SqlTypeName.DECIMAL), -1);
case JSON:
- // TODO: support JSON, JSON should be supported using a special
RelDataType as it is not a simple String,
- // nor can it be easily parsed as a STRUCT.
+ return createSqlType(SqlTypeName.VARCHAR);
case LIST:
// TODO: support LIST, MV column should go fall into this category.
case STRUCT:
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
index 0e1d55d646..f02925186a 100644
---
a/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/type/TypeFactoryTest.java
@@ -43,6 +43,7 @@ public class TypeFactoryTest {
.addSingleValueDimension("DOUBLE_COL", FieldSpec.DataType.DOUBLE)
.addSingleValueDimension("STRING_COL", FieldSpec.DataType.STRING)
.addSingleValueDimension("BYTES_COL", FieldSpec.DataType.BYTES)
+ .addSingleValueDimension("JSON_COL", FieldSpec.DataType.JSON)
.addMultiValueDimension("INT_ARRAY_COL", FieldSpec.DataType.INT)
.addMultiValueDimension("LONG_ARRAY_COL", FieldSpec.DataType.LONG)
.addMultiValueDimension("FLOAT_ARRAY_COL", FieldSpec.DataType.FLOAT)
@@ -67,6 +68,7 @@ public class TypeFactoryTest {
Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.DOUBLE));
break;
case "STRING_COL":
+ case "JSON_COL":
Assert.assertEquals(field.getType(), new BasicSqlType(TYPE_SYSTEM,
SqlTypeName.VARCHAR));
break;
case "BYTES_COL":
diff --git a/pinot-query-runtime/src/test/resources/queries/JsonType.json
b/pinot-query-runtime/src/test/resources/queries/JsonType.json
new file mode 100644
index 0000000000..e32bca7b63
--- /dev/null
+++ b/pinot-query-runtime/src/test/resources/queries/JsonType.json
@@ -0,0 +1,64 @@
+{
+ "json_data": {
+ "tables": {
+ "jsonTbl": {
+ "schema": [
+ {"name": "jsonCol", "type": "JSON"},
+ {"name": "stringCol", "type": "STRING"}
+ ],
+ "inputs": [
+ ["{\"key1\":\"val1\",\"key2\":\"val2\"}", "str1"],
+ ["{\"key11\":\"val11\",\"key22\":\"val22\"}", "str22"],
+ ["{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}",
"str33"]
+ ]
+ }
+ },
+ "queries": [
+ {
+ "sql": "SELECT jsonCol FROM {jsonTbl}",
+ "outputs": [
+ ["{\"key1\":\"val1\",\"key2\":\"val2\"}"],
+ ["{\"key11\":\"val11\",\"key22\":\"val22\"}"],
+ ["{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}"]
+ ]
+ },
+ {
+ "sql": "SELECT jsonCol, stringCol FROM {jsonTbl}",
+ "outputs": [
+ ["{\"key1\":\"val1\",\"key2\":\"val2\"}", "str1"],
+ ["{\"key11\":\"val11\",\"key22\":\"val22\"}", "str22"],
+ ["{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}",
"str33"]
+ ]
+ },
+ {
+ "sql": "SELECT jsonCol, stringCol FROM {jsonTbl} where
stringCol='str33'",
+ "outputs": [
+ ["{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}",
"str33"]
+ ]
+ },
+ {
+ "sql": "SELECT stringCol FROM {jsonTbl} where
jsonCol='{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}'",
+ "outputs": [
+ ["str33"]
+ ]
+ },
+ {
+ "description": "Like clause checking specific key.",
+ "sql": "SELECT jsonCol, stringCol FROM {jsonTbl} where jsonCol like
'%key111%' OR jsonCol like '%val22%'",
+ "outputs": [
+ ["{\"key11\":\"val11\",\"key22\":\"val22\"}", "str22"],
+ ["{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}",
"str33"]
+ ]
+ },
+ {
+ "description": "Like clause is very general, should return all rows.",
+ "sql": "SELECT jsonCol, stringCol FROM {jsonTbl} where jsonCol like
'%key%' ",
+ "outputs": [
+ ["{\"key1\":\"val1\",\"key2\":\"val2\"}", "str1"],
+ ["{\"key11\":\"val11\",\"key22\":\"val22\"}", "str22"],
+ ["{\"key111\":\"val111\",\"key222\":{\"key222_a\":\"val222_a\"}}",
"str33"]
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
