This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d8427037be [Bug](doe) Fix some bug (#11594)
d8427037be is described below
commit d8427037be49a1e3e7c447d378356fc99e5fbecc
Author: Stalary <stal...@163.com>
AuthorDate: Wed Aug 10 21:00:05 2022 +0800
[Bug](doe) Fix some bug (#11594)
---
.../apache/doris/common/proc/EsShardProcDir.java | 2 +-
.../doris/datasource/EsExternalDataSource.java | 31 +++----
.../doris/external/elasticsearch/EsRestClient.java | 5 +-
.../external/elasticsearch/EsShardPartitions.java | 22 ++---
.../external/elasticsearch/EsShardRouting.java | 36 +-------
.../doris/external/elasticsearch/EsUtil.java | 48 +++++++----
.../java/org/apache/doris/planner/EsScanNode.java | 13 ++-
.../doris/external/elasticsearch/EsUtilTest.java | 97 +++++++++++++---------
8 files changed, 127 insertions(+), 127 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java
index 3c0bc26632..8d4d3bf89c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsShardProcDir.java
@@ -63,7 +63,7 @@ public class EsShardProcDir implements ProcDirInterface {
for (EsShardRouting esShardRouting : shardRoutings) {
List<Comparable> shardInfo = new
ArrayList<Comparable>();
shardInfo.add(shardId);
- shardInfo.add(esShardRouting.getAddress().toString());
+
shardInfo.add(esShardRouting.getHttpAddress().toString());
shardInfo.add(esShardRouting.isPrimary());
shardInfos.add(shardInfo);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
index bfa4b56010..fd5b3fb077 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
@@ -98,6 +98,21 @@ public class EsExternalDataSource extends ExternalDataSource
{
throw new DdlException("Hosts of ES table is null.");
}
nodes = properties.get(PROP_HOSTS).trim().split(",");
+ // check protocol
+ for (String seed : nodes) {
+ if (!seed.startsWith("http")) {
+ throw new DdlException("the protocol must be used");
+ }
+ if (properties.containsKey(PROP_SSL)) {
+ enableSsl = EsUtil.getBoolean(properties, PROP_SSL);
+ if (enableSsl && seed.startsWith("http://";)) {
+ throw new DdlException("if ssl_enabled is true, the https
protocol must be used");
+ }
+ if (!enableSsl && seed.startsWith("https://";)) {
+ throw new DdlException("if ssl_enabled is false, the http
protocol must be used");
+ }
+ }
+ }
if (StringUtils.isNotBlank(properties.get(PROP_USERNAME))) {
username = properties.get(PROP_USERNAME).trim();
@@ -119,18 +134,6 @@ public class EsExternalDataSource extends
ExternalDataSource {
enableNodesDiscovery = EsUtil.getBoolean(properties,
PROP_NODES_DISCOVERY);
}
- if (properties.containsKey(PROP_SSL)) {
- enableSsl = EsUtil.getBoolean(properties, PROP_SSL);
- // check protocol
- for (String seed : nodes) {
- if (enableSsl && seed.startsWith("http://";)) {
- throw new DdlException("if ssl_enabled is true, the https
protocol must be used");
- }
- if (!enableSsl && seed.startsWith("https://";)) {
- throw new DdlException("if ssl_enabled is false, the http
protocol must be used");
- }
- }
- }
}
/**
@@ -155,7 +158,7 @@ public class EsExternalDataSource extends
ExternalDataSource {
this.esRestClient = new EsRestClient(this.nodes, this.username,
this.password, this.enableSsl);
long defaultDbId = Env.getCurrentEnv().getNextId();
dbNameToId.put(DEFAULT_DB, defaultDbId);
- idToDb.put(defaultDbId, new EsExternalDatabase(this, defaultDbId,
"default"));
+ idToDb.put(defaultDbId, new EsExternalDatabase(this, defaultDbId,
DEFAULT_DB));
}
@Override
@@ -177,7 +180,7 @@ public class EsExternalDataSource extends
ExternalDataSource {
if (!dbNameToId.containsKey(realDbName)) {
return null;
}
- return new EsExternalDatabase(this, dbNameToId.get(realDbName),
realDbName);
+ return idToDb.get(dbNameToId.get(realDbName));
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
index eb79f4453b..1627895936 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRestClient.java
@@ -189,12 +189,11 @@ public class EsRestClient {
**/
public List<String> listTable() {
List<String> indices =
getIndices().stream().distinct().collect(Collectors.toList());
- getAliases().entrySet().stream().filter(e ->
indices.contains(e.getKey()))
- .flatMap(e ->
e.getValue().stream()).distinct().forEach(indices::add);
+ getAliases().entrySet().stream().filter(e ->
indices.contains(e.getKey())).flatMap(e -> e.getValue().stream())
+ .distinct().forEach(indices::add);
return indices;
}
-
/**
* Get Shard location.
**/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java
index 7b967b0fcc..a889540909 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardPartitions.java
@@ -54,12 +54,11 @@ public class EsShardPartitions {
/**
* Parse shardRoutings from the json
*
- * @param indexName indexName(alias or really name)
+ * @param indexName indexName(alias or really name)
* @param searchShards the return value of _search_shards
* @return shardRoutings is used for searching
*/
public static EsShardPartitions findShardPartitions(String indexName,
String searchShards) throws DorisEsException {
-
EsShardPartitions partitions = new EsShardPartitions(indexName);
JSONObject jsonObject = (JSONObject) JSONValue.parse(searchShards);
JSONArray shards = (JSONArray) jsonObject.get("shards");
@@ -67,23 +66,18 @@ public class EsShardPartitions {
for (int i = 0; i < size; i++) {
List<EsShardRouting> singleShardRouting = Lists.newArrayList();
JSONArray shardsArray = (JSONArray) shards.get(i);
- int arraySize = shardsArray.size();
- for (int j = 0; j < arraySize; j++) {
- JSONObject indexShard = (JSONObject) shardsArray.get(j);
+ for (Object o : shardsArray) {
+ JSONObject indexShard = (JSONObject) o;
String shardState = (String) indexShard.get("state");
if ("STARTED".equalsIgnoreCase(shardState) ||
"RELOCATING".equalsIgnoreCase(shardState)) {
try {
- singleShardRouting.add(
- EsShardRouting.newSearchShard(
- (String) indexShard.get("index"),
- ((Long)
indexShard.get("shard")).intValue(),
- (Boolean) indexShard.get("primary"),
- (String) indexShard.get("node"),
- (JSONObject) jsonObject.get("nodes")));
+ singleShardRouting.add(new EsShardRouting((String)
indexShard.get("index"),
+ ((Long) indexShard.get("shard")).intValue(),
(Boolean) indexShard.get("primary"),
+ (String) indexShard.get("node")));
} catch (Exception e) {
LOG.error("fetch index [{}] shard partitions failure",
indexName, e);
- throw new DorisEsException("fetch [" + indexName
- + "] shard partitions failure [" +
e.getMessage() + "]");
+ throw new DorisEsException(
+ "fetch [" + indexName + "] shard partitions
failure [" + e.getMessage() + "]");
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java
index 7c15e7e0a9..b3a5a8d463 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsShardRouting.java
@@ -19,41 +19,21 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.thrift.TNetworkAddress;
-import com.google.common.base.Strings;
-import org.json.simple.JSONObject;
-
public class EsShardRouting {
private final String indexName;
private final int shardId;
private final boolean isPrimary;
- private final TNetworkAddress address;
-
private TNetworkAddress httpAddress;
private final String nodeId;
- public EsShardRouting(String indexName, int shardId, boolean isPrimary,
TNetworkAddress address, String nodeId) {
+ public EsShardRouting(String indexName, int shardId, boolean isPrimary,
String nodeId) {
this.indexName = indexName;
this.shardId = shardId;
this.isPrimary = isPrimary;
- this.address = address;
this.nodeId = nodeId;
}
- public static EsShardRouting newSearchShard(String indexName, int shardId,
boolean isPrimary,
- String nodeId, JSONObject nodesMap) {
- JSONObject nodeInfo = (JSONObject) nodesMap.get(nodeId);
- String[] transportAddr = ((String)
nodeInfo.get("transport_address")).split(":");
- // get thrift port from node info
- String thriftPort = (String) ((JSONObject)
nodeInfo.get("attributes")).get("thrift_port");
- // In http transport mode, should ignore thrift_port, set address to
null
- TNetworkAddress addr = null;
- if (!Strings.isNullOrEmpty(thriftPort)) {
- addr = new TNetworkAddress(transportAddr[0],
Integer.parseInt(thriftPort));
- }
- return new EsShardRouting(indexName, shardId, isPrimary, addr, nodeId);
- }
-
public int getShardId() {
return shardId;
}
@@ -62,10 +42,6 @@ public class EsShardRouting {
return isPrimary;
}
- public TNetworkAddress getAddress() {
- return address;
- }
-
public String getIndexName() {
return indexName;
}
@@ -84,13 +60,7 @@ public class EsShardRouting {
@Override
public String toString() {
- return "EsShardRouting{"
- + "indexName='" + indexName + '\''
- + ", shardId=" + shardId
- + ", isPrimary=" + isPrimary
- + ", address=" + address
- + ", httpAddress=" + httpAddress
- + ", nodeId='" + nodeId + '\''
- + '}';
+ return "EsShardRouting{" + "indexName='" + indexName + '\'' + ",
shardId=" + shardId + ", isPrimary="
+ + isPrimary + ", httpAddress=" + httpAddress + ", nodeId='" +
nodeId + '\'' + '}';
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
index 4385977216..692b129e77 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsUtil.java
@@ -19,8 +19,8 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.BoolLiteral;
+import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
-import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.DecimalLiteral;
import org.apache.doris.analysis.DistributionDesc;
import org.apache.doris.analysis.Expr;
@@ -35,7 +35,6 @@ import org.apache.doris.analysis.LikePredicate.Operator;
import org.apache.doris.analysis.PartitionDesc;
import org.apache.doris.analysis.RangePartitionDesc;
import org.apache.doris.analysis.SlotRef;
-import org.apache.doris.analysis.StringLiteral;
import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Type;
@@ -192,27 +191,39 @@ public class EsUtil {
return properties;
}
- private static QueryBuilder toCompoundEsDsl(Expr expr) {
+ private static QueryBuilder toCompoundEsDsl(Expr expr, List<Expr>
notPushDownList) {
CompoundPredicate compoundPredicate = (CompoundPredicate) expr;
switch (compoundPredicate.getOp()) {
case AND: {
- QueryBuilder left = toEsDsl(compoundPredicate.getChild(0));
- QueryBuilder right = toEsDsl(compoundPredicate.getChild(1));
+ QueryBuilder left = toEsDsl(compoundPredicate.getChild(0),
notPushDownList);
+ QueryBuilder right = toEsDsl(compoundPredicate.getChild(1),
notPushDownList);
if (left != null && right != null) {
return QueryBuilders.boolQuery().must(left).must(right);
}
return null;
}
case OR: {
- QueryBuilder left = toEsDsl(compoundPredicate.getChild(0));
- QueryBuilder right = toEsDsl(compoundPredicate.getChild(1));
+ int beforeSize = notPushDownList.size();
+ QueryBuilder left = toEsDsl(compoundPredicate.getChild(0),
notPushDownList);
+ QueryBuilder right = toEsDsl(compoundPredicate.getChild(1),
notPushDownList);
+ int afterSize = notPushDownList.size();
if (left != null && right != null) {
return
QueryBuilders.boolQuery().should(left).should(right);
}
+ // One 'or' association cannot be pushed down and the other
cannot be pushed down
+ if (afterSize > beforeSize) {
+ if (left != null) {
+ // add right if right don't pushdown
+ notPushDownList.add(compoundPredicate.getChild(0));
+ } else if (right != null) {
+ // add left if left don't pushdown
+ notPushDownList.add(compoundPredicate.getChild(1));
+ }
+ }
return null;
}
case NOT: {
- QueryBuilder child = toEsDsl(compoundPredicate.getChild(0));
+ QueryBuilder child = toEsDsl(compoundPredicate.getChild(0),
notPushDownList);
if (child != null) {
return QueryBuilders.boolQuery().mustNot(child);
}
@@ -223,18 +234,27 @@ public class EsUtil {
}
}
+ public static QueryBuilder toEsDsl(Expr expr) {
+ return toEsDsl(expr, new ArrayList<>());
+ }
+
/**
* Doris expr to es dsl.
**/
- public static QueryBuilder toEsDsl(Expr expr) {
+ public static QueryBuilder toEsDsl(Expr expr, List<Expr> notPushDownList) {
if (expr == null) {
return null;
}
// CompoundPredicate, `between` also converted to CompoundPredicate.
if (expr instanceof CompoundPredicate) {
- return toCompoundEsDsl(expr);
+ return toCompoundEsDsl(expr, notPushDownList);
}
TExprOpcode opCode = expr.getOpcode();
+ // Cast can not pushdown
+ if (expr.getChild(0) instanceof CastExpr || expr.getChild(1)
instanceof CastExpr) {
+ notPushDownList.add(expr);
+ return null;
+ }
String column = ((SlotRef) expr.getChild(0)).getColumnName();
if (expr instanceof BinaryPredicate) {
Object value = toDorisLiteral(expr.getChild(1));
@@ -386,9 +406,6 @@ public class EsUtil {
if (expr instanceof BoolLiteral) {
BoolLiteral boolLiteral = (BoolLiteral) expr;
return boolLiteral.getValue();
- } else if (expr instanceof DateLiteral) {
- DateLiteral dateLiteral = (DateLiteral) expr;
- return dateLiteral.getStringValue();
} else if (expr instanceof DecimalLiteral) {
DecimalLiteral decimalLiteral = (DecimalLiteral) expr;
return decimalLiteral.getValue();
@@ -401,11 +418,8 @@ public class EsUtil {
} else if (expr instanceof LargeIntLiteral) {
LargeIntLiteral largeIntLiteral = (LargeIntLiteral) expr;
return largeIntLiteral.getLongValue();
- } else if (expr instanceof StringLiteral) {
- StringLiteral stringLiteral = (StringLiteral) expr;
- return stringLiteral.getStringValue();
}
- return null;
+ return expr.getStringValue();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
index 11f3e1cb2b..db0f98a65c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/EsScanNode.java
@@ -338,15 +338,11 @@ public class EsScanNode extends ScanNode {
}
if (!conjuncts.isEmpty()) {
- output.append(prefix).append("PREDICATES:
").append(getExplainString(conjuncts)).append("\n");
- // reserved for later using: LOCAL_PREDICATES is processed by
Doris EsScanNode
- output.append(prefix).append("LOCAL_PREDICATES: ").append("
").append("\n");
- // reserved for later using: REMOTE_PREDICATES is processed by
remote ES Cluster
- output.append(prefix).append("REMOTE_PREDICATES: ").append("
").append("\n");
+ output.append(prefix).append("LOCAL_PREDICATES:
").append(getExplainString(conjuncts)).append("\n");
buildQuery();
- output.append(prefix).append("ES_QUERY_DSL:
").append(queryBuilder.toJson()).append("\n");
+ output.append(prefix).append("REMOTE_PREDICATES:
").append(queryBuilder.toJson()).append("\n");
} else {
- output.append(prefix).append("ES_QUERY_DSL:
").append("{\"match_all\": {}}").append("\n");
+ output.append(prefix).append("REMOTE_PREDICATES:
").append("{\"match_all\": {}}").append("\n");
}
String indexName = table.getIndexName();
String typeName = table.getMappingType();
@@ -360,8 +356,9 @@ public class EsScanNode extends ScanNode {
} else {
boolean hasFilter = false;
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ List<Expr> notPushDownList = new ArrayList<>();
for (Expr expr : conjuncts) {
- QueryBuilder queryBuilder = EsUtil.toEsDsl(expr);
+ QueryBuilder queryBuilder = EsUtil.toEsDsl(expr,
notPushDownList);
if (queryBuilder != null) {
hasFilter = true;
boolQueryBuilder.must(queryBuilder);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
index 432af95c69..435dc7c32e 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/external/elasticsearch/EsUtilTest.java
@@ -19,8 +19,10 @@ package org.apache.doris.external.elasticsearch;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.BinaryPredicate.Operator;
+import org.apache.doris.analysis.CastExpr;
import org.apache.doris.analysis.CompoundPredicate;
import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.FloatLiteral;
import org.apache.doris.analysis.FunctionCallExpr;
import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.IntLiteral;
@@ -28,6 +30,7 @@ import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.analysis.TypeDef;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.PrimitiveType;
@@ -37,7 +40,6 @@ import mockit.Expectations;
import mockit.Injectable;
import org.json.simple.JSONObject;
import org.json.simple.JSONValue;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.jupiter.api.Assertions;
@@ -84,19 +86,19 @@ public class EsUtilTest extends EsTestCase {
EsTable esTableBefore7X = fakeEsTable("fake", "test", "doc", columns);
SearchContext searchContext = new SearchContext(esTableBefore7X);
MappingPhase.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping.json"));
- Assert.assertEquals("k3.keyword",
searchContext.fetchFieldsContext().get("k3"));
- Assert.assertEquals("k3.keyword",
searchContext.docValueFieldsContext().get("k3"));
- Assert.assertEquals("k1",
searchContext.docValueFieldsContext().get("k1"));
- Assert.assertEquals("k2",
searchContext.docValueFieldsContext().get("k2"));
+ Assertions.assertEquals("k3.keyword",
searchContext.fetchFieldsContext().get("k3"));
+ Assertions.assertEquals("k3.keyword",
searchContext.docValueFieldsContext().get("k3"));
+ Assertions.assertEquals("k1",
searchContext.docValueFieldsContext().get("k1"));
+ Assertions.assertEquals("k2",
searchContext.docValueFieldsContext().get("k2"));
// ES version >= 7.0
EsTable esTableAfter7X = fakeEsTable("fake", "test", "_doc", columns);
SearchContext searchContext1 = new SearchContext(esTableAfter7X);
MappingPhase.resolveFields(searchContext1,
loadJsonFromFile("data/es/test_index_mapping_after_7x.json"));
- Assert.assertEquals("k3.keyword",
searchContext1.fetchFieldsContext().get("k3"));
- Assert.assertEquals("k3.keyword",
searchContext1.docValueFieldsContext().get("k3"));
- Assert.assertEquals("k1",
searchContext1.docValueFieldsContext().get("k1"));
- Assert.assertEquals("k2",
searchContext1.docValueFieldsContext().get("k2"));
+ Assertions.assertEquals("k3.keyword",
searchContext1.fetchFieldsContext().get("k3"));
+ Assertions.assertEquals("k3.keyword",
searchContext1.docValueFieldsContext().get("k3"));
+ Assertions.assertEquals("k1",
searchContext1.docValueFieldsContext().get("k1"));
+ Assertions.assertEquals("k2",
searchContext1.docValueFieldsContext().get("k2"));
}
@Test
@@ -114,10 +116,10 @@ public class EsUtilTest extends EsTestCase {
MappingPhase mappingPhase = new MappingPhase(client);
ExceptionChecker.expectThrowsNoException(() ->
mappingPhase.execute(searchContext1));
ExceptionChecker.expectThrowsNoException(() ->
mappingPhase.postProcess(searchContext1));
- Assert.assertEquals("k3.keyword",
searchContext1.fetchFieldsContext().get("k3"));
- Assert.assertEquals("k3.keyword",
searchContext1.docValueFieldsContext().get("k3"));
- Assert.assertEquals("k1",
searchContext1.docValueFieldsContext().get("k1"));
- Assert.assertEquals("k2",
searchContext1.docValueFieldsContext().get("k2"));
+ Assertions.assertEquals("k3.keyword",
searchContext1.fetchFieldsContext().get("k3"));
+ Assertions.assertEquals("k3.keyword",
searchContext1.docValueFieldsContext().get("k3"));
+ Assertions.assertEquals("k1",
searchContext1.docValueFieldsContext().get("k1"));
+ Assertions.assertEquals("k2",
searchContext1.docValueFieldsContext().get("k2"));
}
@@ -127,21 +129,21 @@ public class EsUtilTest extends EsTestCase {
SearchContext searchContext = new SearchContext(esTableAfter7X);
MappingPhase.resolveFields(searchContext,
loadJsonFromFile("data/es/test_index_mapping_field_mult_analyzer.json"));
-
Assert.assertFalse(searchContext.docValueFieldsContext().containsKey("k3"));
+
Assertions.assertFalse(searchContext.docValueFieldsContext().containsKey("k3"));
}
@Test
public void testGetJsonObject() {
JSONObject json = (JSONObject) JSONValue.parse(jsonStr);
JSONObject upperBoundSetting = EsUtil.getJsonObject(json,
"settings.index.bpack.partition", 0);
- Assert.assertTrue(upperBoundSetting.containsKey("upperbound"));
- Assert.assertEquals("12", (String)
upperBoundSetting.get("upperbound"));
+ Assertions.assertTrue(upperBoundSetting.containsKey("upperbound"));
+ Assertions.assertEquals("12", (String)
upperBoundSetting.get("upperbound"));
JSONObject unExistKey = EsUtil.getJsonObject(json, "set", 0);
- Assert.assertNull(unExistKey);
+ Assertions.assertNull(unExistKey);
JSONObject singleKey = EsUtil.getJsonObject(json, "settings", 0);
- Assert.assertTrue(singleKey.containsKey("index"));
+ Assertions.assertTrue(singleKey.containsKey("index"));
}
@Test(expected = ClassCastException.class)
@@ -162,13 +164,13 @@ public class EsUtilTest extends EsTestCase {
Expr ltExpr = new BinaryPredicate(Operator.LT, k1, intLiteral);
Expr gtExpr = new BinaryPredicate(Operator.GT, k1, intLiteral);
Expr efnExpr = new BinaryPredicate(Operator.EQ_FOR_NULL, new
SlotRef(null, "k1"), new IntLiteral(3));
- Assert.assertEquals("{\"term\":{\"k1\":3}}",
EsUtil.toEsDsl(eqExpr).toJson());
- Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
EsUtil.toEsDsl(neExpr).toJson());
- Assert.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}",
EsUtil.toEsDsl(leExpr).toJson());
- Assert.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}",
EsUtil.toEsDsl(geExpr).toJson());
- Assert.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}",
EsUtil.toEsDsl(ltExpr).toJson());
- Assert.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}",
EsUtil.toEsDsl(gtExpr).toJson());
- Assert.assertEquals("{\"term\":{\"k1\":3}}",
EsUtil.toEsDsl(efnExpr).toJson());
+ Assertions.assertEquals("{\"term\":{\"k1\":3}}",
EsUtil.toEsDsl(eqExpr).toJson());
+
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
EsUtil.toEsDsl(neExpr).toJson());
+ Assertions.assertEquals("{\"range\":{\"k1\":{\"lte\":3}}}",
EsUtil.toEsDsl(leExpr).toJson());
+ Assertions.assertEquals("{\"range\":{\"k1\":{\"gte\":3}}}",
EsUtil.toEsDsl(geExpr).toJson());
+ Assertions.assertEquals("{\"range\":{\"k1\":{\"lt\":3}}}",
EsUtil.toEsDsl(ltExpr).toJson());
+ Assertions.assertEquals("{\"range\":{\"k1\":{\"gt\":3}}}",
EsUtil.toEsDsl(gtExpr).toJson());
+ Assertions.assertEquals("{\"term\":{\"k1\":3}}",
EsUtil.toEsDsl(efnExpr).toJson());
}
@Test
@@ -184,11 +186,12 @@ public class EsUtilTest extends EsTestCase {
CompoundPredicate orPredicate = new
CompoundPredicate(CompoundPredicate.Operator.OR, binaryPredicate1,
binaryPredicate2);
CompoundPredicate notPredicate = new
CompoundPredicate(CompoundPredicate.Operator.NOT, binaryPredicate1, null);
-
Assert.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
+
Assertions.assertEquals("{\"bool\":{\"must\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
EsUtil.toEsDsl(andPredicate).toJson());
-
Assert.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
+
Assertions.assertEquals("{\"bool\":{\"should\":[{\"term\":{\"k1\":3}},{\"range\":{\"k2\":{\"gt\":5}}}]}}",
EsUtil.toEsDsl(orPredicate).toJson());
- Assert.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
EsUtil.toEsDsl(notPredicate).toJson());
+
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"term\":{\"k1\":3}}}}",
+ EsUtil.toEsDsl(notPredicate).toJson());
}
@Test
@@ -196,9 +199,9 @@ public class EsUtilTest extends EsTestCase {
SlotRef k1 = new SlotRef(null, "k1");
IsNullPredicate isNullPredicate = new IsNullPredicate(k1, false);
IsNullPredicate isNotNullPredicate = new IsNullPredicate(k1, true);
-
Assert.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}",
+
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"exists\":{\"field\":\"k1\"}}}}",
EsUtil.toEsDsl(isNullPredicate).toJson());
- Assert.assertEquals("{\"exists\":{\"field\":\"k1\"}}",
EsUtil.toEsDsl(isNotNullPredicate).toJson());
+ Assertions.assertEquals("{\"exists\":{\"field\":\"k1\"}}",
EsUtil.toEsDsl(isNotNullPredicate).toJson());
}
@Test
@@ -210,9 +213,9 @@ public class EsUtilTest extends EsTestCase {
LikePredicate likePredicate1 = new
LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral1);
LikePredicate regexPredicate = new
LikePredicate(LikePredicate.Operator.REGEXP, k1, stringLiteral2);
LikePredicate likePredicate2 = new
LikePredicate(LikePredicate.Operator.LIKE, k1, stringLiteral3);
- Assert.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}",
EsUtil.toEsDsl(likePredicate1).toJson());
- Assert.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}",
EsUtil.toEsDsl(regexPredicate).toJson());
- Assert.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}",
EsUtil.toEsDsl(likePredicate2).toJson());
+ Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}",
EsUtil.toEsDsl(likePredicate1).toJson());
+ Assertions.assertEquals("{\"wildcard\":{\"k1\":\"*1*\"}}",
EsUtil.toEsDsl(regexPredicate).toJson());
+ Assertions.assertEquals("{\"wildcard\":{\"k1\":\"1?2\"}}",
EsUtil.toEsDsl(likePredicate2).toJson());
}
@Test
@@ -225,8 +228,8 @@ public class EsUtilTest extends EsTestCase {
intLiterals.add(intLiteral2);
InPredicate isInPredicate = new InPredicate(k1, intLiterals, false);
InPredicate isNotInPredicate = new InPredicate(k1, intLiterals, true);
- Assert.assertEquals("{\"terms\":{\"k1\":[3,5]}}",
EsUtil.toEsDsl(isInPredicate).toJson());
-
Assert.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}",
+ Assertions.assertEquals("{\"terms\":{\"k1\":[3,5]}}",
EsUtil.toEsDsl(isInPredicate).toJson());
+
Assertions.assertEquals("{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}",
EsUtil.toEsDsl(isNotInPredicate).toJson());
}
@@ -239,18 +242,37 @@ public class EsUtilTest extends EsTestCase {
exprs.add(k1);
exprs.add(stringLiteral);
FunctionCallExpr functionCallExpr = new FunctionCallExpr("esquery",
exprs);
- Assert.assertEquals(str, EsUtil.toEsDsl(functionCallExpr).toJson());
+ Assertions.assertEquals(str,
EsUtil.toEsDsl(functionCallExpr).toJson());
SlotRef k2 = new SlotRef(null, "k2");
IntLiteral intLiteral = new IntLiteral(5);
BinaryPredicate binaryPredicate = new BinaryPredicate(Operator.EQ, k2,
intLiteral);
CompoundPredicate compoundPredicate = new
CompoundPredicate(CompoundPredicate.Operator.AND, binaryPredicate,
functionCallExpr);
- Assert.assertEquals(
+ Assertions.assertEquals(
"{\"bool\":{\"must\":[{\"term\":{\"k2\":5}},{\"bool\":{\"must_not\":{\"terms\":{\"k1\":[3,5]}}}}]}}",
EsUtil.toEsDsl(compoundPredicate).toJson());
}
+ @Test
+ public void testCastConvertEsDsl() {
+ SlotRef k1 = new SlotRef(null, "k1");
+ FloatLiteral floatLiteral = new FloatLiteral(3.14);
+ CastExpr castExpr = new CastExpr(TypeDef.create(PrimitiveType.INT),
floatLiteral);
+ BinaryPredicate castPredicate = new BinaryPredicate(Operator.EQ, k1,
castExpr);
+ List<Expr> notPushDownList = new ArrayList<>();
+ Assertions.assertNull(EsUtil.toEsDsl(castPredicate, notPushDownList));
+ Assertions.assertEquals(1, notPushDownList.size());
+
+ SlotRef k2 = new SlotRef(null, "k2");
+ IntLiteral intLiteral = new IntLiteral(5);
+ BinaryPredicate eqPredicate = new BinaryPredicate(Operator.EQ, k2,
intLiteral);
+ CompoundPredicate compoundPredicate = new
CompoundPredicate(CompoundPredicate.Operator.OR, castPredicate,
+ eqPredicate);
+ EsUtil.toEsDsl(compoundPredicate, notPushDownList);
+ Assertions.assertEquals(3, notPushDownList.size());
+ }
+
@Test
public void testEs6Mapping() throws IOException, URISyntaxException {
JSONObject testAliases = EsUtil.getMappingProps("test",
loadJsonFromFile("data/es/es6_aliases_mapping.json"),
@@ -310,4 +332,5 @@ public class EsUtilTest extends EsTestCase {
+
"\"fields\":{\"keyword\":{\"ignore_above\":256,\"type\":\"keyword\"}}},"
+
"\"test3\":{\"type\":\"double\"},\"test1\":{\"type\":\"keyword\"}}",
testIndex.toJSONString());
}
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
