(doris) branch master updated: [feat](Nereids) Optimize Sum Literal Rewriting by Excluding Single Instances (#35559)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9b5a7646238 [feat](Nereids) Optimize Sum Literal Rewriting by
Excluding Single Instances (#35559)
9b5a7646238 is described below
commit 9b5a764623873f3ec3165e9d8eca3980cb67fcd7
Author: 谢健
AuthorDate: Fri Jun 21 13:11:58 2024 +0800
[feat](Nereids) Optimize Sum Literal Rewriting by Excluding Single
Instances (#35559)
## Proposed changes
This PR introduces a change in the method removeOneSumLiteral to enhance
the performance of sum literal rewriting in SQL queries. The
modification ensures that sum literals appearing only once, such as in
expressions like select count(id1 + 1), count(id2 + 1) from t, are not
rewritten.
---
.../nereids/rules/rewrite/SumLiteralRewrite.java | 25 +++--
.../rules/rewrite/SumLiteralRewriteTest.java | 31 ++
2 files changed, 54 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewrite.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewrite.java
index c99071a714e..dcc64ce2c1d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewrite.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewrite.java
@@ -44,6 +44,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
@@ -64,13 +65,33 @@ public class SumLiteralRewrite extends
OneRewriteRuleFactory {
}
sumLiteralMap.put(pel.first, pel.second);
}
-if (sumLiteralMap.isEmpty()) {
+Map>
validSumLiteralMap =
+removeOneSumLiteral(sumLiteralMap);
+if (validSumLiteralMap.isEmpty()) {
return null;
}
-return rewriteSumLiteral(agg, sumLiteralMap);
+return rewriteSumLiteral(agg, validSumLiteralMap);
}).toRule(RuleType.SUM_LITERAL_REWRITE);
}
+// when there only one sum literal like select count(id1 + 1), count(id2 +
1) from t, we don't rewrite them.
+private Map> removeOneSumLiteral(
+Map> sumLiteralMap) {
+Map countSum = new HashMap<>();
+for (Entry> e :
sumLiteralMap.entrySet()) {
+Expression expr = e.getValue().first.expr;
+countSum.merge(expr, 1, Integer::sum);
+}
+Map> validSumLiteralMap = new
HashMap<>();
+for (Entry> e :
sumLiteralMap.entrySet()) {
+Expression expr = e.getValue().first.expr;
+if (countSum.get(expr) > 1) {
+validSumLiteralMap.put(e.getKey(), e.getValue());
+}
+}
+return validSumLiteralMap;
+}
+
private Plan rewriteSumLiteral(
LogicalAggregate agg, Map> sumLiteralMap) {
Set newAggOutput = new HashSet<>();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewriteTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewriteTest.java
index cb2cc77627e..19ea7b864fb 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewriteTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/SumLiteralRewriteTest.java
@@ -112,4 +112,35 @@ class SumLiteralRewriteTest implements
MemoPatternMatchSupported {
.printlnTree()
.matches(logicalAggregate().when(p -> p.getOutputs().size() ==
4));
}
+
+@Test
+void testSumOnce() {
+Slot slot1 = scan1.getOutput().get(0);
+Alias add1 = new Alias(new Sum(false, true, new Add(slot1,
Literal.of(1;
+LogicalAggregate agg = new LogicalAggregate<>(
+ImmutableList.of(scan1.getOutput().get(0)),
ImmutableList.of(add1), scan1);
+PlanChecker.from(MemoTestUtils.createConnectContext(), agg)
+.applyTopDown(ImmutableList.of(new
SumLiteralRewrite().build()))
+.printlnTree()
+.matches(logicalAggregate().when(p -> p.getOutputs().size() ==
1));
+
+Slot slot2 = new Alias(scan1.getOutput().get(0)).toSlot();
+Alias add2 = new Alias(new Sum(false, true, new Add(slot2,
Literal.of(2;
+agg = new LogicalAggregate<>(
+ImmutableList.of(scan1.getOutput().get(0)),
ImmutableList.of(add1, add2), scan1);
+PlanChecker.from(MemoTestUtils.createConnectCo
(doris) branch master updated: [opt](Nereids) Optimize findValidItems method to handle circular dependencies (#36839)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 015f051f73a [opt](Nereids) Optimize findValidItems method to handle
circular dependencies (#36839)
015f051f73a is described below
commit 015f051f73a8fe6d0e50dc643b2fcd114838b465
Author: 谢健
AuthorDate: Wed Jun 26 16:51:39 2024 +0800
[opt](Nereids) Optimize findValidItems method to handle circular
dependencies (#36839)
## Proposed changes
These optimizations allow the findValidItems method to correctly handle
circular dependencies while maintaining the required output slots. The
code is now more efficient and ensures that the necessary edges and
items are preserved during the traversal process.
---
.../apache/doris/nereids/properties/FuncDeps.java | 25 +-
.../nereids/rules/rewrite/EliminateGroupByKey.java | 2 +-
.../doris/nereids/properties/FuncDepsTest.java | 13 +--
.../rules/rewrite/EliminateGroupByKeyTest.java | 21 --
4 files changed, 43 insertions(+), 18 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
index 6c1b302d7dc..c17fd2eee57 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
@@ -27,6 +27,7 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Function dependence items.
@@ -96,11 +97,25 @@ public class FuncDeps {
}
}
-// find item that not in a circle
-private Set findValidItems() {
+// Find items that are not part of a circular dependency.
+// To keep the slots in requireOutputs, we need to always keep the edges
that start with output slots.
+// Note: We reduce the last edge in a circular dependency,
+// so we need to traverse from parents that contain the required output
slots.
+private Set findValidItems(Set requireOutputs) {
Set circleItem = new HashSet<>();
Set> visited = new HashSet<>();
-for (Set parent : edges.keySet()) {
+Set> parentInOutput = edges.keySet().stream()
+.filter(requireOutputs::containsAll)
+.collect(Collectors.toSet());
+for (Set parent : parentInOutput) {
+if (!visited.contains(parent)) {
+dfs(parent, visited, circleItem);
+}
+}
+Set> otherParent = edges.keySet().stream()
+.filter(parent -> !parentInOutput.contains(parent))
+.collect(Collectors.toSet());
+for (Set parent : otherParent) {
if (!visited.contains(parent)) {
dfs(parent, visited, circleItem);
}
@@ -126,10 +141,10 @@ public class FuncDeps {
* @param slots the initial set of slot sets to be reduced
* @return the minimal set of slot sets after applying all possible
reductions
*/
-public Set> eliminateDeps(Set> slots) {
+public Set> eliminateDeps(Set> slots, Set
requireOutputs) {
Set> minSlotSet = Sets.newHashSet(slots);
Set> eliminatedSlots = new HashSet<>();
-Set validItems = findValidItems();
+Set validItems = findValidItems(requireOutputs);
for (FuncDepsItem funcDepsItem : validItems) {
if (minSlotSet.contains(funcDepsItem.dependencies)
&& minSlotSet.contains(funcDepsItem.determinants)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java
index 9e205f85809..fbe0988daff 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateGroupByKey.java
@@ -91,7 +91,7 @@ public class EliminateGroupByKey implements
RewriteRuleFactory {
return null;
}
-Set> minGroupBySlots = funcDeps.eliminateDeps(new
HashSet<>(groupBySlots.values()));
+Set> minGroupBySlots = funcDeps.eliminateDeps(new
HashSet<>(groupBySlots.values()), requireOutput);
Set removeExpression = new HashSet<>();
for (Entry> entry : groupBySlots.entrySet()) {
if (!minGroupBySlots.contains(entry.getValue())
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/properties/FuncDepsTest.java
index 64df33acd60..6b1
(doris) branch master updated: [feat](Nereids) Optimize query by pushing down aggregation through join on foreign key (#36035)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6889225b19e [feat](Nereids) Optimize query by pushing down aggregation
through join on foreign key (#36035)
6889225b19e is described below
commit 6889225b19e5826d74582c518f3d38982a1e3886
Author: 谢健
AuthorDate: Mon Jul 1 14:37:23 2024 +0800
[feat](Nereids) Optimize query by pushing down aggregation through join on
foreign key (#36035)
## Proposed changes
This PR optimizes query performance by pushing down aggregations through
joins when grouped by a foreign key. This adjustment reduces data
processing overhead above the join, improving both speed and resource
efficiency.
Transformation Example:
Before Optimization:
```
Aggregation(group by fk)
|
Join(pk = fk)
/ \
pk fk
```
After Optimization:
```
Join(pk = fk)
/ \
pk Aggregation(group by fk)
|
fk
```
---
.../doris/nereids/jobs/executor/Rewriter.java | 6 +-
.../apache/doris/nereids/properties/FuncDeps.java | 19 ++
.../org/apache/doris/nereids/rules/RuleType.java | 2 +-
.../rewrite/PushDownAggThroughJoinOnPkFk.java | 348 +
.../rewrite/PushDownAggThroughJoinOnPkFkTest.java | 158 ++
.../shape/query38.out | 51 ++-
.../shape/query87.out | 51 ++-
.../noStatsRfPrune/query38.out | 51 ++-
.../noStatsRfPrune/query87.out | 51 ++-
.../no_stats_shape/query38.out | 51 ++-
.../no_stats_shape/query87.out | 51 ++-
.../rf_prune/query38.out | 51 ++-
.../rf_prune/query87.out | 51 ++-
.../nereids_tpcds_shape_sf100_p0/shape/query38.out | 51 ++-
.../nereids_tpcds_shape_sf100_p0/shape/query87.out | 51 ++-
15 files changed, 770 insertions(+), 273 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 9505bdca87d..0a2906ca055 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -110,6 +110,7 @@ import
org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOnPkFk;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOneSide;
import org.apache.doris.nereids.rules.rewrite.PushDownDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject;
@@ -348,8 +349,9 @@ public class Rewriter extends AbstractBatchJobExecutor {
),
// this rule should be invoked after topic "Join pull up"
-topic("eliminate group by keys according to fd items",
-topDown(new EliminateGroupByKey())
+topic("eliminate Aggregate according to fd items",
+topDown(new EliminateGroupByKey()),
+topDown(new PushDownAggThroughJoinOnPkFk())
),
topic("Limit optimization",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
index c17fd2eee57..be7b0853605 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
@@ -62,6 +62,7 @@ public class FuncDeps {
}
private final Set items;
+// determinants -> dependencies
private final Map, Set>> edges;
public FuncDeps() {
@@ -159,6 +160,24 @@ public class FuncDeps {
return items.contains(new FuncDepsItem(dominate, dependency));
}
+public boolean isCircleDeps(Set dominate, Set dependency) {
+return items.contains(new FuncDepsItem(dominate, dependency))
+&& items.contains(new FuncDepsItem(dependency, dominate));
+}
+
+/**
+ * find the determinants of dependencies
+ */
+public Set> findDeterminats(Set dependency) {
+Set> determinants = new HashSet<>();
+for (FuncDepsItem item : items) {
+if (item.dependencies.equals(dependency)) {
+determinants.add(item.
(doris) branch master updated (f5c18ee9321 -> 4b83168943a)
This is an automated email from the ASF dual-hosted git repository. xiejiann pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from f5c18ee9321 [fix](binlog) gc dropped partition records in disable binlog gc (#37326) add 4b83168943a [feat](Nereids) Enhance filter processing in foreign key context to exclude hidden conjuncts (#37179) No new revisions were added by this update. Summary of changes: .../nereids/rules/rewrite/ForeignKeyContext.java | 27 +- .../rules/rewrite/EliminateJoinByFkTest.java | 14 +-- 2 files changed, 28 insertions(+), 13 deletions(-) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated (ed4abc7ff0a -> 7229db0f927)
This is an automated email from the ASF dual-hosted git repository. xiejiann pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/doris.git from ed4abc7ff0a [fix](agg) fix RowsProduced counter is not set (#38271) add 7229db0f927 [opt](Nereids): disable PRUNE_EMPTY_PARTITION rule in SqlTestBase.java (#38246) No new revisions were added by this update. Summary of changes: .../src/test/java/org/apache/doris/nereids/sqltest/JoinOrderJobTest.java | 1 + .../src/test/java/org/apache/doris/nereids/sqltest/SqlTestBase.java | 1 + 2 files changed, 2 insertions(+) - To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [opt](Nereids) fix comment and remove useless function `computeFdItem ` (#38282)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 95270eb1ee5 [opt](Nereids) fix comment and remove useless function
`computeFdItem ` (#38282)
95270eb1ee5 is described below
commit 95270eb1ee5f1aa8d80655e6dd6e38afcd2297bf
Author: 谢健
AuthorDate: Wed Jul 24 13:56:40 2024 +0800
[opt](Nereids) fix comment and remove useless function `computeFdItem `
(#38282)
## Proposed changes
1. fix comment in eliminate func deps
2. remove computeFdItem codes
---
.../apache/doris/nereids/properties/FuncDeps.java | 40 --
.../trees/plans/BlockFuncDepsPropagation.java | 8 --
.../nereids/trees/plans/PropagateFuncDeps.java | 19 ---
.../trees/plans/logical/LogicalAggregate.java | 25
.../trees/plans/logical/LogicalAssertNumRows.java | 7 -
.../plans/logical/LogicalCatalogRelation.java | 5 -
.../plans/logical/LogicalDeferMaterializeTopN.java | 21 ---
.../nereids/trees/plans/logical/LogicalExcept.java | 28
.../nereids/trees/plans/logical/LogicalFilter.java | 11 --
.../trees/plans/logical/LogicalGenerate.java | 7 -
.../nereids/trees/plans/logical/LogicalHaving.java | 11 --
.../trees/plans/logical/LogicalIntersect.java | 30
.../nereids/trees/plans/logical/LogicalJoin.java | 154 -
.../nereids/trees/plans/logical/LogicalLimit.java | 22 ---
.../trees/plans/logical/LogicalOneRowRelation.java | 20 ---
.../nereids/trees/plans/logical/LogicalPlan.java | 5 -
.../trees/plans/logical/LogicalProject.java| 11 --
.../nereids/trees/plans/logical/LogicalRepeat.java | 12 --
.../trees/plans/logical/LogicalSubQueryAlias.java | 8 --
.../nereids/trees/plans/logical/LogicalTopN.java | 22 ---
.../nereids/trees/plans/logical/LogicalUnion.java | 21 ---
.../nereids/trees/plans/logical/LogicalView.java | 7 -
.../nereids/trees/plans/logical/LogicalWindow.java | 13 --
23 files changed, 28 insertions(+), 479 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
index e637af8982c..5eebf08ddd7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/FuncDeps.java
@@ -125,23 +125,39 @@ public class FuncDeps {
}
/**
- * Reduces a given set of slot sets by eliminating dependencies using a
breadth-first search (BFS) approach.
+ * Reduces a given set of slot sets by eliminating dependencies based on
valid functional dependency items.
*
- * Let's assume we have the following sets of slots and functional
dependencies:
- * Slots: {A, B, C}, {D, E}, {F}
- * Dependencies: {A} -> {B}, {D, E} -> {F}
- * The BFS reduction process would look like this:
- * 1. Initial set: [{A, B, C}, {D, E}, {F}]
- * 2. Apply {A} -> {B}:
- *- New set: [{A, C}, {D, E}, {F}]
- * 3. Apply {D, E} -> {F}:
- *- New set: [{A, C}, {D, E}]
- * 4. No more dependencies can be applied, output: [{A, C}, {D, E}]
+ * This method works as follows:
+ * 1. Find valid functional dependency items (those not part of circular
dependencies).
+ * 2. For each valid functional dependency item:
+ *- If both the determinants and dependencies are present in the
current set of slots,
+ * mark the dependencies for elimination.
+ * 3. Remove all marked dependencies from the set of slots.
+ *
+ *
+ * Example:
+ * Given:
+ * - Initial slots: {{A, B, C}, {D, E}, {F, G}}
+ * - Required outputs: {A, D, F}
+ * - Valid functional dependencies: {A} -> {B}, {D, E} -> {G}, {F} -> {G}
+ *
+ * Process:
+ * 1. Start with minSlotSet = {{A, B, C}, {D, E}, {F, G}}
+ * 2. For {A} -> {B}:
+ *- Both {A} and {B} are in minSlotSet, so mark {B} for elimination
+ * 3. For {D, E} -> {G}:
+ *- Both {D, E} and {G} are in minSlotSet, so mark {G} for elimination
+ * 4. For {F} -> {G}:
+ *- Both {F} and {G} are in minSlotSet, but {G} is already marked for
elimination
+ * 5. Remove eliminated slots: {B} and {G}
+ *
+ * Result: {{A, C}, {D, E}, {F}}
*
*
* @param slots the initial set of slot sets to be reduced
+ * @param requireOutputs the set of slots that must be preserved in the
output
* @return the minimal set of slot sets after applying all possible
reductions
- */
+*/
public Set> eliminateDeps(Set> slots, Set
requireOutputs) {
Set> minSlotSet = Sets.newHashSet(slots);
Set> eliminatedSlots = new HashSet<>();
diff --git
a/fe/fe-co
(doris) branch master updated: [fix](Nereids) Add Primary Key by Primary Relation Instead of Foreign Relation (#36008)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 935484ba734 [fix](Nereids) Add Primary Key by Primary Relation Instead
of Foreign Relation (#36008)
935484ba734 is described below
commit 935484ba7348103860ad667cafc364a8a060da17
Author: 谢健
AuthorDate: Wed Jun 12 10:35:59 2024 +0800
[fix](Nereids) Add Primary Key by Primary Relation Instead of Foreign
Relation (#36008)
Ensure the primary key is added by the primary relation, enhancing
efficiency by expiring keys post joins to avoid eliminating redundant
joins based on primary-foreign key relationships.
---
.../apache/doris/nereids/rules/rewrite/ForeignKeyContext.java | 9 -
.../doris/nereids/rules/rewrite/EliminateJoinByFkTest.java| 11 +++
2 files changed, 19 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ForeignKeyContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ForeignKeyContext.java
index 600f3d1ac5a..3668ae68337 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ForeignKeyContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/ForeignKeyContext.java
@@ -67,6 +67,7 @@ public class ForeignKeyContext {
public Void visitLogicalRelation(LogicalRelation relation,
ForeignKeyContext context) {
if (relation instanceof LogicalCatalogRelation) {
context.putAllForeignKeys(((LogicalCatalogRelation)
relation).getTable());
+context.putAllPrimaryKeys(((LogicalCatalogRelation)
relation).getTable());
relation.getOutput().stream()
.filter(SlotReference.class::isInstance)
.map(SlotReference.class::cast)
@@ -101,7 +102,13 @@ public class ForeignKeyContext {
Map constraint = c.getForeignToPrimary(table);
constraints.add(c.getForeignToPrimary(table));
foreignKeys.addAll(constraint.keySet());
-primaryKeys.addAll(constraint.values());
+});
+}
+
+void putAllPrimaryKeys(TableIf table) {
+table.getPrimaryKeyConstraints().forEach(c -> {
+Set primaryKey = c.getPrimaryKeys(table);
+primaryKeys.addAll(primaryKey);
});
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
index 86b37af0169..86d3ef9700a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateJoinByFkTest.java
@@ -57,6 +57,17 @@ class EliminateJoinByFkTest extends TestWithFeService
implements MemoPatternMatc
connectContext.getSessionVariable().setDisableNereidsRules("PRUNE_EMPTY_PARTITION");
}
+@Test
+void testPriWithJoin() {
+String sql = "select pri.id1 from (select p1.id1 from pri as p1 cross
join pri as p2) pri "
++ "inner join foreign_not_null on pri.id1 =
foreign_not_null.id2";
+PlanChecker.from(connectContext)
+.analyze(sql)
+.rewrite()
+.matches(logicalJoin())
+.printlnTree();
+}
+
@Test
void testNotNull() {
String sql = "select pri.id1 from pri inner join foreign_not_null on
pri.id1 = foreign_not_null.id2";
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [fix](Nereids) Enhance LogicalJoin input validation when push down agg (#37343)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9d7d7b9aab6 [fix](Nereids) Enhance LogicalJoin input validation when
push down agg (#37343)
9d7d7b9aab6 is described below
commit 9d7d7b9aab67604ec2077753d6b3456c6371785f
Author: 谢健
AuthorDate: Mon Jul 8 11:24:34 2024 +0800
[fix](Nereids) Enhance LogicalJoin input validation when push down agg
(#37343)
intro by #36035
This PR refines the LogicalJoin class by introducing robust input
validation. Key improvements:
* Implement precise checks for join input validity
* Ensure consistency between input slots and output sets
* Gracefully handle various join scenarios (left/right)
These enhancements bolster query integrity and optimize join operations.
---
.../rules/rewrite/PushDownAggThroughJoinOnPkFk.java| 14 --
.../rules/rewrite/PushDownAggThroughJoinOnPkFkTest.java| 10 ++
2 files changed, 22 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java
index 827f0819637..2aeb59ae9c7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFk.java
@@ -106,9 +106,19 @@ public class PushDownAggThroughJoinOnPkFk implements
RewriteRuleFactory {
LogicalJoin newJoin = innerJoinCluster
.constructJoinWithPrimary(e.getKey(), subJoin,
primaryAndForeign.first);
if (newJoin != null && newJoin.left() == primaryAndForeign.first) {
-return newJoin.withChildren(newJoin.left(),
newAgg.withChildren(newJoin.right()));
+newJoin = (LogicalJoin) newJoin
+.withChildren(newJoin.left(),
newAgg.withChildren(newJoin.right()));
+if (Sets.union(newJoin.left().getOutputSet(),
newJoin.right().getOutputSet())
+.containsAll(newJoin.getInputSlots())) {
+return newJoin;
+}
} else if (newJoin != null && newJoin.right() ==
primaryAndForeign.first) {
-return
newJoin.withChildren(newAgg.withChildren(newJoin.left()), newJoin.right());
+newJoin = (LogicalJoin) newJoin
+.withChildren(newAgg.withChildren(newJoin.left()),
newJoin.right());
+if (Sets.union(newJoin.left().getOutputSet(),
newJoin.right().getOutputSet())
+.containsAll(newJoin.getInputSlots())) {
+return newJoin;
+}
}
}
return null;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFkTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFkTest.java
index 91e66790002..1ef4653e36c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFkTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoinOnPkFkTest.java
@@ -155,4 +155,14 @@ class PushDownAggThroughJoinOnPkFkTest extends
TestWithFeService implements Memo
.matches(logicalJoin(logicalAggregate(), any()))
.printlnTree();
}
+
+@Test
+void testMissSlot() {
+String sql = "select count(pri.name) from pri inner join
foreign_not_null on pri.name = foreign_not_null.name";
+PlanChecker.from(connectContext)
+.analyze(sql)
+.rewrite()
+.matches(logicalAggregate(logicalProject(logicalJoin(
+.printlnTree();
+}
}
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
(doris) branch master updated: [fix](Nereids) add order_qt in fuction groovy (#37542)
This is an automated email from the ASF dual-hosted git repository.
xiejiann pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 24b41fcd0b6 [fix](Nereids) add order_qt in fuction groovy (#37542)
24b41fcd0b6 is described below
commit 24b41fcd0b66eb0a439f6a8f1e48832bb9380d38
Author: 谢健
AuthorDate: Fri Jul 12 19:59:09 2024 +0800
[fix](Nereids) add order_qt in fuction groovy (#37542)
## Proposed changes
use `order_qt` to avoid unstable order of output in the
`function.groovy`
---
regression-test/suites/nereids_syntax_p0/function.groovy | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/regression-test/suites/nereids_syntax_p0/function.groovy
b/regression-test/suites/nereids_syntax_p0/function.groovy
index 41e6309aabe..3ca00abfcd6 100644
--- a/regression-test/suites/nereids_syntax_p0/function.groovy
+++ b/regression-test/suites/nereids_syntax_p0/function.groovy
@@ -86,9 +86,9 @@ suite("nereids_function") {
result([[0L, 0L], [1L, 1L], [2L, 2L], [3L, 3L], [4L, 4L], [5L, 5L],
[6L, 6L], [7L, 7L], [8L, 8L], [9L, 9L]])
}
-qt_subquery1 """ select * from numbers("number" = "10") where number =
(select number from numbers("number" = "10") where number=1); """
-qt_subquery2 """ select * from numbers("number" = "10") where number in
(select number from numbers("number" = "10") where number>5); """
-qt_subquery3 """ select a.number from numbers("number" = "10") a where
number in (select number from numbers("number" = "10") b where
a.number=b.number); """
+order_qt_subquery1 """ select * from numbers("number" = "10") where number
= (select number from numbers("number" = "10") where number=1); """
+order_qt_subquery2 """ select * from numbers("number" = "10") where number
in (select number from numbers("number" = "10") where number>5); """
+order_qt_subquery3 """ select a.number from numbers("number" = "10") a
where number in (select number from numbers("number" = "10") b where
a.number=b.number); """
test {
sql """select `number` from numbers("number" = "-1")"""
-
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
