github-actions[bot] commented on code in PR #65569:
URL: https://github.com/apache/doris/pull/65569#discussion_r3576005356
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalLazyMaterializeFileScan.java:
##########
@@ -17,35 +17,48 @@
package org.apache.doris.nereids.trees.plans.physical;
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.AbstractPlan;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.statistics.Statistics;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Optional;
-/**
- wrapper for FileScan used for lazy materialization
- */
+/** File scan wrapper that replaces deferred columns with a row-id used by the
materialize node. */
public class PhysicalLazyMaterializeFileScan extends PhysicalFileScan {
- private PhysicalFileScan scan;
- private SlotReference rowId;
+ private final PhysicalFileScan scan;
+ private final SlotReference rowId;
private final List<Slot> lazySlots;
- private List<Slot> output;
- /**
- * PhysicalLazyMaterializeFileScan
- */
public PhysicalLazyMaterializeFileScan(PhysicalFileScan scan,
SlotReference rowId, List<Slot> lazySlots) {
+ this(scan, rowId, lazySlots, scan.getGroupExpression(), null,
+ scan.getPhysicalProperties(), scan.getStats());
+ }
+
+ private PhysicalLazyMaterializeFileScan(PhysicalFileScan scan,
SlotReference rowId, List<Slot> lazySlots,
+ Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics) {
super(scan.getRelationId(), scan.getTable(), scan.getQualifier(),
scan.getDistributionSpec(),
- Optional.empty(), null, null, scan.getStats(),
- scan.selectedPartitions, scan.getTableSample(),
+ groupExpression, logicalProperties, physicalProperties,
statistics,
+ scan.getSelectedPartitions(), scan.getTableSample(),
scan.getTableSnapshot(), scan.getOperativeSlots(),
scan.getScanParams());
this.scan = scan;
this.rowId = rowId;
- this.lazySlots = lazySlots;
+ this.lazySlots = ImmutableList.copyOf(lazySlots);
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitPhysicalLazyMaterializeFileScan(this, context);
}
Review Comment:
`LazyMaterializeTopN` runs before `TopNScanOpt` and
`RuntimeFilterGenerator`, so a file scan that this PR wraps reaches those later
visitors as `PhysicalLazyMaterializeFileScan`. This `accept()` dispatches to
`visitPhysicalLazyMaterializeFileScan`, but both pushdown visitors only
delegate the lazy OLAP wrapper back to `visitPhysicalRelation`; the default
file-wrapper visitor is just generic leaf traversal. Normal `PhysicalFileScan`
is supported by TopN filters and can push runtime filters, so the lazy file
path silently loses those filters. Please add lazy-file overrides in
`TopnFilterPushDownVisitor` and `RuntimeFilterPushDownVisitor` that delegate to
the scan-level handling, and cover a lazy file-scan shape.
##########
regression-test/suites/query_p0/topn_lazy/topn_lazy_slot_roles.groovy:
##########
@@ -0,0 +1,569 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("topn_lazy_slot_roles") {
+ sql """
+ set topn_lazy_materialization_threshold = 5;
+ set topn_lazy_materialization_using_index = false;
+ set enable_segment_limit_pushdown = false;
+ set disable_join_reorder = true;
+
+ drop table if exists topn_lazy_slot_roles;
+ create table topn_lazy_slot_roles (
+ id int not null,
+ k int not null,
+ v int null,
+ pad varchar(32) null,
+ extra int null,
+ index idx_v(v) using inverted
+ )
+ duplicate key(id, k)
+ distributed by hash(id) buckets 1
+ properties (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ insert into topn_lazy_slot_roles values
+ (1, 10, null, 'a', 101),
+ (2, 20, 1, 'b', 102),
+ (3, 30, 2, 'c', 103),
+ (4, 40, null, 'd', 104),
+ (5, 50, 3, 'e', 105);
+ """
+
+ explain {
+ sql """
+ select pad
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+ contains("VMaterializeNode")
+ }
+
+ order_qt_slot_roles_direct_base """
+ select pad
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+
+ explain {
+ sql """
+ select pad as pad_alias
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+ contains("VMaterializeNode")
+ }
+
+ order_qt_slot_roles_alias_only """
+ select pad as pad_alias
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+
+ order_qt_slot_roles_base_and_alias """
+ select pad, pad as pad_alias
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+
+ order_qt_slot_roles_two_aliases """
+ select pad as pad_alias_1, pad as pad_alias_2
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+
+ order_qt_slot_roles_chained_projects """
+ select pad_alias_2
+ from (
+ select pad_alias_1 as pad_alias_2, k
+ from (
+ select pad as pad_alias_1, k
+ from topn_lazy_slot_roles
+ ) t1
+ ) t2
+ order by k
+ limit 2
+ """
+
+ order_qt_slot_roles_derived_expression """
+ select concat(pad, '!') as pad_expr
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+
+ order_qt_slot_roles_literal_and_null """
+ select pad, 7 as literal_value, null as null_value
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_predicate_only """
+ select pad
+ from topn_lazy_slot_roles
+ where v is null
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_predicate_and_order_key """
+ select pad
+ from topn_lazy_slot_roles
+ where k < 50
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_predicate_and_output """
+ select v, pad
+ from topn_lazy_slot_roles
+ where v is null
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_predicate_output_order_key """
+ select k, pad
+ from topn_lazy_slot_roles
+ where k < 50
+ order by k
+ limit 2
+ """
+
+ explain {
+ sql """
+ select uuid()
+ from topn_lazy_slot_roles
+ order by k
+ limit 2
+ """
+ notContains("VMaterializeNode")
+ }
+
+ sql "set topn_lazy_materialization_using_index = true"
+
+ order_qt_filter_roles_index_predicate_only """
+ select pad
+ from topn_lazy_slot_roles
+ where v is null
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_index_or_predicate """
+ select pad
+ from topn_lazy_slot_roles
+ where v is null or k < 30
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_index_predicate_and_output """
+ select v, pad
+ from topn_lazy_slot_roles
+ where v is null
+ order by k
+ limit 2
+ """
+
+ order_qt_filter_roles_index_predicate_output_order_key """
+ select v, pad
+ from topn_lazy_slot_roles
+ where v is not null
+ order by v, k
+ limit 2
+ """
+
+ order_qt_filter_roles_index_expression_fallback """
+ select v, pad
+ from topn_lazy_slot_roles
+ where v + 1 > 2
+ order by k
+ limit 2
+ """
+
+ sql "set topn_lazy_materialization_using_index = false"
+
+ order_qt_window_roles_partition_and_order_keys """
+ select k, pad, row_number() over (partition by v order by k) as rn
+ from topn_lazy_slot_roles
+ qualify rn = 1
+ order by k
+ limit 3
+ """
+
+ order_qt_window_roles_alias_and_outer_order """
+ select k, k as k_alias, pad, row_number() over (partition by v order
by k) as rn
+ from topn_lazy_slot_roles
+ qualify rn = 1
+ order by k
+ limit 3
+ """
+
+ order_qt_window_roles_qualify_alias_is_outer_order_key """
+ select k, pad, row_number() over (partition by v order by k) as rn
+ from topn_lazy_slot_roles
+ qualify rn <= 2
+ order by rn, k
+ limit 3
+ """
+
+ order_qt_window_roles_nested_topn """
+ select pad
+ from (
+ select id, k, pad
+ from topn_lazy_slot_roles
+ order by k
+ limit 4
+ ) t
+ order by id
+ limit 2
+ """
+
+ sql """
+ drop table if exists topn_lazy_join_left;
+ create table topn_lazy_join_left (
+ id int not null,
+ k int not null,
+ ts datetime not null,
+ pad varchar(32) null
+ )
+ duplicate key(id, k)
+ distributed by hash(id) buckets 1
+ properties (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ insert into topn_lazy_join_left values
+ (1, 10, '2026-01-01 10:00:00', 'l1'),
+ (2, 20, '2026-01-01 11:00:00', 'l2'),
+ (3, 30, '2026-01-01 12:00:00', 'l3'),
+ (4, 40, '2026-01-01 13:00:00', 'l4');
+
+ drop table if exists topn_lazy_join_right;
+ create table topn_lazy_join_right (
+ id int not null,
+ k int not null,
+ ts datetime not null,
+ pad varchar(32) null
+ )
+ duplicate key(id, k)
+ distributed by hash(id) buckets 1
+ properties (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+
+ insert into topn_lazy_join_right values
+ (2, 200, '2026-01-01 10:30:00', 'r2'),
+ (3, 300, '2026-01-01 11:30:00', 'r3'),
+ (4, 400, '2026-01-01 12:30:00', 'r4'),
+ (5, 500, '2026-01-01 13:30:00', 'r5');
+ """
+
+ order_qt_join_roles_left_relation_lazy """
+ select l.pad
+ from topn_lazy_join_left l
+ inner join topn_lazy_join_right r on l.id = r.id
+ order by l.k
+ limit 3
+ """
+
+ order_qt_join_roles_both_relations_lazy """
+ select l.pad, r.pad
+ from topn_lazy_join_left l
+ inner join topn_lazy_join_right r on l.id = r.id
+ order by l.k
+ limit 3
+ """
+
+ order_qt_join_roles_join_key_is_order_key """
+ select l.pad, r.pad
+ from topn_lazy_join_left l
+ inner join topn_lazy_join_right r on l.id = r.id
+ order by l.id
+ limit 3
+ """
+
+ order_qt_join_roles_predicate_slot_not_output """
+ select l.pad
+ from topn_lazy_join_left l
+ inner join topn_lazy_join_right r on l.id = r.id
+ where r.k >= 300
+ order by l.k
+ limit 2
+ """
+
+ order_qt_join_roles_left_outer_nullable_rowid """
+ select l.pad, r.pad
+ from topn_lazy_join_left l
+ left outer join topn_lazy_join_right r on l.id = r.id
+ order by l.k
+ limit 4
+ """
+
+ order_qt_join_roles_right_outer_nullable_rowid """
+ select l.pad, r.pad
+ from topn_lazy_join_left l
+ right outer join topn_lazy_join_right r on l.id = r.id
+ order by r.k
+ limit 4
+ """
+
+ order_qt_join_roles_full_outer_nullable_rowids """
+ select l.id, r.id, l.pad, r.pad
+ from topn_lazy_join_left l
+ full outer join topn_lazy_join_right r on l.id = r.id
+ order by coalesce(l.id, r.id)
+ limit 5
+ """
+
+ order_qt_join_roles_self_join_relation_identity """
+ select l.pad, r.pad
+ from topn_lazy_join_left l
Review Comment:
This self-join does not actually catch relation-id/source mixups. Since `l`
and `r` are the same table joined on `l.id = r.id`, both selected lazy columns
should be the same value for every row (`l1/l1`, `l2/l2`, ...). A bug that
reused one side's row id/source for both lazy fetches would still satisfy this
expected output. Please make the two sides distinguishable, for example by
joining different ids (`l.id + 1 = r.id`) or otherwise using different expected
left/right pads, and add an explain assertion that this exact case still uses
lazy materialization.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
