Skip to content

Commit e0ae2d8

Browse files
seawindeseawinde
committed
mv poc code
1 parent 7f1ddf8 commit e0ae2d8

21 files changed

+459
-238
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java

+4-5
Original file line numberDiff line numberDiff line change
@@ -238,9 +238,8 @@ public PhysicalPlanTranslator(PlanTranslatorContext context, StatsErrorEstimator
238238
public PlanFragment translatePlan(PhysicalPlan physicalPlan) {
239239
PlanFragment rootFragment = physicalPlan.accept(this, context);
240240
List<Expr> outputExprs = Lists.newArrayList();
241-
physicalPlan.getOutput().stream().forEach(slotReference -> outputExprs.add(
242-
context.findSlotRef((SlotReference) slotReference) == null ? context.findSlotRef(
243-
slotReference.getExprId()) : context.findSlotRef((SlotReference) slotReference)));
241+
physicalPlan.getOutput().stream().map(Slot::getExprId)
242+
.forEach(exprId -> outputExprs.add(context.findSlotRef(exprId)));
244243
rootFragment.setOutputExprs(outputExprs);
245244
Collections.reverse(context.getPlanFragments());
246245
// TODO: maybe we need to trans nullable directly? and then we could remove call computeMemLayout
@@ -961,7 +960,7 @@ public PlanFragment visitPhysicalCTEConsumer(PhysicalCTEConsumer cteConsumer,
961960
Slot consumerSlot = cteConsumer.getProducerToConsumerSlotMap().get(producerSlot);
962961
SlotRef slotRef = context.findSlotRef(producerSlot.getExprId());
963962
tupleDescriptor = slotRef.getDesc().getParent();
964-
context.addExprIdSlotRefPair((SlotReference) consumerSlot, slotRef);
963+
context.addExprIdSlotRefPair(consumerSlot.getExprId(), slotRef);
965964
}
966965
CTEScanNode cteScanNode = new CTEScanNode(tupleDescriptor);
967966
context.getRuntimeTranslator().ifPresent(runtimeFilterTranslator ->
@@ -1646,7 +1645,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject<? extends Plan> project
16461645
inputPlanNode.setOutputTupleDesc(projectionTuple);
16471646
} else {
16481647
for (int i = 0; i < slots.size(); ++i) {
1649-
context.addExprIdSlotRefPair((SlotReference) slots.get(i),
1648+
context.addExprIdSlotRefPair(slots.get(i).getExprId(),
16501649
(SlotRef) projectionExprs.get(i));
16511650
slotIdsByOrder.add(((SlotRef) projectionExprs.get(i)).getSlotId());
16521651
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java

+4-10
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,6 @@ public class PlanTranslatorContext {
7575
* index from Nereids' slot to legacy slot.
7676
*/
7777
private final Map<ExprId, SlotRef> exprIdToSlotRef = Maps.newHashMap();
78-
private final Map<SlotReference, SlotRef> slotReferenceToSlotRef = Maps.newHashMap();
7978

8079
/**
8180
* Inverted index from legacy slot to Nereids' slot.
@@ -200,10 +199,9 @@ public void addPlanFragment(PlanFragment planFragment) {
200199
this.planFragments.add(planFragment);
201200
}
202201

203-
public void addExprIdSlotRefPair(SlotReference slotReference, SlotRef slotRef) {
204-
exprIdToSlotRef.put(slotReference.getExprId(), slotRef);
205-
slotIdToExprId.put(slotRef.getDesc().getId(), slotReference.getExprId());
206-
slotReferenceToSlotRef.put(slotReference, slotRef);
202+
public void addExprIdSlotRefPair(ExprId exprId, SlotRef slotRef) {
203+
exprIdToSlotRef.put(exprId, slotRef);
204+
slotIdToExprId.put(slotRef.getDesc().getId(), exprId);
207205
}
208206

209207
public void addExprIdColumnRefPair(ExprId exprId, ColumnRefExpr columnRefExpr) {
@@ -220,10 +218,6 @@ public SlotRef findSlotRef(ExprId exprId) {
220218
return exprIdToSlotRef.get(exprId);
221219
}
222220

223-
public SlotRef findSlotRef(SlotReference slotReference) {
224-
return slotReferenceToSlotRef.get(slotReference);
225-
}
226-
227221
public ColumnRefExpr findColumnRef(ExprId exprId) {
228222
return exprIdToColumnRef.get(exprId);
229223
}
@@ -279,7 +273,7 @@ public SlotDescriptor createSlotDesc(TupleDescriptor tupleDesc, SlotReference sl
279273
}
280274
slotRef.setTable(table);
281275
slotRef.setLabel(slotReference.getName());
282-
this.addExprIdSlotRefPair(slotReference, slotRef);
276+
this.addExprIdSlotRefPair(slotReference.getExprId(), slotRef);
283277
slotDescriptor.setIsNullable(slotReference.nullable());
284278
return slotDescriptor;
285279
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java

+5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@
2828

2929
/**
3030
* HyperGraph Node.
31+
* Jc
32+
* \
33+
* F
34+
* \
35+
* JC
3136
*/
3237
public class Node {
3338
private final int index;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java

+12
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.nereids.cost.Cost;
2222
import org.apache.doris.nereids.properties.LogicalProperties;
2323
import org.apache.doris.nereids.properties.PhysicalProperties;
24+
import org.apache.doris.nereids.rules.rewrite.mv.StructInfo;
2425
import org.apache.doris.nereids.trees.expressions.literal.Literal;
2526
import org.apache.doris.nereids.trees.plans.JoinType;
2627
import org.apache.doris.nereids.trees.plans.Plan;
@@ -74,6 +75,8 @@ public class Group {
7475

7576
private int chosenGroupExpressionId = -1;
7677

78+
private Optional<StructInfo> structInfo = Optional.empty();
79+
7780
/**
7881
* Constructor for Group.
7982
*
@@ -152,6 +155,7 @@ public GroupExpression logicalExpressionsAt(int index) {
152155
* @return the first logical group expression in this group
153156
*/
154157
public GroupExpression getLogicalExpression() {
158+
// poc tmp
155159
Preconditions.checkArgument(logicalExpressions.size() == 1,
156160
"There should be only one Logical Expression in Group");
157161
return logicalExpressions.get(0);
@@ -532,4 +536,12 @@ public String treeString() {
532536

533537
return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan);
534538
}
539+
540+
public Optional<StructInfo> getStructInfo() {
541+
return structInfo;
542+
}
543+
544+
public void setStructInfo(StructInfo structInfo) {
545+
this.structInfo = Optional.ofNullable(structInfo);
546+
}
535547
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java

+17
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ private Group init(Plan plan) {
314314
plan = replaceChildrenToGroupPlan(plan, childrenGroups);
315315
GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups);
316316
Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties());
317+
// PoC add struct info to group
317318

318319
groups.put(group.getGroupId(), group);
319320
if (groupExpressions.containsKey(newGroupExpression)) {
@@ -323,6 +324,22 @@ private Group init(Plan plan) {
323324
return group;
324325
}
325326

327+
/** initPoC */
328+
public Group initPoC(Plan plan) {
329+
Preconditions.checkArgument(!(plan instanceof GroupPlan), "Cannot init memo by a GroupPlan");
330+
331+
/* initialize children recursively */
332+
List<Group> childrenGroups = new ArrayList<>(plan.arity());
333+
for (Plan child : plan.children()) {
334+
childrenGroups.add(initPoC(child));
335+
}
336+
337+
plan = replaceChildrenToGroupPlan(plan, childrenGroups);
338+
GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups);
339+
Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties());
340+
return group;
341+
}
342+
326343
/**
327344
* add or replace the plan into the target group.
328345
* <p>

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ private Plan deferMaterialize(LogicalResultSink<? extends Plan> logicalResultSin
8383
LogicalTopN<? extends Plan> logicalTopN, Optional<LogicalFilter<? extends Plan>> logicalFilter,
8484
LogicalOlapScan logicalOlapScan) {
8585
Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column");
86-
SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier());
86+
SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier(), null);
8787
Set<ExprId> deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet());
8888
logicalFilter.ifPresent(filter -> filter.getConjuncts()
8989
.forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds())));

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewJoinRule.java

+63-11
Original file line numberDiff line numberDiff line change
@@ -21,19 +21,27 @@
2121
import org.apache.doris.nereids.trees.expressions.Expression;
2222
import org.apache.doris.nereids.trees.expressions.NamedExpression;
2323
import org.apache.doris.nereids.trees.expressions.Slot;
24+
import org.apache.doris.nereids.trees.expressions.SlotReference;
2425
import org.apache.doris.nereids.trees.plans.GroupPlan;
2526
import org.apache.doris.nereids.trees.plans.Plan;
27+
import org.apache.doris.nereids.trees.plans.RelationId;
2628
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
2729
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
2830
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
2931
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
3032
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
3133
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor;
34+
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors;
35+
import org.apache.doris.nereids.trees.plans.visitor.PlanVisitors.SlotReferenceReplacer.ExprReplacer;
3236

3337
import com.google.common.collect.BiMap;
3438

39+
import java.util.HashMap;
40+
import java.util.HashSet;
3541
import java.util.List;
3642
import java.util.Map;
43+
import java.util.Objects;
44+
import java.util.Set;
3745
import java.util.stream.Collectors;
3846

3947
/**
@@ -45,26 +53,70 @@ public abstract class AbstractMaterializedViewJoinRule extends AbstractMateriali
4553
protected Plan rewriteView(MatchMode matchMode,
4654
StructInfo queryStructInfo,
4755
StructInfo viewStructInfo,
48-
BiMap<Long, Long> queryToViewTableMappings,
49-
Plan temporaryRewrite) {
56+
BiMap<RelationId, RelationId> queryToViewTableMappings,
57+
Plan tempRewritedPlan) {
5058

5159
List<Expression> expressions = rewriteExpression(queryStructInfo.getTopExpressions(),
5260
queryStructInfo,
5361
viewStructInfo,
5462
queryToViewTableMappings,
55-
temporaryRewrite
63+
tempRewritedPlan
5664
);
5765
if (expressions == null) {
5866
return queryStructInfo.getPlan();
5967
}
60-
// TODO add rewrited project correctly
61-
Map<Slot, Slot> mvOutputSet = temporaryRewrite.getLogicalProperties().getOutputMap();
62-
// Simplify implement
63-
List<NamedExpression> namedExpressions = queryStructInfo.getPlan().getLogicalProperties().getOutput()
64-
.stream()
65-
.map(slot -> (NamedExpression) mvOutputSet.get(slot))
66-
.collect(Collectors.toList());
67-
return new LogicalProject<>(namedExpressions, temporaryRewrite);
68+
// PoC Generate mapping from query slot reference to mv slot reference, note: clone
69+
// if any slot can not map then bail out
70+
// simplfy implement
71+
Set<SlotReference> querySlotSet = new HashSet<>();
72+
queryStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, querySlotSet);
73+
74+
Set<SlotReference> viewSlotSet = new HashSet<>();
75+
viewStructInfo.getPlan().accept(PlanVisitors.SLOT_REFERENCE_COLLECTOR, viewSlotSet);
76+
77+
Map<SlotReference, SlotReference> queryToViewSlotMapping = new HashMap<>();
78+
for (SlotReference querySlot : querySlotSet) {
79+
for (SlotReference viewSlot : viewSlotSet) {
80+
if (Objects.equals(querySlot.getName(), viewSlot.getName())
81+
&& Objects.equals(querySlot.getQualifier(), viewSlot.getQualifier())) {
82+
queryToViewSlotMapping.put(querySlot, viewSlot);
83+
}
84+
}
85+
}
86+
// PoC Generate mapping from mv sql output to mv scan out put
87+
Map<SlotReference, SlotReference> mvToMvScanMapping = new HashMap<>();
88+
List<Slot> mvScanSlotList = tempRewritedPlan.getOutput();
89+
List<Slot> mvSlotList = viewStructInfo.getPlan().getOutput();
90+
for (int i = 0; i < mvSlotList.size(); i++) {
91+
mvToMvScanMapping.put((SlotReference) mvSlotList.get(i), (SlotReference) mvScanSlotList.get(i));
92+
}
93+
94+
// TODO check if the query expr can derive from the view
95+
// PoC If the query expression can get from mv sql, so replace the mv scan slot reference
96+
// PoC according to the mapping above. Simplify implement
97+
Map<Slot, Slot> mvScanToQueryMapping = new HashMap<>();
98+
List<Slot> output = queryStructInfo.getPlan().getOutput();
99+
for (Slot querySlot : output) {
100+
Slot mvSlot = queryToViewSlotMapping.get(querySlot);
101+
if (mvSlot == null) {
102+
return null;
103+
}
104+
SlotReference mvScanSlot = mvToMvScanMapping.get(mvSlot);
105+
if (mvScanSlot == null) {
106+
return null;
107+
}
108+
mvScanToQueryMapping.put(mvScanSlot, querySlot);
109+
}
110+
// Replace the mv scan output with query slot, lazy before add filter and other project
111+
112+
// tempRewritedPlan.accept(SlotReferenceReplacer.INSTANCE, mvScanToQueryMapping);
113+
114+
tempRewritedPlan.getOutput().stream()
115+
.forEach(slot -> slot.accept(ExprReplacer.INSTANCE, mvScanToQueryMapping));
116+
LogicalProject<Plan> planLogicalProject = new LogicalProject<>(
117+
output.stream().map(NamedExpression.class::cast).collect(Collectors.toList()),
118+
tempRewritedPlan);
119+
return planLogicalProject;
68120
}
69121

70122
protected boolean isPatternSupport(LogicalProject topProject, Plan plan) {

0 commit comments

Comments
 (0)