Skip to content

Commit d6a9c6e

Browse files
seawindeseawinde
committed
[planner](poc)Poc
1 parent 5d45579 commit d6a9c6e

34 files changed

+1828
-41
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java

+12
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.apache.doris.nereids.rules.RuleFactory;
4545
import org.apache.doris.nereids.rules.RuleSet;
4646
import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver;
47+
import org.apache.doris.nereids.rules.rewrite.mv.MaterializationContext;
4748
import org.apache.doris.nereids.trees.expressions.CTEId;
4849
import org.apache.doris.nereids.trees.expressions.Expression;
4950
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -112,6 +113,8 @@ public class CascadesContext implements ScheduleContext {
112113
private final Optional<CTEId> currentTree;
113114
private final Optional<CascadesContext> parent;
114115

116+
private List<MaterializationContext> materializationContexts;
117+
115118
/**
116119
* Constructor of OptimizerContext.
117120
*
@@ -133,6 +136,7 @@ private CascadesContext(Optional<CascadesContext> parent, Optional<CTEId> curren
133136
this.currentJobContext = new JobContext(this, requireProperties, Double.MAX_VALUE);
134137
this.subqueryExprIsAnalyzed = new HashMap<>();
135138
this.runtimeFilterContext = new RuntimeFilterContext(getConnectContext().getSessionVariable());
139+
this.materializationContexts = new ArrayList<>();
136140
}
137141

138142
/**
@@ -309,6 +313,14 @@ public void setOuterScope(@Nullable Scope outerScope) {
309313
this.outerScope = Optional.ofNullable(outerScope);
310314
}
311315

316+
public List<MaterializationContext> getMaterializationContexts() {
317+
return materializationContexts;
318+
}
319+
320+
public void addMaterializationContext(MaterializationContext materializationContext) {
321+
this.materializationContexts.add(materializationContext);
322+
}
323+
312324
/**
313325
* getAndCacheSessionVariable
314326
*/

‎fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java

+11-3
Original file line numberDiff line numberDiff line change
@@ -257,9 +257,12 @@ private LogicalPlan preprocess(LogicalPlan logicalPlan) {
257257
}
258258

259259
private void initCascadesContext(LogicalPlan plan, PhysicalProperties requireProperties) {
260-
cascadesContext = CascadesContext.initContext(statementContext, plan, requireProperties);
261-
if (statementContext.getConnectContext().getTables() != null) {
262-
cascadesContext.setTables(statementContext.getConnectContext().getTables());
260+
// should call statementContext.getConnectContext().getEnv().getMgr
261+
if (cascadesContext == null) {
262+
cascadesContext = CascadesContext.initContext(statementContext, plan, requireProperties);
263+
if (statementContext.getConnectContext().getTables() != null) {
264+
cascadesContext.setTables(statementContext.getConnectContext().getTables());
265+
}
263266
}
264267
}
265268

@@ -477,6 +480,11 @@ public CascadesContext getCascadesContext() {
477480
return cascadesContext;
478481
}
479482

483+
@VisibleForTesting
484+
public void setCascadesContext(CascadesContext cascadesContext) {
485+
this.cascadesContext = cascadesContext;
486+
}
487+
480488
public static PhysicalProperties buildInitRequireProperties() {
481489
return PhysicalProperties.GATHER;
482490
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java

+11-7
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
import org.apache.doris.nereids.memo.GroupExpression;
2424
import org.apache.doris.nereids.rules.Rule;
2525

26-
import java.util.Collections;
26+
import java.util.ArrayList;
2727
import java.util.List;
2828

2929
/**
@@ -69,20 +69,24 @@ private List<Rule> getExplorationRules() {
6969
.isEnableBushyTree();
7070
int joinNumBushyTree = context.getCascadesContext().getConnectContext()
7171
.getSessionVariable().getMaxJoinNumBushyTree();
72+
73+
List<Rule> exploreRules = new ArrayList<>(getRuleSet().getMaterializedViewRules());
74+
7275
if (isDisableJoinReorder) {
73-
return Collections.emptyList();
76+
//todo
7477
} else if (isDpHyp) {
7578
if (isOtherJoinReorder) {
76-
return getRuleSet().getDPHypReorderRules();
79+
exploreRules.addAll(getRuleSet().getDPHypReorderRules());
7780
} else {
78-
return Collections.emptyList();
81+
//todo
7982
}
8083
} else if (isEnableBushyTree) {
81-
return getRuleSet().getBushyTreeJoinReorder();
84+
exploreRules.addAll(getRuleSet().getBushyTreeJoinReorder());
8285
} else if (context.getCascadesContext().getStatementContext().getMaxNAryInnerJoin() <= joinNumBushyTree) {
83-
return getRuleSet().getBushyTreeJoinReorder();
86+
exploreRules.addAll(getRuleSet().getBushyTreeJoinReorder());
8487
} else {
85-
return getRuleSet().getZigZagTreeJoinReorder();
88+
exploreRules.addAll(getRuleSet().getZigZagTreeJoinReorder());
8689
}
90+
return exploreRules;
8791
}
8892
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/joinorder/hypergraph/Node.java

+5
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,11 @@
2828

2929
/**
3030
* HyperGraph Node.
31+
* Jc
32+
* \
33+
* F
34+
* \
35+
* JC
3136
*/
3237
public class Node {
3338
private final int index;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java

+12
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.doris.nereids.cost.Cost;
2222
import org.apache.doris.nereids.properties.LogicalProperties;
2323
import org.apache.doris.nereids.properties.PhysicalProperties;
24+
import org.apache.doris.nereids.rules.rewrite.mv.StructInfo;
2425
import org.apache.doris.nereids.trees.expressions.literal.Literal;
2526
import org.apache.doris.nereids.trees.plans.JoinType;
2627
import org.apache.doris.nereids.trees.plans.Plan;
@@ -74,6 +75,8 @@ public class Group {
7475

7576
private int chosenGroupExpressionId = -1;
7677

78+
private Optional<StructInfo> structInfo = Optional.empty();
79+
7780
/**
7881
* Constructor for Group.
7982
*
@@ -152,6 +155,7 @@ public GroupExpression logicalExpressionsAt(int index) {
152155
* @return the first logical group expression in this group
153156
*/
154157
public GroupExpression getLogicalExpression() {
158+
// poc tmp
155159
Preconditions.checkArgument(logicalExpressions.size() == 1,
156160
"There should be only one Logical Expression in Group");
157161
return logicalExpressions.get(0);
@@ -532,4 +536,12 @@ public String treeString() {
532536

533537
return TreeStringUtils.treeString(this, toString, getChildren, getExtraPlans, displayExtraPlan);
534538
}
539+
540+
public Optional<StructInfo> getStructInfo() {
541+
return structInfo;
542+
}
543+
544+
public void setStructInfo(StructInfo structInfo) {
545+
this.structInfo = Optional.ofNullable(structInfo);
546+
}
535547
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java

+17
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ private Group init(Plan plan) {
314314
plan = replaceChildrenToGroupPlan(plan, childrenGroups);
315315
GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups);
316316
Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties());
317+
// PoC add struct info to group
317318

318319
groups.put(group.getGroupId(), group);
319320
if (groupExpressions.containsKey(newGroupExpression)) {
@@ -323,6 +324,22 @@ private Group init(Plan plan) {
323324
return group;
324325
}
325326

327+
/** initPoC */
328+
public Group initPoC(Plan plan) {
329+
Preconditions.checkArgument(!(plan instanceof GroupPlan), "Cannot init memo by a GroupPlan");
330+
331+
/* initialize children recursively */
332+
List<Group> childrenGroups = new ArrayList<>(plan.arity());
333+
for (Plan child : plan.children()) {
334+
childrenGroups.add(initPoC(child));
335+
}
336+
337+
plan = replaceChildrenToGroupPlan(plan, childrenGroups);
338+
GroupExpression newGroupExpression = new GroupExpression(plan, childrenGroups);
339+
Group group = new Group(groupIdGenerator.getNextId(), newGroupExpression, plan.getLogicalProperties());
340+
return group;
341+
}
342+
326343
/**
327344
* add or replace the plan into the target group.
328345
* <p>

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java

+9
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
import org.apache.doris.nereids.rules.rewrite.PushdownFilterThroughWindow;
9393
import org.apache.doris.nereids.rules.rewrite.PushdownJoinOtherCondition;
9494
import org.apache.doris.nereids.rules.rewrite.PushdownProjectThroughLimit;
95+
import org.apache.doris.nereids.rules.rewrite.mv.MaterializedViewProjectJoinRule;
9596

9697
import com.google.common.collect.ImmutableList;
9798
import com.google.common.collect.ImmutableList.Builder;
@@ -212,6 +213,10 @@ public class RuleSet {
212213
.add(JoinCommute.BUSHY.build())
213214
.build();
214215

216+
public static final List<Rule> MATERIALIZED_VIEW_RULES = planRuleFactories()
217+
.add(MaterializedViewProjectJoinRule.INSTANCE)
218+
.build();
219+
215220
public List<Rule> getDPHypReorderRules() {
216221
return DPHYP_REORDER_RULES;
217222
}
@@ -228,6 +233,10 @@ public List<Rule> getImplementationRules() {
228233
return IMPLEMENTATION_RULES;
229234
}
230235

236+
public List<Rule> getMaterializedViewRules() {
237+
return MATERIALIZED_VIEW_RULES;
238+
}
239+
231240
public static RuleFactories planRuleFactories() {
232241
return new RuleFactories();
233242
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java

+18
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,24 @@ public enum RuleType {
227227
MATERIALIZED_INDEX_PROJECT_SCAN(RuleTypeClass.REWRITE),
228228
MATERIALIZED_INDEX_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE),
229229
MATERIALIZED_INDEX_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE),
230+
231+
MATERIALIZED_VIEW_PROJECT_JOIN(RuleTypeClass.REWRITE),
232+
MATERIALIZED_VIEW_FILTER_JOIN(RuleTypeClass.REWRITE),
233+
MATERIALIZED_VIEW_PROJECT_FILTER_JOIN(RuleTypeClass.REWRITE),
234+
MATERIALIZED_VIEW_FILTER_PROJECT_JOIN(RuleTypeClass.REWRITE),
235+
MATERIALIZED_VIEW_ONLY_JOIN(RuleTypeClass.REWRITE),
236+
237+
MATERIALIZED_VIEW_PROJECT_AGGREGATE(RuleTypeClass.REWRITE),
238+
MATERIALIZED_VIEW_FILTER_AGGREGATE(RuleTypeClass.REWRITE),
239+
MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE(RuleTypeClass.REWRITE),
240+
MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.REWRITE),
241+
MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.REWRITE),
242+
243+
MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.REWRITE),
244+
MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.REWRITE),
245+
MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE),
246+
MATERIALIZED_VIEW_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE),
247+
230248
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
231249
FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
232250
PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/DeferMaterializeTopNResult.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ private Plan deferMaterialize(LogicalResultSink<? extends Plan> logicalResultSin
8383
LogicalTopN<? extends Plan> logicalTopN, Optional<LogicalFilter<? extends Plan>> logicalFilter,
8484
LogicalOlapScan logicalOlapScan) {
8585
Column rowId = new Column(Column.ROWID_COL, Type.STRING, false, null, false, "", "rowid column");
86-
SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier());
86+
SlotReference columnId = SlotReference.fromColumn(rowId, logicalOlapScan.getQualifier(), null);
8787
Set<ExprId> deferredMaterializedExprIds = Sets.newHashSet(logicalOlapScan.getOutputExprIdSet());
8888
logicalFilter.ifPresent(filter -> filter.getConjuncts()
8989
.forEach(e -> deferredMaterializedExprIds.removeAll(e.getInputSlotExprIds())));

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractMaterializedViewAggregateRule.java

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.rules.rewrite.mv;
19+
20+
/**
21+
* AbstractMaterializedViewAggregateRule
22+
* */
23+
public abstract class AbstractMaterializedViewAggregateRule extends AbstractMaterializedViewRule {
24+
}

0 commit comments

Comments
 (0)