From e618b998b81d8ef538d2c0a3a587c2fd73eeedc0 Mon Sep 17 00:00:00 2001 From: Seaven Date: Wed, 15 Jan 2025 21:03:50 +0800 Subject: [PATCH] xxx Signed-off-by: Seaven --- .../starrocks/sql/optimizer/Optimizer.java | 27 ++++----- .../sql/optimizer/OptimizerContext.java | 58 +++++++++---------- .../validate/MVRewriteValidator.java | 18 ++++-- .../MvRewritePreprocessorTest.java | 25 ++++---- 4 files changed, 62 insertions(+), 66 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java index a336f48cefbee..86291f8c17175 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/Optimizer.java @@ -124,7 +124,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.Set; @@ -143,18 +142,16 @@ public class Optimizer { private final OptimizerContext context; private OptimizerConfig optimizerConfig; private MvRewriteStrategy mvRewriteStrategy = new MvRewriteStrategy(); - private TaskScheduler scheduler = new TaskScheduler(); + private final TaskScheduler scheduler = new TaskScheduler(); private Memo memo; + // collect all LogicalOlapScanOperators in the query before any optimization + private final List allLogicalOlapScanOperators = Lists.newArrayList(); + Optimizer(OptimizerContext context) { this.context = context; } - @VisibleForTesting - public OptimizerConfig getOptimizerConfig() { - return optimizerConfig; - } - public OptimizerContext getContext() { return context; } @@ -167,14 +164,15 @@ public MvRewriteStrategy getMvRewriteStrategy() { private void prepare(OptExpression logicOperatorTree) { optimizerConfig = context.getOptimizerConfig(); - if (!context.getOptimizerConfig().isRuleBased()) { + if (!optimizerConfig.isRuleBased()) { memo = new Memo(); context.setMemo(memo); } context.setTaskScheduler(scheduler); // collect all olap scan operator - collectAllLogicalOlapScanOperators(logicOperatorTree, context); + Utils.extractOperator(logicOperatorTree, allLogicalOlapScanOperators, + op -> op instanceof LogicalOlapScanOperator); } public OptExpression optimize(OptExpression logicOperatorTree, ColumnRefSet requiredColumns) { @@ -307,9 +305,10 @@ private OptExpression optimizeByCost(ConnectContext connectContext, // valid the final plan PlanValidator.getInstance().validatePlan(finalPlan, rootTaskContext); // validate mv and log tracer if needed - MVRewriteValidator.getInstance().validateMV(connectContext, finalPlan, rootTaskContext); + MVRewriteValidator mvRewriteValidator = new MVRewriteValidator(allLogicalOlapScanOperators); + mvRewriteValidator.validateMV(connectContext, finalPlan, rootTaskContext); // audit mv - MVRewriteValidator.getInstance().auditMv(connectContext, finalPlan, rootTaskContext); + mvRewriteValidator.auditMv(connectContext, finalPlan, rootTaskContext); return finalPlan; } } @@ -995,12 +994,6 @@ private OptExpression extractBestPlan(PhysicalPropertySet requiredProperty, return expression; } - private void collectAllLogicalOlapScanOperators(OptExpression tree, OptimizerContext optimizerContext) { - List list = Lists.newArrayList(); - Utils.extractOperator(tree, list, op -> op instanceof LogicalOlapScanOperator); - optimizerContext.setAllLogicalOlapScanOperators(Collections.unmodifiableList(list)); - } - private List collectAllPhysicalOlapScanOperators(OptExpression tree) { List list = Lists.newArrayList(); Utils.extractOperator(tree, list, op -> op instanceof PhysicalOlapScanOperator); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java index 13c3e0e074156..6ca3ec3952dca 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/OptimizerContext.java @@ -26,7 +26,6 @@ import com.starrocks.sql.common.StarRocksPlannerException; import com.starrocks.sql.optimizer.base.ColumnRefFactory; import com.starrocks.sql.optimizer.dump.DumpInfo; -import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.operator.scalar.IsNullPredicateOperator; import com.starrocks.sql.optimizer.rewrite.JoinPredicatePushdown; import com.starrocks.sql.optimizer.rule.RuleSet; @@ -49,14 +48,15 @@ public class OptimizerContext { private Set queryTables; private long updateTableId = -1; + private OptimizerConfig optimizerConfig; + private VectorSearchOptions vectorSearchOptions; + // ============================ Optimizer ============================ private Memo memo; - private final RuleSet ruleSet = new RuleSet(); + private final RuleSet ruleSet; private TaskScheduler taskScheduler; - private OptimizerConfig optimizerConfig; - private VectorSearchOptions vectorSearchOptions = new VectorSearchOptions(); - private final CTEContext cteContext = new CTEContext(); + private final CTEContext cteContext; private TaskContext currentTaskContext; private final QueryMaterializationContext queryMaterializationContext = new QueryMaterializationContext(); @@ -82,15 +82,17 @@ public class OptimizerContext { // which should be kept to be used to convert outer join into inner join. private final List pushdownNotNullPredicates = Lists.newArrayList(); - // collect all LogicalOlapScanOperators in the query before any optimization - private List allLogicalOlapScanOperators; - OptimizerContext(ConnectContext context) { this.connectContext = context; + this.ruleSet = new RuleSet(); + this.cteContext = new CTEContext(); cteContext.reset(); this.cteContext.setEnableCTE(getSessionVariable().isCboCteReuse()); this.cteContext.setInlineCTERatio(getSessionVariable().getCboCTERuseRatio()); this.cteContext.setMaxCTELimit(getSessionVariable().getCboCTEMaxLimit()); + + this.vectorSearchOptions = new VectorSearchOptions(); + this.optimizerConfig = OptimizerConfig.defaultConfig(); } // @VisibleForTesting @@ -178,6 +180,22 @@ public long getUpdateTableId() { return updateTableId; } + public OptimizerConfig getOptimizerConfig() { + return optimizerConfig; + } + + public void setOptimizerConfig(OptimizerConfig optimizerConfig) { + this.optimizerConfig = optimizerConfig; + } + + public void setVectorSearchOptions(VectorSearchOptions vectorSearchOptions) { + this.vectorSearchOptions = vectorSearchOptions; + } + + public VectorSearchOptions getVectorSearchOptions() { + return vectorSearchOptions; + } + // ============================ Optimizer ============================ public Memo getMemo() { return memo; @@ -199,22 +217,6 @@ public void setTaskScheduler(TaskScheduler taskScheduler) { this.taskScheduler = taskScheduler; } - public OptimizerConfig getOptimizerConfig() { - return optimizerConfig; - } - - public void setOptimizerConfig(OptimizerConfig optimizerConfig) { - this.optimizerConfig = optimizerConfig; - } - - public void setVectorSearchOptions(VectorSearchOptions vectorSearchOptions) { - this.vectorSearchOptions = vectorSearchOptions; - } - - public VectorSearchOptions getVectorSearchOptions() { - return vectorSearchOptions; - } - public CTEContext getCteContext() { return cteContext; } @@ -268,14 +270,6 @@ public boolean isInMemoPhase() { return this.inMemoPhase; } - public void setAllLogicalOlapScanOperators(List allScanOperators) { - this.allLogicalOlapScanOperators = allScanOperators; - } - - public List getAllLogicalOlapScanOperators() { - return allLogicalOlapScanOperators; - } - /** * Get all valid candidate materialized views for the query: * - The materialized view is valid to rewrite by rule(SPJG) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java index 8d141a99e9bb1..ba06194d0f2fd 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/validate/MVRewriteValidator.java @@ -24,6 +24,7 @@ import com.starrocks.sql.optimizer.MaterializationContext; import com.starrocks.sql.optimizer.OptExpression; import com.starrocks.sql.optimizer.OptimizerContext; +import com.starrocks.sql.optimizer.operator.logical.LogicalOlapScanOperator; import com.starrocks.sql.optimizer.rule.transformation.materialization.MaterializedViewRewriter; import com.starrocks.sql.optimizer.task.TaskContext; import org.apache.commons.collections.CollectionUtils; @@ -36,10 +37,15 @@ import static com.starrocks.sql.optimizer.rule.transformation.materialization.MvUtils.collectMaterializedViews; public class MVRewriteValidator { - private static final MVRewriteValidator INSTANCE = new MVRewriteValidator(); - - public static MVRewriteValidator getInstance() { - return INSTANCE; + // private static final MVRewriteValidator INSTANCE = new MVRewriteValidator(); + // + // public static MVRewriteValidator getInstance() { + // return INSTANCE; + // } + private List allLogicalOlapScanOperators; + + public MVRewriteValidator(List allLogicalOlapScanOperators) { + this.allLogicalOlapScanOperators = allLogicalOlapScanOperators; } private static boolean isUpdateMaterializedViewMetrics(ConnectContext connectContext) { @@ -79,7 +85,7 @@ public void auditMv(ConnectContext connectContext, OptExpression physicalPlan, T List mvs = collectMaterializedViews(physicalPlan); // To avoid queries that query the materialized view directly, only consider materialized views // that are not used in rewriting before. - Set beforeTableIds = optimizerContext.getAllLogicalOlapScanOperators().stream() + Set beforeTableIds = allLogicalOlapScanOperators.stream() .map(op -> op.getTable().getId()) .collect(Collectors.toSet()); if (CollectionUtils.isNotEmpty(mvs)) { @@ -130,7 +136,7 @@ public void validateMV(ConnectContext connectContext, OptExpression physicalPlan } List mvs = collectMaterializedViews(physicalPlan); - Set beforeTableIds = taskContext.getOptimizerContext().getAllLogicalOlapScanOperators().stream() + Set beforeTableIds = allLogicalOlapScanOperators.stream() .map(op -> op.getTable().getId()) .collect(Collectors.toSet()); diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java index 9e8ffede2a975..8fcdcae3bba98 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/rule/transformation/materialization/MvRewritePreprocessorTest.java @@ -104,26 +104,29 @@ public void testOptimizer() throws Exception { LogicalPlan logicalPlan = new RelationTransformer(columnRefFactory, connectContext) .transformWithSelectLimit(query.getQueryRelation()); - Optimizer optimizer = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory)); - Assert.assertFalse(optimizer.getOptimizerConfig().isRuleBased()); - Assert.assertFalse(optimizer.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); - Assert.assertFalse(optimizer.getOptimizerConfig().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE)); + OptimizerContext optimizerContext = OptimizerFactory.mockContext(connectContext, columnRefFactory); + Optimizer optimizer = OptimizerFactory.create(optimizerContext); + Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleBased()); + Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); + Assert.assertFalse(optimizerContext.getOptimizerConfig().isRuleDisable(RuleType.GP_AGGREGATE_REWRITE)); OptExpression expr = optimizer.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn())); Assert.assertTrue(expr.getInputs().get(0).getOp() instanceof PhysicalOlapScanOperator); Assert.assertNotNull(expr.getInputs().get(0).getOp().getPredicate()); + OptimizerConfig optimizerConfig = new OptimizerConfig(OptimizerConfig.OptimizerAlgorithm.RULE_BASED); optimizerConfig.disableRule(RuleType.TF_MERGE_TWO_PROJECT); optimizerConfig.disableRule(RuleType.GP_PUSH_DOWN_PREDICATE); - Optimizer optimizer1 = OptimizerFactory.create(OptimizerFactory.mockContext(connectContext, columnRefFactory, - optimizerConfig)); - Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleBased()); - Assert.assertFalse(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE)); - Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); - Assert.assertFalse(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.GP_COLLECT_CTE)); - Assert.assertTrue(optimizer1.getOptimizerConfig().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE)); + OptimizerContext optimizerContext1 = OptimizerFactory.mockContext(connectContext, columnRefFactory, + optimizerConfig); + Optimizer optimizer1 = OptimizerFactory.create(optimizerContext1); + Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleBased()); + Assert.assertFalse(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_AGG_RULE)); + Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.TF_MERGE_TWO_PROJECT)); + Assert.assertFalse(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.GP_COLLECT_CTE)); + Assert.assertTrue(optimizerContext1.getOptimizerConfig().isRuleDisable(RuleType.GP_PUSH_DOWN_PREDICATE)); OptExpression expr1 = optimizer1.optimize(logicalPlan.getRoot(), new PhysicalPropertySet(), new ColumnRefSet(logicalPlan.getOutputColumn()));