Skip to content

[pull] master from prestodb:master #950

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pull merged 1 commit into from
Aug 26, 2026
Merged

[pull] master from prestodb:master #950

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 58 additions & 111 deletions presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,19 +330,29 @@ public static AggregationNode.Aggregation createArrayAggregation(FunctionAndType
return new AggregationNode.Aggregation(call, Optional.empty(), Optional.empty(), false, Optional.empty());
}

private static PlanNode cloneFilterNode(FilterNode filterNode, Session session, Metadata metadata, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> variablesToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap, PlanNodeIdAllocator idAllocator)
private static Optional<PlanNode> copyFilterNode(FilterNode filterNode, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> variablesToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap, DeterminismEvaluator determinismEvaluator)
{
PlanNode newSource = clonePlanNode(filterNode.getSource(), session, metadata, planNodeIdAllocator, variablesToKeep, varMap);
return new FilterNode(
filterNode.getSourceLocation(),
idAllocator.getNextId(),
newSource,
RowExpressionVariableInliner.inlineVariables(varMap, filterNode.getPredicate()));
if (!determinismEvaluator.isDeterministic(filterNode.getPredicate())) {
return Optional.empty();
}
return copyDeterministicScanNodes(filterNode.getSource(), planNodeIdAllocator, variablesToKeep, varMap, determinismEvaluator)
.map(newSource -> new FilterNode(
filterNode.getSourceLocation(),
planNodeIdAllocator.getNextId(),
newSource,
RowExpressionVariableInliner.inlineVariables(varMap, filterNode.getPredicate())));
}

private static PlanNode cloneProjectNode(ProjectNode projectNode, Session session, Metadata metadata, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap, PlanNodeIdAllocator idAllocator)
private static Optional<PlanNode> copyProjectNode(ProjectNode projectNode, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap, DeterminismEvaluator determinismEvaluator)
{
PlanNode newSource = clonePlanNode(projectNode.getSource(), session, metadata, planNodeIdAllocator, fieldsToKeep, varMap);
if (!projectNode.getAssignments().getExpressions().stream().allMatch(determinismEvaluator::isDeterministic)) {
return Optional.empty();
}

Optional<PlanNode> newSource = copyDeterministicScanNodes(projectNode.getSource(), planNodeIdAllocator, fieldsToKeep, varMap, determinismEvaluator);
if (!newSource.isPresent()) {
return Optional.empty();
}

Assignments.Builder newAssignments = Assignments.builder();

Expand All @@ -354,13 +364,13 @@ private static PlanNode cloneProjectNode(ProjectNode projectNode, Session sessio
newAssignments.put(varMap.getOrDefault(var, var), RowExpressionVariableInliner.inlineVariables(varMap, entry.getValue()));
}

return new ProjectNode(
idAllocator.getNextId(),
newSource,
newAssignments.build());
return Optional.of(new ProjectNode(
planNodeIdAllocator.getNextId(),
newSource.get(),
newAssignments.build()));
}

private static TableScanNode cloneTableScan(TableScanNode scanNode, Session session, Metadata metadata, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap)
private static TableScanNode copyTableScan(TableScanNode scanNode, PlanNodeIdAllocator planNodeIdAllocator, Map<VariableReferenceExpression, VariableReferenceExpression> varMap)
{
Map<VariableReferenceExpression, ColumnHandle> assignments = scanNode.getAssignments();

Expand Down Expand Up @@ -396,30 +406,44 @@ private static TableScanNode cloneTableScan(TableScanNode scanNode, Session sess
scanNode.getEnforcedConstraint(), scanNode.getCteMaterializationInfo());
}

public static PlanNode clonePlanNode(PlanNode planNode, Session session, Metadata metadata, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap)
/**
* Copies a scan/filter/project subtree (or a UNION ALL of such subtrees), giving every copied node a
* freshly allocated plan node id and renaming the variables listed in {@code varMap}. Sharing one
* subtree between two parents instead of copying it leaves the plan with duplicated plan node ids,
* which the plan checker rejects.
* <p>
* Returns {@link Optional#empty()} when the subtree must not be duplicated: it contains a node type
* this method cannot rebuild, or a non-deterministic expression, which would make the copy produce
* different rows than the original (e.g. a rand() filter). Callers are expected to abandon their
* rewrite in that case, so they do not need to check determinism themselves.
*/
public static Optional<PlanNode> copyDeterministicScanNodes(PlanNode planNode, Metadata metadata, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap)
{
return copyDeterministicScanNodes(planNode, planNodeIdAllocator, fieldsToKeep, varMap, new RowExpressionDeterminismEvaluator(metadata.getFunctionAndTypeManager()));
}

private static Optional<PlanNode> copyDeterministicScanNodes(PlanNode planNode, PlanNodeIdAllocator planNodeIdAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap, DeterminismEvaluator determinismEvaluator)
{
if (planNode instanceof TableScanNode) {
TableScanNode scanNode = (TableScanNode) planNode;
return cloneTableScan(scanNode, session, metadata, planNodeIdAllocator, fieldsToKeep, varMap);
return Optional.of(copyTableScan((TableScanNode) planNode, planNodeIdAllocator, varMap));
}
else if (planNode instanceof FilterNode) {
return cloneFilterNode((FilterNode) planNode, session, metadata, planNodeIdAllocator, fieldsToKeep, varMap, planNodeIdAllocator);
return copyFilterNode((FilterNode) planNode, planNodeIdAllocator, fieldsToKeep, varMap, determinismEvaluator);
}
else if (planNode instanceof ProjectNode) {
return cloneProjectNode((ProjectNode) planNode, session, metadata, planNodeIdAllocator, fieldsToKeep, varMap, planNodeIdAllocator);
return copyProjectNode((ProjectNode) planNode, planNodeIdAllocator, fieldsToKeep, varMap, determinismEvaluator);
}
else if (planNode instanceof UnionNode) {
return cloneUnionNode((UnionNode) planNode, session, metadata, planNodeIdAllocator, fieldsToKeep, varMap);
return copyUnionNode((UnionNode) planNode, planNodeIdAllocator, fieldsToKeep, varMap, determinismEvaluator);
}

checkState(false, "Currently cannot clone: " + planNode.getClass().getName() + " nodes.");
return null;
return Optional.empty();
}

private static PlanNode cloneUnionNode(UnionNode unionNode, Session session, Metadata metadata, PlanNodeIdAllocator idAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap)
private static Optional<PlanNode> copyUnionNode(UnionNode unionNode, PlanNodeIdAllocator idAllocator, List<VariableReferenceExpression> fieldsToKeep, Map<VariableReferenceExpression, VariableReferenceExpression> varMap, DeterminismEvaluator determinismEvaluator)
{
int numSources = unionNode.getSources().size();
List<PlanNode> clonedSources = new ArrayList<>();
List<PlanNode> copiedSources = new ArrayList<>();
List<Map<VariableReferenceExpression, VariableReferenceExpression>> legVarMaps = new ArrayList<>();

for (int i = 0; i < numSources; i++) {
Expand All @@ -431,8 +455,11 @@ private static PlanNode cloneUnionNode(UnionNode unionNode, Session session, Met
.filter(v -> v != null)
.collect(toImmutableList());

PlanNode clonedLeg = clonePlanNode(unionNode.getSources().get(i), session, metadata, idAllocator, legFieldsToKeep, legVarMap);
clonedSources.add(clonedLeg);
Optional<PlanNode> copiedLeg = copyDeterministicScanNodes(unionNode.getSources().get(i), idAllocator, legFieldsToKeep, legVarMap, determinismEvaluator);
if (!copiedLeg.isPresent()) {
return Optional.empty();
}
copiedSources.add(copiedLeg.get());
legVarMaps.add(legVarMap);
}

Expand All @@ -447,18 +474,18 @@ private static PlanNode cloneUnionNode(UnionNode unionNode, Session session, Met
List<VariableReferenceExpression> originalInputs = unionNode.getVariableMapping().get(outputVar);
for (int i = 0; i < numSources; i++) {
VariableReferenceExpression originalInput = originalInputs.get(i);
VariableReferenceExpression clonedInput = legVarMaps.get(i).getOrDefault(originalInput, originalInput);
newOutputToInputs.put(newOutputVar, clonedInput);
VariableReferenceExpression copiedInput = legVarMaps.get(i).getOrDefault(originalInput, originalInput);
newOutputToInputs.put(newOutputVar, copiedInput);
}
}

ListMultimap<VariableReferenceExpression, VariableReferenceExpression> multimap = newOutputToInputs.build();
return new UnionNode(
return Optional.of(new UnionNode(
unionNode.getSourceLocation(),
idAllocator.getNextId(),
clonedSources,
copiedSources,
newOutputVars.build(),
fromListMultimap(multimap));
fromListMultimap(multimap)));
}

public static String getPlanString(PlanNode planNode, Session session, TypeProvider types, Metadata metadata, boolean isVerboseOptimizerInfoEnabled)
Expand Down Expand Up @@ -607,53 +634,6 @@ public static boolean isScanFilterProjectOrUnion(PlanNode node)
node instanceof UnionNode && ((UnionNode) node).getSources().stream().allMatch(PlannerUtils::isScanFilterProjectOrUnion);
}

/**
* Returns true if the scan-filter-project plan subtree contains only deterministic
* expressions in all filters and projections. This check is critical for optimizations
* that clone the subtree (e.g., JoinPrefilter), because cloning a subtree with
* non-deterministic expressions (like rand()) produces different results from each
* clone, leading to incorrect query results.
*/
public static boolean isDeterministicScanFilterProject(PlanNode node, FunctionAndTypeManager functionAndTypeManager)
{
DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager);
return isDeterministicPlanSubtree(node, determinismEvaluator, false);
}

/**
* Like {@link #isDeterministicScanFilterProject}, but additionally accepts a UnionNode
* whose every source is itself a deterministic scan/filter/project (or another such Union)
* subtree. Pair with {@link #isScanFilterProjectOrUnion} when the caller specifically
* supports cloning UNION ALL probe sides.
*/
public static boolean isDeterministicScanFilterProjectOrUnion(PlanNode node, FunctionAndTypeManager functionAndTypeManager)
{
DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager);
return isDeterministicPlanSubtree(node, determinismEvaluator, true);
}

private static boolean isDeterministicPlanSubtree(PlanNode node, DeterminismEvaluator determinismEvaluator, boolean allowUnion)
{
if (node instanceof TableScanNode) {
return true;
}
else if (node instanceof FilterNode) {
FilterNode filterNode = (FilterNode) node;
return determinismEvaluator.isDeterministic(filterNode.getPredicate())
&& isDeterministicPlanSubtree(filterNode.getSource(), determinismEvaluator, allowUnion);
}
else if (node instanceof ProjectNode) {
ProjectNode projectNode = (ProjectNode) node;
return projectNode.getAssignments().getExpressions().stream().allMatch(determinismEvaluator::isDeterministic)
&& isDeterministicPlanSubtree(projectNode.getSource(), determinismEvaluator, allowUnion);
}
else if (allowUnion && node instanceof UnionNode) {
return ((UnionNode) node).getSources().stream()
.allMatch(source -> isDeterministicPlanSubtree(source, determinismEvaluator, allowUnion));
}
return false;
}

public static CallExpression equalityPredicate(FunctionResolution functionResolution, RowExpression leftExpr, RowExpression rightExpr)
{
return new CallExpression(EQUAL.name(),
Expand Down Expand Up @@ -959,37 +939,4 @@ public static Optional<PlanNode> addPassThroughVariable(
}
return Optional.empty();
}

/**
* Returns true if the plan subtree (expected to be a Filter/Project chain
* above a TableScanNode) contains any non-deterministic expressions.
* Useful for optimizations that clone the subtree — non-deterministic
* expressions would produce different values in the clone vs the original.
*/
public static boolean containsNonDeterministicExpression(PlanNode node, FunctionAndTypeManager functionAndTypeManager)
{
DeterminismEvaluator determinismEvaluator = new RowExpressionDeterminismEvaluator(functionAndTypeManager);
PlanNode current = node;
while (current != null) {
if (current instanceof ProjectNode) {
for (RowExpression expression : ((ProjectNode) current).getAssignments().getExpressions()) {
if (!determinismEvaluator.isDeterministic(expression)) {
return true;
}
}
current = ((ProjectNode) current).getSource();
}
else if (current instanceof FilterNode) {
if (!determinismEvaluator.isDeterministic(((FilterNode) current).getPredicate())) {
return true;
}
current = ((FilterNode) current).getSource();
}
else {
// TableScanNode or other leaf
break;
}
}
return false;
}
}
Loading
Loading