From b7e59f76d7730f0f2da5363b3dedbd4cf23d9a9a Mon Sep 17 00:00:00 2001 From: Matthew Gapp <61894094+matthewgapp@users.noreply.github.com> Date: Sat, 18 Nov 2023 14:07:28 -0800 Subject: [PATCH] refactor fixes --- .../enforce_distribution.rs | 48 +++++++++---------- datafusion/proto/src/physical_plan/mod.rs | 8 ---- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_distribution.rs b/datafusion/core/src/physical_optimizer/enforce_distribution.rs index 5201e2ef8f6dc..d6374eec1e2d0 100644 --- a/datafusion/core/src/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/src/physical_optimizer/enforce_distribution.rs @@ -1114,29 +1114,29 @@ fn remove_dist_changing_operators( }) } -/// Changes each child of the `dist_context.plan` such that they no longer -/// use order preserving variants, if no ordering is required at the output -/// of the physical plan (there is no global ordering requirement by the query). -fn update_plan_to_remove_unnecessary_final_order( - dist_context: DistributionContext, -) -> Result> { - let DistributionContext { - plan, - distribution_onwards, - .. - } = dist_context; - let new_children = izip!(plan.children(), distribution_onwards) - .map(|(mut child, mut dist_onward)| { - replace_order_preserving_variants(&mut child, &mut dist_onward)?; - Ok(child) - }) - .collect::>>()?; - if !new_children.is_empty() { - plan.with_new_children(new_children) - } else { - Ok(plan) - } -} +// /// Changes each child of the `dist_context.plan` such that they no longer +// /// use order preserving variants, if no ordering is required at the output +// /// of the physical plan (there is no global ordering requirement by the query). +// fn update_plan_to_remove_unnecessary_final_order( +// dist_context: DistributionContext, +// ) -> Result> { +// let DistributionContext { +// plan, +// distribution_onwards, +// .. +// } = dist_context; +// let new_children = izip!(plan.children(), distribution_onwards) +// .map(|(mut child, mut dist_onward)| { +// replace_order_preserving_variants(&mut child, &mut dist_onward)?; +// Ok(child) +// }) +// .collect::>>()?; +// if !new_children.is_empty() { +// plan.with_new_children(new_children) +// } else { +// Ok(plan) +// } +// } /// Updates the physical plan `input` by using `dist_onward` replace order preserving operator variants /// with their corresponding operators that do not preserve order. It is a wrapper for `replace_order_preserving_variants_helper` @@ -1228,7 +1228,7 @@ fn ensure_distribution( mut plan, mut distribution_onwards, has_recursive_ancestor, - } = remove_unnecessary_repartition(dist_context)?; + } = remove_dist_changing_operators(dist_context)?; if let Some(exec) = plan.as_any().downcast_ref::() { if let Some(updated_window) = get_best_fitting_window( diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index cb509473efbda..1eedbe987ec14 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -53,14 +53,6 @@ use datafusion::physical_plan::windows::{ use datafusion::physical_plan::{ udaf, AggregateExpr, ExecutionPlan, Partitioning, PhysicalExpr, WindowExpr, }; -<<<<<<< HEAD -<<<<<<< HEAD -======= -// use datafusion::physical_plan::con -======= ->>>>>>> f9614d09e (Continuing implementation with fixes and improvements) -use datafusion_common::FileCompressionType; ->>>>>>> 7d3565a4c (partway through porting over isidentical's work) use datafusion_common::{internal_err, not_impl_err, DataFusionError, Result}; use prost::bytes::BufMut; use prost::Message;