merge main and resolve conflicts

.github/actions/setup-rust-runtime/action.yaml

@@ -30,12 +30,9 @@ runs:
       # 
       # Set debuginfo=line-tables-only as debuginfo=0 causes immensely slow build
       # See for more details: https://github.com/rust-lang/rust/issues/119560
-      #
-      # set RUST_MIN_STACK to avoid rust stack overflows on tpc-ds tests
       run: |
         echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV  
         echo "SCCACHE_GHA_ENABLED=true" >> $GITHUB_ENV 
         echo "RUST_BACKTRACE=1" >> $GITHUB_ENV
-        echo "RUST_MIN_STACK=3000000" >> $GITHUB_ENV
         echo "RUSTFLAGS=-C debuginfo=line-tables-only -C incremental=false" >> $GITHUB_ENV
      

Cargo.toml

@@ -23,9 +23,11 @@ members = [
     "datafusion/core",
     "datafusion/expr",
     "datafusion/execution",
+    "datafusion/functions-aggregate",
     "datafusion/functions",
     "datafusion/functions-array",
     "datafusion/optimizer",
+    "datafusion/physical-expr-common",
     "datafusion/physical-expr",
     "datafusion/physical-plan",
     "datafusion/proto",
@@ -48,7 +50,7 @@ homepage = "https://github.com/apache/arrow-datafusion"
 license = "Apache-2.0"
 readme = "README.md"
 repository = "https://github.com/apache/arrow-datafusion"
-rust-version = "1.72"
+rust-version = "1.73"
 version = "37.0.0"
 
 [workspace.dependencies]
@@ -77,9 +79,11 @@ datafusion-common-runtime = { path = "datafusion/common-runtime", version = "37.
 datafusion-execution = { path = "datafusion/execution", version = "37.0.0" }
 datafusion-expr = { path = "datafusion/expr", version = "37.0.0" }
 datafusion-functions = { path = "datafusion/functions", version = "37.0.0" }
+datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "37.0.0" }
 datafusion-functions-array = { path = "datafusion/functions-array", version = "37.0.0" }
 datafusion-optimizer = { path = "datafusion/optimizer", version = "37.0.0", default-features = false }
 datafusion-physical-expr = { path = "datafusion/physical-expr", version = "37.0.0", default-features = false }
+datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "37.0.0", default-features = false }
 datafusion-physical-plan = { path = "datafusion/physical-plan", version = "37.0.0" }
 datafusion-proto = { path = "datafusion/proto", version = "37.0.0" }
 datafusion-sql = { path = "datafusion/sql", version = "37.0.0" }
@@ -97,7 +101,7 @@ object_store = { version = "0.9.1", default-features = false }
 parking_lot = "0.12"
 parquet = { version = "51.0.0", default-features = false, features = ["arrow", "async", "object_store"] }
 rand = "0.8"
-rstest = "0.18.0"
+rstest = "0.19.0"
 serde_json = "1"
 sqlparser = { version = "0.44.0", features = ["visitor"] }
 tempfile = "3"

benchmarks/queries/clickbench/README.md

@@ -63,7 +63,7 @@ LIMIT 10;
 Here are some interesting statistics about the data used in the queries
 Max length of `"SearchPhrase"` is 1113 characters
 ```sql
-❯ select min(length("SearchPhrase")) as "SearchPhrase_len_min", max(length("SearchPhrase")) "SearchPhrase_len_max" from 'hits.parquet' limit 10;
+> select min(length("SearchPhrase")) as "SearchPhrase_len_min", max(length("SearchPhrase")) "SearchPhrase_len_max" from 'hits.parquet' limit 10;
 +----------------------+----------------------+
 | SearchPhrase_len_min | SearchPhrase_len_max |
 +----------------------+----------------------+
@@ -74,7 +74,7 @@ Max length of `"SearchPhrase"` is 1113 characters
 
 Here is the schema of the data
 ```sql
-❯ describe 'hits.parquet';
+> describe 'hits.parquet';
 +-----------------------+-----------+-------------+
 | column_name           | data_type | is_nullable |
 +-----------------------+-----------+-------------+

datafusion-cli/Cargo.toml

@@ -26,7 +26,7 @@ license = "Apache-2.0"
 homepage = "https://github.com/apache/arrow-datafusion"
 repository = "https://github.com/apache/arrow-datafusion"
 # Specify MSRV here as `cargo msrv` doesn't support workspace version
-rust-version = "1.72"
+rust-version = "1.73"
 readme = "README.md"
 
 [dependencies]
@@ -45,7 +45,6 @@ datafusion = { path = "../datafusion/core", version = "37.0.0", features = [
     "unicode_expressions",
     "compression",
 ] }
-datafusion-common = { path = "../datafusion/common" }
 dirs = "4.0.0"
 env_logger = "0.9"
 futures = "0.3"

datafusion-cli/Dockerfile

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-FROM rust:1.72-bullseye as builder
+FROM rust:1.73-bullseye as builder
 
 COPY . /usr/src/arrow-datafusion
 COPY ./datafusion /usr/src/arrow-datafusion/datafusion

datafusion-cli/src/command.rs

@@ -26,9 +26,9 @@ use datafusion::arrow::array::{ArrayRef, StringArray};
 use datafusion::arrow::datatypes::{DataType, Field, Schema};
 use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::common::exec_err;
+use datafusion::common::instant::Instant;
 use datafusion::error::{DataFusionError, Result};
 use datafusion::prelude::SessionContext;
-use datafusion_common::instant::Instant;
 use std::fs::File;
 use std::io::BufReader;
 use std::str::FromStr;

datafusion-cli/src/exec.rs

@@ -41,7 +41,7 @@ use datafusion::prelude::SessionContext;
 use datafusion::sql::parser::{DFParser, Statement};
 use datafusion::sql::sqlparser::dialect::dialect_from_str;
 
-use datafusion_common::FileType;
+use datafusion::common::FileType;
 use rustyline::error::ReadlineError;
 use rustyline::Editor;
 use tokio::signal;
@@ -131,7 +131,7 @@ pub async fn exec_from_repl(
     rl.load_history(".history").ok();
 
     loop {
-        match rl.readline("❯ ") {
+        match rl.readline("> ") {
             Ok(line) if line.starts_with('\\') => {
                 rl.add_history_entry(line.trim_end())?;
                 let command = line.split_whitespace().collect::<Vec<_>>().join(" ");
@@ -350,8 +350,8 @@ pub(crate) async fn register_object_store_and_config_extensions(
 mod tests {
     use super::*;
 
-    use datafusion_common::config::FormatOptions;
-    use datafusion_common::plan_err;
+    use datafusion::common::config::FormatOptions;
+    use datafusion::common::plan_err;
 
     use url::Url;
 

datafusion-cli/src/helper.rs

@@ -20,7 +20,7 @@
 
 use std::borrow::Cow;
 
-use datafusion::common::sql_err;
+use datafusion::common::sql_datafusion_err;
 use datafusion::error::DataFusionError;
 use datafusion::sql::parser::{DFParser, Statement};
 use datafusion::sql::sqlparser::dialect::dialect_from_str;
@@ -189,10 +189,9 @@ pub fn unescape_input(input: &str) -> datafusion::error::Result<String> {
                     't' => '\t',
                     '\\' => '\\',
                     _ => {
-                        return sql_err!(ParserError::TokenizerError(format!(
-                            "unsupported escape char: '\\{}'",
-                            next_char
-                        ),))
+                        return Err(sql_datafusion_err!(ParserError::TokenizerError(
+                            format!("unsupported escape char: '\\{}'", next_char)
+                        )))
                     }
                 });
             }

datafusion-cli/src/object_storage.rs

@@ -19,13 +19,13 @@ use std::any::Any;
 use std::fmt::{Debug, Display};
 use std::sync::Arc;
 
+use datafusion::common::config::{
+    ConfigEntry, ConfigExtension, ConfigField, ExtensionOptions, TableOptions, Visit,
+};
 use datafusion::common::{exec_datafusion_err, exec_err, internal_err};
 use datafusion::error::{DataFusionError, Result};
 use datafusion::execution::context::SessionState;
 use datafusion::prelude::SessionContext;
-use datafusion_common::config::{
-    ConfigEntry, ConfigExtension, ConfigField, ExtensionOptions, TableOptions, Visit,
-};
 
 use async_trait::async_trait;
 use aws_credential_types::provider::ProvideCredentials;

datafusion-cli/src/print_options.rs

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-use datafusion_common::instant::Instant;
+use datafusion::common::instant::Instant;
 use std::fmt::{Display, Formatter};
 use std::io::Write;
 use std::pin::Pin;

datafusion-examples/examples/rewrite_expr.rs

@@ -59,7 +59,7 @@ pub fn main() -> Result<()> {
 
     // then run the optimizer with our custom rule
     let optimizer = Optimizer::with_rules(vec![Arc::new(MyOptimizerRule {})]);
-    let optimized_plan = optimizer.optimize(&analyzed_plan, &config, observe)?;
+    let optimized_plan = optimizer.optimize(analyzed_plan, &config, observe)?;
     println!(
         "Optimized Logical Plan:\n\n{}\n",
         optimized_plan.display_indent()

datafusion/common/src/dfschema.rs

@@ -319,7 +319,7 @@ impl DFSchema {
         &self,
         qualifier: Option<&TableReference>,
         name: &str,
-    ) -> Result<Option<usize>> {
+    ) -> Option<usize> {
         let mut matches = self
             .iter()
             .enumerate()
@@ -345,19 +345,19 @@ impl DFSchema {
                 (None, Some(_)) | (None, None) => f.name() == name,
             })
             .map(|(idx, _)| idx);
-        Ok(matches.next())
+        matches.next()
     }
 
     /// Find the index of the column with the given qualifier and name
     pub fn index_of_column(&self, col: &Column) -> Result<usize> {
-        self.index_of_column_by_name(col.relation.as_ref(), &col.name)?
+        self.index_of_column_by_name(col.relation.as_ref(), &col.name)
             .ok_or_else(|| field_not_found(col.relation.clone(), &col.name, self))
     }
 
     /// Check if the column is in the current schema
-    pub fn is_column_from_schema(&self, col: &Column) -> Result<bool> {
+    pub fn is_column_from_schema(&self, col: &Column) -> bool {
         self.index_of_column_by_name(col.relation.as_ref(), &col.name)
-            .map(|idx| idx.is_some())
+            .is_some()
     }
 
     /// Find the field with the given name
@@ -381,7 +381,7 @@ impl DFSchema {
     ) -> Result<(Option<&TableReference>, &Field)> {
         if let Some(qualifier) = qualifier {
             let idx = self
-                .index_of_column_by_name(Some(qualifier), name)?
+                .index_of_column_by_name(Some(qualifier), name)
                 .ok_or_else(|| field_not_found(Some(qualifier.clone()), name, self))?;
             Ok((self.field_qualifiers[idx].as_ref(), self.field(idx)))
         } else {
@@ -519,7 +519,7 @@ impl DFSchema {
         name: &str,
     ) -> Result<&Field> {
         let idx = self
-            .index_of_column_by_name(Some(qualifier), name)?
+            .index_of_column_by_name(Some(qualifier), name)
             .ok_or_else(|| field_not_found(Some(qualifier.clone()), name, self))?;
 
         Ok(self.field(idx))
@@ -1190,11 +1190,8 @@ mod tests {
                 .to_string(),
             expected_help
         );
-        assert!(schema.index_of_column_by_name(None, "y").unwrap().is_none());
-        assert!(schema
-            .index_of_column_by_name(None, "t1.c0")
-            .unwrap()
-            .is_none());
+        assert!(schema.index_of_column_by_name(None, "y").is_none());
+        assert!(schema.index_of_column_by_name(None, "t1.c0").is_none());
 
         Ok(())
     }
@@ -1284,28 +1281,28 @@ mod tests {
         {
             let col = Column::from_qualified_name("t1.c0");
             let schema = DFSchema::try_from_qualified_schema("t1", &test_schema_1())?;
-            assert!(schema.is_column_from_schema(&col)?);
+            assert!(schema.is_column_from_schema(&col));
         }
 
         // qualified not exists
         {
             let col = Column::from_qualified_name("t1.c2");
             let schema = DFSchema::try_from_qualified_schema("t1", &test_schema_1())?;
-            assert!(!schema.is_column_from_schema(&col)?);
+            assert!(!schema.is_column_from_schema(&col));
         }
 
         // unqualified exists
         {
             let col = Column::from_name("c0");
             let schema = DFSchema::try_from_qualified_schema("t1", &test_schema_1())?;
-            assert!(schema.is_column_from_schema(&col)?);
+            assert!(schema.is_column_from_schema(&col));
         }
 
         // unqualified not exists
         {
             let col = Column::from_name("c2");
             let schema = DFSchema::try_from_qualified_schema("t1", &test_schema_1())?;
-            assert!(!schema.is_column_from_schema(&col)?);
+            assert!(!schema.is_column_from_schema(&col));
         }
 
         Ok(())