diff --git a/Cargo.toml b/Cargo.toml index cfdb12efe8..4358d912ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ debug = true debug = "line-tables-only" [workspace.dependencies] -delta_kernel = { version = "0.5.0", features = ["default-engine"] } +delta_kernel = { version = "0.6.0", features = ["default-engine"] } #delta_kernel = { path = "../delta-kernel-rs/kernel", features = ["sync-engine"] } # arrow @@ -45,16 +45,16 @@ object_store = { version = "0.11.2" } parquet = { version = "53" } # datafusion -datafusion = { version = "43" } -datafusion-expr = { version = "43" } -datafusion-common = { version = "43" } -datafusion-ffi = { version = "43" } -datafusion-functions = { version = "43" } -datafusion-functions-aggregate = { version = "43" } -datafusion-physical-expr = { version = "43" } -datafusion-physical-plan = { version = "43" } -datafusion-proto = { version = "43" } -datafusion-sql = { version = "43" } +datafusion = { version = "44" } +datafusion-expr = { version = "44" } +datafusion-common = { version = "44" } +datafusion-ffi = { version = "44" } +datafusion-functions = { version = "44" } +datafusion-functions-aggregate = { version = "44" } +datafusion-physical-expr = { version = "44" } +datafusion-physical-plan = { version = "44" } +datafusion-proto = { version = "44" } +datafusion-sql = { version = "44" } # serde serde = { version = "1.0.194", features = ["derive"] } @@ -69,7 +69,6 @@ thiserror = { version = "2" } url = { version = "2" } urlencoding = "2.1.3" uuid = { version = "1" } -path-tree = { version = "0.8.1"} # pin to 0.8.1 due to nightly features # runtime / async async-trait = { version = "0.1" } @@ -77,16 +76,3 @@ futures = { version = "0.3" } tokio = { version = "1" } num_cpus = { version = "1" } -# temporary datafusion patches -[patch.crates-io] -datafusion = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } diff --git a/crates/aws/src/lib.rs b/crates/aws/src/lib.rs index e32aefcf8b..154b9de5bc 100644 --- a/crates/aws/src/lib.rs +++ b/crates/aws/src/lib.rs @@ -78,7 +78,7 @@ impl LogStoreFactory for S3LogStoreFactory { store, )?)); } - Ok(default_logstore(store, location, &options)) + Ok(default_logstore(store, location, options)) } } @@ -141,6 +141,7 @@ impl std::fmt::Debug for DynamoDbLockClient { impl DynamoDbLockClient { /// Creates a new DynamoDbLockClient from the supplied storage options. + #[allow(clippy::too_many_arguments)] pub fn try_new( sdk_config: &SdkConfig, lock_table_name: Option, diff --git a/crates/azure/Cargo.toml b/crates/azure/Cargo.toml index 20fd7c7e66..6a31103393 100644 --- a/crates/azure/Cargo.toml +++ b/crates/azure/Cargo.toml @@ -14,7 +14,7 @@ rust-version.workspace = true [dependencies] deltalake-core = { version = "0.23.0", path = "../core", features = [ "datafusion", -] } +]} lazy_static = "1" # workspace depenndecies diff --git a/crates/catalog-unity/Cargo.toml b/crates/catalog-unity/Cargo.toml index 4ed44d9050..670c8953ae 100644 --- a/crates/catalog-unity/Cargo.toml +++ b/crates/catalog-unity/Cargo.toml @@ -17,17 +17,19 @@ tokio.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true -deltalake-core = { version = "0.23", path = "../core" } +deltalake-core = { version = "0.23", path = "../core", features = [ + "datafusion", +]} reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "json", "http2"] } reqwest-retry = "0.7" reqwest-middleware = "0.4.0" rand = "0.8" -futures = "0.3" -chrono = "0.4" +futures = { workspace = true } +chrono = { workspace = true } dashmap = "6" -tracing = "0.1" -datafusion = { version = "43", optional = true } -datafusion-common = { version = "43", optional = true } +tracing = { workspace = true } +datafusion = { workspace = true, optional = true } +datafusion-common = { workspace = true, optional = true } [dev-dependencies] tokio = { version = "1", features = ["macros", "rt-multi-thread"] } diff --git a/crates/catalog-unity/src/credential.rs b/crates/catalog-unity/src/credential.rs index b6b21b47eb..2e0be5ea51 100644 --- a/crates/catalog-unity/src/credential.rs +++ b/crates/catalog-unity/src/credential.rs @@ -205,8 +205,7 @@ impl TokenCredential for AzureCliCredential { "got unexpected token type from azure cli: {0}", token_response.token_type ), - } - .into()); + }); } let duration = token_response.expires_on.naive_local() - chrono::Local::now().naive_local(); @@ -224,18 +223,15 @@ impl TokenCredential for AzureCliCredential { let message = String::from_utf8_lossy(&az_output.stderr); Err(UnityCatalogError::AzureCli { message: message.into(), - } - .into()) + }) } Err(e) => match e.kind() { std::io::ErrorKind::NotFound => Err(UnityCatalogError::AzureCli { message: "Azure Cli not installed".into(), - } - .into()), + }), error_kind => Err(UnityCatalogError::AzureCli { message: format!("io error: {error_kind:?}"), - } - .into()), + }), }, } } diff --git a/crates/catalog-unity/src/lib.rs b/crates/catalog-unity/src/lib.rs index f5b8d1d08a..816376eba8 100644 --- a/crates/catalog-unity/src/lib.rs +++ b/crates/catalog-unity/src/lib.rs @@ -644,8 +644,7 @@ impl DataCatalog for UnityCatalog { GetTableResponse::Error(err) => Err(UnityCatalogError::InvalidTable { error_code: err.error_code, message: err.message, - } - .into()), + }), } } } diff --git a/crates/core/src/table/state_arrow.rs b/crates/core/src/table/state_arrow.rs index 0258109859..4ea67cd9ef 100644 --- a/crates/core/src/table/state_arrow.rs +++ b/crates/core/src/table/state_arrow.rs @@ -14,7 +14,7 @@ use arrow_array::{ use arrow_cast::cast; use arrow_cast::parse::Parser; use arrow_schema::{DataType, Field, Fields, TimeUnit}; -use delta_kernel::table_features::ColumnMappingMode; +use delta_kernel::table_features::{validate_schema_column_mapping, ColumnMappingMode}; use itertools::Itertools; use super::state::DeltaTableState; @@ -171,6 +171,8 @@ impl DeltaTableState { }) .collect::>(); + validate_schema_column_mapping(self.schema(), column_mapping_mode)?; + let physical_name_to_logical_name = match column_mapping_mode { ColumnMappingMode::None => HashMap::with_capacity(0), // No column mapping, no need for this HashMap ColumnMappingMode::Id | ColumnMappingMode::Name => metadata @@ -184,7 +186,7 @@ impl DeltaTableState { "Invalid partition column {0}", name )))? - .physical_name(column_mapping_mode)? + .physical_name() .to_string(); Ok((physical_name, name.as_str())) }) diff --git a/python/src/lib.rs b/python/src/lib.rs index 0135864c7e..259b3d8a05 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -670,6 +670,7 @@ impl RawDeltaTable { } #[pyo3(signature = (starting_version = 0, ending_version = None, starting_timestamp = None, ending_timestamp = None, columns = None, allow_out_of_range = false))] + #[allow(clippy::too_many_arguments)] pub fn load_cdf( &mut self, py: Python,