Fix some compilation issues

Cargo.toml

@@ -17,7 +17,7 @@
 
 [package]
 name = "datafusion-python"
-version = "33.0.0"
+version = "34.0.0"
 homepage = "https://github.com/apache/arrow-datafusion-python"
 repository = "https://github.com/apache/arrow-datafusion-python"
 authors = ["Apache Arrow <dev@arrow.apache.org>"]
@@ -37,12 +37,12 @@ substrait = ["dep:datafusion-substrait"]
 tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync"] }
 rand = "0.8"
 pyo3 = { version = "0.20", features = ["extension-module", "abi3", "abi3-py38"] }
-datafusion = { version = "33.0.0", features = ["pyarrow", "avro"] }
-datafusion-common = { version = "33.0.0", features = ["pyarrow"] }
-datafusion-expr = { version = "33.0.0" }
-datafusion-optimizer = { version = "33.0.0" }
-datafusion-sql = { version = "33.0.0" }
-datafusion-substrait = { version = "33.0.0", optional = true }
+datafusion = { version = "34.0.0", features = ["pyarrow", "avro"] }
+datafusion-common = { version = "34.0.0", features = ["pyarrow"] }
+datafusion-expr = { version = "34.0.0" }
+datafusion-optimizer = { version = "34.0.0" }
+datafusion-sql = { version = "34.0.0" }
+datafusion-substrait = { version = "34.0.0", optional = true }
 prost = "0.12"
 prost-types = "0.12"
 uuid = { version = "1.3", features = ["v4"] }

src/common/data_type.rs

@@ -291,9 +291,10 @@ impl DataTypeMap {
             ScalarValue::List(arr) => Ok(arr.data_type().to_owned()),
             ScalarValue::Struct(_, fields) => Ok(DataType::Struct(fields.to_owned())),
             ScalarValue::FixedSizeBinary(size, _) => Ok(DataType::FixedSizeBinary(*size)),
-            ScalarValue::Fixedsizelist(_, field_ref, size) => {
-                Ok(DataType::FixedSizeList(field_ref.to_owned(), *size))
-            }
+            ScalarValue::FixedSizeList(array_ref) => Ok(DataType::FixedSizeList(
+                array_ref.to_owned(),
+                array_ref.len() as i32,
+            )),
             ScalarValue::DurationSecond(_) => Ok(DataType::Duration(TimeUnit::Second)),
             ScalarValue::DurationMillisecond(_) => Ok(DataType::Duration(TimeUnit::Millisecond)),
             ScalarValue::DurationMicrosecond(_) => Ok(DataType::Duration(TimeUnit::Microsecond)),

src/common/schema.rs

@@ -21,7 +21,7 @@ use datafusion::arrow::datatypes::SchemaRef;
 use datafusion_expr::{Expr, TableProviderFilterPushDown, TableSource};
 use pyo3::prelude::*;
 
-use datafusion_optimizer::utils::split_conjunction;
+use datafusion_expr::utils::split_conjunction;
 
 use super::{data_type::DataTypeMap, function::SqlFunction};
 

src/dataset_exec.rs

@@ -38,8 +38,8 @@ use datafusion::physical_plan::{
     DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream,
     Statistics,
 };
+use datafusion_expr::utils::conjunction;
 use datafusion_expr::Expr;
-use datafusion_optimizer::utils::conjunction;
 
 use crate::errors::DataFusionError;
 use crate::pyarrow_filter_expression::PyArrowFilterExpression;

src/expr.rs

@@ -24,10 +24,7 @@ use datafusion::scalar::ScalarValue;
 use datafusion_common::DFField;
 use datafusion_expr::{
     col,
-    expr::{
-        AggregateFunction, AggregateUDF, InList, InSubquery, ScalarFunction, ScalarUDF, Sort,
-        WindowFunction,
-    },
+    expr::{AggregateFunction, InList, InSubquery, ScalarFunction, Sort, WindowFunction},
     lit,
     utils::exprlist_to_fields,
     Between, BinaryExpr, Case, Cast, Expr, GetFieldAccess, GetIndexedField, Like, LogicalPlan,
@@ -283,10 +280,8 @@ impl PyExpr {
             | Expr::ScalarFunction { .. }
             | Expr::AggregateFunction { .. }
             | Expr::WindowFunction { .. }
-            | Expr::AggregateUDF { .. }
             | Expr::InList { .. }
             | Expr::Wildcard { .. }
-            | Expr::ScalarUDF { .. }
             | Expr::Exists { .. }
             | Expr::InSubquery { .. }
             | Expr::GroupingSet(..)
@@ -351,7 +346,7 @@ impl PyExpr {
                 ScalarValue::DurationMillisecond(v) => v.into_py(py),
                 ScalarValue::Struct(_, _) => todo!(),
                 ScalarValue::Dictionary(_, _) => todo!(),
-                ScalarValue::Fixedsizelist(_, _, _) => todo!(),
+                ScalarValue::FixedSizeList(_, _, _) => todo!(),
             }),
             _ => Err(py_type_err(format!(
                 "Non Expr::Literal encountered in types: {:?}",
@@ -391,9 +386,7 @@ impl PyExpr {
 
             // Expr variants containing a collection of Expr(s) for operands
             Expr::AggregateFunction(AggregateFunction { args, .. })
-            | Expr::AggregateUDF(AggregateUDF { args, .. })
             | Expr::ScalarFunction(ScalarFunction { args, .. })
-            | Expr::ScalarUDF(ScalarUDF { args, .. })
             | Expr::WindowFunction(WindowFunction { args, .. }) => {
                 Ok(args.iter().map(|arg| PyExpr::from(arg.clone())).collect())
             }
@@ -480,8 +473,9 @@ impl PyExpr {
                 op,
                 right: _,
             }) => format!("{op}"),
-            Expr::ScalarFunction(ScalarFunction { fun, args: _ }) => format!("{fun}"),
-            Expr::ScalarUDF(ScalarUDF { fun, .. }) => fun.name.clone(),
+            Expr::ScalarFunction(ScalarFunction { func_def, args: _ }) => {
+                format!("{}", func_def.name())
+            }
             Expr::Cast { .. } => "cast".to_string(),
             Expr::Between { .. } => "between".to_string(),
             Expr::Case { .. } => "case".to_string(),

src/expr/aggregate.rs

@@ -16,7 +16,7 @@
 // under the License.
 
 use datafusion_common::DataFusionError;
-use datafusion_expr::expr::{AggregateFunction, AggregateUDF, Alias};
+use datafusion_expr::expr::{AggregateFunction, Alias};
 use datafusion_expr::logical_plan::Aggregate;
 use datafusion_expr::Expr;
 use pyo3::prelude::*;
@@ -126,10 +126,9 @@ impl PyAggregate {
         match expr {
             // TODO: This Alias logic seems to be returning some strange results that we should investigate
             Expr::Alias(Alias { expr, .. }) => self._aggregation_arguments(expr.as_ref()),
-            Expr::AggregateFunction(AggregateFunction { fun: _, args, .. })
-            | Expr::AggregateUDF(AggregateUDF { fun: _, args, .. }) => {
-                Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect())
-            }
+            Expr::AggregateFunction(AggregateFunction {
+                func_def: _, args, ..
+            }) => Ok(args.iter().map(|e| PyExpr::from(e.clone())).collect()),
             _ => Err(py_type_err(
                 "Encountered a non Aggregate type in aggregation_arguments",
             )),
@@ -139,8 +138,9 @@ impl PyAggregate {
     fn _agg_func_name(expr: &Expr) -> PyResult<String> {
         match expr {
             Expr::Alias(Alias { expr, .. }) => Self::_agg_func_name(expr.as_ref()),
-            Expr::AggregateFunction(AggregateFunction { fun, .. }) => Ok(fun.to_string()),
-            Expr::AggregateUDF(AggregateUDF { fun, .. }) => Ok(fun.name.clone()),
+            Expr::AggregateFunction(AggregateFunction { func_def, .. }) => {
+                Ok(func_def.name().to_owned())
+            }
             _ => Err(py_type_err(
                 "Encountered a non Aggregate type in agg_func_name",
             )),

src/expr/aggregate_expr.rs

@@ -41,15 +41,15 @@ impl From<AggregateFunction> for PyAggregateFunction {
 impl Display for PyAggregateFunction {
     fn fmt(&self, f: &mut Formatter) -> std::fmt::Result {
         let args: Vec<String> = self.aggr.args.iter().map(|expr| expr.to_string()).collect();
-        write!(f, "{}({})", self.aggr.fun, args.join(", "))
+        write!(f, "{}({})", self.aggr.func_def.name(), args.join(", "))
     }
 }
 
 #[pymethods]
 impl PyAggregateFunction {
     /// Get the aggregate type, such as "MIN", or "MAX"
     fn aggregate_type(&self) -> String {
-        format!("{}", self.aggr.fun)
+        format!("{}", self.aggr.func_def.name())
     }
 
     /// is this a distinct aggregate such as `COUNT(DISTINCT expr)`

src/udaf.rs

@@ -172,6 +172,6 @@ impl PyAggregateUDF {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("AggregateUDF({})", self.function.name))
+        Ok(format!("AggregateUDF({})", self.function.name()))
     }
 }

src/udf.rs

@@ -95,6 +95,6 @@ impl PyScalarUDF {
     }
 
     fn __repr__(&self) -> PyResult<String> {
-        Ok(format!("ScalarUDF({})", self.function.name))
+        Ok(format!("ScalarUDF({})", self.function.name()))
     }
 }