From 51ed994a3f35bf4404e7c275a87b693fd72def29 Mon Sep 17 00:00:00 2001
From: Jacob Sherin <jacob.sherin@massive.com>
Date: Wed, 26 Nov 2025 19:12:05 +0530
Subject: [PATCH 1/2] feat: upgrade to internal DF v51

---
 Cargo.toml | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 36d414f..439e9ce 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -32,15 +32,15 @@ arrow = "56.0.0"
 arrow-schema = "56.0.0"
 async-trait = "0.1.89"
 dashmap = "6"
-datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-common = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-expr = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-functions = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-functions-aggregate = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-optimizer = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-physical-expr = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-physical-plan = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
-datafusion-sql = { git = "https://github.com/massive-com/arrow-datafusion", rev = "b5a4d29" }
+datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-common = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-expr = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-functions = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-functions-aggregate = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-optimizer = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-physical-expr = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-physical-plan = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
+datafusion-sql = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
 futures = "0.3"
 itertools = "0.14"
 log = "0.4"

From 2f67ef1133093d16c8e9714270af89634b6326e3 Mon Sep 17 00:00:00 2001
From: Jacob Sherin <jacob.sherin@massive.com>
Date: Wed, 26 Nov 2025 19:25:02 +0530
Subject: [PATCH 2/2] Upgrade DF51.0.0 (#104)

Partial cherry-pick of datafusion-contrib commit f1f7ad8
* fixes conflicting dependecies in Cargo.toml
---
 Cargo.toml                          | 5 +++--
 src/rewrite/normal_form.rs          | 4 ++--
 tests/materialized_listing_table.rs | 8 +++++---
 3 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 439e9ce..65fca48 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -28,8 +28,9 @@ keywords = ["arrow", "arrow-rs", "datafusion"]
 rust-version = "1.88.0"
 
 [dependencies]
-arrow = "56.0.0"
-arrow-schema = "56.0.0"
+aquamarine = "0.6.0"
+arrow = "57.0.0"
+arrow-schema = "57.0.0"
 async-trait = "0.1.89"
 dashmap = "6"
 datafusion = { git = "https://github.com/massive-com/arrow-datafusion", rev = "c958b4e" }
diff --git a/src/rewrite/normal_form.rs b/src/rewrite/normal_form.rs
index 7efd53d..d7d9c3e 100644
--- a/src/rewrite/normal_form.rs
+++ b/src/rewrite/normal_form.rs
@@ -1099,7 +1099,7 @@ mod test {
         assert_eq!(rewritten.schema().as_ref(), query_plan.schema().as_ref());
 
         let expected = concat_batches(
-            &query_plan.schema().as_ref().clone().into(),
+            &query_plan.schema().inner().clone(),
             &context
                 .execute_logical_plan(query_plan)
                 .await?
@@ -1108,7 +1108,7 @@ mod test {
         )?;
 
         let result = concat_batches(
-            &rewritten.schema().as_ref().clone().into(),
+            &rewritten.schema().inner().clone(),
             &context
                 .execute_logical_plan(rewritten)
                 .await?
diff --git a/tests/materialized_listing_table.rs b/tests/materialized_listing_table.rs
index 5d91a67..7bba358 100644
--- a/tests/materialized_listing_table.rs
+++ b/tests/materialized_listing_table.rs
@@ -32,7 +32,9 @@ use datafusion::{
     },
     prelude::{SessionConfig, SessionContext},
 };
-use datafusion_common::{Constraints, DataFusionError, ParamValues, ScalarValue, Statistics};
+use datafusion_common::{
+    metadata::ScalarAndMetadata, Constraints, DataFusionError, ParamValues, ScalarValue, Statistics,
+};
 use datafusion_expr::{
     col, dml::InsertOp, Expr, JoinType, LogicalPlan, LogicalPlanBuilder, SortExpr,
     TableProviderFilterPushDown, TableType,
@@ -550,7 +552,7 @@ impl TableProvider for MaterializedListingTable {
 fn parse_partition_values(
     path: &ObjectPath,
     partition_columns: &[(String, DataType)],
-) -> Result<Vec<ScalarValue>, DataFusionError> {
+) -> Result<Vec<ScalarAndMetadata>, DataFusionError> {
     let parts = path.parts().map(|part| part.to_owned()).collect::<Vec<_>>();
 
     let pairs = parts
@@ -562,7 +564,7 @@ fn parse_partition_values(
         .iter()
         .map(|(column, datatype)| {
             let value = pairs.get(column.as_str()).copied().map(String::from);
-            ScalarValue::Utf8(value).cast_to(datatype)
+            ScalarAndMetadata::from(ScalarValue::Utf8(value)).cast_storage_to(datatype)
         })
         .collect::<Result<Vec<_>, _>>()?;