From 9707a8a26b00a02e3cad4eae56cc79f9fd4692cd Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 4 Nov 2024 19:40:09 -0700 Subject: [PATCH 001/177] bump version and generate changelog --- Cargo.toml | 50 ++-- datafusion-cli/Cargo.lock | 377 +++++++++++++++++++++----- datafusion-cli/Cargo.toml | 4 +- dev/changelog/43.0.0.md | 545 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 885 insertions(+), 91 deletions(-) create mode 100644 dev/changelog/43.0.0.md diff --git a/Cargo.toml b/Cargo.toml index 21079c484ce03..dd70535be88af 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ license = "Apache-2.0" readme = "README.md" repository = "https://github.com/apache/datafusion" rust-version = "1.79" -version = "42.2.0" +version = "43.0.0" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can @@ -93,30 +93,30 @@ bytes = "1.4" chrono = { version = "0.4.38", default-features = false } ctor = "0.2.0" dashmap = "6.0.1" -datafusion = { path = "datafusion/core", version = "42.2.0", default-features = false } -datafusion-catalog = { path = "datafusion/catalog", version = "42.2.0" } -datafusion-common = { path = "datafusion/common", version = "42.2.0", default-features = false } -datafusion-common-runtime = { path = "datafusion/common-runtime", version = "42.2.0" } -datafusion-execution = { path = "datafusion/execution", version = "42.2.0" } -datafusion-expr = { path = "datafusion/expr", version = "42.2.0" } -datafusion-expr-common = { path = "datafusion/expr-common", version = "42.2.0" } -datafusion-ffi = { path = "datafusion/ffi", version = "42.2.0" } -datafusion-functions = { path = "datafusion/functions", version = "42.2.0" } -datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "42.2.0" } -datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "42.2.0" } -datafusion-functions-nested = { path = "datafusion/functions-nested", version = "42.2.0" } -datafusion-functions-window = { path = "datafusion/functions-window", version = "42.2.0" } -datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "42.2.0" } -datafusion-optimizer = { path = "datafusion/optimizer", version = "42.2.0", default-features = false } -datafusion-physical-expr = { path = "datafusion/physical-expr", version = "42.2.0", default-features = false } -datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "42.2.0", default-features = false } -datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "42.2.0" } -datafusion-physical-plan = { path = "datafusion/physical-plan", version = "42.2.0" } -datafusion-proto = { path = "datafusion/proto", version = "42.2.0" } -datafusion-proto-common = { path = "datafusion/proto-common", version = "42.2.0" } -datafusion-sql = { path = "datafusion/sql", version = "42.2.0" } -datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "42.2.0" } -datafusion-substrait = { path = "datafusion/substrait", version = "42.2.0" } +datafusion = { path = "datafusion/core", version = "43.0.0", default-features = false } +datafusion-catalog = { path = "datafusion/catalog", version = "43.0.0" } +datafusion-common = { path = "datafusion/common", version = "43.0.0", default-features = false } +datafusion-common-runtime = { path = "datafusion/common-runtime", version = "43.0.0" } +datafusion-execution = { path = "datafusion/execution", version = "43.0.0" } +datafusion-expr = { path = "datafusion/expr", version = "43.0.0" } +datafusion-expr-common = { path = "datafusion/expr-common", version = "43.0.0" } +datafusion-ffi = { path = "datafusion/ffi", version = "43.0.0" } +datafusion-functions = { path = "datafusion/functions", version = "43.0.0" } +datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "43.0.0" } +datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "43.0.0" } +datafusion-functions-nested = { path = "datafusion/functions-nested", version = "43.0.0" } +datafusion-functions-window = { path = "datafusion/functions-window", version = "43.0.0" } +datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "43.0.0" } +datafusion-optimizer = { path = "datafusion/optimizer", version = "43.0.0", default-features = false } +datafusion-physical-expr = { path = "datafusion/physical-expr", version = "43.0.0", default-features = false } +datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "43.0.0", default-features = false } +datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "43.0.0" } +datafusion-physical-plan = { path = "datafusion/physical-plan", version = "43.0.0" } +datafusion-proto = { path = "datafusion/proto", version = "43.0.0" } +datafusion-proto-common = { path = "datafusion/proto-common", version = "43.0.0" } +datafusion-sql = { path = "datafusion/sql", version = "43.0.0" } +datafusion-sqllogictest = { path = "datafusion/sqllogictest", version = "43.0.0" } +datafusion-substrait = { path = "datafusion/substrait", version = "43.0.0" } doc-comment = "0.3" env_logger = "0.11" futures = "0.3" diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 541d464d381fb..b37253d1a135d 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.17" +version = "0.6.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23a1e53f0f5d86382dafe1cf314783b2044280f406e7e1506368220ad11b1338" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" dependencies = [ "anstyle", "anstyle-parse", @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8365de52b16c035ff4fcafe0092ba9390540e3e352870ac09933bebcaa2c8c56" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" [[package]] name = "anstyle-parse" @@ -523,9 +523,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.47.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8776850becacbd3a82a4737a9375ddb5c6832a51379f24443a98e61513f852c" +checksum = "ded855583fa1d22e88fe39fd6062b062376e50a8211989e07cf5e38d52eb3453" dependencies = [ "aws-credential-types", "aws-runtime", @@ -545,9 +545,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.48.0" +version = "1.49.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0007b5b8004547133319b6c4e87193eee2a0bcb3e4c18c75d09febe9dab7b383" +checksum = "9177ea1192e6601ae16c7273385690d88a7ed386a00b74a6bc894d12103cd933" dependencies = [ "aws-credential-types", "aws-runtime", @@ -567,9 +567,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.47.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fffaa356e7f1c725908b75136d53207fa714e348f365671df14e95a60530ad3" +checksum = "823ef553cf36713c97453e2ddff1eb8f62be7f4523544e2a5db64caf80100f0a" dependencies = [ "aws-credential-types", "aws-runtime", @@ -917,9 +917,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.1.31" +version = "1.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2e7962b54006dcfcc61cb72735f4d89bb97061dd6a7ed882ec6b8ee53714c6f" +checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" dependencies = [ "jobserver", "libc", @@ -1188,7 +1188,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "apache-avro", @@ -1245,7 +1245,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow-schema", "async-trait", @@ -1258,7 +1258,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "assert_cmd", @@ -1288,7 +1288,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "apache-avro", @@ -1312,7 +1312,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "42.2.0" +version = "43.0.0" dependencies = [ "log", "tokio", @@ -1320,7 +1320,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "chrono", @@ -1339,7 +1339,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1361,7 +1361,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "datafusion-common", @@ -1371,7 +1371,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-buffer", @@ -1396,7 +1396,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1415,7 +1415,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1427,7 +1427,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-array", @@ -1448,7 +1448,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "42.2.0" +version = "43.0.0" dependencies = [ "datafusion-common", "datafusion-expr", @@ -1461,7 +1461,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1469,7 +1469,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "async-trait", @@ -1487,7 +1487,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1513,7 +1513,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1525,7 +1525,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-schema", @@ -1539,7 +1539,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "42.2.0" +version = "43.0.0" dependencies = [ "ahash", "arrow", @@ -1572,7 +1572,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "42.2.0" +version = "43.0.0" dependencies = [ "arrow", "arrow-array", @@ -1633,6 +1633,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "doc-comment" version = "0.3.3" @@ -1952,9 +1963,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.0" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -2192,14 +2203,143 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "idna" -version = "0.5.0" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", ] [[package]] @@ -2209,7 +2349,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.15.0", + "hashbrown 0.15.1", ] [[package]] @@ -2407,6 +2547,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "litemap" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" + [[package]] name = "lock_api" version = "0.4.12" @@ -3188,9 +3334,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.38" +version = "0.38.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" +checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" dependencies = [ "bitflags 2.6.0", "errno", @@ -3553,6 +3699,12 @@ dependencies = [ "syn", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "static_assertions" version = "1.1.0" @@ -3614,9 +3766,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.85" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5023162dfcd14ef8f32034d8bcd4cc5ddc61ef7a247c024a33e24e1f24d21b56" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", @@ -3632,6 +3784,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tempfile" version = "3.13.0" @@ -3653,18 +3816,18 @@ checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" [[package]] name = "thiserror" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d11abd9594d9b38965ef50805c5e469ca9cc6f197f883f717e0269a3057b3d5" +checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.65" +version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae71770322cbd277e69d762a16c444af02aa0575ac0d174f0b9562d3b37f8602" +checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" dependencies = [ "proc-macro2", "quote", @@ -3721,6 +3884,16 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + [[package]] name = "tinyvec" version = "1.8.0" @@ -3895,27 +4068,12 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "unicode-bidi" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab17db44d7388991a428b2ee655ce0c212e862eff1768a455c58f9aad6e7893" - [[package]] name = "unicode-ident" version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -[[package]] -name = "unicode-normalization" -version = "0.1.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" -dependencies = [ - "tinyvec", -] - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -3936,9 +4094,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.2" +version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" dependencies = [ "form_urlencoded", "idna", @@ -3951,6 +4109,18 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "utf8parse" version = "0.2.2" @@ -4308,6 +4478,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + [[package]] name = "xmlparser" version = "0.13.6" @@ -4323,6 +4505,30 @@ dependencies = [ "lzma-sys", ] +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -4344,12 +4550,55 @@ dependencies = [ "syn", ] +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + [[package]] name = "zeroize" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zstd" version = "0.12.4" diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 049f87f08e696..784d47220c7c9 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -18,7 +18,7 @@ [package] name = "datafusion-cli" description = "Command Line Client for DataFusion query engine." -version = "42.2.0" +version = "43.0.0" authors = ["Apache DataFusion "] edition = "2021" keywords = ["arrow", "datafusion", "query", "sql"] @@ -39,7 +39,7 @@ aws-sdk-sts = "1.43.0" # end pin aws-sdk crates aws-credential-types = "1.2.0" clap = { version = "4.5.16", features = ["derive", "cargo"] } -datafusion = { path = "../datafusion/core", version = "42.2.0", features = [ +datafusion = { path = "../datafusion/core", version = "43.0.0", features = [ "avro", "crypto_expressions", "datetime_expressions", diff --git a/dev/changelog/43.0.0.md b/dev/changelog/43.0.0.md new file mode 100644 index 0000000000000..e1fcc55b4b91d --- /dev/null +++ b/dev/changelog/43.0.0.md @@ -0,0 +1,545 @@ + + +# Apache DataFusion 43.0.0 Changelog + +This release consists of 403 commits from 96 contributors. See credits at the end of this changelog for more information. + +**Breaking changes:** + +- Remove Arc wrapping from create_udf's return_type [#12489](https://github.com/apache/datafusion/pull/12489) (findepi) +- Make make_scalar_function() result candidate for inlining, by removing the `Arc` [#12477](https://github.com/apache/datafusion/pull/12477) (findepi) +- Bump MSRV to 1.78 [#12398](https://github.com/apache/datafusion/pull/12398) (comphead) +- fix: DataFusion panics with "No candidates provided" [#12469](https://github.com/apache/datafusion/pull/12469) (Weijun-H) +- Implement PartialOrd for Expr and sub fields/structs without using hash values [#12481](https://github.com/apache/datafusion/pull/12481) (ngli-me) +- Add `field` trait method to `WindowUDFImpl`, remove `return_type`/`nullable` [#12374](https://github.com/apache/datafusion/pull/12374) (jcsherin) +- parquet: Make page_index/pushdown metrics consistent with row_group metrics [#12545](https://github.com/apache/datafusion/pull/12545) (progval) +- Make SessionContext::enable_url_table consume self [#12573](https://github.com/apache/datafusion/pull/12573) (alamb) +- LexRequirement as a struct, instead of a type [#12583](https://github.com/apache/datafusion/pull/12583) (ngli-me) +- Require `Debug` for `AnalyzerRule`, `FunctionRewriter`, and `OptimizerRule` [#12556](https://github.com/apache/datafusion/pull/12556) (alamb) +- Require `Debug` for `TableProvider`, `TableProviderFactory` and `PartitionStream` [#12557](https://github.com/apache/datafusion/pull/12557) (alamb) +- Require `Debug` for `PhysicalOptimizerRule` [#12624](https://github.com/apache/datafusion/pull/12624) (AnthonyZhOon) +- Rename aggregation modules, GroupColumn [#12619](https://github.com/apache/datafusion/pull/12619) (alamb) +- Update `register_table` functions args to take `Into` [#12630](https://github.com/apache/datafusion/pull/12630) (JasonLi-cn) +- Derive `Debug` for `SessionStateBuilder`, adding `Debug` requirements to fields [#12632](https://github.com/apache/datafusion/pull/12632) (AnthonyZhOon) +- Support REPLACE INTO for INSERT statements [#12516](https://github.com/apache/datafusion/pull/12516) (fmeringdal) +- Add `PartitionEvaluatorArgs` to `WindowUDFImpl::partition_evaluator` [#12804](https://github.com/apache/datafusion/pull/12804) (jcsherin) +- Convert `rank` / `dense_rank` and `percent_rank` builtin functions to UDWF [#12718](https://github.com/apache/datafusion/pull/12718) (jatin510) +- Bug-fix: MemoryExec sort expressions do NOT refer to the projected schema [#12876](https://github.com/apache/datafusion/pull/12876) (berkaysynnada) +- Minor: add flags for temporary ddl [#12561](https://github.com/apache/datafusion/pull/12561) (hailelagi) +- Convert `BuiltInWindowFunction::{Lead, Lag}` to a user defined window function [#12857](https://github.com/apache/datafusion/pull/12857) (jcsherin) +- Improve performance for physical plan creation with many columns [#12950](https://github.com/apache/datafusion/pull/12950) (askalt) +- Improve recursive `unnest` options API [#12836](https://github.com/apache/datafusion/pull/12836) (duongcongtoai) +- fix(substrait): disallow union with a single input [#13023](https://github.com/apache/datafusion/pull/13023) (tokoko) +- feat: support arbitrary expressions in `LIMIT` plan [#13028](https://github.com/apache/datafusion/pull/13028) (jonahgao) +- Remove unused `LogicalPlan::CrossJoin` as it is unused [#13076](https://github.com/apache/datafusion/pull/13076) (buraksenn) +- Minor: make `Expr::volatile` infallible [#13206](https://github.com/apache/datafusion/pull/13206) (alamb) +- Convert LexOrdering `type` to `struct`. [#13146](https://github.com/apache/datafusion/pull/13146) (ngli-me) + +**Implemented enhancements:** + +- feat(unparser): adding alias for table scan filter in sql unparser [#12453](https://github.com/apache/datafusion/pull/12453) (Lordworms) +- feat(substrait): set ProjectRel output_mapping in producer [#12495](https://github.com/apache/datafusion/pull/12495) (vbarua) +- feat:Support applying parquet bloom filters to StringView columns [#12503](https://github.com/apache/datafusion/pull/12503) (my-vegetable-has-exploded) +- feat: Support adding a single new table factory to SessionStateBuilder [#12563](https://github.com/apache/datafusion/pull/12563) (Weijun-H) +- feat(planner): Allowing setting sort order of parquet files without specifying the schema [#12466](https://github.com/apache/datafusion/pull/12466) (devanbenz) +- feat: add support for Substrait ExtendedExpression [#12728](https://github.com/apache/datafusion/pull/12728) (westonpace) +- feat(substrait): add intersect support to consumer [#12830](https://github.com/apache/datafusion/pull/12830) (tokoko) +- feat: Implement grouping function using grouping id [#12704](https://github.com/apache/datafusion/pull/12704) (eejbyfeldt) +- feat(substrait): add set operations to consumer, update substrait to `0.45.0` [#12863](https://github.com/apache/datafusion/pull/12863) (tokoko) +- feat(substrait): add wildcard handling to producer [#12987](https://github.com/apache/datafusion/pull/12987) (tokoko) +- feat: Add regexp_count function [#12970](https://github.com/apache/datafusion/pull/12970) (Omega359) +- feat: Decorrelate more predicate subqueries [#12945](https://github.com/apache/datafusion/pull/12945) (eejbyfeldt) +- feat: Run (logical) optimizers on subqueries [#13066](https://github.com/apache/datafusion/pull/13066) (eejbyfeldt) +- feat: Convert CumeDist to UDWF [#13051](https://github.com/apache/datafusion/pull/13051) (jonathanc-n) +- feat: Migrate Map Functions [#13047](https://github.com/apache/datafusion/pull/13047) (jonathanc-n) +- feat: improve type inference for WindowFrame [#13059](https://github.com/apache/datafusion/pull/13059) (notfilippo) +- feat: Move subquery check from analyzer to PullUpCorrelatedExpr (Fix TPC-DS q41) [#13091](https://github.com/apache/datafusion/pull/13091) (eejbyfeldt) +- feat: Add `Date32`/`Date64` in aggregate fuzz testing [#13041](https://github.com/apache/datafusion/pull/13041) (LeslieKid) +- feat(substrait): support order_by in aggregate functions [#13114](https://github.com/apache/datafusion/pull/13114) (bvolpato) +- feat: Support Substrait's IntervalCompound type/literal instead of interval-month-day-nano UDT [#12112](https://github.com/apache/datafusion/pull/12112) (Blizzara) +- feat: Implement LeftMark join to fix subquery correctness issue [#13134](https://github.com/apache/datafusion/pull/13134) (eejbyfeldt) +- feat: support logical plan for `EXECUTE` statement [#13194](https://github.com/apache/datafusion/pull/13194) (jonahgao) +- feat(substrait): handle emit_kind when consuming Substrait plans [#13127](https://github.com/apache/datafusion/pull/13127) (vbarua) +- feat(substrait): AggregateRel grouping_expressions support [#13173](https://github.com/apache/datafusion/pull/13173) (akoshchiy) + +**Fixed bugs:** + +- fix: Panic/correctness issue in variance GroupsAccumulator [#12615](https://github.com/apache/datafusion/pull/12615) (eejbyfeldt) +- fix: coalesce schema issues [#12308](https://github.com/apache/datafusion/pull/12308) (mesejo) +- fix: Correct results for grouping sets when columns contain nulls [#12571](https://github.com/apache/datafusion/pull/12571) (eejbyfeldt) +- fix(substrait): remove optimize calls from substrait consumer [#12800](https://github.com/apache/datafusion/pull/12800) (tokoko) +- fix(substrait): consuming AggregateRel as last node [#12875](https://github.com/apache/datafusion/pull/12875) (tokoko) +- fix: Update TO_DATE, TO_TIMESTAMP scalar functions to support LargeUtf8, Utf8View [#12929](https://github.com/apache/datafusion/pull/12929) (Omega359) +- fix: Add Int32 type override for Dialects [#12916](https://github.com/apache/datafusion/pull/12916) (peasee) +- fix: using simple string match replace regex match for contains udf [#12931](https://github.com/apache/datafusion/pull/12931) (zhuliquan) +- fix: Dialect requires derived table alias [#12994](https://github.com/apache/datafusion/pull/12994) (peasee) +- fix: join swap for projected semi/anti joins [#13022](https://github.com/apache/datafusion/pull/13022) (korowa) +- fix: Verify supported type for Unary::Plus in sql planner [#13019](https://github.com/apache/datafusion/pull/13019) (eejbyfeldt) +- fix: Do NOT preserve names (aliases) of Exprs for simplification in TableScan filters [#13048](https://github.com/apache/datafusion/pull/13048) (eejbyfeldt) +- fix: planning of prepare statement with limit clause [#13088](https://github.com/apache/datafusion/pull/13088) (jonahgao) +- fix: add missing `NotExpr::evaluate_bounds` [#13082](https://github.com/apache/datafusion/pull/13082) (crepererum) +- fix: Order by mentioning missing column multiple times [#13158](https://github.com/apache/datafusion/pull/13158) (eejbyfeldt) +- fix: import JoinTestType without triggering unused_qualifications lint [#13170](https://github.com/apache/datafusion/pull/13170) (smarticen) +- fix: default UDWFImpl::expressions returns all expressions [#13169](https://github.com/apache/datafusion/pull/13169) (Michael-J-Ward) +- fix: date_bin() on timstamps before 1970 [#13204](https://github.com/apache/datafusion/pull/13204) (mhilton) +- fix: array_resize null fix [#13209](https://github.com/apache/datafusion/pull/13209) (jonathanc-n) +- fix: CSV Infer Schema now properly supports escaped characters. [#13214](https://github.com/apache/datafusion/pull/13214) (mnorfolk03) + +**Documentation updates:** + +- chore: Prepare 42.0.0 Release [#12465](https://github.com/apache/datafusion/pull/12465) (andygrove) +- Minor: improve ParquetOpener docs [#12456](https://github.com/apache/datafusion/pull/12456) (alamb) +- Improve doc wording around scalar authoring [#12478](https://github.com/apache/datafusion/pull/12478) (findepi) +- Minor: improve `GroupsAccumulator` docs [#12501](https://github.com/apache/datafusion/pull/12501) (alamb) +- Minor: improve `GroupsAccumulatorAdapter` docs [#12502](https://github.com/apache/datafusion/pull/12502) (alamb) +- Improve flamegraph profiling instructions [#12521](https://github.com/apache/datafusion/pull/12521) (alamb) +- docs: :memo: Add expected answers to `DataFrame` method examples [#12564](https://github.com/apache/datafusion/pull/12564) (Eason0729) +- parquet: Add finer metrics on operations covered by `time_elapsed_opening` [#12585](https://github.com/apache/datafusion/pull/12585) (progval) +- Update scalar_functions.md [#12627](https://github.com/apache/datafusion/pull/12627) (Abdullahsab3) +- Move `kurtosis_pop` to datafusion-functions-extra and out of core [#12647](https://github.com/apache/datafusion/pull/12647) (dharanad) +- Update introduction.md for `blaze` project [#12577](https://github.com/apache/datafusion/pull/12577) (liyuance) +- docs: improve the documentation for Aggregate code [#12617](https://github.com/apache/datafusion/pull/12617) (alamb) +- doc: Fix malformed hex string literal in user guide [#12708](https://github.com/apache/datafusion/pull/12708) (kawadakk) +- docs: Update DataFusion introduction to clarify that DataFusion does provide an "out of the box" query engine [#12666](https://github.com/apache/datafusion/pull/12666) (andygrove) +- Framework for generating function docs from embedded code documentation [#12668](https://github.com/apache/datafusion/pull/12668) (Omega359) +- Fix misformatted links on project index page [#12750](https://github.com/apache/datafusion/pull/12750) (amoeba) +- Add `DocumentationBuilder::with_standard_argument` to reduce copy/paste [#12747](https://github.com/apache/datafusion/pull/12747) (alamb) +- Minor: doc how field name is to be set for `WindowUDF` [#12757](https://github.com/apache/datafusion/pull/12757) (jcsherin) +- Port / Add Documentation for `VarianceSample` and `VariancePopulation` [#12742](https://github.com/apache/datafusion/pull/12742) (alamb) +- Transformed::new_transformed: Fix documentation formatting [#12787](https://github.com/apache/datafusion/pull/12787) (progval) +- Migrate documentation for all string functions from scalar_functions.md to code [#12775](https://github.com/apache/datafusion/pull/12775) (Omega359) +- Minor: add README to Catalog Folder [#12797](https://github.com/apache/datafusion/pull/12797) (jonathanc-n) +- Remove redundant aggregate/window/scalar function documentation [#12745](https://github.com/apache/datafusion/pull/12745) (alamb) +- Improve description of function migration [#12743](https://github.com/apache/datafusion/pull/12743) (alamb) +- Crypto Function Migration [#12840](https://github.com/apache/datafusion/pull/12840) (jonathanc-n) +- Minor: more doc to `MemoryPool` module [#12849](https://github.com/apache/datafusion/pull/12849) (2010YOUY01) +- Migrate documentation for all core functions from scalar_functions.md to code [#12854](https://github.com/apache/datafusion/pull/12854) (Omega359) +- Migrate documentation for Aggregate Functions to code [#12861](https://github.com/apache/datafusion/pull/12861) (jonathanc-n) +- Wordsmith project description [#12778](https://github.com/apache/datafusion/pull/12778) (matthewmturner) +- Migrate Regex Functions from static docs [#12886](https://github.com/apache/datafusion/pull/12886) (jonathanc-n) +- Migrate documentation for all math functions from scalar_functions.md to code [#12908](https://github.com/apache/datafusion/pull/12908) (juroberttyb) +- Combine the logic of rank, dense_rank and percent_rank udwf to reduce duplications [#12893](https://github.com/apache/datafusion/pull/12893) (jatin510) +- Migrate Array function Documentation to code [#12948](https://github.com/apache/datafusion/pull/12948) (jonathanc-n) +- Minor: fix Aggregation Docs from review [#12880](https://github.com/apache/datafusion/pull/12880) (jonathanc-n) +- Minor: expr-doc small fixes [#12960](https://github.com/apache/datafusion/pull/12960) (jonathanc-n) +- docs: Add documentation about conventional commits [#12971](https://github.com/apache/datafusion/pull/12971) (andygrove) +- Migrate datetime documentation to code [#12966](https://github.com/apache/datafusion/pull/12966) (jatin510) +- Fix CI on main ( regenerate function docs) [#12991](https://github.com/apache/datafusion/pull/12991) (alamb) +- Split output batches of joins that do not respect batch size [#12969](https://github.com/apache/datafusion/pull/12969) (alihan-synnada) +- Minor: Fixed regexpr_match docs [#13008](https://github.com/apache/datafusion/pull/13008) (jonathanc-n) +- Minor: Fix spelling in regexpr_count docs [#13014](https://github.com/apache/datafusion/pull/13014) (jonathanc-n) +- Update version to 42.1.0, add CHANGELOG (#12986) [#12989](https://github.com/apache/datafusion/pull/12989) (alamb) +- Added expresion to "with_standard_argument" [#12926](https://github.com/apache/datafusion/pull/12926) (jonathanc-n) +- Migrate documentation for `regr*` aggregate functions to code [#12871](https://github.com/apache/datafusion/pull/12871) (alamb) +- Minor: Add documentation for `cot` [#13069](https://github.com/apache/datafusion/pull/13069) (alamb) +- Documentation: Add API deprecation policy [#13083](https://github.com/apache/datafusion/pull/13083) (comphead) +- docs: Fixed generate_series docs [#13097](https://github.com/apache/datafusion/pull/13097) (jonathanc-n) +- [docs]: migrate lead/lag window function docs to new docs [#13095](https://github.com/apache/datafusion/pull/13095) (buraksenn) +- minor: Add deprecated policy to the contributor guide contents [#13100](https://github.com/apache/datafusion/pull/13100) (comphead) +- Introduce `binary_as_string` parquet option, upgrade to arrow/parquet `53.2.0` [#12816](https://github.com/apache/datafusion/pull/12816) (goldmedal) +- Convert `ntile` builtIn function to UDWF [#13040](https://github.com/apache/datafusion/pull/13040) (jatin510) +- docs: Added Special Functions Page [#13102](https://github.com/apache/datafusion/pull/13102) (jonathanc-n) +- [docs]: added `alternative_syntax` function for docs [#13140](https://github.com/apache/datafusion/pull/13140) (jonathanc-n) +- Minor: Delete old cume_dist and percent_rank docs [#13137](https://github.com/apache/datafusion/pull/13137) (jonathanc-n) +- docs: Add alternative syntax for extract, trim and substring. [#13143](https://github.com/apache/datafusion/pull/13143) (Omega359) +- docs: switch completely to generated docs for scalar and aggregate functions [#13161](https://github.com/apache/datafusion/pull/13161) (Omega359) +- Minor: improve testing docs, mention `cargo nextest` [#13160](https://github.com/apache/datafusion/pull/13160) (alamb) +- minor: Update HOWTO to help with updating new docs [#13172](https://github.com/apache/datafusion/pull/13172) (jonathanc-n) +- Add config option `skip_physical_aggregate_schema_check ` [#13176](https://github.com/apache/datafusion/pull/13176) (alamb) +- Enable reading `StringViewArray` by default from Parquet (8% improvement for entire ClickBench suite) [#13101](https://github.com/apache/datafusion/pull/13101) (alamb) +- Forward port changes for `42.2.0` release (#13191) [#13193](https://github.com/apache/datafusion/pull/13193) (alamb) +- [minor] overload from_unixtime func to have optional timezone parameter [#13130](https://github.com/apache/datafusion/pull/13130) (buraksenn) + +**Other:** + +- Impl `convert_to_state` for `GroupsAccumulatorAdapter` (faster median for high cardinality aggregates) [#11827](https://github.com/apache/datafusion/pull/11827) (Rachelint) +- Upgrade sqlparser-rs to 0.51.0, support new interval logic from `sqlparse-rs` [#12222](https://github.com/apache/datafusion/pull/12222) (samuelcolvin) +- Do not silently ignore unsupported `CREATE TABLE` and `CREATE VIEW` syntax [#12450](https://github.com/apache/datafusion/pull/12450) (alamb) +- use FileFormat::get_ext as the default file extension filter [#12417](https://github.com/apache/datafusion/pull/12417) (waruto210) +- fix interval units parsing [#12448](https://github.com/apache/datafusion/pull/12448) (samuelcolvin) +- test(substrait): update TPCH tests [#12462](https://github.com/apache/datafusion/pull/12462) (vbarua) +- Add "Extended Clickbench" benchmark for median and approx_median for high cardinality aggregates [#12438](https://github.com/apache/datafusion/pull/12438) (alamb) +- date_trunc small update for readability [#12479](https://github.com/apache/datafusion/pull/12479) (findepi) +- cleanup `array_has` [#12460](https://github.com/apache/datafusion/pull/12460) (samuelcolvin) +- chore: bump chrono to 0.4.38 [#12485](https://github.com/apache/datafusion/pull/12485) (my-vegetable-has-exploded) +- Remove deprecated ScalarUDF::new [#12487](https://github.com/apache/datafusion/pull/12487) (findepi) +- Remove deprecated config setup functions [#12486](https://github.com/apache/datafusion/pull/12486) (findepi) +- Remove unnecessary shifts in gcd() [#12480](https://github.com/apache/datafusion/pull/12480) (findepi) +- Return TableProviderFilterPushDown::Exact when Parquet Pushdown Enabled [#12135](https://github.com/apache/datafusion/pull/12135) (itsjunetime) +- Update substrait requirement from 0.41 to 0.42, `prost-build` to `0.13.2` [#12483](https://github.com/apache/datafusion/pull/12483) (dependabot[bot]) +- Faster strpos() string function for ASCII-only case [#12401](https://github.com/apache/datafusion/pull/12401) (goldmedal) +- Specialize ASCII case for substr() [#12444](https://github.com/apache/datafusion/pull/12444) (2010YOUY01) +- Improve SQLite subquery tables aliasing unparsing [#12482](https://github.com/apache/datafusion/pull/12482) (sgrebnov) +- Minor: use Option rather than Result for not found suggestion [#12512](https://github.com/apache/datafusion/pull/12512) (alamb) +- Remove deprecated datafusion_physical_expr::functions module [#12505](https://github.com/apache/datafusion/pull/12505) (findepi) +- Remove deprecated AggregateUDF::new [#12508](https://github.com/apache/datafusion/pull/12508) (findepi) +- Make `required_guarantees` output to be deterministic [#12484](https://github.com/apache/datafusion/pull/12484) (austin362667) +- Deprecate unused ScalarUDF::fun [#12506](https://github.com/apache/datafusion/pull/12506) (findepi) +- Remove deprecated WindowUDF::new [#12507](https://github.com/apache/datafusion/pull/12507) (findepi) +- Preserve the order of right table in NestedLoopJoinExec [#12504](https://github.com/apache/datafusion/pull/12504) (alihan-synnada) +- Improve benchmark for ltrim [#12513](https://github.com/apache/datafusion/pull/12513) (Rachelint) +- Fix: check ambiguous column reference [#12467](https://github.com/apache/datafusion/pull/12467) (HuSen8891) +- Minor: move imports to top in `row_hash.rs` [#12530](https://github.com/apache/datafusion/pull/12530) (Rachelint) +- tests: Fix typo in config setting name [#12535](https://github.com/apache/datafusion/pull/12535) (progval) +- Expose DataFrame select_exprs method [#12520](https://github.com/apache/datafusion/pull/12520) (milenkovicm) +- Replace some usages of `Expr::to_field` with `Expr::qualified_name` [#12522](https://github.com/apache/datafusion/pull/12522) (jonahgao) +- Bump aws-sdk-sso to 1.43.0, aws-sdk-sts to 1.43.0 and aws-sdk-ssooidc from 1.40.0 to 1.44.0 in /datafusion-cli [#12409](https://github.com/apache/datafusion/pull/12409) (dependabot[bot]) +- Fix NestedLoopJoin performance regression [#12531](https://github.com/apache/datafusion/pull/12531) (alihan-synnada) +- Produce informative error message on insert plan type mismatch [#12540](https://github.com/apache/datafusion/pull/12540) (findepi) +- Fix unparse table scan with the projection pushdown [#12534](https://github.com/apache/datafusion/pull/12534) (goldmedal) +- Automate sqllogictest for String, LargeString and StringView behavior [#12525](https://github.com/apache/datafusion/pull/12525) (goldmedal) +- Fix unparsing offset [#12539](https://github.com/apache/datafusion/pull/12539) (Stazer) +- support EXTRACT on intervals and durations [#12514](https://github.com/apache/datafusion/pull/12514) (nrc) +- Support List type coercion for CASE-WHEN-THEN expression [#12490](https://github.com/apache/datafusion/pull/12490) (goldmedal) +- Sort metrics alphabetically in EXPLAIN ANALYZE output [#12568](https://github.com/apache/datafusion/pull/12568) (progval) +- Add `RuntimeEnv::try_new` and deprecate `RuntimeEnv::new` [#12566](https://github.com/apache/datafusion/pull/12566) (OussamaSaoudi) +- Reorgnize the StringView tests in sqllogictests [#12572](https://github.com/apache/datafusion/pull/12572) (goldmedal) +- fix parquet infer statistics for BinaryView types [#12575](https://github.com/apache/datafusion/pull/12575) (XiangpengHao) +- Minor: add example to of assert_batches_eq [#12580](https://github.com/apache/datafusion/pull/12580) (alamb) +- Use qualified aliases to simplify searching DFSchema [#12546](https://github.com/apache/datafusion/pull/12546) (jonahgao) +- return absent stats when filters are pushed down [#12471](https://github.com/apache/datafusion/pull/12471) (waruto210) +- Minor: add new() function for ParquetReadOptions [#12579](https://github.com/apache/datafusion/pull/12579) (Smith-Cruise) +- make `Debug` for `MemoryExec` prettier [#12582](https://github.com/apache/datafusion/pull/12582) (samuelcolvin) +- Add `SessionStateBuilder::with_object_store` method [#12578](https://github.com/apache/datafusion/pull/12578) (OussamaSaoudi) +- Fix and Improve Sort Pushdown for Nested Loop and Hash Join [#12559](https://github.com/apache/datafusion/pull/12559) (berkaysynnada) +- Add Docs and Examples and helper methods to `PhysicalSortExpr` [#12589](https://github.com/apache/datafusion/pull/12589) (alamb) +- Warn instead of error for unused imports [#12588](https://github.com/apache/datafusion/pull/12588) (samuelcolvin) +- Update prost-build requirement from =0.13.2 to =0.13.3 [#12587](https://github.com/apache/datafusion/pull/12587) (dependabot[bot]) +- Add JOB benchmark dataset [1/N] (imdb dataset) [#12497](https://github.com/apache/datafusion/pull/12497) (doupache) +- Improve documentation and add `Display` impl to `EquivalenceProperties` [#12590](https://github.com/apache/datafusion/pull/12590) (alamb) +- physical-plan: Cast nested group values back to dictionary if necessary [#12586](https://github.com/apache/datafusion/pull/12586) (brancz) +- Support `Date32` for `date_trunc` function [#12603](https://github.com/apache/datafusion/pull/12603) (goldmedal) +- Avoid RowConverter for multi column grouping (10% faster clickbench queries) [#12269](https://github.com/apache/datafusion/pull/12269) (jayzhan211) +- Refactor to support recursive unnest in physical plan [#11577](https://github.com/apache/datafusion/pull/11577) (duongcongtoai) +- Use original value when comparing with dictionary column in unparser [#12610](https://github.com/apache/datafusion/pull/12610) (Sevenannn) +- Fix to unparse the plan with multiple UNION statements into an SQL string [#12605](https://github.com/apache/datafusion/pull/12605) (goldmedal) +- Keep the float information in scalar_to_sql [#12609](https://github.com/apache/datafusion/pull/12609) (Sevenannn) +- Add Dictionary String (UTF8) type to String sqllogictests [#12621](https://github.com/apache/datafusion/pull/12621) (goldmedal) +- Improve SanityChecker error message [#12595](https://github.com/apache/datafusion/pull/12595) (alamb) +- Improve performance of `trim` for string view (10%) [#12395](https://github.com/apache/datafusion/pull/12395) (Rachelint) +- Simplify `update_skip_aggregation_probe` method [#12332](https://github.com/apache/datafusion/pull/12332) (lewiszlw) +- Minor: Encapsulate type check in GroupValuesColumn, avoid panic [#12620](https://github.com/apache/datafusion/pull/12620) (alamb) +- Fix sort node deserialization from proto [#12626](https://github.com/apache/datafusion/pull/12626) (palaska) +- Minor: improve documentation to StringView trim [#12629](https://github.com/apache/datafusion/pull/12629) (alamb) +- [MINOR]: Simplifications Sort Operator [#12639](https://github.com/apache/datafusion/pull/12639) (akurmustafa) +- [Minor] Remove redundant member from RepartitionExec [#12638](https://github.com/apache/datafusion/pull/12638) (akurmustafa) +- implement nested identifier access [#12614](https://github.com/apache/datafusion/pull/12614) (Lordworms) +- [MINOR]: Rename get_arrayref_at_indices to take_arrays [#12654](https://github.com/apache/datafusion/pull/12654) (akurmustafa) +- [MINOR]: Use take_arrays in repartition , fix build [#12657](https://github.com/apache/datafusion/pull/12657) (doupache) +- Add binary_view to string_view coercion [#12643](https://github.com/apache/datafusion/pull/12643) (doupache) +- [Minor] Improve error message when bitwise\_\* operator takes wrong unsupported type [#12646](https://github.com/apache/datafusion/pull/12646) (dharanad) +- Minor: Add github link to code that was upstreamed [#12660](https://github.com/apache/datafusion/pull/12660) (alamb) +- Minor: Improve documentation on execution error handling [#12651](https://github.com/apache/datafusion/pull/12651) (alamb) +- Adds `WindowUDFImpl::reverse_expr`trait method + Support for `IGNORE NULLS` [#12662](https://github.com/apache/datafusion/pull/12662) (jcsherin) +- Fill in missing `Debug` fields for `SessionState` [#12663](https://github.com/apache/datafusion/pull/12663) (AnthonyZhOon) +- Minor: add partial assertion for skip aggregation probe [#12640](https://github.com/apache/datafusion/pull/12640) (Rachelint) +- Add more functions for string sqllogictests [#12665](https://github.com/apache/datafusion/pull/12665) (goldmedal) +- Update rstest requirement from 0.22.0 to 0.23.0 [#12678](https://github.com/apache/datafusion/pull/12678) (dependabot[bot]) +- Minor: Change LiteralGuarantee try_new to new [#12669](https://github.com/apache/datafusion/pull/12669) (pgwhalen) +- Refactor PrimitiveGroupValueBuilder to use `MaybeNullBufferBuilder` [#12623](https://github.com/apache/datafusion/pull/12623) (alamb) +- Add `value_from_statisics` to AggregateUDFImpl, remove special case for min/max/count aggregate statistics [#12296](https://github.com/apache/datafusion/pull/12296) (edmondop) +- Provide field and schema metadata missing on distinct aggregations. [#12691](https://github.com/apache/datafusion/pull/12691) (wiedld) +- [MINOR]: Simplify required_input_ordering of BoundedWindowAggExec [#12656](https://github.com/apache/datafusion/pull/12656) (akurmustafa) +- handle 0 and NULL value of NTH_VALUE function [#12676](https://github.com/apache/datafusion/pull/12676) (thinh2) +- Improve documentation for AggregateUDFImpl::value_from_stats [#12689](https://github.com/apache/datafusion/pull/12689) (alamb) +- Add support for external tables with qualified names [#12645](https://github.com/apache/datafusion/pull/12645) (OussamaSaoudi) +- Fix Regex signature types [#12690](https://github.com/apache/datafusion/pull/12690) (blaginin) +- Refactor `ByteGroupValueBuilder` to use `MaybeNullBufferBuilder` [#12681](https://github.com/apache/datafusion/pull/12681) (alamb) +- Simplify match patterns in coercion rules [#12711](https://github.com/apache/datafusion/pull/12711) (findepi) +- Remove aggregate functions dependency on frontend [#12715](https://github.com/apache/datafusion/pull/12715) (findepi) +- Minor: Remove clone in `transform_to_states` [#12707](https://github.com/apache/datafusion/pull/12707) (jayzhan211) +- Refactor tests for union sorting properties, add tests for unions and constants [#12702](https://github.com/apache/datafusion/pull/12702) (alamb) +- Fix: support Qualified Wildcard in count aggregate function [#12673](https://github.com/apache/datafusion/pull/12673) (HuSen8891) +- Reduce code duplication in `PrimitiveGroupValueBuilder` with const generics [#12703](https://github.com/apache/datafusion/pull/12703) (alamb) +- Disallow duplicated qualified field names [#12608](https://github.com/apache/datafusion/pull/12608) (eejbyfeldt) +- Optimize base64/hex decoding by pre-allocating output buffers (~2x faster) [#12675](https://github.com/apache/datafusion/pull/12675) (simonvandel) +- Allow DynamicFileCatalog support to query partitioned file [#12683](https://github.com/apache/datafusion/pull/12683) (goldmedal) +- Support `LIMIT` Push-down logical plan optimization for `Extension` nodes [#12685](https://github.com/apache/datafusion/pull/12685) (austin362667) +- Fix AvroReader: Add union resolving for nested struct arrays [#12686](https://github.com/apache/datafusion/pull/12686) (JonasDev1) +- Adds macros for creating `WindowUDF` and `WindowFunction` expression [#12693](https://github.com/apache/datafusion/pull/12693) (jcsherin) +- Support unparsing plans with both Aggregation and Window functions [#12705](https://github.com/apache/datafusion/pull/12705) (sgrebnov) +- Fix strpos invocation with dictionary and null [#12712](https://github.com/apache/datafusion/pull/12712) (findepi) +- Add IMDB(JOB) Benchmark [2/N] (imdb queries) [#12529](https://github.com/apache/datafusion/pull/12529) (austin362667) +- Minor: avoid clone while calculating union equivalence properties [#12722](https://github.com/apache/datafusion/pull/12722) (alamb) +- Simplify streaming_merge function parameters [#12719](https://github.com/apache/datafusion/pull/12719) (mertak-synnada) +- Provide field and schema metadata missing on cross joins, and union with null fields. [#12729](https://github.com/apache/datafusion/pull/12729) (wiedld) +- Minor: Update string tests for strpos [#12739](https://github.com/apache/datafusion/pull/12739) (alamb) +- Apply `type_union_resolution` to array and values [#12753](https://github.com/apache/datafusion/pull/12753) (jayzhan211) +- fix `equal_to` in `PrimitiveGroupValueBuilder` [#12758](https://github.com/apache/datafusion/pull/12758) (Rachelint) +- Fix `equal_to` in `ByteGroupValueBuilder` [#12770](https://github.com/apache/datafusion/pull/12770) (alamb) +- Allow boolean Expr simplification even when nullable [#12746](https://github.com/apache/datafusion/pull/12746) (eejbyfeldt) +- Fix unnest conjunction with selecting wildcard expression [#12760](https://github.com/apache/datafusion/pull/12760) (goldmedal) +- Improve `round` scalar function unparsing for Postgres [#12744](https://github.com/apache/datafusion/pull/12744) (sgrebnov) +- Fix stack overflow calculating projected orderings [#12759](https://github.com/apache/datafusion/pull/12759) (alamb) +- Upgrade arrow/parquet to `53.1.0` / fix clippy [#12724](https://github.com/apache/datafusion/pull/12724) (alamb) +- Account for constant equivalence properties in union, tests [#12562](https://github.com/apache/datafusion/pull/12562) (alamb) +- Minor: clarify comment about empty dependencies [#12786](https://github.com/apache/datafusion/pull/12786) (alamb) +- Introduce Signature::String and return error if input of `strpos` is integer [#12751](https://github.com/apache/datafusion/pull/12751) (jayzhan211) +- Minor: improve docs on MovingMin/MovingMax [#12790](https://github.com/apache/datafusion/pull/12790) (alamb) +- Add union sorting equivalence end to end tests [#12721](https://github.com/apache/datafusion/pull/12721) (alamb) +- Fix bug in TopK aggregates [#12766](https://github.com/apache/datafusion/pull/12766) (avantgardnerio) +- Minor: clean up TODO comments in unnest.slt [#12795](https://github.com/apache/datafusion/pull/12795) (goldmedal) +- Refactor `DependencyMap` and `Dependencies` into structs [#12761](https://github.com/apache/datafusion/pull/12761) (alamb) +- Remove unnecessary `DFSchema::check_ambiguous_name` [#12805](https://github.com/apache/datafusion/pull/12805) (jonahgao) +- API from `ParquetExec` to `ParquetExecBuilder` [#12799](https://github.com/apache/datafusion/pull/12799) (alamb) +- Minor: add documentation note about `NullState` [#12791](https://github.com/apache/datafusion/pull/12791) (alamb) +- Chore: Move `aggregate statistics` optimizer test from core to optimizer crate [#12783](https://github.com/apache/datafusion/pull/12783) (jayzhan211) +- Clarify documentation on ArrowBytesMap and ArrowBytesViewMap [#12789](https://github.com/apache/datafusion/pull/12789) (alamb) +- Bump cookie and express in /datafusion/wasmtest/datafusion-wasm-app [#12825](https://github.com/apache/datafusion/pull/12825) (dependabot[bot]) +- Remove unused dependencies and features [#12808](https://github.com/apache/datafusion/pull/12808) (jonahgao) +- Add Aggregation fuzzer framework [#12667](https://github.com/apache/datafusion/pull/12667) (Rachelint) +- Retry apt-get and rustup on CI [#12714](https://github.com/apache/datafusion/pull/12714) (findepi) +- Support creating tables via SQL with `FixedSizeList` column (e.g. `a int[3]`) [#12810](https://github.com/apache/datafusion/pull/12810) (jandremarais) +- Make HashJoinExec::join_schema public [#12807](https://github.com/apache/datafusion/pull/12807) (progval) +- Fix convert_to_state bug in `GroupsAccumulatorAdapter` [#12834](https://github.com/apache/datafusion/pull/12834) (alamb) +- Fix: approx_percentile_cont_with_weight Panic [#12823](https://github.com/apache/datafusion/pull/12823) (jonathanc-n) +- Fix clippy error on wasmtest [#12844](https://github.com/apache/datafusion/pull/12844) (jonahgao) +- Fix panic on wrong number of arguments to substr [#12837](https://github.com/apache/datafusion/pull/12837) (eejbyfeldt) +- Fix Bug in Display for ScalarValue::Struct [#12856](https://github.com/apache/datafusion/pull/12856) (avantgardnerio) +- Support DictionaryString for Regex matching operators [#12768](https://github.com/apache/datafusion/pull/12768) (blaginin) +- Minor: Small comment changes in sql folder [#12838](https://github.com/apache/datafusion/pull/12838) (jonathanc-n) +- Add DuckDB struct test and row as alias [#12841](https://github.com/apache/datafusion/pull/12841) (jayzhan211) +- Support struct coercion in `type_union_resolution` [#12839](https://github.com/apache/datafusion/pull/12839) (jayzhan211) +- Added check for aggregate functions in optimizer rules [#12860](https://github.com/apache/datafusion/pull/12860) (jonathanc-n) +- Optimize `iszero` function (3-5x faster) [#12881](https://github.com/apache/datafusion/pull/12881) (simonvandel) +- Macro for creating record batch from literal slice [#12846](https://github.com/apache/datafusion/pull/12846) (timsaucer) +- Implement special min/max accumulator for Strings and Binary (10% faster for Clickbench Q28) [#12792](https://github.com/apache/datafusion/pull/12792) (alamb) +- Make PruningPredicate's rewrite public [#12850](https://github.com/apache/datafusion/pull/12850) (adriangb) +- octet_length + string view == ❤️ [#12900](https://github.com/apache/datafusion/pull/12900) (Omega359) +- Remove Expr clones in `select_to_plan` [#12887](https://github.com/apache/datafusion/pull/12887) (jonahgao) +- Minor: added to docs in expr folder [#12882](https://github.com/apache/datafusion/pull/12882) (jonathanc-n) +- Print undocumented functions to console while generating docs [#12874](https://github.com/apache/datafusion/pull/12874) (alamb) +- Fix: handle NULL offset of NTH_VALUE window function [#12851](https://github.com/apache/datafusion/pull/12851) (HuSen8891) +- Optimize `signum` function (3-25x faster) [#12890](https://github.com/apache/datafusion/pull/12890) (simonvandel) +- re-export PartitionEvaluatorArgs from datafusion_expr::function [#12878](https://github.com/apache/datafusion/pull/12878) (Michael-J-Ward) +- Unparse Sort with pushdown limit to SQL string [#12873](https://github.com/apache/datafusion/pull/12873) (goldmedal) +- Add spilling related metrics for aggregation [#12888](https://github.com/apache/datafusion/pull/12888) (2010YOUY01) +- Move equivalence fuzz testing to fuzz test binary [#12767](https://github.com/apache/datafusion/pull/12767) (alamb) +- Remove unused `math_expressions.rs` [#12917](https://github.com/apache/datafusion/pull/12917) (jonahgao) +- Improve AggregationFuzzer error reporting [#12832](https://github.com/apache/datafusion/pull/12832) (alamb) +- Import Arc consistently [#12899](https://github.com/apache/datafusion/pull/12899) (findepi) +- Optimize `isnan` (2-5x faster) [#12889](https://github.com/apache/datafusion/pull/12889) (simonvandel) +- Minor: Move StringArrayType, StringViewArrayBuilder, etc outside of string module [#12912](https://github.com/apache/datafusion/pull/12912) (Omega359) +- Remove redundant unsafe in test [#12914](https://github.com/apache/datafusion/pull/12914) (findepi) +- Ensure that math functions fulfil the ColumnarValue contract [#12922](https://github.com/apache/datafusion/pull/12922) (joroKr21) +- Optimization: support push down limit when full join [#12963](https://github.com/apache/datafusion/pull/12963) (JasonLi-cn) +- Implement `GroupColumn` support for `StringView` / `ByteView` (faster grouping performance) [#12809](https://github.com/apache/datafusion/pull/12809) (Rachelint) +- Implement native support StringView for `REGEXP_LIKE` [#12897](https://github.com/apache/datafusion/pull/12897) (tlm365) +- Minor: Refactor benchmark imports to use `util` module [#12885](https://github.com/apache/datafusion/pull/12885) (loloxwg) +- Fix zero data type in `expr % 1` simplification [#12913](https://github.com/apache/datafusion/pull/12913) (eejbyfeldt) +- Optimize performance of `math::cot` (~2x faster) [#12910](https://github.com/apache/datafusion/pull/12910) (tlm365) +- Expand wildcard expressions in distinct on [#12941](https://github.com/apache/datafusion/pull/12941) (epsio-banay) +- chores: remove redundant clone [#12964](https://github.com/apache/datafusion/pull/12964) (JasonLi-cn) +- Fix: handle NULL input in lead/lag window function [#12811](https://github.com/apache/datafusion/pull/12811) (HuSen8891) +- Fix logical vs physical schema mismatch for aliased `now()` [#12951](https://github.com/apache/datafusion/pull/12951) (wiedld) +- Optimize performance of `math::trunc` (~2.5x faster) [#12909](https://github.com/apache/datafusion/pull/12909) (tlm365) +- Minor: Add slt test for `DISTINCT ON` with wildcard [#12968](https://github.com/apache/datafusion/pull/12968) (alamb) +- Fix 'Too many open files' on fuzz test. [#12961](https://github.com/apache/datafusion/pull/12961) (dhegberg) +- Increase minimum supported Rust version (MSRV) to 1.79 [#12962](https://github.com/apache/datafusion/pull/12962) (findepi) +- Unparse `SubqueryAlias` without projections to SQL [#12896](https://github.com/apache/datafusion/pull/12896) (goldmedal) +- Fix 2 bugs related to push down partition filters [#12902](https://github.com/apache/datafusion/pull/12902) (eejbyfeldt) +- Move TableConstraint to Constraints conversion [#12953](https://github.com/apache/datafusion/pull/12953) (findepi) +- Added current_timestamp alias [#12958](https://github.com/apache/datafusion/pull/12958) (jonathanc-n) +- Improve unparsing for `ORDER BY`, `UNION`, Windows functions with Aggregation [#12946](https://github.com/apache/datafusion/pull/12946) (sgrebnov) +- Handle one-element array return value in ScalarFunctionExpr [#12965](https://github.com/apache/datafusion/pull/12965) (joroKr21) +- Add links to new_constraint_from_table_constraints doc [#12995](https://github.com/apache/datafusion/pull/12995) (findepi) +- Fix：fix HashJoin projection swap [#12967](https://github.com/apache/datafusion/pull/12967) (my-vegetable-has-exploded) +- refactor(substrait): refactor ReadRel consumer [#12983](https://github.com/apache/datafusion/pull/12983) (tokoko) +- Move SMJ join filtered part out of join_output stage. LeftOuter, LeftSemi [#12764](https://github.com/apache/datafusion/pull/12764) (comphead) +- Remove logical cross join in planning [#12985](https://github.com/apache/datafusion/pull/12985) (Dandandan) +- [MINOR]: Use arrow take_arrays, remove datafusion take_arrays [#13013](https://github.com/apache/datafusion/pull/13013) (akurmustafa) +- Don't preserve functional dependency when generating UNION logical plan [#12979](https://github.com/apache/datafusion/pull/12979) (Sevenannn) +- [Minor]: Add data based sort expression test [#12992](https://github.com/apache/datafusion/pull/12992) (akurmustafa) +- Removed last usages of scalar_inputs, scalar_input_types and inputs2 to use arrow unary/binary for performance [#12972](https://github.com/apache/datafusion/pull/12972) (buraksenn) +- Minor: Update release instructions to include new crates [#13024](https://github.com/apache/datafusion/pull/13024) (alamb) +- Extract CSE logic to `datafusion_common` [#13002](https://github.com/apache/datafusion/pull/13002) (peter-toth) +- Enhance table scan unparsing to avoid unnamed subqueries. [#13006](https://github.com/apache/datafusion/pull/13006) (goldmedal) +- Fix count on all null `VALUES` clause [#13029](https://github.com/apache/datafusion/pull/13029) (findepi) +- Support filter in cross join elimination [#13025](https://github.com/apache/datafusion/pull/13025) (Dandandan) +- [minor]: remove same util functions from the code base. [#13026](https://github.com/apache/datafusion/pull/13026) (akurmustafa) +- Improve `AggregateFuzz` testing: generate random queries [#12847](https://github.com/apache/datafusion/pull/12847) (alamb) +- Fix functions with Volatility::Volatile and parameters [#13001](https://github.com/apache/datafusion/pull/13001) (agscpp) +- refactor: Incorporate RewriteDisjunctivePredicate rule into SimplifyExpressions [#13032](https://github.com/apache/datafusion/pull/13032) (eejbyfeldt) +- Move filtered SMJ right join out of `join_partial` phase [#13053](https://github.com/apache/datafusion/pull/13053) (comphead) +- Remove functions and types deprecated since 37 [#13056](https://github.com/apache/datafusion/pull/13056) (findepi) +- Minor: Cleaned physical-plan Comments [#13055](https://github.com/apache/datafusion/pull/13055) (jonathanc-n) +- improve the condition checking for unparsing table_scan [#13062](https://github.com/apache/datafusion/pull/13062) (goldmedal) +- minor: simplify associated item bound of `hash_array_primitive` [#13070](https://github.com/apache/datafusion/pull/13070) (jonahgao) +- extended log.rs tests for unary/binary and f32/f64 casting [#13034](https://github.com/apache/datafusion/pull/13034) (buraksenn) +- Fix check_not_null_constraints null detection [#13033](https://github.com/apache/datafusion/pull/13033) (findepi) +- [Minor] Update info/list of TPC-DS queries [#13075](https://github.com/apache/datafusion/pull/13075) (Dandandan) +- Fix logical vs physical schema mismatch for UNION where some inputs are constants [#12954](https://github.com/apache/datafusion/pull/12954) (wiedld) +- Improve CSE stats [#13080](https://github.com/apache/datafusion/pull/13080) (peter-toth) +- Infer data type from schema for `Values` and add struct coercion to `coalesce` [#12864](https://github.com/apache/datafusion/pull/12864) (jayzhan211) +- [minor]: use arrow take_batch instead of get_record_batch_indices [#13084](https://github.com/apache/datafusion/pull/13084) (akurmustafa) +- chore: Added a number of physical planning join benchmarks [#13085](https://github.com/apache/datafusion/pull/13085) (mnorfolk03) +- Fix more instances of schema missing metadata [#13068](https://github.com/apache/datafusion/pull/13068) (itsjunetime) +- Bug-fix / Limit with_new_exprs() [#13109](https://github.com/apache/datafusion/pull/13109) (berkaysynnada) +- Minor: doc IMDB in benchmark README [#13107](https://github.com/apache/datafusion/pull/13107) (2010YOUY01) +- removed --prefer_hash_join option from parquet_filter command. [#13106](https://github.com/apache/datafusion/pull/13106) (neyama) +- Make CI error if a function has no documentation [#12938](https://github.com/apache/datafusion/pull/12938) (alamb) +- Allow using `cargo nextest` for running tests [#13045](https://github.com/apache/datafusion/pull/13045) (alamb) +- Add benchmark for memory-limited aggregation [#13090](https://github.com/apache/datafusion/pull/13090) (2010YOUY01) +- Add clickbench parquet based queries to sql_planner benchmark [#13103](https://github.com/apache/datafusion/pull/13103) (Omega359) +- Improve documentation and examples for `SchemaAdapterFactory`, make `record_batch` "hygenic" [#13063](https://github.com/apache/datafusion/pull/13063) (alamb) +- Move filtered SMJ Left Anti filtered join out of `join_partial` phase [#13111](https://github.com/apache/datafusion/pull/13111) (comphead) +- Improve TableScan with filters pushdown unparsing (multiple filters) [#13131](https://github.com/apache/datafusion/pull/13131) (sgrebnov) +- Raise a plan error on union if column count is not the same between plans [#13117](https://github.com/apache/datafusion/pull/13117) (Omega359) +- Add basic support for `unnest` unparsing [#13129](https://github.com/apache/datafusion/pull/13129) (sgrebnov) +- Improve TableScan with filters pushdown unparsing (joins) [#13132](https://github.com/apache/datafusion/pull/13132) (sgrebnov) +- Report offending plan node when In/Exist subquery misused [#13155](https://github.com/apache/datafusion/pull/13155) (findepi) +- Remove unused assert_analyzed_plan_ne test helper [#13121](https://github.com/apache/datafusion/pull/13121) (findepi) +- Fix Utf8View as Join Key [#13115](https://github.com/apache/datafusion/pull/13115) (demetribu) +- Add Support for `modulus` operation in substrait [#13108](https://github.com/apache/datafusion/pull/13108) (LatrecheYasser) +- unify cast_to function of ScalarValue [#13122](https://github.com/apache/datafusion/pull/13122) (JasonLi-cn) +- Add unused_qualifications rustic lint with deny lint level. [#13086](https://github.com/apache/datafusion/pull/13086) (dhegberg) +- [Optimization] Infer predicate under all JoinTypes [#13081](https://github.com/apache/datafusion/pull/13081) (JasonLi-cn) +- Support `negate` arithmetic expression in substrait [#13112](https://github.com/apache/datafusion/pull/13112) (LatrecheYasser) +- Fix to_char signature ordering [#13126](https://github.com/apache/datafusion/pull/13126) (Omega359) +- chore: re-export functions_window_common::ExpressionArgs [#13149](https://github.com/apache/datafusion/pull/13149) (Michael-J-Ward) +- minor: Fix build on main [#13159](https://github.com/apache/datafusion/pull/13159) (eejbyfeldt) +- minor: Update test case for issue #5771 showing it is resolved [#13180](https://github.com/apache/datafusion/pull/13180) (eejbyfeldt) +- Test LIKE with dynamic pattern [#13141](https://github.com/apache/datafusion/pull/13141) (findepi) +- Increase fuzz testing of streaming group by / low cardinality columns [#12990](https://github.com/apache/datafusion/pull/12990) (alamb) +- FFI initial implementation [#12920](https://github.com/apache/datafusion/pull/12920) (timsaucer) +- Report file location and offset when CSV schema mismatch [#13185](https://github.com/apache/datafusion/pull/13185) (findepi) +- Round robin polling between tied winners in sort preserving merge [#13133](https://github.com/apache/datafusion/pull/13133) (jayzhan211) +- Fix rendering of dictionary empty string values in SLT tests [#13198](https://github.com/apache/datafusion/pull/13198) (findepi) +- Improve push down filter of join [#13184](https://github.com/apache/datafusion/pull/13184) (JasonLi-cn) +- Minor: Reduce indirection for finding changlog [#13199](https://github.com/apache/datafusion/pull/13199) (alamb) +- Support `DictionaryArray` in `OVER` clause [#13153](https://github.com/apache/datafusion/pull/13153) (adriangb) +- Allow testing records with sibling whitespace in SLT tests and add more string tests [#13197](https://github.com/apache/datafusion/pull/13197) (findepi) +- Use single file write when an extension is present in the path. [#13079](https://github.com/apache/datafusion/pull/13079) (dhegberg) +- Deprecate ScalarUDF::invoke and invoke_no_args for invoke_batch [#13179](https://github.com/apache/datafusion/pull/13179) (findepi) +- consider volatile function in simply_expression [#13128](https://github.com/apache/datafusion/pull/13128) (Lordworms) +- Fix CI compile failure due to merge conflict [#13219](https://github.com/apache/datafusion/pull/13219) (alamb) +- Revert "Improve push down filter of join (#13184)" [#13229](https://github.com/apache/datafusion/pull/13229) (eejbyfeldt) +- Derive `Clone` for more ExecutionPlans [#13203](https://github.com/apache/datafusion/pull/13203) (alamb) +- feat(logical-types): add NativeType and LogicalType [#12853](https://github.com/apache/datafusion/pull/12853) (notfilippo) +- Apply projection to `Statistics` in `FilterExec` [#13187](https://github.com/apache/datafusion/pull/13187) (alamb) +- Minor: make LeftJoinData into a struct in CrossJoinExec [#13227](https://github.com/apache/datafusion/pull/13227) (alamb) +- Deprecate invoke and invoke_no_args in favor of invoke_batch [#13174](https://github.com/apache/datafusion/pull/13174) (findepi) +- Support timestamp(n) SQL type [#13231](https://github.com/apache/datafusion/pull/13231) (findepi) +- Remove elements deprecated since v 38. [#13245](https://github.com/apache/datafusion/pull/13245) (findepi) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 68 Andrew Lamb + 34 Piotr Findeisen + 24 Jonathan Chen + 19 Emil Ejbyfeldt + 17 Jax Liu + 12 Bruce Ritchie + 11 Jonah Gao + 9 Jay Zhan + 8 Mustafa Akur + 8 kamille + 7 Sergei Grebnov + 7 Tornike Gurgenidze + 6 JasonLi + 6 Oleks V + 6 Val Lorentz + 6 jcsherin + 5 Burak Şen + 5 Samuel Colvin + 5 Yongting You + 5 dependabot[bot] + 4 HuSen + 4 Jagdish Parihar + 4 Simon Vandel Sillesen + 4 wiedld + 3 Alihan Çelikcan + 3 Andy Grove + 3 AnthonyZhOon + 3 Austin Liu + 3 Berkay Şahin + 3 Daniel Hegberg + 3 Daniël Heres + 3 Lordworms + 3 Michael J Ward + 3 OussamaSaoudi + 3 Qianqian + 3 Tai Le Manh + 3 Victor Barua + 3 doupache + 3 ngli-me + 3 yi wang + 2 Adrian Garcia Badaracco + 2 Alex Huang + 2 Brent Gardner + 2 Dharan Aditya + 2 Dmitrii Blaginin + 2 Duong Cong Toai + 2 Filippo Rossi + 2 Georgi Krastev + 2 June + 2 Max Norfolk + 2 Peter Toth + 2 Tim Saucer + 2 Yasser Latreche + 2 peasee + 2 waruto + 1 Abdullah Sabaa Allil + 1 Agaev Guseyn + 1 Albert Skalt + 1 Andrey Koshchiy + 1 Arttu + 1 Baris Palaska + 1 Bruno Volpato + 1 Bryce Mecum + 1 Daniel Mesejo + 1 Dmitry Bugakov + 1 Eason + 1 Edmondo Porcu + 1 Eduard Karacharov + 1 Frederic Branczyk + 1 Fredrik Meringdal + 1 Haile + 1 Jan + 1 JonasDev1 + 1 Justus Flerlage + 1 Leslie Su + 1 Marco Neumann + 1 Marko Milenković + 1 Martin Hilton + 1 Matthew Turner + 1 Nick Cameron + 1 Paul + 1 Smith Cruise + 1 Tomoaki Kawada + 1 WeblWabl + 1 Weston Pace + 1 Xiangpeng Hao + 1 Xwg + 1 Yuance.Li + 1 epsio-banay + 1 iamthinh + 1 juroberttyb + 1 mertak-synnada + 1 neyama + 1 smarticen + 1 zhuliquan + 1 张林伟 +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. From 88f58bf929167c5c5e2250ad87caa88d4dff11e5 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Mon, 4 Nov 2024 19:40:26 -0700 Subject: [PATCH 002/177] bump version and generate changelog --- docs/source/user-guide/configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index bd8591b5d7234..6a49fda668a9f 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -67,7 +67,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 42.2.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 43.0.0 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | | datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | From 2d5364e9f1d79d3e8dd8a6af48966716e1ea1c43 Mon Sep 17 00:00:00 2001 From: Matthew Turner Date: Mon, 23 Dec 2024 18:23:18 -0500 Subject: [PATCH 003/177] Downgrade tonic --- Cargo.toml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index dd70535be88af..54bc68aa63296 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,22 +71,22 @@ version = "43.0.0" ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } -arrow = { version = "53.2.0", features = [ +arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", features = [ "prettyprint", ] } -arrow-array = { version = "53.2.0", default-features = false, features = [ +arrow-array = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false, features = [ "chrono-tz", ] } -arrow-buffer = { version = "53.2.0", default-features = false } -arrow-flight = { version = "53.2.0", features = [ +arrow-buffer = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } +arrow-flight = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "53.2.0", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "53.2.0", default-features = false } -arrow-schema = { version = "53.2.0", default-features = false } -arrow-string = { version = "53.2.0", default-features = false } +arrow-ord = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } +arrow-schema = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } +arrow-string = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } async-trait = "0.1.73" bigdecimal = "=0.4.1" bytes = "1.4" @@ -128,15 +128,15 @@ log = "^0.4" num_cpus = "1.13.0" object_store = { version = "0.11.0", default-features = false } parking_lot = "0.12" -parquet = { version = "53.2.0", default-features = false, features = [ +parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false, features = [ "arrow", "async", "object_store", ] } pbjson = { version = "0.7.0" } # Should match arrow-flight's version of prost. -prost = "0.13.1" -prost-derive = "0.13.1" +prost = "0.12.3" +prost-derive = "0.12.3" rand = "0.8" regex = "1.8" rstest = "0.23.0" From 2c35f172ecea269e4b2616f22fe03a47882cb20b Mon Sep 17 00:00:00 2001 From: Mustafa Akur <33904309+akurmustafa@users.noreply.github.com> Date: Sun, 24 Nov 2024 03:56:03 -0800 Subject: [PATCH 004/177] [bug]: Fix wrong order by removal from plan (#13497) * Initial commit * Fix formatting * Add across partitions check * Add new test case Add a new test case * Fix buggy test --- .../src/equivalence/properties.rs | 37 ++++++++++++++- datafusion/sqllogictest/test_files/order.slt | 46 +++++++++++++++++++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties.rs b/datafusion/physical-expr/src/equivalence/properties.rs index 55c99e93d040c..1eb88d8a26f0a 100644 --- a/datafusion/physical-expr/src/equivalence/properties.rs +++ b/datafusion/physical-expr/src/equivalence/properties.rs @@ -887,9 +887,11 @@ impl EquivalenceProperties { if self.is_expr_constant(source) && !const_exprs_contains(&projected_constants, target) { + let across_partitions = self.is_expr_constant_accross_partitions(source); // Expression evaluates to single value - projected_constants - .push(ConstExpr::from(target).with_across_partitions(true)); + projected_constants.push( + ConstExpr::from(target).with_across_partitions(across_partitions), + ); } } projected_constants @@ -1018,6 +1020,37 @@ impl EquivalenceProperties { is_constant_recurse(&normalized_constants, &normalized_expr) } + /// This function determines whether the provided expression is constant + /// across partitions based on the known constants. + /// + /// # Arguments + /// + /// - `expr`: A reference to a `Arc` representing the + /// expression to be checked. + /// + /// # Returns + /// + /// Returns `true` if the expression is constant across all partitions according + /// to equivalence group, `false` otherwise. + pub fn is_expr_constant_accross_partitions( + &self, + expr: &Arc, + ) -> bool { + // As an example, assume that we know columns `a` and `b` are constant. + // Then, `a`, `b` and `a + b` will all return `true` whereas `c` will + // return `false`. + let const_exprs = self.constants.iter().flat_map(|const_expr| { + if const_expr.across_partitions() { + Some(Arc::clone(const_expr.expr())) + } else { + None + } + }); + let normalized_constants = self.eq_group.normalize_exprs(const_exprs); + let normalized_expr = self.eq_group.normalize_expr(Arc::clone(expr)); + is_constant_recurse(&normalized_constants, &normalized_expr) + } + /// Retrieves the properties for a given physical expression. /// /// This function constructs an [`ExprProperties`] object for the given diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt index d5f0521407c54..a46040aa532ed 100644 --- a/datafusion/sqllogictest/test_files/order.slt +++ b/datafusion/sqllogictest/test_files/order.slt @@ -1260,3 +1260,49 @@ limit 2; statement ok drop table ordered_table; + +query TT +EXPLAIN SELECT + CASE + WHEN name = 'name1' THEN 0.0 + WHEN name = 'name2' THEN 0.5 + END AS a +FROM ( + SELECT 'name1' AS name + UNION ALL + SELECT 'name2' +) +ORDER BY a DESC; +---- +logical_plan +01)Sort: a DESC NULLS FIRST +02)--Projection: CASE WHEN name = Utf8("name1") THEN Float64(0) WHEN name = Utf8("name2") THEN Float64(0.5) END AS a +03)----Union +04)------Projection: Utf8("name1") AS name +05)--------EmptyRelation +06)------Projection: Utf8("name2") AS name +07)--------EmptyRelation +physical_plan +01)SortPreservingMergeExec: [a@0 DESC] +02)--ProjectionExec: expr=[CASE WHEN name@0 = name1 THEN 0 WHEN name@0 = name2 THEN 0.5 END as a] +03)----UnionExec +04)------ProjectionExec: expr=[name1 as name] +05)--------PlaceholderRowExec +06)------ProjectionExec: expr=[name2 as name] +07)--------PlaceholderRowExec + +query R +SELECT + CASE + WHEN name = 'name1' THEN 0.0 + WHEN name = 'name2' THEN 0.5 + END AS a +FROM ( + SELECT 'name1' AS name + UNION ALL + SELECT 'name2' +) +ORDER BY a DESC; +---- +0.5 +0 From 608ee580fb48cb52943f799dada89a4e251ae292 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 28 Dec 2024 10:16:34 -0500 Subject: [PATCH 005/177] Correct return type for initcap scalar function with utf8view (#13909) (#13934) * Set utf8view as return type when input type is the same * Verify that the returned type from call to scalar function matches the return type specified in the return_type function * Match return type to utf8view Co-authored-by: Tim Saucer --- datafusion/functions/src/unicode/initcap.rs | 18 +++++++++++------- datafusion/functions/src/utils.rs | 1 + 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/datafusion/functions/src/unicode/initcap.rs b/datafusion/functions/src/unicode/initcap.rs index e9f966b958683..c21fb77c9eca9 100644 --- a/datafusion/functions/src/unicode/initcap.rs +++ b/datafusion/functions/src/unicode/initcap.rs @@ -63,7 +63,11 @@ impl ScalarUDFImpl for InitcapFunc { } fn return_type(&self, arg_types: &[DataType]) -> Result { - utf8_to_str_type(&arg_types[0], "initcap") + if let DataType::Utf8View = arg_types[0] { + Ok(DataType::Utf8View) + } else { + utf8_to_str_type(&arg_types[0], "initcap") + } } fn invoke_batch( @@ -188,7 +192,7 @@ mod tests { use crate::unicode::initcap::InitcapFunc; use crate::utils::test::test_function; use arrow::array::{Array, StringArray, StringViewArray}; - use arrow::datatypes::DataType::Utf8; + use arrow::datatypes::DataType::{Utf8, Utf8View}; use datafusion_common::{Result, ScalarValue}; use datafusion_expr::{ColumnarValue, ScalarUDFImpl}; @@ -247,7 +251,7 @@ mod tests { )))], Ok(Some("Hi Thomas")), &str, - Utf8, + Utf8View, StringViewArray ); test_function!( @@ -257,7 +261,7 @@ mod tests { )))], Ok(Some("Hi Thomas With M0re Than 12 Chars")), &str, - Utf8, + Utf8View, StringViewArray ); test_function!( @@ -270,7 +274,7 @@ mod tests { "Đẹp Đẽ Êm Ả Ñandú Árbol Олег Иванович Íslensku Þjóðarinnar Ελληνική" )), &str, - Utf8, + Utf8View, StringViewArray ); test_function!( @@ -280,7 +284,7 @@ mod tests { )))], Ok(Some("")), &str, - Utf8, + Utf8View, StringViewArray ); test_function!( @@ -288,7 +292,7 @@ mod tests { vec![ColumnarValue::Scalar(ScalarValue::Utf8View(None))], Ok(None), &str, - Utf8, + Utf8View, StringViewArray ); diff --git a/datafusion/functions/src/utils.rs b/datafusion/functions/src/utils.rs index 53f607492266f..39d8aeeda460c 100644 --- a/datafusion/functions/src/utils.rs +++ b/datafusion/functions/src/utils.rs @@ -154,6 +154,7 @@ pub mod test { let result = result.unwrap().to_array(cardinality).expect("Failed to convert to array"); let result = result.as_any().downcast_ref::<$ARRAY_TYPE>().expect("Failed to convert to type"); + assert_eq!(result.data_type(), &$EXPECTED_DATA_TYPE); // value is correct match expected { From 3cc3fca31e6edc2d953e663bfd7f856bcb70d8c4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 28 Dec 2024 10:33:02 -0500 Subject: [PATCH 006/177] Update CHANGELOG --- dev/changelog/44.0.0.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dev/changelog/44.0.0.md b/dev/changelog/44.0.0.md index 233e302e50e69..b3f10f6794b53 100644 --- a/dev/changelog/44.0.0.md +++ b/dev/changelog/44.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 44.0.0 Changelog -This release consists of 332 commits from 94 contributors. See credits at the end of this changelog for more information. +This release consists of 329 commits from 94 contributors. See credits at the end of this changelog for more information. **Breaking changes:** @@ -110,6 +110,7 @@ This release consists of 332 commits from 94 contributors. See credits at the en - Support unicode character for `initcap` function [#13752](https://github.com/apache/datafusion/pull/13752) (tlm365) - [minor] make recursive package dependency optional [#13778](https://github.com/apache/datafusion/pull/13778) (buraksenn) - Fix `recursive-protection` feature flag [#13887](https://github.com/apache/datafusion/pull/13887) (alamb) +- Prepare for 44.0.0 release: version and changelog [#13882](https://github.com/apache/datafusion/pull/13882) (alamb) **Other:** @@ -362,13 +363,15 @@ This release consists of 332 commits from 94 contributors. See credits at the en - Minor: change the sort merge join emission as incremental [#13894](https://github.com/apache/datafusion/pull/13894) (berkaysynnada) - Minor: change visibility of hash join utils [#13893](https://github.com/apache/datafusion/pull/13893) (berkaysynnada) - Fix visibility of `swap_hash_join` to be `pub` [#13899](https://github.com/apache/datafusion/pull/13899) (alamb) +- Minor: Avoid emitting empty batches in partial sort [#13895](https://github.com/apache/datafusion/pull/13895) (berkaysynnada) +- BACKPORT: Correct return type for initcap scalar function with utf8view (#13909) [#13934](https://github.com/apache/datafusion/pull/13934) (alamb) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` - 59 Andrew Lamb + 55 Andrew Lamb 35 Piotr Findeisen 16 Jonathan Chen 14 Jonah Gao @@ -383,13 +386,13 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co 5 Dmitrii Blaginin 5 Qianqian 4 Adrian Garcia Badaracco + 4 Berkay Şahin 4 Marco Neumann 4 Tai Le Manh 4 Tim Saucer 4 zhuliquan 3 Andy Grove 3 Arttu - 3 Berkay Şahin 3 Burak Şen 3 Onur Satici 3 Qi Zhu From 5383d30db11a9c1c795b2d94f6dabba8a7c614cc Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Mon, 30 Dec 2024 13:44:22 +0100 Subject: [PATCH 007/177] enforce_distribution: fix for limits getting lost --- .../src/physical_optimizer/enforce_distribution.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/datafusion/core/src/physical_optimizer/enforce_distribution.rs b/datafusion/core/src/physical_optimizer/enforce_distribution.rs index 6cd902db72449..06d62c8a93816 100644 --- a/datafusion/core/src/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/src/physical_optimizer/enforce_distribution.rs @@ -55,6 +55,7 @@ use datafusion_physical_expr::{ use datafusion_physical_expr_common::sort_expr::LexOrdering; use datafusion_physical_optimizer::output_requirements::OutputRequirementExec; use datafusion_physical_optimizer::PhysicalOptimizerRule; +use datafusion_physical_plan::limit::GlobalLimitExec; use datafusion_physical_plan::windows::{get_best_fitting_window, BoundedWindowAggExec}; use datafusion_physical_plan::ExecutionPlanProperties; @@ -1169,6 +1170,8 @@ fn ensure_distribution( children, } = remove_dist_changing_operators(dist_context)?; + let fetch = plan.fetch(); + if let Some(exec) = plan.as_any().downcast_ref::() { if let Some(updated_window) = get_best_fitting_window( exec.window_expr(), @@ -1336,6 +1339,14 @@ fn ensure_distribution( plan.with_new_children(children_plans)? }; + // NOTE: Workaround for limits getting lost + // + // Add a GlobalLimitExec if the plan had a fetch orignally. + // This makes sure `LimitPushdown` pushes down the limit. + if fetch.is_some() { + plan = Arc::new(GlobalLimitExec::new(plan, 0, fetch)); + } + Ok(Transformed::yes(DistributionContext::new( plan, data, children, ))) From 13f6aca6fd09582b1d5f6af497618465def244bc Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:58:31 +0100 Subject: [PATCH 008/177] set default-features=false for datafusion in proto crate --- datafusion/proto/Cargo.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index 9e4b331a01bfa..cd6c385b89187 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -44,7 +44,7 @@ parquet = ["datafusion/parquet", "datafusion-common/parquet"] [dependencies] arrow = { workspace = true } chrono = { workspace = true } -datafusion = { workspace = true, default-features = true } +datafusion = { workspace = true, default-features = false } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } datafusion-proto-common = { workspace = true } @@ -55,6 +55,7 @@ serde = { version = "1.0", optional = true } serde_json = { workspace = true, optional = true } [dev-dependencies] +datafusion = { workspace = true, default-features = true } datafusion-functions = { workspace = true, default-features = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-window-common = { workspace = true } From d357c7aff1944c7b5a6a61e00b53e06e79d56c92 Mon Sep 17 00:00:00 2001 From: Matt Green Date: Wed, 11 Sep 2024 15:08:06 -0700 Subject: [PATCH 009/177] Adding node_id patch to our fork --- datafusion-examples/examples/planner_api.rs | 34 ++++++++--- .../datasource/physical_plan/arrow_file.rs | 16 ++++++ .../core/src/datasource/physical_plan/avro.rs | 16 ++++++ .../core/src/datasource/physical_plan/csv.rs | 22 ++++++++ .../core/src/datasource/physical_plan/json.rs | 15 +++++ .../datasource/physical_plan/parquet/mod.rs | 23 ++++++++ .../core/src/execution/session_state.rs | 10 +++- .../physical-plan/src/aggregates/mod.rs | 17 ++++++ datafusion/physical-plan/src/analyze.rs | 15 +++++ .../physical-plan/src/coalesce_batches.rs | 10 ++++ .../physical-plan/src/coalesce_partitions.rs | 9 +++ datafusion/physical-plan/src/display.rs | 15 ++++- .../physical-plan/src/execution_plan.rs | 34 +++++++++++ datafusion/physical-plan/src/filter.rs | 11 ++++ datafusion/physical-plan/src/insert.rs | 15 +++++ .../physical-plan/src/joins/cross_join.rs | 10 ++++ .../physical-plan/src/joins/hash_join.rs | 19 +++++++ .../src/joins/sort_merge_join.rs | 18 ++++++ .../src/joins/symmetric_hash_join.rs | 20 +++++++ datafusion/physical-plan/src/lib.rs | 2 +- datafusion/physical-plan/src/limit.rs | 11 ++++ datafusion/physical-plan/src/memory.rs | 14 +++++ datafusion/physical-plan/src/node_id.rs | 56 +++++++++++++++++++ .../physical-plan/src/placeholder_row.rs | 10 ++++ datafusion/physical-plan/src/projection.rs | 10 ++++ .../physical-plan/src/recursive_query.rs | 15 +++++ .../physical-plan/src/repartition/mod.rs | 10 ++++ .../physical-plan/src/sorts/partial_sort.rs | 14 +++++ datafusion/physical-plan/src/sorts/sort.rs | 9 +++ .../src/sorts/sort_preserving_merge.rs | 11 ++++ datafusion/physical-plan/src/streaming.rs | 19 +++++++ datafusion/physical-plan/src/union.rs | 10 ++++ datafusion/physical-plan/src/unnest.rs | 16 ++++++ datafusion/physical-plan/src/values.rs | 13 +++++ .../src/windows/window_agg_exec.rs | 14 +++++ datafusion/physical-plan/src/work_table.rs | 10 ++++ 36 files changed, 562 insertions(+), 11 deletions(-) create mode 100644 datafusion/physical-plan/src/node_id.rs diff --git a/datafusion-examples/examples/planner_api.rs b/datafusion-examples/examples/planner_api.rs index 35cf766ba1afe..07fe79399d5cf 100644 --- a/datafusion-examples/examples/planner_api.rs +++ b/datafusion-examples/examples/planner_api.rs @@ -15,10 +15,10 @@ // specific language governing permissions and limitations // under the License. -use datafusion::error::Result; use datafusion::physical_plan::displayable; use datafusion::physical_planner::DefaultPhysicalPlanner; use datafusion::prelude::*; +use datafusion::{error::Result, physical_plan::ExecutionPlan}; use datafusion_expr::{LogicalPlan, PlanType}; /// This example demonstrates the process of converting logical plan @@ -82,9 +82,35 @@ async fn to_physical_plan_in_one_api_demo( .plan ); + let traversal = extract_node_ids_from_execution_plan_tree(physical_plan.as_ref()); + let expected_traversal = vec![ + Some(0), + Some(1), + Some(2), + Some(3), + Some(4), + Some(5), + Some(6), + Some(7), + Some(8), + Some(9), + ]; + assert_eq!(expected_traversal, traversal); Ok(()) } +fn extract_node_ids_from_execution_plan_tree( + physical_plan: &dyn ExecutionPlan, +) -> Vec> { + let mut traversed_nodes: Vec> = vec![]; + for child in physical_plan.children() { + let node_ids = extract_node_ids_from_execution_plan_tree(child.as_ref()); + traversed_nodes.extend(node_ids); + } + traversed_nodes.push(physical_plan.properties().node_id()); + traversed_nodes +} + /// Converts a logical plan into a physical plan by utilizing the analyzer, /// optimizer, and query planner APIs separately. This flavor gives more /// control over the planning process. @@ -117,12 +143,6 @@ async fn to_physical_plan_step_by_step_demo( .query_planner() .create_physical_plan(&optimized_logical_plan, &ctx.state()) .await?; - println!( - "Final physical plan:\n\n{}\n\n", - displayable(physical_plan.as_ref()) - .to_stringified(false, PlanType::InitialPhysicalPlan) - .plan - ); // Call the physical optimizer with an existing physical plan (in this // case the plan is already optimized, but an unoptimized plan would diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/core/src/datasource/physical_plan/arrow_file.rs index 39625a55ca15e..4f6f1cd7d95bd 100644 --- a/datafusion/core/src/datasource/physical_plan/arrow_file.rs +++ b/datafusion/core/src/datasource/physical_plan/arrow_file.rs @@ -213,6 +213,22 @@ impl ExecutionPlan for ArrowExec { cache: self.cache.clone(), })) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let new_cache = self.cache.clone().with_node_id(_node_id); + + Ok(Some(Arc::new(Self { + base_config: self.base_config.clone(), + projected_statistics: self.projected_statistics.clone(), + projected_schema: self.projected_schema.clone(), + projected_output_ordering: self.projected_output_ordering.clone(), + metrics: self.metrics.clone(), + cache: new_cache, + }))) + } } pub struct ArrowOpener { diff --git a/datafusion/core/src/datasource/physical_plan/avro.rs b/datafusion/core/src/datasource/physical_plan/avro.rs index ce72c4087424e..aeab0f8c9a43d 100644 --- a/datafusion/core/src/datasource/physical_plan/avro.rs +++ b/datafusion/core/src/datasource/physical_plan/avro.rs @@ -181,6 +181,22 @@ impl ExecutionPlan for AvroExec { cache: self.cache.clone(), })) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let new_cache = self.cache.clone().with_node_id(_node_id); + + Ok(Some(Arc::new(Self { + base_config: self.base_config.clone(), + projected_statistics: self.projected_statistics.clone(), + projected_schema: self.projected_schema.clone(), + projected_output_ordering: self.projected_output_ordering.clone(), + metrics: self.metrics.clone(), + cache: new_cache, + }))) + } } #[cfg(feature = "avro")] diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rs index 5beffc3b0581d..23451bfce020f 100644 --- a/datafusion/core/src/datasource/physical_plan/csv.rs +++ b/datafusion/core/src/datasource/physical_plan/csv.rs @@ -469,6 +469,28 @@ impl ExecutionPlan for CsvExec { cache: self.cache.clone(), })) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let new_cache = self.cache.clone().with_node_id(_node_id); + + Ok(Some(Arc::new(Self { + base_config: self.base_config.clone(), + projected_statistics: self.projected_statistics.clone(), + has_header: self.has_header, + delimiter: self.delimiter, + quote: self.quote, + escape: self.escape, + terminator: self.terminator, + comment: self.comment, + newlines_in_values: self.newlines_in_values, + metrics: self.metrics.clone(), + file_compression_type: self.file_compression_type, + cache: new_cache, + }))) + } } /// A Config for [`CsvOpener`] diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rs index cf8f129a50369..c0a2ecf51fdbb 100644 --- a/datafusion/core/src/datasource/physical_plan/json.rs +++ b/datafusion/core/src/datasource/physical_plan/json.rs @@ -222,6 +222,21 @@ impl ExecutionPlan for NdJsonExec { cache: self.cache.clone(), })) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let new_cache = self.cache.clone().with_node_id(_node_id); + + Ok(Some(Arc::new(Self { + base_config: self.base_config.clone(), + projected_statistics: self.projected_statistics.clone(), + metrics: self.metrics.clone(), + file_compression_type: self.file_compression_type, + cache: new_cache, + }))) + } } /// A [`FileOpener`] that opens a JSON file and yields a [`FileOpenFuture`] diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs index 059f86ce110f4..8fc0f868d7d45 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs @@ -864,6 +864,29 @@ impl ExecutionPlan for ParquetExec { schema_adapter_factory: self.schema_adapter_factory.clone(), })) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let new_cache = self.cache.clone().with_node_id(_node_id); + + let new_plan = Self { + base_config: self.base_config.clone(), + projected_statistics: self.projected_statistics.clone(), + metrics: self.metrics.clone(), + predicate: self.predicate.clone(), + pruning_predicate: self.pruning_predicate.clone(), + page_pruning_predicate: self.page_pruning_predicate.clone(), + metadata_size_hint: self.metadata_size_hint, + parquet_file_reader_factory: self.parquet_file_reader_factory.clone(), + cache: new_cache, + table_parquet_options: self.table_parquet_options.clone(), + schema_adapter_factory: self.schema_adapter_factory.clone(), + }; + + Ok(Some(Arc::new(new_plan))) + } } fn should_enable_page_index( diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index d50c912dd2fdc..9b8b697315332 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -63,6 +63,9 @@ use datafusion_optimizer::{ use datafusion_physical_expr::create_physical_expr; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; use datafusion_physical_optimizer::PhysicalOptimizerRule; +use datafusion_physical_plan::node_id::{ + annotate_node_id_for_execution_plan, NodeIdAnnotator, +}; use datafusion_physical_plan::ExecutionPlan; use datafusion_sql::parser::{DFParser, Statement}; use datafusion_sql::planner::{ContextProvider, ParserOptions, PlannerContext, SqlToRel}; @@ -710,9 +713,12 @@ impl SessionState { logical_plan: &LogicalPlan, ) -> datafusion_common::Result> { let logical_plan = self.optimize(logical_plan)?; - self.query_planner + let physical_plan = self + .query_planner .create_physical_plan(&logical_plan, self) - .await + .await?; + let mut id_annotator = NodeIdAnnotator::new(); + annotate_node_id_for_execution_plan(&physical_plan, &mut id_annotator) } /// Create a [`PhysicalExpr`] from an [`Expr`] after applying type diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 5ffe797c5c26e..a0b70028e6a19 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -874,6 +874,23 @@ impl ExecutionPlan for AggregateExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::LowerEqual } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = AggregateExec::try_new_with_schema( + self.mode, + self.group_by.clone(), + self.aggr_expr.clone(), + self.filter_expr.clone(), + self.input().clone(), + Arc::clone(&self.input_schema), + Arc::clone(&self.schema), + )?; + let new_props: PlanProperties = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } fn create_schema( diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs index c8b329fabdaab..92dde7d72dee1 100644 --- a/datafusion/physical-plan/src/analyze.rs +++ b/datafusion/physical-plan/src/analyze.rs @@ -204,6 +204,21 @@ impl ExecutionPlan for AnalyzeExec { futures::stream::once(output), ))) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = AnalyzeExec::new( + self.verbose, + self.show_statistics, + self.input.clone(), + self.schema.clone(), + ); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Creates the output of AnalyzeExec as a RecordBatch diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs index 11678e7a46961..9dcb537525a1f 100644 --- a/datafusion/physical-plan/src/coalesce_batches.rs +++ b/datafusion/physical-plan/src/coalesce_batches.rs @@ -204,6 +204,16 @@ impl ExecutionPlan for CoalesceBatchesExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = + CoalesceBatchesExec::new(self.input.clone(), self.target_batch_size); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Stream for [`CoalesceBatchesExec`]. See [`CoalesceBatchesExec`] for more details. diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index 3da101d6092f0..ae2ebef63b3d5 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -183,6 +183,15 @@ impl ExecutionPlan for CoalescePartitionsExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = CoalescePartitionsExec::new(self.input.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs index 9f3a76e285777..abad41815b199 100644 --- a/datafusion/physical-plan/src/display.rs +++ b/datafusion/physical-plan/src/display.rs @@ -273,6 +273,11 @@ impl<'a, 'b> ExecutionPlanVisitor for IndentVisitor<'a, 'b> { fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result { write!(self.f, "{:indent$}", "", indent = self.indent * 2)?; plan.fmt_as(self.t, self.f)?; + let node_id = plan + .properties() + .node_id() + .map_or("None".to_string(), |id| format!(", node_id={}", id)); + write!(self.f, "{node_id}")?; match self.show_metrics { ShowMetrics::None => {} ShowMetrics::Aggregated => { @@ -393,11 +398,19 @@ impl ExecutionPlanVisitor for GraphvizVisitor<'_, '_> { "" }; + let node_id = plan + .properties() + .node_id() + .map_or("node_id=None".to_string(), |id| format!("node_id={}", id)); + self.graphviz_builder.add_node( self.f, id, &label, - Some(&format!("{}{}{}", metrics, delimiter, statistics)), + Some(&format!( + "{}{}{}{}", + metrics, delimiter, statistics, node_id + )), )?; if let Some(parent_node_id) = self.parents.last() { diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index d65320dbab68d..80a8dcbbf169a 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -422,6 +422,14 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Unknown } + /// If supported, returns a copy of this `ExecutionPlan` node with the specified + /// node_id. Returns `None` otherwise. + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + Ok(None) + } } /// Extension trait provides an easy API to fetch various properties of @@ -463,6 +471,11 @@ pub trait ExecutionPlanProperties { /// See also [`ExecutionPlan::maintains_input_order`] and [`Self::output_ordering`] /// for related concepts. fn equivalence_properties(&self) -> &EquivalenceProperties; + + // Node Id of this ExecutionPlan node. See also [`ExecutionPlan::with_node_id`] + fn node_id(&self) -> Option { + None + } } impl ExecutionPlanProperties for Arc { @@ -481,6 +494,10 @@ impl ExecutionPlanProperties for Arc { fn equivalence_properties(&self) -> &EquivalenceProperties { self.properties().equivalence_properties() } + + fn node_id(&self) -> Option { + self.properties().node_id() + } } impl ExecutionPlanProperties for &dyn ExecutionPlan { @@ -499,6 +516,10 @@ impl ExecutionPlanProperties for &dyn ExecutionPlan { fn equivalence_properties(&self) -> &EquivalenceProperties { self.properties().equivalence_properties() } + + fn node_id(&self) -> Option { + self.properties().node_id() + } } /// Describes the execution mode of the result of calling @@ -595,6 +616,8 @@ pub struct PlanProperties { pub execution_mode: ExecutionMode, /// See [ExecutionPlanProperties::output_ordering] output_ordering: Option, + /// See [ExecutionPlanProperties::node_id] + node_id: Option, } impl PlanProperties { @@ -611,6 +634,7 @@ impl PlanProperties { partitioning, execution_mode, output_ordering, + node_id: None, } } @@ -635,6 +659,12 @@ impl PlanProperties { self } + /// Overwrite node id with its new value. + pub fn with_node_id(mut self, node_id: usize) -> Self { + self.node_id = Some(node_id); + self + } + pub fn equivalence_properties(&self) -> &EquivalenceProperties { &self.eq_properties } @@ -651,6 +681,10 @@ impl PlanProperties { self.execution_mode } + pub fn node_id(&self) -> Option { + self.node_id + } + /// Get schema of the node. fn schema(&self) -> &SchemaRef { self.eq_properties.schema() diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 07898e8d22d85..df0cc2f117324 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -382,6 +382,17 @@ impl ExecutionPlan for FilterExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::LowerEqual } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = + FilterExec::try_new(self.predicate.clone(), self.input.clone())?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// This function ensures that all bounds in the `ExprBoundaries` vector are diff --git a/datafusion/physical-plan/src/insert.rs b/datafusion/physical-plan/src/insert.rs index ae8a2acce696b..35e176187740f 100644 --- a/datafusion/physical-plan/src/insert.rs +++ b/datafusion/physical-plan/src/insert.rs @@ -247,6 +247,21 @@ impl ExecutionPlan for DataSinkExec { fn metrics(&self) -> Option { self.sink.metrics() } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = DataSinkExec::new( + self.input.clone(), + self.sink.clone(), + self.sink_schema.clone(), + self.sort_order.clone(), + ); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Create a output record batch with a count diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 8c8921eba6a1d..12a0fe5a7363d 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -308,6 +308,16 @@ impl ExecutionPlan for CrossJoinExec { self.right.statistics()?, )) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = CrossJoinExec::new(self.left.clone(), self.right.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// [left/right]_col_count are required in case the column statistics are None diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index ae872e13a9f63..1188c6b5cbc8a 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -795,6 +795,25 @@ impl ExecutionPlan for HashJoinExec { // Project statistics if there is a projection Ok(stats.project(self.projection.as_ref())) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = HashJoinExec::try_new( + self.left.clone(), + self.right.clone(), + self.on.clone(), + self.filter.clone(), + self.join_type(), + self.projection.clone(), + self.partition_mode().clone(), + self.null_equals_null, + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Reads the left (build) side of the input, buffering it in memory, to build a diff --git a/datafusion/physical-plan/src/joins/sort_merge_join.rs b/datafusion/physical-plan/src/joins/sort_merge_join.rs index 3ad892c880f61..5bed14e4c94e6 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join.rs @@ -409,6 +409,24 @@ impl ExecutionPlan for SortMergeJoinExec { &self.schema, ) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = SortMergeJoinExec::try_new( + self.left.clone(), + self.right.clone(), + self.on.clone(), + self.filter.clone(), + self.join_type(), + self.sort_options.clone(), + self.null_equals_null, + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Metrics for SortMergeJoinExec diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index 5b6dc2cd2ae9a..06abef3eff22b 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -457,6 +457,26 @@ impl ExecutionPlan for SymmetricHashJoinExec { Ok(Statistics::new_unknown(&self.schema())) } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = SymmetricHashJoinExec::try_new( + self.left.clone(), + self.right.clone(), + self.on.clone(), + self.filter.clone(), + self.join_type(), + self.null_equals_null, + self.left_sort_exprs.clone(), + self.right_sort_exprs.clone(), + self.mode, + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } + fn execute( &self, partition: usize, diff --git a/datafusion/physical-plan/src/lib.rs b/datafusion/physical-plan/src/lib.rs index 845a74eaea48e..7bbb8c8e7fd80 100644 --- a/datafusion/physical-plan/src/lib.rs +++ b/datafusion/physical-plan/src/lib.rs @@ -66,6 +66,7 @@ pub mod joins; pub mod limit; pub mod memory; pub mod metrics; +pub mod node_id; pub mod placeholder_row; pub mod projection; pub mod recursive_query; @@ -80,7 +81,6 @@ pub mod unnest; pub mod values; pub mod windows; pub mod work_table; - pub mod udaf { pub use datafusion_expr::StatisticsArgs; pub use datafusion_physical_expr::aggregate::AggregateFunctionExpr; diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index ab1e6cb37bc8c..431b00751b4f8 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -201,6 +201,17 @@ impl ExecutionPlan for GlobalLimitExec { fn supports_limit_pushdown(&self) -> bool { true } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = + GlobalLimitExec::new(self.input.clone(), self.skip, self.fetch); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// LocalLimitExec applies a limit to a single partition diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs index c9ada345afc7f..c8f54ffce1d84 100644 --- a/datafusion/physical-plan/src/memory.rs +++ b/datafusion/physical-plan/src/memory.rs @@ -148,6 +148,20 @@ impl ExecutionPlan for MemoryExec { self.projection.clone(), )) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = MemoryExec::try_new( + &self.partitions.clone(), + self.schema.clone(), + self.projection.clone(), + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } impl MemoryExec { diff --git a/datafusion/physical-plan/src/node_id.rs b/datafusion/physical-plan/src/node_id.rs new file mode 100644 index 0000000000000..a03c747f7c15d --- /dev/null +++ b/datafusion/physical-plan/src/node_id.rs @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +use std::sync::Arc; + +use crate::ExecutionPlan; + +use datafusion_common::DataFusionError; + +// Util for traversing ExecutionPlan tree and annotating node_id +pub struct NodeIdAnnotator { + next_id: usize, +} + +impl NodeIdAnnotator { + pub fn new() -> Self { + NodeIdAnnotator { next_id: 0 } + } + + fn annotate_execution_plan_with_node_id( + &mut self, + plan: Arc, + ) -> Result, DataFusionError> { + let plan_with_id = plan.clone().with_node_id(self.next_id)?.unwrap_or(plan); + self.next_id += 1; + Ok(plan_with_id) + } +} + +pub fn annotate_node_id_for_execution_plan( + plan: &Arc, + annotator: &mut NodeIdAnnotator, +) -> Result, DataFusionError> { + let mut new_children: Vec> = vec![]; + for child in plan.children() { + let new_child: Arc = + annotate_node_id_for_execution_plan(child, annotator)?; + new_children.push(new_child); + } + let new_plan = plan.clone().with_new_children(new_children)?; + let new_plan_with_id = annotator.annotate_execution_plan_with_node_id(new_plan)?; + Ok(new_plan_with_id) +} diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs index f9437f46f8a6c..52cb683729216 100644 --- a/datafusion/physical-plan/src/placeholder_row.rs +++ b/datafusion/physical-plan/src/placeholder_row.rs @@ -175,6 +175,16 @@ impl ExecutionPlan for PlaceholderRowExec { None, )) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = PlaceholderRowExec::new(self.schema.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index c1d3f368366f6..f246344bc91cc 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -238,6 +238,16 @@ impl ExecutionPlan for ProjectionExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = + ProjectionExec::try_new(self.expr.clone(), self.input.clone())?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// If 'e' is a direct column reference, returns the field level diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs index cbf22a4b392ff..3e93df1b6e9bd 100644 --- a/datafusion/physical-plan/src/recursive_query.rs +++ b/datafusion/physical-plan/src/recursive_query.rs @@ -185,6 +185,21 @@ impl ExecutionPlan for RecursiveQueryExec { fn statistics(&self) -> Result { Ok(Statistics::new_unknown(&self.schema())) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = RecursiveQueryExec::try_new( + self.name.clone(), + self.static_term.clone(), + self.recursive_term.clone(), + self.is_distinct, + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } impl DisplayAs for RecursiveQueryExec { diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index bc65b251561bd..2724490576de3 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -672,6 +672,16 @@ impl ExecutionPlan for RepartitionExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::Equal } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = + RepartitionExec::try_new(self.input.clone(), self.partitioning().clone())?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } impl RepartitionExec { diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index 8f853464c9bdd..0ee0d793c844d 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -308,6 +308,20 @@ impl ExecutionPlan for PartialSortExec { fn statistics(&self) -> Result { self.input.statistics() } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = PartialSortExec::new( + self.expr.clone(), + self.input.clone(), + self.common_prefix_length, + ); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } struct PartialSortStream { diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index d90d0f64ceb41..1002789168cdf 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -977,6 +977,15 @@ impl ExecutionPlan for SortExec { CardinalityEffect::LowerEqual } } + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = SortExec::new(self.expr.clone(), self.input.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index 9ee0faaa0a44a..9aa30bef29571 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -315,6 +315,17 @@ impl ExecutionPlan for SortPreservingMergeExec { fn supports_limit_pushdown(&self) -> bool { true } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = + SortPreservingMergeExec::new(self.expr.clone(), self.input.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs index 7ccef32480696..43288a7f5ab45 100644 --- a/datafusion/physical-plan/src/streaming.rs +++ b/datafusion/physical-plan/src/streaming.rs @@ -283,6 +283,25 @@ impl ExecutionPlan for StreamingTableExec { metrics: self.metrics.clone(), })) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = StreamingTableExec { + partitions: self.partitions.clone(), + projection: self.projection.clone(), + projected_schema: Arc::clone(&self.projected_schema), + projected_output_ordering: self.projected_output_ordering.clone(), + infinite: self.infinite, + limit: self.limit, + cache: self.cache.clone(), + metrics: self.metrics.clone(), + }; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index bd36753880eb2..340f758b103d1 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -264,6 +264,16 @@ impl ExecutionPlan for UnionExec { fn supports_limit_pushdown(&self) -> bool { true } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = UnionExec::new(self.inputs.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Combines multiple input streams by interleaving them. diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index b7b9f17eb1b63..27e1c105adffe 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -199,6 +199,22 @@ impl ExecutionPlan for UnnestExec { fn metrics(&self) -> Option { Some(self.metrics.clone_inner()) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = UnnestExec::new( + self.input.clone(), + self.list_column_indices.clone(), + self.struct_column_indices.clone(), + self.schema.clone(), + self.options.clone(), + ); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[derive(Clone, Debug)] diff --git a/datafusion/physical-plan/src/values.rs b/datafusion/physical-plan/src/values.rs index edadf98cb10c1..d8830931e0dee 100644 --- a/datafusion/physical-plan/src/values.rs +++ b/datafusion/physical-plan/src/values.rs @@ -210,6 +210,19 @@ impl ExecutionPlan for ValuesExec { None, )) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = ValuesExec::try_new_from_batches( + Arc::clone(&self.schema), + self.data.clone(), + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs index f71a0b9fd095c..156e1d660169d 100644 --- a/datafusion/physical-plan/src/windows/window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs @@ -261,6 +261,20 @@ impl ExecutionPlan for WindowAggExec { total_byte_size: Precision::Absent, }) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = WindowAggExec::try_new( + self.window_expr.clone(), + self.input.clone(), + self.partition_keys.clone(), + )?; + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } /// Compute the window aggregate columns diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs index 61d444171cc72..e0364acf5d40a 100644 --- a/datafusion/physical-plan/src/work_table.rs +++ b/datafusion/physical-plan/src/work_table.rs @@ -214,6 +214,16 @@ impl ExecutionPlan for WorkTableExec { fn statistics(&self) -> Result { Ok(Statistics::new_unknown(&self.schema())) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = WorkTableExec::new(self.name.clone(), self.schema.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] From cbd3dbc31614fa1b9f8d72ef9d49d12be3a68a99 Mon Sep 17 00:00:00 2001 From: Amey Chaugule Date: Thu, 2 May 2024 11:56:28 -0700 Subject: [PATCH 010/177] Changes to make streaming work --- datafusion/expr-common/src/accumulator.rs | 11 +++++++++++ datafusion/expr/src/lib.rs | 1 + 2 files changed, 12 insertions(+) diff --git a/datafusion/expr-common/src/accumulator.rs b/datafusion/expr-common/src/accumulator.rs index 7155c7993f8c9..bd286fbf95cad 100644 --- a/datafusion/expr-common/src/accumulator.rs +++ b/datafusion/expr-common/src/accumulator.rs @@ -311,4 +311,15 @@ pub trait Accumulator: Send + Sync + Debug { fn supports_retract_batch(&self) -> bool { false } + + fn as_serializable(&self) -> Option<&dyn SerializableAccumulator> { + None + } +} + +pub trait SerializableAccumulator: Accumulator { + fn serialize(&self) -> Result>; + fn deserialize(bytes: &[u8]) -> Result> + where + Self: Sized; } diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs index 849d9604808ca..05f5ef6ddbbc9 100644 --- a/datafusion/expr/src/lib.rs +++ b/datafusion/expr/src/lib.rs @@ -69,6 +69,7 @@ pub mod window_state; pub use built_in_window_function::BuiltInWindowFunction; pub use datafusion_expr_common::accumulator::Accumulator; +pub use datafusion_expr_common::accumulator::SerializableAccumulator; pub use datafusion_expr_common::columnar_value::ColumnarValue; pub use datafusion_expr_common::groups_accumulator::{EmitTo, GroupsAccumulator}; pub use datafusion_expr_common::operator::Operator; From deecef104c6f23cba72be1778040f92cbec943a8 Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Wed, 11 Dec 2024 14:13:02 +0700 Subject: [PATCH 011/177] only output node_id in display if it exists --- datafusion/physical-plan/src/display.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs index abad41815b199..4c5bbc567679c 100644 --- a/datafusion/physical-plan/src/display.rs +++ b/datafusion/physical-plan/src/display.rs @@ -273,11 +273,11 @@ impl<'a, 'b> ExecutionPlanVisitor for IndentVisitor<'a, 'b> { fn pre_visit(&mut self, plan: &dyn ExecutionPlan) -> Result { write!(self.f, "{:indent$}", "", indent = self.indent * 2)?; plan.fmt_as(self.t, self.f)?; - let node_id = plan - .properties() - .node_id() - .map_or("None".to_string(), |id| format!(", node_id={}", id)); - write!(self.f, "{node_id}")?; + + if let Some(node_id) = plan.properties().node_id() { + write!(self.f, ", node_id={}", node_id)?; + } + match self.show_metrics { ShowMetrics::None => {} ShowMetrics::Aggregated => { From 57bf8d680fa0fe8e7a2839dd2aab3f67edd8383c Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:30:43 +0100 Subject: [PATCH 012/177] include projection in FilterExec::with_node_id --- datafusion/physical-plan/src/filter.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index df0cc2f117324..1885ace0d3c66 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -388,7 +388,8 @@ impl ExecutionPlan for FilterExec { _node_id: usize, ) -> Result>> { let mut new_plan = - FilterExec::try_new(self.predicate.clone(), self.input.clone())?; + FilterExec::try_new(self.predicate.clone(), self.input.clone())? + .with_projection(self.projection.clone())?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) From c431f0f129b45e22c307bc76cb2f8b5314110cd8 Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:46:40 +0100 Subject: [PATCH 013/177] add missing with_fetch calls to with_node_id method --- datafusion/physical-plan/src/coalesce_batches.rs | 3 ++- datafusion/physical-plan/src/sorts/partial_sort.rs | 3 ++- datafusion/physical-plan/src/sorts/sort.rs | 3 ++- datafusion/physical-plan/src/sorts/sort_preserving_merge.rs | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs index 9dcb537525a1f..cfe18d648e507 100644 --- a/datafusion/physical-plan/src/coalesce_batches.rs +++ b/datafusion/physical-plan/src/coalesce_batches.rs @@ -209,7 +209,8 @@ impl ExecutionPlan for CoalesceBatchesExec { _node_id: usize, ) -> Result>> { let mut new_plan = - CoalesceBatchesExec::new(self.input.clone(), self.target_batch_size); + CoalesceBatchesExec::new(self.input.clone(), self.target_batch_size) + .with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index 0ee0d793c844d..e520fad3826e0 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -317,7 +317,8 @@ impl ExecutionPlan for PartialSortExec { self.expr.clone(), self.input.clone(), self.common_prefix_length, - ); + ) + .with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 1002789168cdf..692f66211983e 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -981,7 +981,8 @@ impl ExecutionPlan for SortExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = SortExec::new(self.expr.clone(), self.input.clone()); + let mut new_plan = + SortExec::new(self.expr.clone(), self.input.clone()).with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index 9aa30bef29571..46023d858a293 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -321,7 +321,8 @@ impl ExecutionPlan for SortPreservingMergeExec { _node_id: usize, ) -> Result>> { let mut new_plan = - SortPreservingMergeExec::new(self.expr.clone(), self.input.clone()); + SortPreservingMergeExec::new(self.expr.clone(), self.input.clone()) + .with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) From fa581d01d44c04650927c020ba9eaf3429edff8a Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Wed, 8 Jan 2025 14:57:57 +0100 Subject: [PATCH 014/177] rework SortExec::with_node_id to not drop preserve_partitioning --- datafusion/physical-plan/src/sorts/sort.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 692f66211983e..91c24185681e8 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -981,10 +981,14 @@ impl ExecutionPlan for SortExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = - SortExec::new(self.expr.clone(), self.input.clone()).with_fetch(self.fetch()); - let new_props = new_plan.cache.clone().with_node_id(_node_id); - new_plan.cache = new_props; + let new_plan = SortExec { + input: self.input.clone(), + expr: self.expr.clone(), + fetch: self.fetch, + metrics_set: self.metrics_set.clone(), + preserve_partitioning: self.preserve_partitioning, + cache: self.cache.clone().with_node_id(_node_id), + }; Ok(Some(Arc::new(new_plan))) } } From 555ef6be820f0d3e830e059a3ab48268182ede20 Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Thu, 9 Jan 2025 18:21:40 +0100 Subject: [PATCH 015/177] set schema_force_view_types to false in ParquetOptions --- datafusion/common/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 3365130350367..1fa32aefb33f0 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -399,7 +399,7 @@ config_namespace! { /// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, /// and `Binary/BinaryLarge` with `BinaryView`. - pub schema_force_view_types: bool, default = true + pub schema_force_view_types: bool, default = false /// (reading) If true, parquet reader will read columns of /// `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. From 0e3c9e0affd41f47cbfcb24a8182b3ce389a97b8 Mon Sep 17 00:00:00 2001 From: suremarc <8771538+suremarc@users.noreply.github.com> Date: Tue, 14 Jan 2025 20:47:32 +0000 Subject: [PATCH 016/177] Revert "enforce_distribution: fix for limits getting lost" This reverts commit 5383d30db11a9c1c795b2d94f6dabba8a7c614cc. --- .../src/physical_optimizer/enforce_distribution.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_distribution.rs b/datafusion/core/src/physical_optimizer/enforce_distribution.rs index 06d62c8a93816..6cd902db72449 100644 --- a/datafusion/core/src/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/src/physical_optimizer/enforce_distribution.rs @@ -55,7 +55,6 @@ use datafusion_physical_expr::{ use datafusion_physical_expr_common::sort_expr::LexOrdering; use datafusion_physical_optimizer::output_requirements::OutputRequirementExec; use datafusion_physical_optimizer::PhysicalOptimizerRule; -use datafusion_physical_plan::limit::GlobalLimitExec; use datafusion_physical_plan::windows::{get_best_fitting_window, BoundedWindowAggExec}; use datafusion_physical_plan::ExecutionPlanProperties; @@ -1170,8 +1169,6 @@ fn ensure_distribution( children, } = remove_dist_changing_operators(dist_context)?; - let fetch = plan.fetch(); - if let Some(exec) = plan.as_any().downcast_ref::() { if let Some(updated_window) = get_best_fitting_window( exec.window_expr(), @@ -1339,14 +1336,6 @@ fn ensure_distribution( plan.with_new_children(children_plans)? }; - // NOTE: Workaround for limits getting lost - // - // Add a GlobalLimitExec if the plan had a fetch orignally. - // This makes sure `LimitPushdown` pushes down the limit. - if fetch.is_some() { - plan = Arc::new(GlobalLimitExec::new(plan, 0, fetch)); - } - Ok(Transformed::yes(DistributionContext::new( plan, data, children, ))) From a4153bf5cc6fc27e9a7abbb9dd0c7d78cea345c2 Mon Sep 17 00:00:00 2001 From: suremarc <8771538+suremarc@users.noreply.github.com> Date: Tue, 14 Jan 2025 21:08:56 +0000 Subject: [PATCH 017/177] update sqllogictests after disabling view types --- .../sqllogictest/test_files/describe.slt | 4 +- .../sqllogictest/test_files/explain.slt | 12 ++--- .../test_files/information_schema.slt | 4 +- datafusion/sqllogictest/test_files/map.slt | 2 +- .../sqllogictest/test_files/parquet.slt | 48 +++++++++---------- 5 files changed, 35 insertions(+), 35 deletions(-) diff --git a/datafusion/sqllogictest/test_files/describe.slt b/datafusion/sqllogictest/test_files/describe.slt index e4cb30628eec5..077e8e6474d1f 100644 --- a/datafusion/sqllogictest/test_files/describe.slt +++ b/datafusion/sqllogictest/test_files/describe.slt @@ -81,8 +81,8 @@ int_col Int32 YES bigint_col Int64 YES float_col Float32 YES double_col Float64 YES -date_string_col Utf8View YES -string_col Utf8View YES +date_string_col Utf8 YES +string_col Utf8 YES timestamp_col Timestamp(Nanosecond, None) YES year Int32 YES month Int32 YES diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt index 54658f36ca14b..1340fd490e06f 100644 --- a/datafusion/sqllogictest/test_files/explain.slt +++ b/datafusion/sqllogictest/test_files/explain.slt @@ -305,8 +305,8 @@ initial_physical_plan 01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] 02)--ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] initial_physical_plan_with_schema -01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] -02)--ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] +02)--ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] physical_plan after OutputRequirements 01)OutputRequirementExec, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] 02)--GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] @@ -328,7 +328,7 @@ physical_plan after ProjectionPushdown SAME TEXT AS ABOVE physical_plan after LimitPushdown ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] physical_plan after SanityCheckPlan SAME TEXT AS ABOVE physical_plan ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] -physical_plan_with_schema ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +physical_plan_with_schema ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] statement ok @@ -345,8 +345,8 @@ initial_physical_plan_with_stats 01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] 02)--ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] initial_physical_plan_with_schema -01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] -02)--ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] +02)--ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] physical_plan after OutputRequirements 01)OutputRequirementExec 02)--GlobalLimitExec: skip=0, fetch=10 @@ -369,7 +369,7 @@ physical_plan after LimitPushdown ParquetExec: file_groups={1 group: [[WORKSPACE physical_plan after SanityCheckPlan SAME TEXT AS ABOVE physical_plan ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10 physical_plan_with_stats ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, statistics=[Rows=Exact(8), Bytes=Absent, [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] -physical_plan_with_schema ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +physical_plan_with_schema ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] statement ok diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 84d18233d5726..03ab4a090e67e 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -202,7 +202,7 @@ datafusion.execution.parquet.metadata_size_hint NULL datafusion.execution.parquet.pruning true datafusion.execution.parquet.pushdown_filters false datafusion.execution.parquet.reorder_filters false -datafusion.execution.parquet.schema_force_view_types true +datafusion.execution.parquet.schema_force_view_types false datafusion.execution.parquet.skip_metadata true datafusion.execution.parquet.statistics_enabled page datafusion.execution.parquet.write_batch_size 1024 @@ -295,7 +295,7 @@ datafusion.execution.parquet.metadata_size_hint NULL (reading) If specified, the datafusion.execution.parquet.pruning true (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file datafusion.execution.parquet.pushdown_filters false (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". datafusion.execution.parquet.reorder_filters false (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query -datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. +datafusion.execution.parquet.schema_force_view_types false (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata datafusion.execution.parquet.statistics_enabled page (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting datafusion.execution.parquet.write_batch_size 1024 (writing) Sets write_batch_size in bytes diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt index 10ca3ae881bfd..d24b66aa5c30a 100644 --- a/datafusion/sqllogictest/test_files/map.slt +++ b/datafusion/sqllogictest/test_files/map.slt @@ -42,7 +42,7 @@ describe data; ---- ints Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO strings Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO -timestamp Utf8View NO +timestamp Utf8 NO query ??T SELECT * FROM data ORDER by ints['bytes'] DESC LIMIT 10; diff --git a/datafusion/sqllogictest/test_files/parquet.slt b/datafusion/sqllogictest/test_files/parquet.slt index 253ebb9ea0ac7..65341e67be873 100644 --- a/datafusion/sqllogictest/test_files/parquet.slt +++ b/datafusion/sqllogictest/test_files/parquet.slt @@ -384,15 +384,15 @@ select arrow_typeof(binaryview_col), binaryview_col FROM binary_as_string_default; ---- -BinaryView 616161 BinaryView 616161 BinaryView 616161 -BinaryView 626262 BinaryView 626262 BinaryView 626262 -BinaryView 636363 BinaryView 636363 BinaryView 636363 -BinaryView 646464 BinaryView 646464 BinaryView 646464 -BinaryView 656565 BinaryView 656565 BinaryView 656565 -BinaryView 666666 BinaryView 666666 BinaryView 666666 -BinaryView 676767 BinaryView 676767 BinaryView 676767 -BinaryView 686868 BinaryView 686868 BinaryView 686868 -BinaryView 696969 BinaryView 696969 BinaryView 696969 +Binary 616161 Binary 616161 Binary 616161 +Binary 626262 Binary 626262 Binary 626262 +Binary 636363 Binary 636363 Binary 636363 +Binary 646464 Binary 646464 Binary 646464 +Binary 656565 Binary 656565 Binary 656565 +Binary 666666 Binary 666666 Binary 666666 +Binary 676767 Binary 676767 Binary 676767 +Binary 686868 Binary 686868 Binary 686868 +Binary 696969 Binary 696969 Binary 696969 # Run an explain plan to show the cast happens in the plan (a CAST is needed for the predicates) query TT @@ -405,13 +405,13 @@ EXPLAIN binaryview_col LIKE '%a%'; ---- logical_plan -01)Filter: CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%") -02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%")] +01)Filter: CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8) LIKE Utf8("%a%") +02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8) LIKE Utf8("%a%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 -02)--FilterExec: CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% +02)--FilterExec: CAST(binary_col@0 AS Utf8) LIKE %a% AND CAST(largebinary_col@1 AS Utf8) LIKE %a% AND CAST(binaryview_col@2 AS Utf8) LIKE %a% 03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 -04)------ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/binary_as_string.parquet]]}, projection=[binary_col, largebinary_col, binaryview_col], predicate=CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% +04)------ParquetExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/binary_as_string.parquet]]}, projection=[binary_col, largebinary_col, binaryview_col], predicate=CAST(binary_col@0 AS Utf8) LIKE %a% AND CAST(largebinary_col@1 AS Utf8) LIKE %a% AND CAST(binaryview_col@2 AS Utf8) LIKE %a% statement ok @@ -432,15 +432,15 @@ select arrow_typeof(binaryview_col), binaryview_col FROM binary_as_string_option; ---- -Utf8View aaa Utf8View aaa Utf8View aaa -Utf8View bbb Utf8View bbb Utf8View bbb -Utf8View ccc Utf8View ccc Utf8View ccc -Utf8View ddd Utf8View ddd Utf8View ddd -Utf8View eee Utf8View eee Utf8View eee -Utf8View fff Utf8View fff Utf8View fff -Utf8View ggg Utf8View ggg Utf8View ggg -Utf8View hhh Utf8View hhh Utf8View hhh -Utf8View iii Utf8View iii Utf8View iii +Utf8 aaa Utf8 aaa Utf8 aaa +Utf8 bbb Utf8 bbb Utf8 bbb +Utf8 ccc Utf8 ccc Utf8 ccc +Utf8 ddd Utf8 ddd Utf8 ddd +Utf8 eee Utf8 eee Utf8 eee +Utf8 fff Utf8 fff Utf8 fff +Utf8 ggg Utf8 ggg Utf8 ggg +Utf8 hhh Utf8 hhh Utf8 hhh +Utf8 iii Utf8 iii Utf8 iii # Run an explain plan to show the cast happens in the plan (there should be no casts) query TT @@ -453,8 +453,8 @@ EXPLAIN binaryview_col LIKE '%a%'; ---- logical_plan -01)Filter: binary_as_string_option.binary_col LIKE Utf8View("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8View("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8View("%a%") -02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8View("%a%"), binary_as_string_option.largebinary_col LIKE Utf8View("%a%"), binary_as_string_option.binaryview_col LIKE Utf8View("%a%")] +01)Filter: binary_as_string_option.binary_col LIKE Utf8("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8("%a%") +02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8("%a%"), binary_as_string_option.largebinary_col LIKE Utf8("%a%"), binary_as_string_option.binaryview_col LIKE Utf8("%a%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 02)--FilterExec: binary_col@0 LIKE %a% AND largebinary_col@1 LIKE %a% AND binaryview_col@2 LIKE %a% From 8ae4a9503f8503f9989cd6a2484b0e0bcc975b92 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 15 Jan 2025 17:24:27 +0800 Subject: [PATCH 018/177] fix fetch missed in EnforceDistribution --- .../enforce_distribution.rs | 102 ++++++++++++------ 1 file changed, 69 insertions(+), 33 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_distribution.rs b/datafusion/core/src/physical_optimizer/enforce_distribution.rs index 6cd902db72449..eb6deafd5a312 100644 --- a/datafusion/core/src/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/src/physical_optimizer/enforce_distribution.rs @@ -923,7 +923,10 @@ fn add_hash_on_top( /// /// Updated node with an execution plan, where desired single /// distribution is satisfied by adding [`SortPreservingMergeExec`]. -fn add_spm_on_top(input: DistributionContext) -> DistributionContext { +fn add_spm_on_top( + input: DistributionContext, + fetch: &mut Option, +) -> DistributionContext { // Add SortPreservingMerge only when partition count is larger than 1. if input.plan.output_partitioning().partition_count() > 1 { // When there is an existing ordering, we preserve ordering @@ -935,10 +938,13 @@ fn add_spm_on_top(input: DistributionContext) -> DistributionContext { let should_preserve_ordering = input.plan.output_ordering().is_some(); let new_plan = if should_preserve_ordering { - Arc::new(SortPreservingMergeExec::new( - LexOrdering::from_ref(input.plan.output_ordering().unwrap_or(&[])), - input.plan.clone(), - )) as _ + Arc::new( + SortPreservingMergeExec::new( + LexOrdering::from_ref(input.plan.output_ordering().unwrap_or(&[])), + input.plan.clone(), + ) + .with_fetch(fetch.take()), + ) as _ } else { Arc::new(CoalescePartitionsExec::new(input.plan.clone())) as _ }; @@ -968,18 +974,28 @@ fn add_spm_on_top(input: DistributionContext) -> DistributionContext { /// ``` fn remove_dist_changing_operators( mut distribution_context: DistributionContext, -) -> Result { +) -> Result<(DistributionContext, Option)> { + let mut fetch = None; while is_repartition(&distribution_context.plan) || is_coalesce_partitions(&distribution_context.plan) || is_sort_preserving_merge(&distribution_context.plan) { + if is_sort_preserving_merge(&distribution_context.plan) { + if let Some(child_fetch) = distribution_context.plan.fetch() { + if fetch.is_none() { + fetch = Some(child_fetch); + } else { + fetch = Some(fetch.unwrap().min(child_fetch)); + } + } + } // All of above operators have a single child. First child is only child. // Remove any distribution changing operators at the beginning: distribution_context = distribution_context.children.swap_remove(0); // Note that they will be re-inserted later on if necessary or helpful. } - Ok(distribution_context) + Ok((distribution_context, fetch)) } /// Updates the [`DistributionContext`] if preserving ordering while changing partitioning is not helpful or desirable. @@ -1002,23 +1018,25 @@ fn remove_dist_changing_operators( /// ``` fn replace_order_preserving_variants( mut context: DistributionContext, -) -> Result { - context.children = context - .children - .into_iter() - .map(|child| { - if child.data { - replace_order_preserving_variants(child) - } else { - Ok(child) - } - }) - .collect::>>()?; - +) -> Result<(DistributionContext, Option)> { + let mut children = vec![]; + let mut fetch = None; + for child in context.children.into_iter() { + if child.data { + let (child, inner_fetch) = replace_order_preserving_variants(child)?; + children.push(child); + fetch = inner_fetch; + } else { + children.push(child); + } + } + context.children = children; if is_sort_preserving_merge(&context.plan) { + // Keep the fetch value of the SortPreservingMerge operator, maybe it will be used later. + let fetch = context.plan.fetch(); let child_plan = context.children[0].plan.clone(); context.plan = Arc::new(CoalescePartitionsExec::new(child_plan)); - return Ok(context); + return Ok((context, fetch)); } else if let Some(repartition) = context.plan.as_any().downcast_ref::() { @@ -1027,11 +1045,11 @@ fn replace_order_preserving_variants( context.children[0].plan.clone(), repartition.partitioning().clone(), )?); - return Ok(context); + return Ok((context, None)); } } - context.update_plan_from_children() + Ok((context.update_plan_from_children()?, fetch)) } /// A struct to keep track of repartition requirements for each child node. @@ -1163,11 +1181,14 @@ fn ensure_distribution( is_unbounded || config.optimizer.prefer_existing_sort; // Remove unnecessary repartition from the physical plan if any - let DistributionContext { - mut plan, - data, - children, - } = remove_dist_changing_operators(dist_context)?; + let ( + DistributionContext { + mut plan, + data, + children, + }, + mut fetch, + ) = remove_dist_changing_operators(dist_context)?; if let Some(exec) = plan.as_any().downcast_ref::() { if let Some(updated_window) = get_best_fitting_window( @@ -1232,7 +1253,7 @@ fn ensure_distribution( // Satisfy the distribution requirement if it is unmet. match &requirement { Distribution::SinglePartition => { - child = add_spm_on_top(child); + child = add_spm_on_top(child, &mut fetch); } Distribution::HashPartitioned(exprs) => { if add_roundrobin { @@ -1267,7 +1288,9 @@ fn ensure_distribution( if (!ordering_satisfied || !order_preserving_variants_desirable) && child.data { - child = replace_order_preserving_variants(child)?; + let (replaced_child, fetch) = + replace_order_preserving_variants(child)?; + child = replaced_child; // If ordering requirements were satisfied before repartitioning, // make sure ordering requirements are still satisfied after. if ordering_satisfied { @@ -1275,7 +1298,7 @@ fn ensure_distribution( child = add_sort_above_with_check( child, required_input_ordering.clone(), - None, + fetch, ); } } @@ -1287,12 +1310,12 @@ fn ensure_distribution( // Operator requires specific distribution. Distribution::SinglePartition | Distribution::HashPartitioned(_) => { // Since there is no ordering requirement, preserving ordering is pointless - child = replace_order_preserving_variants(child)?; + child = replace_order_preserving_variants(child)?.0; } Distribution::UnspecifiedDistribution => { // Since ordering is lost, trying to preserve ordering is pointless if !maintains || plan.as_any().is::() { - child = replace_order_preserving_variants(child)?; + child = replace_order_preserving_variants(child)?.0; } } } @@ -1336,6 +1359,19 @@ fn ensure_distribution( plan.with_new_children(children_plans)? }; + // If `fetch` was not consumed, it means that there was `SortPreservingMergeExec` with fetch before + // It was removed by `remove_dist_changing_operators` + // and we need to add it back. + if fetch.is_some() { + plan = Arc::new( + SortPreservingMergeExec::new( + LexOrdering::from_ref(plan.output_ordering().unwrap_or(&[])), + plan.clone(), + ) + .with_fetch(fetch.take()), + ) + } + Ok(Transformed::yes(DistributionContext::new( plan, data, children, ))) From 1ae2702ff9e33ff25363bcaeb6dc0b517a10f2a0 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 17 Jan 2025 15:18:16 +0800 Subject: [PATCH 019/177] fix enforcesorting missing fetch --- .../replace_with_order_preserving_variants.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs b/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs index 930ce52e6fa27..17b41fd3598ea 100644 --- a/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs +++ b/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs @@ -99,6 +99,7 @@ fn plan_with_order_preserving_variants( // Flag indicating that it is desirable to replace `CoalescePartitionsExec`s // with `SortPreservingMergeExec`s: is_spm_better: bool, + fetch: Option, ) -> Result { sort_input.children = sort_input .children @@ -106,7 +107,12 @@ fn plan_with_order_preserving_variants( .map(|node| { // Update descendants in the given tree if there is a connection: if node.data { - plan_with_order_preserving_variants(node, is_spr_better, is_spm_better) + plan_with_order_preserving_variants( + node, + is_spr_better, + is_spm_better, + fetch, + ) } else { Ok(node) } @@ -133,7 +139,8 @@ fn plan_with_order_preserving_variants( // When the input of a `CoalescePartitionsExec` has an ordering, // replace it with a `SortPreservingMergeExec` if appropriate: let spm = - SortPreservingMergeExec::new(LexOrdering::new(ordering), child.clone()); + SortPreservingMergeExec::new(LexOrdering::new(ordering), child.clone()) + .with_fetch(fetch); sort_input.plan = Arc::new(spm) as _; sort_input.children[0].data = true; return Ok(sort_input); @@ -251,6 +258,7 @@ pub(crate) fn replace_with_order_preserving_variants( requirements.children.swap_remove(0), is_spr_better || use_order_preserving_variant, is_spm_better || use_order_preserving_variant, + requirements.plan.fetch(), )?; // If the alternate plan makes this sort unnecessary, accept the alternate: From 38f39f5c81af92ebb83ec41fb1bae9c4be1d915a Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 17 Jan 2025 15:40:39 +0800 Subject: [PATCH 020/177] fix more fetch missing in enforcesorting --- .../core/src/physical_optimizer/enforce_sorting.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_sorting.rs b/datafusion/core/src/physical_optimizer/enforce_sorting.rs index 7b111cddc6fdb..1df56e593da82 100644 --- a/datafusion/core/src/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/src/physical_optimizer/enforce_sorting.rs @@ -399,7 +399,10 @@ fn analyze_immediate_sort_removal( { // Replace the sort with a sort-preserving merge: let expr = LexOrdering::new(sort_exec.expr().to_vec()); - Arc::new(SortPreservingMergeExec::new(expr, sort_input.clone())) as _ + Arc::new( + SortPreservingMergeExec::new(expr, sort_input.clone()) + .with_fetch(node.plan.fetch()), + ) as _ } else { // Remove the sort: node.children = node.children.swap_remove(0).children; @@ -618,11 +621,12 @@ fn remove_corresponding_sort_from_sub_plan( // If there is existing ordering, to preserve ordering use // `SortPreservingMergeExec` instead of a `CoalescePartitionsExec`. let plan = node.plan.clone(); + let fetch = plan.fetch(); let plan = if let Some(ordering) = plan.output_ordering() { - Arc::new(SortPreservingMergeExec::new( - LexOrdering::new(ordering.to_vec()), - plan, - )) as _ + Arc::new( + SortPreservingMergeExec::new(LexOrdering::new(ordering.to_vec()), plan) + .with_fetch(fetch), + ) as _ } else { Arc::new(CoalescePartitionsExec::new(plan)) as _ }; From f7740af335fa1ac627091bf47c1e2a8a9adf6904 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Tue, 21 Jan 2025 06:49:02 +0800 Subject: [PATCH 021/177] fix: fetch is missed in the EnforceSorting (#14192) * fix: fetch is missed in the EnfoceSorting * fix conflict * resolve comments from alamb * update --- .../core/src/physical_optimizer/enforce_sorting.rs | 2 -- .../core/src/physical_optimizer/sort_pushdown.rs | 10 +++++++--- datafusion/sqllogictest/test_files/topk.slt | 7 ++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_sorting.rs b/datafusion/core/src/physical_optimizer/enforce_sorting.rs index 1df56e593da82..cfabb4f927c02 100644 --- a/datafusion/core/src/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/src/physical_optimizer/enforce_sorting.rs @@ -186,13 +186,11 @@ impl PhysicalOptimizerRule for EnforceSorting { ) }) .data()?; - // Execute a top-down traversal to exploit sort push-down opportunities // missed by the bottom-up traversal: let mut sort_pushdown = SortPushDown::new_default(updated_plan.plan); assign_initial_requirements(&mut sort_pushdown); let adjusted = pushdown_sorts(sort_pushdown)?; - adjusted .plan .transform_up(|plan| Ok(Transformed::yes(replace_with_partial_sort(plan)?))) diff --git a/datafusion/core/src/physical_optimizer/sort_pushdown.rs b/datafusion/core/src/physical_optimizer/sort_pushdown.rs index 9eb200f534db6..0f6eb89d6ceb4 100644 --- a/datafusion/core/src/physical_optimizer/sort_pushdown.rs +++ b/datafusion/core/src/physical_optimizer/sort_pushdown.rs @@ -64,7 +64,9 @@ pub fn assign_initial_requirements(node: &mut SortPushDown) { for (child, requirement) in node.children.iter_mut().zip(reqs) { child.data = ParentRequirements { ordering_requirement: requirement, - fetch: None, + // If the parent has a fetch value, assign it to the children + // Or use the fetch value of the child. + fetch: child.plan.fetch(), }; } } @@ -95,6 +97,7 @@ fn pushdown_sorts_helper( .equivalence_properties() .ordering_satisfy_requirement(parent_reqs); if is_sort(plan) { + let sort_fetch = plan.fetch(); let required_ordering = plan .output_ordering() .map(PhysicalSortRequirement::from_sort_exprs) @@ -102,7 +105,8 @@ fn pushdown_sorts_helper( if !satisfy_parent { // Make sure this `SortExec` satisfies parent requirements: let sort_reqs = requirements.data.ordering_requirement.unwrap_or_default(); - let fetch = requirements.data.fetch; + // It's possible current plan (`SortExec`) has a fetch value. + let fetch = requirements.data.fetch.or(sort_fetch); requirements = requirements.children.swap_remove(0); requirements = add_sort_above(requirements, sort_reqs, fetch); }; @@ -112,7 +116,7 @@ fn pushdown_sorts_helper( if let Some(adjusted) = pushdown_requirement_to_children(&child.plan, &required_ordering)? { - let fetch = child.plan.fetch(); + let fetch = sort_fetch.or_else(|| child.plan.fetch()); for (grand_child, order) in child.children.iter_mut().zip(adjusted) { grand_child.data = ParentRequirements { ordering_requirement: order, diff --git a/datafusion/sqllogictest/test_files/topk.slt b/datafusion/sqllogictest/test_files/topk.slt index 1dbce79e0f1a6..57a4dd95f5228 100644 --- a/datafusion/sqllogictest/test_files/topk.slt +++ b/datafusion/sqllogictest/test_files/topk.slt @@ -49,7 +49,12 @@ select * from topk order by x desc limit 3; 8 5 - +query I +select * from (select * from topk limit 8) order by x limit 3; +---- +0 +1 +2 statement ok From 22473d9c7d3859433a2dadd6da6d5c9cb89b4e61 Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:13:45 +0100 Subject: [PATCH 022/177] fix remaining test issues regarding with_node_id --- .../physical-plan/src/aggregates/mod.rs | 24 ++++++++++++------- datafusion/physical-plan/src/display.rs | 17 ++++--------- datafusion/physical-plan/src/memory.rs | 14 +++++++---- .../physical-plan/src/repartition/mod.rs | 9 +++++-- .../physical-plan/src/sorts/partial_sort.rs | 15 +++++++----- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index a0b70028e6a19..c92b37c30bc77 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -878,15 +878,21 @@ impl ExecutionPlan for AggregateExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = AggregateExec::try_new_with_schema( - self.mode, - self.group_by.clone(), - self.aggr_expr.clone(), - self.filter_expr.clone(), - self.input().clone(), - Arc::clone(&self.input_schema), - Arc::clone(&self.schema), - )?; + let mut new_plan = AggregateExec { + mode: self.mode, + group_by: self.group_by.clone(), + aggr_expr: self.aggr_expr.clone(), + filter_expr: self.filter_expr.clone(), + input_order_mode: self.input_order_mode.clone(), + input: Arc::clone(&self.input), + input_schema: Arc::clone(&self.input_schema), + schema: Arc::clone(&self.schema), + cache: self.cache.clone(), + limit: self.limit, + required_input_ordering: self.required_input_ordering.clone(), + metrics: self.metrics.clone(), + }; + let new_props: PlanProperties = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/display.rs b/datafusion/physical-plan/src/display.rs index 4c5bbc567679c..7354dbdc18aea 100644 --- a/datafusion/physical-plan/src/display.rs +++ b/datafusion/physical-plan/src/display.rs @@ -274,9 +274,10 @@ impl<'a, 'b> ExecutionPlanVisitor for IndentVisitor<'a, 'b> { write!(self.f, "{:indent$}", "", indent = self.indent * 2)?; plan.fmt_as(self.t, self.f)?; - if let Some(node_id) = plan.properties().node_id() { - write!(self.f, ", node_id={}", node_id)?; - } + // MAX: disable this for now since we don't need it displayed + it fails many DF tests + //if let Some(node_id) = plan.properties().node_id() { + // write!(self.f, ", node_id={}", node_id)?; + //} match self.show_metrics { ShowMetrics::None => {} @@ -398,19 +399,11 @@ impl ExecutionPlanVisitor for GraphvizVisitor<'_, '_> { "" }; - let node_id = plan - .properties() - .node_id() - .map_or("node_id=None".to_string(), |id| format!("node_id={}", id)); - self.graphviz_builder.add_node( self.f, id, &label, - Some(&format!( - "{}{}{}{}", - metrics, delimiter, statistics, node_id - )), + Some(&format!("{}{}{}", metrics, delimiter, statistics)), )?; if let Some(parent_node_id) = self.parents.last() { diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs index c8f54ffce1d84..b8f8a598d9049 100644 --- a/datafusion/physical-plan/src/memory.rs +++ b/datafusion/physical-plan/src/memory.rs @@ -153,11 +153,15 @@ impl ExecutionPlan for MemoryExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = MemoryExec::try_new( - &self.partitions.clone(), - self.schema.clone(), - self.projection.clone(), - )?; + let mut new_plan = MemoryExec { + partitions: self.partitions.clone(), + schema: Arc::clone(&self.schema), + projected_schema: Arc::clone(&self.projected_schema), + cache: self.cache.clone(), + projection: self.projection.clone(), + show_sizes: self.show_sizes, + sort_information: self.sort_information.clone(), + }; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 2724490576de3..512ebd5fe1cc9 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -676,8 +676,13 @@ impl ExecutionPlan for RepartitionExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = - RepartitionExec::try_new(self.input.clone(), self.partitioning().clone())?; + let mut new_plan = RepartitionExec { + input: Arc::clone(&self.input), + state: Arc::clone(&self.state), + metrics: self.metrics.clone(), + preserve_order: self.preserve_order, + cache: self.cache.clone(), + }; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index e520fad3826e0..7565116b52713 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -313,12 +313,15 @@ impl ExecutionPlan for PartialSortExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = PartialSortExec::new( - self.expr.clone(), - self.input.clone(), - self.common_prefix_length, - ) - .with_fetch(self.fetch()); + let mut new_plan = PartialSortExec { + expr: self.expr.clone(), + input: Arc::clone(&self.input), + common_prefix_length: self.common_prefix_length, + metrics_set: self.metrics_set.clone(), + preserve_partitioning: self.preserve_partitioning, + fetch: self.fetch, + cache: self.cache.clone(), + }; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) From f0f6e814d7befd00dff938a20179a5cad21d3a3b Mon Sep 17 00:00:00 2001 From: Max Meldrum <11488530+Max-Meldrum@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:49:36 +0100 Subject: [PATCH 023/177] use new_utf8 instead of new_utf8view in page_pruning test as we have it disabled by default --- datafusion/core/tests/parquet/page_pruning.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rs index d201ed3a841fd..25a99eea8cc7e 100644 --- a/datafusion/core/tests/parquet/page_pruning.rs +++ b/datafusion/core/tests/parquet/page_pruning.rs @@ -151,7 +151,7 @@ async fn page_index_filter_one_col() { // 5.create filter date_string_col == "01/01/09"`; // Note this test doesn't apply type coercion so the literal must match the actual view type - let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8view("01/01/09"))); + let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8("01/01/09"))); let parquet_exec = get_parquet_exec(&state, filter).await; let mut results = parquet_exec.execute(0, task_ctx.clone()).unwrap(); let batch = results.next().await.unwrap().unwrap(); From f3e700455985cf0573a41bcf96f738c23d0ddf47 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Thu, 23 Jan 2025 23:58:41 +0800 Subject: [PATCH 024/177] Expose more components from sqllogictest (#14249) --- datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs | 1 + .../sqllogictest/src/engines/datafusion_engine/normalize.rs | 4 ++-- datafusion/sqllogictest/src/engines/mod.rs | 3 +++ datafusion/sqllogictest/src/engines/output.rs | 2 +- datafusion/sqllogictest/src/lib.rs | 3 +++ 5 files changed, 10 insertions(+), 3 deletions(-) diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs index 8e2bbbfe4f697..663bbdd5a3c7c 100644 --- a/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_engine/mod.rs @@ -21,4 +21,5 @@ mod normalize; mod runner; pub use error::*; +pub use normalize::*; pub use runner::*; diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs index 8337d2e9a39c9..5fcef1cbc14bb 100644 --- a/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs @@ -28,8 +28,8 @@ use std::sync::OnceLock; use super::super::conversion::*; use super::error::{DFSqlLogicTestError, Result}; -/// Converts `batches` to a result as expected by sqllogicteset. -pub(crate) fn convert_batches(batches: Vec) -> Result>> { +/// Converts `batches` to a result as expected by sqllogictest. +pub fn convert_batches(batches: Vec) -> Result>> { if batches.is_empty() { Ok(vec![]) } else { diff --git a/datafusion/sqllogictest/src/engines/mod.rs b/datafusion/sqllogictest/src/engines/mod.rs index a6a0886332ed7..7b65c0aa77cb0 100644 --- a/datafusion/sqllogictest/src/engines/mod.rs +++ b/datafusion/sqllogictest/src/engines/mod.rs @@ -20,7 +20,10 @@ mod conversion; mod datafusion_engine; mod output; +pub use datafusion_engine::convert_batches; pub use datafusion_engine::DataFusion; +pub use output::DFColumnType; +pub use output::DFOutput; #[cfg(feature = "postgres")] mod postgres_engine; diff --git a/datafusion/sqllogictest/src/engines/output.rs b/datafusion/sqllogictest/src/engines/output.rs index 24299856e00d5..0682f5df97c19 100644 --- a/datafusion/sqllogictest/src/engines/output.rs +++ b/datafusion/sqllogictest/src/engines/output.rs @@ -54,4 +54,4 @@ impl ColumnType for DFColumnType { } } -pub(crate) type DFOutput = DBOutput; +pub type DFOutput = DBOutput; diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rs index 1bcfd71af0fd0..30a882011dd54 100644 --- a/datafusion/sqllogictest/src/lib.rs +++ b/datafusion/sqllogictest/src/lib.rs @@ -19,6 +19,9 @@ mod engines; +pub use engines::convert_batches; +pub use engines::DFColumnType; +pub use engines::DFOutput; pub use engines::DataFusion; #[cfg(feature = "postgres")] From c976a89f348415ba231ddf477cb0404530e5ea42 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Sat, 25 Jan 2025 19:26:25 +0800 Subject: [PATCH 025/177] Extract useful methods from sqllogictest bin (#14267) --- datafusion/sqllogictest/bin/sqllogictests.rs | 109 +++++++++---------- datafusion/sqllogictest/src/lib.rs | 3 + datafusion/sqllogictest/src/util.rs | 108 ++++++++++++++++++ 3 files changed, 161 insertions(+), 59 deletions(-) create mode 100644 datafusion/sqllogictest/src/util.rs diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index c3e739d146c6c..8739a208f2397 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -15,8 +15,32 @@ // specific language governing permissions and limitations // under the License. +use clap::Parser; +use datafusion_common::instant::Instant; +use datafusion_common::utils::get_available_parallelism; +use datafusion_common::{exec_err, DataFusionError, Result}; +use datafusion_common_runtime::SpawnedTask; +use datafusion_sqllogictest::{ + df_value_validator, read_dir_recursive, setup_scratch_dir, value_normalizer, + DataFusion, TestContext, +}; +use futures::stream::StreamExt; +use indicatif::{ + HumanDuration, MultiProgress, ProgressBar, ProgressDrawTarget, ProgressStyle, +}; +use itertools::Itertools; +use log::Level::Info; +use log::{info, log_enabled}; +use sqllogictest::{ + parse_file, strict_column_validator, AsyncDB, Condition, Normalizer, Record, + Validator, +}; + +#[cfg(feature = "postgres")] +use crate::postgres_container::{ + initialize_postgres_container, terminate_postgres_container, +}; use std::ffi::OsStr; -use std::fs; use std::path::{Path, PathBuf}; use clap::Parser; @@ -40,39 +64,33 @@ pub fn main() -> Result<()> { .block_on(run_tests()) } -fn value_validator(actual: &[Vec], expected: &[String]) -> bool { - let expected = expected +fn sqlite_value_validator( + normalizer: Normalizer, + actual: &[Vec], + expected: &[String], +) -> bool { + let normalized_expected = expected.iter().map(normalizer).collect::>(); + let normalized_actual = actual .iter() - // Trailing whitespace from lines in SLT will typically be removed, but do not fail if it is not - // If particular test wants to cover trailing whitespace on a value, - // it should project additional non-whitespace column on the right. - .map(|s| s.trim_end().to_owned()) - .collect::>(); - let actual = actual - .iter() - .map(|strs| strs.iter().join(" ")) - // Editors do not preserve trailing whitespace, so expected may or may not lack it included - .map(|s| s.trim_end().to_owned()) - .collect::>(); - actual == expected -} - -/// Sets up an empty directory at test_files/scratch/ -/// creating it if needed and clearing any file contents if it exists -/// This allows tests for inserting to external tables or copy to -/// to persist data to disk and have consistent state when running -/// a new test -fn setup_scratch_dir(name: &Path) -> Result<()> { - // go from copy.slt --> copy - let file_stem = name.file_stem().expect("File should have a stem"); - let path = PathBuf::from("test_files").join("scratch").join(file_stem); - - info!("Creating scratch dir in {path:?}"); - if path.exists() { - fs::remove_dir_all(&path)?; + .map(|strs| strs.iter().map(normalizer).join(" ")) + .collect_vec(); + + if log_enabled!(Info) && normalized_actual != normalized_expected { + info!("sqlite validation failed. actual vs expected:"); + for i in 0..normalized_actual.len() { + info!("[{i}] {}", normalized_actual[i]); + info!( + "[{i}] {}", + if normalized_expected.len() >= i { + &normalized_expected[i] + } else { + "No more results" + } + ); + } } - fs::create_dir_all(&path)?; - Ok(()) + + normalized_actual == normalized_expected } async fn run_tests() -> Result<()> { @@ -275,33 +293,6 @@ fn read_test_files<'a>( )) } -fn read_dir_recursive>(path: P) -> Result> { - let mut dst = vec![]; - read_dir_recursive_impl(&mut dst, path.as_ref())?; - Ok(dst) -} - -/// Append all paths recursively to dst -fn read_dir_recursive_impl(dst: &mut Vec, path: &Path) -> Result<()> { - let entries = fs::read_dir(path) - .map_err(|e| exec_datafusion_err!("Error reading directory {path:?}: {e}"))?; - for entry in entries { - let path = entry - .map_err(|e| { - exec_datafusion_err!("Error reading entry in directory {path:?}: {e}") - })? - .path(); - - if path.is_dir() { - read_dir_recursive_impl(dst, &path)?; - } else { - dst.push(path); - } - } - - Ok(()) -} - /// Parsed command line options /// /// This structure attempts to mimic the command line options of the built in rust test runner diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rs index 30a882011dd54..82f194321a8e1 100644 --- a/datafusion/sqllogictest/src/lib.rs +++ b/datafusion/sqllogictest/src/lib.rs @@ -28,4 +28,7 @@ pub use engines::DataFusion; pub use engines::Postgres; mod test_context; +mod util; + pub use test_context::TestContext; +pub use util::*; diff --git a/datafusion/sqllogictest/src/util.rs b/datafusion/sqllogictest/src/util.rs new file mode 100644 index 0000000000000..1bdfdd03360f3 --- /dev/null +++ b/datafusion/sqllogictest/src/util.rs @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_common::{exec_datafusion_err, Result}; +use itertools::Itertools; +use log::Level::Warn; +use log::{info, log_enabled, warn}; +use sqllogictest::Normalizer; +use std::fs; +use std::path::{Path, PathBuf}; + +/// Sets up an empty directory at `test_files/scratch/` +/// creating it if needed and clearing any file contents if it exists +/// This allows tests for inserting to external tables or copy to +/// persist data to disk and have consistent state when running +/// a new test +pub fn setup_scratch_dir(name: &Path) -> Result<()> { + // go from copy.slt --> copy + let file_stem = name.file_stem().expect("File should have a stem"); + let path = PathBuf::from("test_files").join("scratch").join(file_stem); + + info!("Creating scratch dir in {path:?}"); + if path.exists() { + fs::remove_dir_all(&path)?; + } + fs::create_dir_all(&path)?; + Ok(()) +} + +/// Trailing whitespace from lines in SLT will typically be removed, but do not fail if it is not +/// If particular test wants to cover trailing whitespace on a value, +/// it should project additional non-whitespace column on the right. +#[allow(clippy::ptr_arg)] +pub fn value_normalizer(s: &String) -> String { + s.trim_end().to_string() +} + +pub fn read_dir_recursive>(path: P) -> Result> { + let mut dst = vec![]; + read_dir_recursive_impl(&mut dst, path.as_ref())?; + Ok(dst) +} + +/// Append all paths recursively to dst +fn read_dir_recursive_impl(dst: &mut Vec, path: &Path) -> Result<()> { + let entries = fs::read_dir(path) + .map_err(|e| exec_datafusion_err!("Error reading directory {path:?}: {e}"))?; + for entry in entries { + let path = entry + .map_err(|e| { + exec_datafusion_err!("Error reading entry in directory {path:?}: {e}") + })? + .path(); + + if path.is_dir() { + read_dir_recursive_impl(dst, &path)?; + } else { + dst.push(path); + } + } + + Ok(()) +} + +/// Validate the actual and expected values. +pub fn df_value_validator( + normalizer: Normalizer, + actual: &[Vec], + expected: &[String], +) -> bool { + let normalized_expected = expected.iter().map(normalizer).collect::>(); + let normalized_actual = actual + .iter() + .map(|strs| strs.iter().join(" ")) + .map(|str| str.trim_end().to_string()) + .collect_vec(); + + if log_enabled!(Warn) && normalized_actual != normalized_expected { + warn!("df validation failed. actual vs expected:"); + for i in 0..normalized_actual.len() { + warn!("[{i}] {}", normalized_actual[i]); + warn!( + "[{i}] {}", + if normalized_expected.len() >= i { + &normalized_expected[i] + } else { + "No more results" + } + ); + } + } + + normalized_actual == normalized_expected +} From ffff7a153b5552edb9a465f6d82d5852c2c639ab Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 27 Jan 2025 10:21:03 +0800 Subject: [PATCH 026/177] expose df sqllogictest error --- .../sqllogictest/src/engines/datafusion_engine/normalize.rs | 2 +- datafusion/sqllogictest/src/engines/mod.rs | 2 ++ datafusion/sqllogictest/src/lib.rs | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs index 5fcef1cbc14bb..25a394558efd8 100644 --- a/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs +++ b/datafusion/sqllogictest/src/engines/datafusion_engine/normalize.rs @@ -253,7 +253,7 @@ pub fn cell_to_string(col: &ArrayRef, row: usize) -> Result { } /// Converts columns to a result as expected by sqllogicteset. -pub(crate) fn convert_schema_to_types(columns: &Fields) -> Vec { +pub fn convert_schema_to_types(columns: &Fields) -> Vec { columns .iter() .map(|f| f.data_type()) diff --git a/datafusion/sqllogictest/src/engines/mod.rs b/datafusion/sqllogictest/src/engines/mod.rs index 7b65c0aa77cb0..3569dea701761 100644 --- a/datafusion/sqllogictest/src/engines/mod.rs +++ b/datafusion/sqllogictest/src/engines/mod.rs @@ -21,6 +21,8 @@ mod datafusion_engine; mod output; pub use datafusion_engine::convert_batches; +pub use datafusion_engine::convert_schema_to_types; +pub use datafusion_engine::DFSqlLogicTestError; pub use datafusion_engine::DataFusion; pub use output::DFColumnType; pub use output::DFOutput; diff --git a/datafusion/sqllogictest/src/lib.rs b/datafusion/sqllogictest/src/lib.rs index 82f194321a8e1..0ea55782d34e6 100644 --- a/datafusion/sqllogictest/src/lib.rs +++ b/datafusion/sqllogictest/src/lib.rs @@ -20,8 +20,10 @@ mod engines; pub use engines::convert_batches; +pub use engines::convert_schema_to_types; pub use engines::DFColumnType; pub use engines::DFOutput; +pub use engines::DFSqlLogicTestError; pub use engines::DataFusion; #[cfg(feature = "postgres")] From 63bad1183053fda6aaf0017a4f38098b592a103a Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 27 Jan 2025 20:14:39 +0800 Subject: [PATCH 027/177] update sqllogictest --- datafusion/sqllogictest/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index 07dbc60e86bc8..6bf8fa296b2dd 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -51,7 +51,7 @@ object_store = { workspace = true } postgres-protocol = { version = "0.6.4", optional = true } postgres-types = { version = "0.2.4", optional = true } rust_decimal = { version = "1.27.0" } -sqllogictest = "0.22.0" +sqllogictest = "0.26.3" sqlparser = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } From e3ea7d13b7b2d946ad3279357e834fe1838864a4 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Feb 2025 14:57:18 -0500 Subject: [PATCH 028/177] chore: Upgrade to `arrow`/`parquet` `54.1.0` and fix clippy/ci (#14415) (#14453) * chore: Fixed CI * chore * chore: Fixed clippy * chore Co-authored-by: Alex Huang --- Cargo.toml | 16 +- datafusion-cli/Cargo.lock | 67 +- datafusion-cli/Cargo.toml | 4 +- .../datasource/physical_plan/arrow_file.rs | 11 +- .../expr-common/src/interval_arithmetic.rs | 714 +----------------- .../src/unwrap_cast_in_comparison.rs | 7 +- 6 files changed, 56 insertions(+), 763 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c17f4f7affaac..55855d09d50ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,21 +77,21 @@ version = "45.0.0" ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } -arrow = { version = "54.0.0", features = [ +arrow = { version = "54.1.0", features = [ "prettyprint", ] } -arrow-array = { version = "54.0.0", default-features = false, features = [ +arrow-array = { version = "54.1.0", default-features = false, features = [ "chrono-tz", ] } -arrow-buffer = { version = "54.0.0", default-features = false } -arrow-flight = { version = "54.0.0", features = [ +arrow-buffer = { version = "54.1.0", default-features = false } +arrow-flight = { version = "54.1.0", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "54.0.0", default-features = false, features = [ +arrow-ipc = { version = "54.1.0", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "54.0.0", default-features = false } -arrow-schema = { version = "54.0.0", default-features = false } +arrow-ord = { version = "54.1.0", default-features = false } +arrow-schema = { version = "54.1.0", default-features = false } async-trait = "0.1.73" bigdecimal = "0.4.7" bytes = "1.4" @@ -133,7 +133,7 @@ itertools = "0.14" log = "^0.4" object_store = { version = "0.11.0", default-features = false } parking_lot = "0.12" -parquet = { version = "54.0.0", default-features = false, features = [ +parquet = { version = "54.1.0", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 4a4d0157c620c..a5cf71426607b 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -175,9 +175,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ccdcc8fb14508ca20aaec7076032e5c0b0751b906036d4496786e2f227a37a" +checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc" dependencies = [ "arrow-arith", "arrow-array", @@ -196,9 +196,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1aad8e27f32e411a0fc0bf5a625a35f0bf9b9f871cf4542abe31f7cef4beea2" +checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248" dependencies = [ "arrow-array", "arrow-buffer", @@ -210,9 +210,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6ed90c28c6f73a706c55799b8cc3a094e89257238e5b1d65ca7c70bd3ae23f" +checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" dependencies = [ "ahash", "arrow-buffer", @@ -227,9 +227,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4a40bdc1552ea10fbdeae4e5a945d8572c32f66bce457b96c13d9c46b80447" +checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89" dependencies = [ "bytes", "half", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "430c0a21aa7f81bcf0f97c57216d7127795ea755f494d27bae2bd233be43c2cc" +checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870" dependencies = [ "arrow-array", "arrow-buffer", @@ -259,9 +259,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4444c8f8c57ac00e6a679ede67d1ae8872c170797dc45b46f75702437a77888" +checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6" dependencies = [ "arrow-array", "arrow-cast", @@ -275,9 +275,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09af476cfbe9879937e50b1334c73189de6039186e025b1b1ac84b283b87b20e" +checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754" dependencies = [ "arrow-buffer", "arrow-schema", @@ -287,9 +287,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136296e8824333a8a4c4a6e508e4aa65d5678b801246d0408825ae7b2523c628" +checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e" dependencies = [ "arrow-array", "arrow-buffer", @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e222ad0e419ab8276818c5605a5bb1e35ed86fa8c5e550726433cc63b09c3c78" +checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c" dependencies = [ "arrow-array", "arrow-buffer", @@ -321,9 +321,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddf14c5f03b679ec8ceac4dfac43f63cdc4ed54dab3cc120a4ef46af38481eb" +checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da" dependencies = [ "arrow-array", "arrow-buffer", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9acdc58da19f383f4ba381fa0e3583534ae2ceb31269aaf4a03f08ff13e8443" +checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c" dependencies = [ "arrow-array", "arrow-buffer", @@ -347,15 +347,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a1822a1a952955637e85e8f9d6b0e04dd75d65492b87ec548dd593d3a1f772b" +checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" [[package]] name = "arrow-select" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4172e9a12dfe15303d3926269f9ead471ea93bdd067d113abc65cb6c48e246" +checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" dependencies = [ "ahash", "arrow-array", @@ -367,9 +367,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73683040445f4932342781926189901c9521bb1a787c35dbe628a3ce51372d3c" +checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0" dependencies = [ "arrow-array", "arrow-buffer", @@ -2901,9 +2901,9 @@ dependencies = [ [[package]] name = "parquet" -version = "54.0.0" +version = "54.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3334c50239d9f4951653d84fa6f636da86f53742e5e5849a30fbe852f3ff4383" +checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" dependencies = [ "ahash", "arrow-array", @@ -2927,6 +2927,7 @@ dependencies = [ "object_store", "paste", "seq-macro", + "simdutf8", "snap", "thrift", "tokio", @@ -3716,6 +3717,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "siphasher" version = "1.0.1" diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 6b6b52fce7430..7daa32562173b 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.81.0" readme = "README.md" [dependencies] -arrow = { version = "54.0.0" } +arrow = { version = "54.1.0" } async-trait = "0.1.0" aws-config = "1.5.0" aws-credential-types = "1.2.0" @@ -57,7 +57,7 @@ home = "=0.5.11" mimalloc = { version = "0.1", default-features = false } object_store = { version = "0.11.0", features = ["aws", "gcp", "http"] } parking_lot = { version = "0.12" } -parquet = { version = "54.0.0", default-features = false } +parquet = { version = "54.1.0", default-features = false } regex = "1.8" rustyline = "15.0" tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] } diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/core/src/datasource/physical_plan/arrow_file.rs index 54344d55bbd11..82735334c7f80 100644 --- a/datafusion/core/src/datasource/physical_plan/arrow_file.rs +++ b/datafusion/core/src/datasource/physical_plan/arrow_file.rs @@ -309,10 +309,8 @@ impl FileOpener for ArrowOpener { for (dict_block, dict_result) in footer.dictionaries().iter().flatten().zip(dict_results) { - decoder.read_dictionary( - dict_block, - &Buffer::from_bytes(dict_result.into()), - )?; + decoder + .read_dictionary(dict_block, &Buffer::from(dict_result))?; } // filter recordbatches according to range @@ -348,10 +346,7 @@ impl FileOpener for ArrowOpener { .zip(recordbatch_results) .filter_map(move |(block, data)| { decoder - .read_record_batch( - &block, - &Buffer::from_bytes(data.into()), - ) + .read_record_batch(&block, &Buffer::from(data)) .transpose() }), ) diff --git a/datafusion/expr-common/src/interval_arithmetic.rs b/datafusion/expr-common/src/interval_arithmetic.rs index 993051eaeee15..fc1955705ee08 100644 --- a/datafusion/expr-common/src/interval_arithmetic.rs +++ b/datafusion/expr-common/src/interval_arithmetic.rs @@ -26,6 +26,8 @@ use std::ops::{AddAssign, SubAssign}; use arrow::compute::{cast_with_options, CastOptions}; use arrow::datatypes::{ DataType, IntervalDayTime, IntervalMonthDayNano, IntervalUnit, TimeUnit, + MAX_DECIMAL128_FOR_EACH_PRECISION, MAX_DECIMAL256_FOR_EACH_PRECISION, + MIN_DECIMAL128_FOR_EACH_PRECISION, MIN_DECIMAL256_FOR_EACH_PRECISION, }; use datafusion_common::rounding::{alter_fp_rounding_mode, next_down, next_up}; use datafusion_common::{internal_err, Result, ScalarValue}; @@ -97,718 +99,6 @@ macro_rules! get_extreme_value { }; } -/// The maximum `i128` value that can be stored in a `Decimal128` value of precision `p`. -/// -/// Remove this once is available -const MAX_DECIMAL128_FOR_EACH_PRECISION: [i128; 39] = [ - 0, // unused first element - 9, - 99, - 999, - 9999, - 99999, - 999999, - 9999999, - 99999999, - 999999999, - 9999999999, - 99999999999, - 999999999999, - 9999999999999, - 99999999999999, - 999999999999999, - 9999999999999999, - 99999999999999999, - 999999999999999999, - 9999999999999999999, - 99999999999999999999, - 999999999999999999999, - 9999999999999999999999, - 99999999999999999999999, - 999999999999999999999999, - 9999999999999999999999999, - 99999999999999999999999999, - 999999999999999999999999999, - 9999999999999999999999999999, - 99999999999999999999999999999, - 999999999999999999999999999999, - 9999999999999999999999999999999, - 99999999999999999999999999999999, - 999999999999999999999999999999999, - 9999999999999999999999999999999999, - 99999999999999999999999999999999999, - 999999999999999999999999999999999999, - 9999999999999999999999999999999999999, - 99999999999999999999999999999999999999, -]; - -/// The minimum `i128` value that can be stored in a `Decimal128` value of precision `p`. -/// -/// Remove this once is available -const MIN_DECIMAL128_FOR_EACH_PRECISION: [i128; 39] = [ - 0, // unused first element - -9, - -99, - -999, - -9999, - -99999, - -999999, - -9999999, - -99999999, - -999999999, - -9999999999, - -99999999999, - -999999999999, - -9999999999999, - -99999999999999, - -999999999999999, - -9999999999999999, - -99999999999999999, - -999999999999999999, - -9999999999999999999, - -99999999999999999999, - -999999999999999999999, - -9999999999999999999999, - -99999999999999999999999, - -999999999999999999999999, - -9999999999999999999999999, - -99999999999999999999999999, - -999999999999999999999999999, - -9999999999999999999999999999, - -99999999999999999999999999999, - -999999999999999999999999999999, - -9999999999999999999999999999999, - -99999999999999999999999999999999, - -999999999999999999999999999999999, - -9999999999999999999999999999999999, - -99999999999999999999999999999999999, - -999999999999999999999999999999999999, - -9999999999999999999999999999999999999, - -99999999999999999999999999999999999999, -]; - -/// The maximum `i256` value that can be stored in a `Decimal256` value of precision `p`. -/// -/// Remove this once is available -const MAX_DECIMAL256_FOR_EACH_PRECISION: [arrow::datatypes::i256; 77] = [ - arrow::datatypes::i256::from_i128(0_i128), // unused first element - arrow::datatypes::i256::from_le_bytes([ - 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 231, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 15, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 159, 134, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 63, 66, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 127, 150, 152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 224, 245, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 201, 154, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 227, 11, 84, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 231, 118, 72, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 15, 165, 212, 232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 159, 114, 78, 24, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 63, 122, 16, 243, 90, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 127, 198, 164, 126, 141, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 192, 111, 242, 134, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 137, 93, 120, 69, 99, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 99, 167, 179, 182, 224, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 231, 137, 4, 35, 199, 138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 15, 99, 45, 94, 199, 107, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 159, 222, 197, 173, 201, 53, 54, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 63, 178, 186, 201, 224, 25, 30, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 127, 246, 74, 225, 199, 2, 45, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 160, 237, 204, 206, 27, 194, 211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 73, 72, 1, 20, 22, 149, 69, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 227, 210, 12, 200, 220, 210, 183, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 231, 60, 128, 208, 159, 60, 46, 59, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 15, 97, 2, 37, 62, 94, 206, 79, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 159, 202, 23, 114, 109, 174, 15, 30, 67, 1, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 63, 234, 237, 116, 70, 208, 156, 44, 159, 12, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 127, 38, 75, 145, 192, 34, 32, 190, 55, 126, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 128, 239, 172, 133, 91, 65, 109, 45, 238, 4, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 9, 91, 193, 56, 147, 141, 68, 198, 77, 49, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 99, 142, 141, 55, 192, 135, 173, 190, 9, 237, 1, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 231, 143, 135, 43, 130, 77, 199, 114, 97, 66, 19, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 15, 159, 75, 179, 21, 7, 201, 123, 206, 151, 192, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 159, 54, 244, 0, 217, 70, 218, 213, 16, 238, 133, 7, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 63, 34, 138, 9, 122, 196, 134, 90, 168, 76, 59, 75, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 127, 86, 101, 95, 196, 172, 67, 137, 147, 254, 80, 240, 2, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 96, 245, 185, 171, 191, 164, 92, 195, 241, 41, 99, 29, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 201, 149, 67, 181, 124, 111, 158, 161, 113, 163, 223, - 37, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 227, 217, 163, 20, 223, 90, 48, 80, 112, 98, 188, 122, - 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 231, 130, 102, 206, 182, 140, 227, 33, 99, 216, 91, 203, - 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 15, 29, 1, 16, 36, 127, 227, 82, 223, 115, 150, 241, - 123, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 159, 34, 11, 160, 104, 247, 226, 60, 185, 134, 224, 111, - 215, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 63, 90, 111, 64, 22, 170, 221, 96, 60, 67, 197, 94, 106, - 192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 127, 134, 89, 132, 222, 164, 168, 200, 91, 160, 180, - 179, 39, 132, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 64, 127, 43, 177, 112, 150, 214, 149, 67, 14, 5, - 141, 41, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 137, 248, 178, 235, 102, 224, 97, 218, 163, 142, - 50, 130, 159, 215, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 99, 181, 253, 52, 5, 196, 210, 135, 102, 146, 249, - 21, 59, 108, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 231, 21, 233, 17, 52, 168, 59, 78, 1, 184, 191, - 219, 78, 58, 172, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 15, 219, 26, 179, 8, 146, 84, 14, 13, 48, 125, 149, - 20, 71, 186, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 159, 142, 12, 255, 86, 180, 77, 143, 130, 224, 227, - 214, 205, 198, 70, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 63, 146, 125, 246, 101, 11, 9, 153, 25, 197, 230, - 100, 10, 196, 195, 112, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 127, 182, 231, 160, 251, 113, 90, 250, 255, 178, 3, - 241, 103, 168, 165, 103, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 32, 13, 73, 212, 115, 136, 199, 255, 253, 36, - 106, 15, 148, 120, 12, 20, 4, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 73, 131, 218, 74, 134, 84, 203, 253, 235, 113, - 37, 154, 200, 181, 124, 200, 40, 0, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 227, 32, 137, 236, 62, 77, 241, 233, 55, 115, - 118, 5, 214, 25, 223, 212, 151, 1, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 231, 72, 91, 61, 117, 4, 109, 35, 47, 128, - 160, 54, 92, 2, 183, 80, 238, 15, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 15, 217, 144, 101, 148, 44, 66, 98, 215, 1, - 69, 34, 154, 23, 38, 39, 79, 159, 0, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 159, 122, 168, 247, 203, 189, 149, 214, 105, - 18, 178, 86, 5, 236, 124, 135, 23, 57, 6, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 63, 202, 148, 172, 247, 105, 217, 97, 34, 184, - 244, 98, 53, 56, 225, 74, 235, 58, 62, 0, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 127, 230, 207, 189, 172, 35, 126, 210, 87, 49, - 143, 221, 21, 50, 204, 236, 48, 77, 110, 2, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 0, 31, 106, 191, 100, 237, 56, 110, 237, - 151, 167, 218, 244, 249, 63, 233, 3, 79, 24, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 9, 54, 37, 122, 239, 69, 57, 78, 70, 239, - 139, 138, 144, 195, 127, 28, 39, 22, 243, 0, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 99, 28, 116, 197, 90, 187, 60, 14, 191, - 88, 119, 105, 165, 163, 253, 28, 135, 221, 126, 9, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 231, 27, 137, 182, 139, 81, 95, 142, 118, - 119, 169, 30, 118, 100, 232, 33, 71, 167, 244, 94, 0, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 15, 23, 91, 33, 117, 47, 185, 143, 161, - 170, 158, 50, 157, 236, 19, 83, 199, 136, 142, 181, 3, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 159, 230, 142, 77, 147, 218, 59, 157, 79, - 170, 50, 250, 35, 62, 199, 62, 201, 87, 145, 23, 37, 0, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 63, 2, 149, 7, 193, 137, 86, 36, 28, 167, - 250, 197, 103, 109, 200, 115, 220, 109, 173, 235, 114, 1, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 127, 22, 210, 75, 138, 97, 97, 107, 25, - 135, 202, 187, 13, 70, 212, 133, 156, 74, 198, 52, 125, 14, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 224, 52, 246, 102, 207, 205, 49, - 254, 70, 233, 85, 137, 188, 74, 58, 29, 234, 190, 15, 228, 144, 0, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 201, 16, 158, 5, 26, 10, 242, 237, - 197, 28, 91, 93, 93, 235, 70, 36, 37, 117, 157, 232, 168, 5, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 227, 167, 44, 56, 4, 101, 116, 75, - 187, 31, 143, 165, 165, 49, 197, 106, 115, 147, 38, 22, 153, 56, 0, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 231, 142, 190, 49, 42, 242, 139, - 242, 80, 61, 151, 119, 120, 240, 179, 43, 130, 194, 129, 221, 250, 53, 2, - ]), - arrow::datatypes::i256::from_le_bytes([ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 15, 149, 113, 241, 165, 117, 119, - 121, 41, 101, 232, 171, 180, 100, 7, 181, 21, 153, 17, 167, 204, 27, 22, - ]), -]; - -/// The minimum `i256` value that can be stored in a `Decimal256` value of precision `p`. -/// -/// Remove this once is available -const MIN_DECIMAL256_FOR_EACH_PRECISION: [arrow::datatypes::i256; 77] = [ - arrow::datatypes::i256::from_i128(0_i128), // unused first element - arrow::datatypes::i256::from_le_bytes([ - 247, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 157, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 25, 252, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 241, 216, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 97, 121, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 193, 189, 240, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 129, 105, 103, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 31, 10, 250, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 54, 101, 196, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 28, 244, 171, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 24, 137, 183, 232, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 240, 90, 43, 23, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 96, 141, 177, 231, 246, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 192, 133, 239, 12, 165, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 128, 57, 91, 129, 114, 252, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 63, 144, 13, 121, 220, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 118, 162, 135, 186, 156, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 156, 88, 76, 73, 31, 242, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 24, 118, 251, 220, 56, 117, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 240, 156, 210, 161, 56, 148, 250, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 96, 33, 58, 82, 54, 202, 201, 255, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 192, 77, 69, 54, 31, 230, 225, 253, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 128, 9, 181, 30, 56, 253, 210, 234, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 95, 18, 51, 49, 228, 61, 44, 255, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 182, 183, 254, 235, 233, 106, 186, 247, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 28, 45, 243, 55, 35, 45, 72, 173, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 24, 195, 127, 47, 96, 195, 209, 196, 252, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 240, 158, 253, 218, 193, 161, 49, 176, 223, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 96, 53, 232, 141, 146, 81, 240, 225, 188, 254, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 192, 21, 18, 139, 185, 47, 99, 211, 96, 243, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 128, 217, 180, 110, 63, 221, 223, 65, 200, 129, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 127, 16, 83, 122, 164, 190, 146, 210, 17, 251, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 246, 164, 62, 199, 108, 114, 187, 57, 178, 206, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 156, 113, 114, 200, 63, 120, 82, 65, 246, 18, 254, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 24, 112, 120, 212, 125, 178, 56, 141, 158, 189, 236, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 240, 96, 180, 76, 234, 248, 54, 132, 49, 104, 63, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 96, 201, 11, 255, 38, 185, 37, 42, 239, 17, 122, 248, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 192, 221, 117, 246, 133, 59, 121, 165, 87, 179, 196, 180, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 128, 169, 154, 160, 59, 83, 188, 118, 108, 1, 175, 15, 253, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 159, 10, 70, 84, 64, 91, 163, 60, 14, 214, 156, 226, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 54, 106, 188, 74, 131, 144, 97, 94, 142, 92, 32, 218, 254, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 28, 38, 92, 235, 32, 165, 207, 175, 143, 157, 67, 133, 244, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 24, 125, 153, 49, 73, 115, 28, 222, 156, 39, 164, 52, 141, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 240, 226, 254, 239, 219, 128, 28, 173, 32, 140, 105, 14, 132, 251, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 96, 221, 244, 95, 151, 8, 29, 195, 70, 121, 31, 144, 40, 211, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 192, 165, 144, 191, 233, 85, 34, 159, 195, 188, 58, 161, 149, 63, - 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 128, 121, 166, 123, 33, 91, 87, 55, 164, 95, 75, 76, 216, 123, - 238, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 191, 128, 212, 78, 143, 105, 41, 106, 188, 241, 250, 114, 214, - 80, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 118, 7, 77, 20, 153, 31, 158, 37, 92, 113, 205, 125, 96, 40, - 249, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 156, 74, 2, 203, 250, 59, 45, 120, 153, 109, 6, 234, 196, 147, - 187, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 24, 234, 22, 238, 203, 87, 196, 177, 254, 71, 64, 36, 177, 197, - 83, 253, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 240, 36, 229, 76, 247, 109, 171, 241, 242, 207, 130, 106, 235, - 184, 69, 229, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 96, 113, 243, 0, 169, 75, 178, 112, 125, 31, 28, 41, 50, 57, - 185, 244, 254, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 192, 109, 130, 9, 154, 244, 246, 102, 230, 58, 25, 155, 245, - 59, 60, 143, 245, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 128, 73, 24, 95, 4, 142, 165, 5, 0, 77, 252, 14, 152, 87, 90, - 152, 151, 255, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 223, 242, 182, 43, 140, 119, 56, 0, 2, 219, 149, 240, 107, - 135, 243, 235, 251, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 182, 124, 37, 181, 121, 171, 52, 2, 20, 142, 218, 101, 55, - 74, 131, 55, 215, 255, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 28, 223, 118, 19, 193, 178, 14, 22, 200, 140, 137, 250, 41, - 230, 32, 43, 104, 254, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 24, 183, 164, 194, 138, 251, 146, 220, 208, 127, 95, 201, - 163, 253, 72, 175, 17, 240, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 240, 38, 111, 154, 107, 211, 189, 157, 40, 254, 186, 221, - 101, 232, 217, 216, 176, 96, 255, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 96, 133, 87, 8, 52, 66, 106, 41, 150, 237, 77, 169, 250, 19, - 131, 120, 232, 198, 249, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 192, 53, 107, 83, 8, 150, 38, 158, 221, 71, 11, 157, 202, - 199, 30, 181, 20, 197, 193, 255, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 128, 25, 48, 66, 83, 220, 129, 45, 168, 206, 112, 34, 234, - 205, 51, 19, 207, 178, 145, 253, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 255, 224, 149, 64, 155, 18, 199, 145, 18, 104, 88, 37, - 11, 6, 192, 22, 252, 176, 231, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 246, 201, 218, 133, 16, 186, 198, 177, 185, 16, 116, 117, - 111, 60, 128, 227, 216, 233, 12, 255, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 156, 227, 139, 58, 165, 68, 195, 241, 64, 167, 136, 150, - 90, 92, 2, 227, 120, 34, 129, 246, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 24, 228, 118, 73, 116, 174, 160, 113, 137, 136, 86, 225, - 137, 155, 23, 222, 184, 88, 11, 161, 255, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 240, 232, 164, 222, 138, 208, 70, 112, 94, 85, 97, 205, - 98, 19, 236, 172, 56, 119, 113, 74, 252, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 96, 25, 113, 178, 108, 37, 196, 98, 176, 85, 205, 5, 220, - 193, 56, 193, 54, 168, 110, 232, 218, 255, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 192, 253, 106, 248, 62, 118, 169, 219, 227, 88, 5, 58, - 152, 146, 55, 140, 35, 146, 82, 20, 141, 254, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 128, 233, 45, 180, 117, 158, 158, 148, 230, 120, 53, 68, - 242, 185, 43, 122, 99, 181, 57, 203, 130, 241, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 0, 31, 203, 9, 153, 48, 50, 206, 1, 185, 22, 170, 118, - 67, 181, 197, 226, 21, 65, 240, 27, 111, 255, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 0, 54, 239, 97, 250, 229, 245, 13, 18, 58, 227, 164, 162, - 162, 20, 185, 219, 218, 138, 98, 23, 87, 250, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 0, 28, 88, 211, 199, 251, 154, 139, 180, 68, 224, 112, - 90, 90, 206, 58, 149, 140, 108, 217, 233, 102, 199, 255, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 0, 24, 113, 65, 206, 213, 13, 116, 13, 175, 194, 104, - 136, 135, 15, 76, 212, 125, 61, 126, 34, 5, 202, 253, - ]), - arrow::datatypes::i256::from_le_bytes([ - 1, 0, 0, 0, 0, 0, 0, 0, 0, 240, 106, 142, 14, 90, 138, 136, 134, 214, 154, 23, - 84, 75, 155, 248, 74, 234, 102, 238, 88, 51, 228, 233, - ]), -]; - macro_rules! value_transition { ($bound:ident, $direction:expr, $value:expr) => { match $value { diff --git a/datafusion/optimizer/src/unwrap_cast_in_comparison.rs b/datafusion/optimizer/src/unwrap_cast_in_comparison.rs index 892d450ba85bc..e2b8a966cb920 100644 --- a/datafusion/optimizer/src/unwrap_cast_in_comparison.rs +++ b/datafusion/optimizer/src/unwrap_cast_in_comparison.rs @@ -26,7 +26,8 @@ use crate::{OptimizerConfig, OptimizerRule}; use crate::utils::NamePreserver; use arrow::datatypes::{ - DataType, TimeUnit, MAX_DECIMAL_FOR_EACH_PRECISION, MIN_DECIMAL_FOR_EACH_PRECISION, + DataType, TimeUnit, MAX_DECIMAL128_FOR_EACH_PRECISION, + MIN_DECIMAL128_FOR_EACH_PRECISION, }; use arrow::temporal_conversions::{MICROSECONDS, MILLISECONDS, NANOSECONDS}; use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRewriter}; @@ -369,8 +370,8 @@ fn try_cast_numeric_literal( // Different precision for decimal128 can store different range of value. // For example, the precision is 3, the max of value is `999` and the min // value is `-999` - MIN_DECIMAL_FOR_EACH_PRECISION[*precision as usize - 1], - MAX_DECIMAL_FOR_EACH_PRECISION[*precision as usize - 1], + MIN_DECIMAL128_FOR_EACH_PRECISION[*precision as usize], + MAX_DECIMAL128_FOR_EACH_PRECISION[*precision as usize], ), _ => return None, }; From 8f10fdff4d472c0f9420f32e2bbaf5116391e44d Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Feb 2025 16:06:15 -0500 Subject: [PATCH 029/177] Fix join type coercion (#14387) (#14454) --- datafusion/core/tests/dataframe/mod.rs | 37 ++++++++++++++++++- .../optimizer/src/analyzer/type_coercion.rs | 23 +++++++++--- 2 files changed, 52 insertions(+), 8 deletions(-) diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index 1ebbf92c736e0..eed11f634c9d7 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -30,8 +30,8 @@ use arrow::{ record_batch::RecordBatch, }; use arrow_array::{ - Array, BooleanArray, DictionaryArray, Float32Array, Float64Array, Int8Array, - UnionArray, + record_batch, Array, BooleanArray, DictionaryArray, Float32Array, Float64Array, + Int8Array, UnionArray, }; use arrow_buffer::ScalarBuffer; use arrow_schema::{ArrowError, SchemaRef, UnionFields, UnionMode}; @@ -1121,6 +1121,39 @@ async fn join() -> Result<()> { Ok(()) } +#[tokio::test] +async fn join_coercion_unnnamed() -> Result<()> { + let ctx = SessionContext::new(); + + // Test that join will coerce column types when necessary + // even when the relations don't have unique names + let left = ctx.read_batch(record_batch!( + ("id", Int32, [1, 2, 3]), + ("name", Utf8, ["a", "b", "c"]) + )?)?; + let right = ctx.read_batch(record_batch!( + ("id", Int32, [10, 3]), + ("name", Utf8View, ["d", "c"]) // Utf8View is a different type + )?)?; + let cols = vec!["name", "id"]; + + let filter = None; + let join = right.join(left, JoinType::LeftAnti, &cols, &cols, filter)?; + let results = join.collect().await?; + + assert_batches_sorted_eq!( + [ + "+----+------+", + "| id | name |", + "+----+------+", + "| 10 | d |", + "+----+------+", + ], + &results + ); + Ok(()) +} + #[tokio::test] async fn join_on() -> Result<()> { let left = test_table_with_name("a") diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index 48a5e2f9a07c9..7a41f54c56e15 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -190,7 +190,15 @@ impl<'a> TypeCoercionRewriter<'a> { .map(|(lhs, rhs)| { // coerce the arguments as though they were a single binary equality // expression - let (lhs, rhs) = self.coerce_binary_op(lhs, Operator::Eq, rhs)?; + let left_schema = join.left.schema(); + let right_schema = join.right.schema(); + let (lhs, rhs) = self.coerce_binary_op( + lhs, + left_schema, + Operator::Eq, + rhs, + right_schema, + )?; Ok((lhs, rhs)) }) .collect::>>()?; @@ -275,17 +283,19 @@ impl<'a> TypeCoercionRewriter<'a> { fn coerce_binary_op( &self, left: Expr, + left_schema: &DFSchema, op: Operator, right: Expr, + right_schema: &DFSchema, ) -> Result<(Expr, Expr)> { let (left_type, right_type) = get_input_types( - &left.get_type(self.schema)?, + &left.get_type(left_schema)?, &op, - &right.get_type(self.schema)?, + &right.get_type(right_schema)?, )?; Ok(( - left.cast_to(&left_type, self.schema)?, - right.cast_to(&right_type, self.schema)?, + left.cast_to(&left_type, left_schema)?, + right.cast_to(&right_type, right_schema)?, )) } } @@ -404,7 +414,8 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { )))) } Expr::BinaryExpr(BinaryExpr { left, op, right }) => { - let (left, right) = self.coerce_binary_op(*left, op, *right)?; + let (left, right) = + self.coerce_binary_op(*left, self.schema, op, *right, self.schema)?; Ok(Transformed::yes(Expr::BinaryExpr(BinaryExpr::new( Box::new(left), op, From 755b26abcfa06bf019172ff8df46f2e1b0d473bd Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Feb 2025 16:06:33 -0500 Subject: [PATCH 030/177] Support `Utf8View` to `numeric` coercion (#14377) (#14455) * Test for string / numeric coercion * fix tests * Update tests * Add tests to stringview * add numeric coercion --- .../expr-common/src/type_coercion/binary.rs | 2 + .../test_files/string/dictionary_utf8.slt | 12 +++++ .../test_files/string/init_data.slt.part | 11 +++++ .../test_files/string/large_string.slt | 12 +++++ .../sqllogictest/test_files/string/string.slt | 10 ++++ .../test_files/string/string_query.slt.part | 47 +++++++++++++++++++ .../test_files/string/string_view.slt | 11 +++++ 7 files changed, 105 insertions(+) diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs index 571c17119427a..4811b050907b3 100644 --- a/datafusion/expr-common/src/type_coercion/binary.rs +++ b/datafusion/expr-common/src/type_coercion/binary.rs @@ -684,8 +684,10 @@ fn string_numeric_coercion(lhs_type: &DataType, rhs_type: &DataType) -> Option Some(Utf8), (LargeUtf8, _) if rhs_type.is_numeric() => Some(LargeUtf8), + (Utf8View, _) if rhs_type.is_numeric() => Some(Utf8View), (_, Utf8) if lhs_type.is_numeric() => Some(Utf8), (_, LargeUtf8) if lhs_type.is_numeric() => Some(LargeUtf8), + (_, Utf8View) if lhs_type.is_numeric() => Some(Utf8View), _ => None, } } diff --git a/datafusion/sqllogictest/test_files/string/dictionary_utf8.slt b/datafusion/sqllogictest/test_files/string/dictionary_utf8.slt index 01071f03dce6f..2f12e9c7a39b5 100644 --- a/datafusion/sqllogictest/test_files/string/dictionary_utf8.slt +++ b/datafusion/sqllogictest/test_files/string/dictionary_utf8.slt @@ -34,6 +34,15 @@ statement ok create table test_substr as select arrow_cast(col1, 'Dictionary(Int32, Utf8)') as c1 from test_substr_base; +statement ok +create table test_datetime as +select + arrow_cast(column1, 'Dictionary(Int32, Utf8)') as ts, + arrow_cast(column2, 'Dictionary(Int32, Utf8)') as d, + arrow_cast(column3, 'Dictionary(Int32, Utf8)') as t +from test_datetime_base; + + statement ok drop table test_source @@ -56,3 +65,6 @@ drop table test_basic_operator; statement ok drop table test_substr_base; + +statement ok +drop table test_datetime_base; \ No newline at end of file diff --git a/datafusion/sqllogictest/test_files/string/init_data.slt.part b/datafusion/sqllogictest/test_files/string/init_data.slt.part index 06b65ff8e72a0..7799dd605b907 100644 --- a/datafusion/sqllogictest/test_files/string/init_data.slt.part +++ b/datafusion/sqllogictest/test_files/string/init_data.slt.part @@ -37,3 +37,14 @@ statement ok create table test_substr_base ( col1 VARCHAR ) as values ('foo'), ('hello🌏世界'), ('💩'), ('ThisIsAVeryLongASCIIString'), (''), (NULL); + + +# -------------------------------------- +# Setup test tables with date/time values to test coercion +# -------------------------------------- +statement ok +create table test_datetime_base as values + ('2024-08-09T12:13:14', '2024-08-09', '12:13:14'), + ('2024-08-09T12:13:15', '2024-09-09', '12:14:14'), + (NULL, NULL, NULL) +; diff --git a/datafusion/sqllogictest/test_files/string/large_string.slt b/datafusion/sqllogictest/test_files/string/large_string.slt index 84f1e8382e538..93ec796ec6f05 100644 --- a/datafusion/sqllogictest/test_files/string/large_string.slt +++ b/datafusion/sqllogictest/test_files/string/large_string.slt @@ -34,6 +34,15 @@ statement ok create table test_substr as select arrow_cast(col1, 'LargeUtf8') as c1 from test_substr_base; +statement ok +create table test_datetime as +select + arrow_cast(column1, 'LargeUtf8') as ts, + arrow_cast(column2, 'LargeUtf8') as d, + arrow_cast(column3, 'LargeUtf8') as t +from test_datetime_base; + + # select query TTTT SELECT ascii_1, ascii_2, unicode_1, unicode_2 FROM test_basic_operator @@ -64,3 +73,6 @@ drop table test_basic_operator; statement ok drop table test_substr_base; + +statement ok +drop table test_datetime_base; \ No newline at end of file diff --git a/datafusion/sqllogictest/test_files/string/string.slt b/datafusion/sqllogictest/test_files/string/string.slt index 55f0c034f5f9e..d724e672e0fc6 100644 --- a/datafusion/sqllogictest/test_files/string/string.slt +++ b/datafusion/sqllogictest/test_files/string/string.slt @@ -34,6 +34,13 @@ statement ok create table test_substr as select arrow_cast(col1, 'Utf8') as c1 from test_substr_base; +statement ok +create table test_datetime as +select + arrow_cast(column1, 'Utf8') as ts, + arrow_cast(column2, 'Utf8') as d, + arrow_cast(column3, 'Utf8') as t +from test_datetime_base; # @@ -186,3 +193,6 @@ drop table test_basic_operator; statement ok drop table test_substr; + +statement ok +drop table test_datetime; diff --git a/datafusion/sqllogictest/test_files/string/string_query.slt.part b/datafusion/sqllogictest/test_files/string/string_query.slt.part index 2414e5864c998..a2806859b5bae 100644 --- a/datafusion/sqllogictest/test_files/string/string_query.slt.part +++ b/datafusion/sqllogictest/test_files/string/string_query.slt.part @@ -19,6 +19,10 @@ # with standard values, but different types in string columns # (String, StringView, etc.) +# -------------------------------------- +# Show the input data +# -------------------------------------- + # select query TTTT SELECT ascii_1, ascii_2, unicode_1, unicode_2 FROM test_basic_operator @@ -35,6 +39,49 @@ _ \_ (empty) (empty) NULL % NULL NULL NULL R NULL 🔥 +# -------------------------------------- +# test type coercion (compare to int) +# queries should not error +# -------------------------------------- + +query BB +select ascii_1 = 1 as col1, 1 = ascii_1 as col2 from test_basic_operator; +---- +false false +false false +false false +false false +false false +false false +false false +false false +false false +NULL NULL +NULL NULL + +query BB +select ascii_1 <> 1 as col1, 1 <> ascii_1 as col2 from test_basic_operator; +---- +true true +true true +true true +true true +true true +true true +true true +true true +true true +NULL NULL +NULL NULL + +# Coercion to date/time +query BBB +select ts = '2024-08-09T12:13:14'::timestamp, d = '2024-08-08'::date, t = '12:13:14'::time from test_datetime; +---- +true false true +false false false +NULL NULL NULL + # -------------------------------------- # column comparison as filters # -------------------------------------- diff --git a/datafusion/sqllogictest/test_files/string/string_view.slt b/datafusion/sqllogictest/test_files/string/string_view.slt index 435b4bc3c5a8e..bef0011a20610 100644 --- a/datafusion/sqllogictest/test_files/string/string_view.slt +++ b/datafusion/sqllogictest/test_files/string/string_view.slt @@ -34,6 +34,14 @@ statement ok create table test_substr as select arrow_cast(col1, 'Utf8View') as c1 from test_substr_base; +statement ok +create table test_datetime as +select + arrow_cast(column1, 'Utf8View') as ts, + arrow_cast(column2, 'Utf8View') as d, + arrow_cast(column3, 'Utf8View') as t +from test_datetime_base; + statement ok drop table test_source @@ -51,6 +59,9 @@ drop table test_basic_operator; statement ok drop table test_substr_base; +statement ok +drop table test_datetime_base; + # -------------------------------------- # String_view specific tests From 9d287bdde56967e5a028563bc14dc9040e1e68b2 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Feb 2025 16:08:17 -0500 Subject: [PATCH 031/177] Update REGEXP_MATCH scalar function to support Utf8View (#14449) (#14457) * Update REGEXP_MATCH scalar function to support Utf8View * Cargo fmt fix. Co-authored-by: Bruce Ritchie --- datafusion/functions/benches/regx.rs | 59 ++++++++++++++++- datafusion/functions/src/regex/regexpmatch.rs | 66 +++++++++---------- datafusion/sqllogictest/test_files/regexp.slt | 60 +++++++++++++++-- .../test_files/string/string_view.slt | 2 +- 4 files changed, 144 insertions(+), 43 deletions(-) diff --git a/datafusion/functions/benches/regx.rs b/datafusion/functions/benches/regx.rs index 468d3d548bcf0..1f99cc3a5f0bc 100644 --- a/datafusion/functions/benches/regx.rs +++ b/datafusion/functions/benches/regx.rs @@ -18,7 +18,7 @@ extern crate criterion; use arrow::array::builder::StringBuilder; -use arrow::array::{ArrayRef, AsArray, Int64Array, StringArray}; +use arrow::array::{ArrayRef, AsArray, Int64Array, StringArray, StringViewArray}; use arrow::compute::cast; use arrow::datatypes::DataType; use criterion::{black_box, criterion_group, criterion_main, Criterion}; @@ -141,6 +141,20 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); + c.bench_function("regexp_like_1000 utf8view", |b| { + let mut rng = rand::thread_rng(); + let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap(); + let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap(); + let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap(); + + b.iter(|| { + black_box( + regexp_like(&[Arc::clone(&data), Arc::clone(®ex), Arc::clone(&flags)]) + .expect("regexp_like should work on valid values"), + ) + }) + }); + c.bench_function("regexp_match_1000", |b| { let mut rng = rand::thread_rng(); let data = Arc::new(data(&mut rng)) as ArrayRef; @@ -149,7 +163,25 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - regexp_match::(&[ + regexp_match(&[ + Arc::clone(&data), + Arc::clone(®ex), + Arc::clone(&flags), + ]) + .expect("regexp_match should work on valid values"), + ) + }) + }); + + c.bench_function("regexp_match_1000 utf8view", |b| { + let mut rng = rand::thread_rng(); + let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap(); + let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap(); + let flags = cast(&flags(&mut rng), &DataType::Utf8View).unwrap(); + + b.iter(|| { + black_box( + regexp_match(&[ Arc::clone(&data), Arc::clone(®ex), Arc::clone(&flags), @@ -180,6 +212,29 @@ fn criterion_benchmark(c: &mut Criterion) { ) }) }); + + c.bench_function("regexp_replace_1000 utf8view", |b| { + let mut rng = rand::thread_rng(); + let data = cast(&data(&mut rng), &DataType::Utf8View).unwrap(); + let regex = cast(®ex(&mut rng), &DataType::Utf8View).unwrap(); + // flags are not allowed to be utf8view according to the function + let flags = Arc::new(flags(&mut rng)) as ArrayRef; + let replacement = Arc::new(StringViewArray::from_iter_values( + iter::repeat("XX").take(1000), + )); + + b.iter(|| { + black_box( + regexp_replace::( + data.as_string_view(), + regex.as_string_view(), + &replacement, + Some(&flags), + ) + .expect("regexp_replace should work on valid values"), + ) + }) + }); } criterion_group!(benches, criterion_benchmark); diff --git a/datafusion/functions/src/regex/regexpmatch.rs b/datafusion/functions/src/regex/regexpmatch.rs index 06b9a9d98b47c..57207ecfdacda 100644 --- a/datafusion/functions/src/regex/regexpmatch.rs +++ b/datafusion/functions/src/regex/regexpmatch.rs @@ -16,16 +16,14 @@ // under the License. //! Regex expressions -use arrow::array::{Array, ArrayRef, OffsetSizeTrait}; +use arrow::array::{Array, ArrayRef, AsArray}; use arrow::compute::kernels::regexp; use arrow::datatypes::DataType; use arrow::datatypes::Field; use datafusion_common::exec_err; use datafusion_common::ScalarValue; use datafusion_common::{arrow_datafusion_err, plan_err}; -use datafusion_common::{ - cast::as_generic_string_array, internal_err, DataFusionError, Result, -}; +use datafusion_common::{DataFusionError, Result}; use datafusion_expr::{ColumnarValue, Documentation, TypeSignature}; use datafusion_expr::{ScalarUDFImpl, Signature, Volatility}; use datafusion_macros::user_doc; @@ -86,11 +84,12 @@ impl RegexpMatchFunc { signature: Signature::one_of( vec![ // Planner attempts coercion to the target type starting with the most preferred candidate. - // For example, given input `(Utf8View, Utf8)`, it first tries coercing to `(Utf8, Utf8)`. - // If that fails, it proceeds to `(LargeUtf8, Utf8)`. - // TODO: Native support Utf8View for regexp_match. + // For example, given input `(Utf8View, Utf8)`, it first tries coercing to `(Utf8View, Utf8View)`. + // If that fails, it proceeds to `(Utf8, Utf8)`. + TypeSignature::Exact(vec![Utf8View, Utf8View]), TypeSignature::Exact(vec![Utf8, Utf8]), TypeSignature::Exact(vec![LargeUtf8, LargeUtf8]), + TypeSignature::Exact(vec![Utf8View, Utf8View, Utf8View]), TypeSignature::Exact(vec![Utf8, Utf8, Utf8]), TypeSignature::Exact(vec![LargeUtf8, LargeUtf8, LargeUtf8]), ], @@ -138,7 +137,7 @@ impl ScalarUDFImpl for RegexpMatchFunc { .map(|arg| arg.to_array(inferred_length)) .collect::>>()?; - let result = regexp_match_func(&args); + let result = regexp_match(&args); if is_scalar { // If all inputs are scalar, keeps output as scalar let result = result.and_then(|arr| ScalarValue::try_from_array(&arr, 0)); @@ -153,33 +152,35 @@ impl ScalarUDFImpl for RegexpMatchFunc { } } -fn regexp_match_func(args: &[ArrayRef]) -> Result { - match args[0].data_type() { - DataType::Utf8 => regexp_match::(args), - DataType::LargeUtf8 => regexp_match::(args), - other => { - internal_err!("Unsupported data type {other:?} for function regexp_match") - } - } -} -pub fn regexp_match(args: &[ArrayRef]) -> Result { +pub fn regexp_match(args: &[ArrayRef]) -> Result { match args.len() { 2 => { - let values = as_generic_string_array::(&args[0])?; - let regex = as_generic_string_array::(&args[1])?; - regexp::regexp_match(values, regex, None) + regexp::regexp_match(&args[0], &args[1], None) .map_err(|e| arrow_datafusion_err!(e)) } 3 => { - let values = as_generic_string_array::(&args[0])?; - let regex = as_generic_string_array::(&args[1])?; - let flags = as_generic_string_array::(&args[2])?; - - if flags.iter().any(|s| s == Some("g")) { - return plan_err!("regexp_match() does not support the \"global\" option"); + match args[2].data_type() { + DataType::Utf8View => { + if args[2].as_string_view().iter().any(|s| s == Some("g")) { + return plan_err!("regexp_match() does not support the \"global\" option"); + } + } + DataType::Utf8 => { + if args[2].as_string::().iter().any(|s| s == Some("g")) { + return plan_err!("regexp_match() does not support the \"global\" option"); + } + } + DataType::LargeUtf8 => { + if args[2].as_string::().iter().any(|s| s == Some("g")) { + return plan_err!("regexp_match() does not support the \"global\" option"); + } + } + e => { + return plan_err!("regexp_match was called with unexpected data type {e:?}"); + } } - regexp::regexp_match(values, regex, Some(flags)) + regexp::regexp_match(&args[0], &args[1], Some(&args[2])) .map_err(|e| arrow_datafusion_err!(e)) } other => exec_err!( @@ -211,7 +212,7 @@ mod tests { expected_builder.append(false); let expected = expected_builder.finish(); - let re = regexp_match::(&[Arc::new(values), Arc::new(patterns)]).unwrap(); + let re = regexp_match(&[Arc::new(values), Arc::new(patterns)]).unwrap(); assert_eq!(re.as_ref(), &expected); } @@ -236,9 +237,8 @@ mod tests { expected_builder.append(false); let expected = expected_builder.finish(); - let re = - regexp_match::(&[Arc::new(values), Arc::new(patterns), Arc::new(flags)]) - .unwrap(); + let re = regexp_match(&[Arc::new(values), Arc::new(patterns), Arc::new(flags)]) + .unwrap(); assert_eq!(re.as_ref(), &expected); } @@ -250,7 +250,7 @@ mod tests { let flags = StringArray::from(vec!["g"]); let re_err = - regexp_match::(&[Arc::new(values), Arc::new(patterns), Arc::new(flags)]) + regexp_match(&[Arc::new(values), Arc::new(patterns), Arc::new(flags)]) .expect_err("unsupported flag should have failed"); assert_eq!(re_err.strip_backtrace(), "Error during planning: regexp_match() does not support the \"global\" option"); diff --git a/datafusion/sqllogictest/test_files/regexp.slt b/datafusion/sqllogictest/test_files/regexp.slt index 800026dd766d2..80f94e21d1fe3 100644 --- a/datafusion/sqllogictest/test_files/regexp.slt +++ b/datafusion/sqllogictest/test_files/regexp.slt @@ -193,6 +193,29 @@ NULL [Köln] [إسرائيل] +# test string view +statement ok +CREATE TABLE t_stringview AS +SELECT arrow_cast(str, 'Utf8View') as str, arrow_cast(pattern, 'Utf8View') as pattern, arrow_cast(flags, 'Utf8View') as flags FROM t; + +query ? +SELECT regexp_match(str, pattern, flags) FROM t_stringview; +---- +[a] +[A] +[B] +NULL +NULL +NULL +[010] +[Düsseldorf] +[Москва] +[Köln] +[إسرائيل] + +statement ok +DROP TABLE t_stringview; + query ? SELECT regexp_match('foobarbequebaz', ''); ---- @@ -354,6 +377,29 @@ X X X +# test string view +statement ok +CREATE TABLE t_stringview AS +SELECT arrow_cast(str, 'Utf8View') as str, arrow_cast(pattern, 'Utf8View') as pattern, arrow_cast(flags, 'Utf8View') as flags FROM t; + +query T +SELECT regexp_replace(str, pattern, 'X', concat('g', flags)) FROM t_stringview; +---- +Xbc +X +aXc +AbC +aBC +4000 +X +X +X +X +X + +statement ok +DROP TABLE t_stringview; + query T SELECT regexp_replace('ABCabcABC', '(abc)', 'X', 'gi'); ---- @@ -621,7 +667,7 @@ CREATE TABLE t_stringview AS SELECT arrow_cast(str, 'Utf8View') as str, arrow_cast(pattern, 'Utf8View') as pattern, arrow_cast(start, 'Int64') as start, arrow_cast(flags, 'Utf8View') as flags FROM t; query I -SELECT regexp_count(str, '\w') from t; +SELECT regexp_count(str, '\w') from t_stringview; ---- 3 3 @@ -636,7 +682,7 @@ SELECT regexp_count(str, '\w') from t; 7 query I -SELECT regexp_count(str, '\w{2}', start) from t; +SELECT regexp_count(str, '\w{2}', start) from t_stringview; ---- 1 1 @@ -651,7 +697,7 @@ SELECT regexp_count(str, '\w{2}', start) from t; 3 query I -SELECT regexp_count(str, 'ab', 1, 'i') from t; +SELECT regexp_count(str, 'ab', 1, 'i') from t_stringview; ---- 1 1 @@ -667,7 +713,7 @@ SELECT regexp_count(str, 'ab', 1, 'i') from t; query I -SELECT regexp_count(str, pattern) from t; +SELECT regexp_count(str, pattern) from t_stringview; ---- 1 1 @@ -682,7 +728,7 @@ SELECT regexp_count(str, pattern) from t; 1 query I -SELECT regexp_count(str, pattern, start) from t; +SELECT regexp_count(str, pattern, start) from t_stringview; ---- 1 1 @@ -697,7 +743,7 @@ SELECT regexp_count(str, pattern, start) from t; 1 query I -SELECT regexp_count(str, pattern, start, flags) from t; +SELECT regexp_count(str, pattern, start, flags) from t_stringview; ---- 1 1 @@ -713,7 +759,7 @@ SELECT regexp_count(str, pattern, start, flags) from t; # test type coercion query I -SELECT regexp_count(arrow_cast(str, 'Utf8'), arrow_cast(pattern, 'LargeUtf8'), arrow_cast(start, 'Int32'), flags) from t; +SELECT regexp_count(arrow_cast(str, 'Utf8'), arrow_cast(pattern, 'LargeUtf8'), arrow_cast(start, 'Int32'), flags) from t_stringview; ---- 1 1 diff --git a/datafusion/sqllogictest/test_files/string/string_view.slt b/datafusion/sqllogictest/test_files/string/string_view.slt index bef0011a20610..c54e2aa7002c2 100644 --- a/datafusion/sqllogictest/test_files/string/string_view.slt +++ b/datafusion/sqllogictest/test_files/string/string_view.slt @@ -794,7 +794,7 @@ EXPLAIN SELECT FROM test; ---- logical_plan -01)Projection: regexp_match(CAST(test.column1_utf8view AS Utf8), Utf8("^https?://(?:www\.)?([^/]+)/.*$")) AS k +01)Projection: regexp_match(test.column1_utf8view, Utf8View("^https?://(?:www\.)?([^/]+)/.*$")) AS k 02)--TableScan: test projection=[column1_utf8view] ## Ensure no casts for REGEXP_REPLACE From 61466008592d1b83896982ddf2c39ce57b7bc826 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Feb 2025 16:12:48 -0500 Subject: [PATCH 032/177] Fix regression list Type Coercion List with inner type struct which has large/view types (#14385) (#14456) * Test for coercing inner structs * Fix but, update tests * Update tests --- .../expr-common/src/type_coercion/binary.rs | 15 +++--- datafusion/sqllogictest/test_files/case.slt | 53 +++++++++++++++++++ datafusion/sqllogictest/test_files/struct.slt | 8 +-- 3 files changed, 65 insertions(+), 11 deletions(-) diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs index 4811b050907b3..a760a3a632bc6 100644 --- a/datafusion/expr-common/src/type_coercion/binary.rs +++ b/datafusion/expr-common/src/type_coercion/binary.rs @@ -507,18 +507,19 @@ fn type_union_resolution_coercion( None } - let types = lhs + let coerced_types = lhs .iter() .map(|lhs_field| search_corresponding_coerced_type(lhs_field, rhs)) .collect::>>()?; - let fields = types + // preserve the field name and nullability + let orig_fields = std::iter::zip(lhs.iter(), rhs.iter()); + + let fields: Vec = coerced_types .into_iter() - .enumerate() - .map(|(i, datatype)| { - Arc::new(Field::new(format!("c{i}"), datatype, true)) - }) - .collect::>(); + .zip(orig_fields) + .map(|(datatype, (lhs, rhs))| coerce_fields(datatype, lhs, rhs)) + .collect(); Some(DataType::Struct(fields.into())) } _ => { diff --git a/datafusion/sqllogictest/test_files/case.slt b/datafusion/sqllogictest/test_files/case.slt index 46e9c86c7591c..8e470fe988d3e 100644 --- a/datafusion/sqllogictest/test_files/case.slt +++ b/datafusion/sqllogictest/test_files/case.slt @@ -416,5 +416,58 @@ SELECT end FROM t; +statement ok +drop table t + +# Fix coercion of lists of structs +# https://github.com/apache/datafusion/issues/14154 + +statement ok +create or replace table t as values +( + 100, -- column1 int (so the case isn't constant folded) + [{ 'foo': arrow_cast('baz', 'Utf8View') }], -- column2 has List of Struct w/ Utf8View + [{ 'foo': 'bar' }], -- column3 has List of Struct w/ Utf8 + [{ 'foo': 'blarg' }] -- column4 has List of Struct w/ Utf8 +); + +# This case forces all branches to be coerced to the same type +query ? +SELECT + case + when column1 > 0 then column2 + when column1 < 0 then column3 + else column4 + end +FROM t; +---- +[{foo: baz}] + +# different orders of the branches +query ? +SELECT + case + when column1 > 0 then column3 -- NB different order + when column1 < 0 then column4 + else column2 + end +FROM t; +---- +[{foo: bar}] + +# different orders of the branches +query ? +SELECT + case + when column1 > 0 then column4 -- NB different order + when column1 < 0 then column2 + else column3 + end +FROM t; +---- +[{foo: blarg}] + + + statement ok drop table t diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt index d671798b7d0f7..0afe39de17950 100644 --- a/datafusion/sqllogictest/test_files/struct.slt +++ b/datafusion/sqllogictest/test_files/struct.slt @@ -459,14 +459,14 @@ create table t as values({r: 'a', c: 1}), ({r: 'b', c: 2.3}); query ? select * from t; ---- -{c0: a, c1: 1.0} -{c0: b, c1: 2.3} +{r: a, c: 1.0} +{r: b, c: 2.3} query T select arrow_typeof(column1) from t; ---- -Struct([Field { name: "c0", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "c1", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }]) -Struct([Field { name: "c0", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "c1", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }]) +Struct([Field { name: "r", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "c", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }]) +Struct([Field { name: "r", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "c", data_type: Float64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} }]) statement ok drop table t; From 26058ac024095ad8852eb3a8ab707ac09a02e8d7 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Feb 2025 16:26:35 -0500 Subject: [PATCH 033/177] Update changelog (#14460) --- dev/changelog/45.0.0.md | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/dev/changelog/45.0.0.md b/dev/changelog/45.0.0.md index 2303eee92a1da..ca905c0a1a566 100644 --- a/dev/changelog/45.0.0.md +++ b/dev/changelog/45.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 45.0.0 Changelog -This release consists of 252 commits from 83 contributors. See credits at the end of this changelog for more information. +This release consists of 258 commits from 83 contributors. See credits at the end of this changelog for more information. **Breaking changes:** @@ -94,6 +94,7 @@ This release consists of 252 commits from 83 contributors. See credits at the en - Support arrays_overlap function (alias of `array_has_any`) [#14217](https://github.com/apache/datafusion/pull/14217) (erenavsarogullari) - chore: Adding commit activity badge [#14386](https://github.com/apache/datafusion/pull/14386) (comphead) - docs: Clarify join behavior in `DataFrame::join` [#14393](https://github.com/apache/datafusion/pull/14393) (rkrishn7) +- Prepare for `45.0.0` release: Version and Changelog [#14397](https://github.com/apache/datafusion/pull/14397) (alamb) **Other:** @@ -290,13 +291,18 @@ This release consists of 252 commits from 83 contributors. See credits at the en - FFI support for versions and alternate tokio runtimes [#13937](https://github.com/apache/datafusion/pull/13937) (timsaucer) - Do not rename struct fields when coercing types in `CASE` [#14384](https://github.com/apache/datafusion/pull/14384) (alamb) - Add `TableProvider::insert_into` into FFI Bindings [#14391](https://github.com/apache/datafusion/pull/14391) (davisp) +- [branch-45]: Backport chore: Upgrade to `arrow`/`parquet` `54.1.0` and fix clippy/ci (#14415) [#14453](https://github.com/apache/datafusion/pull/14453) (alamb) +- [release-45] Fix join type coercion (#14387) [#14454](https://github.com/apache/datafusion/pull/14454) (alamb) +- [branch-45] Support `Utf8View` to `numeric` coercion (#14377) [#14455](https://github.com/apache/datafusion/pull/14455) (alamb) +- [branch-45] Update REGEXP_MATCH scalar function to support Utf8View (#14449) [#14457](https://github.com/apache/datafusion/pull/14457) (alamb) +- [branch-45] Fix regression list Type Coercion List with inner type struct which has large/view types (#14385) [#14456](https://github.com/apache/datafusion/pull/14456) (alamb) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` - 46 Andrew Lamb + 52 Andrew Lamb 22 Ian Lai 20 dependabot[bot] 8 Bruce Ritchie From 6e1e0d17b410fef678068b7b3b29f4c830c2bad2 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 6 Feb 2025 17:21:38 +0800 Subject: [PATCH 034/177] fix datafusion-cli --- datafusion-cli/Cargo.lock | 45 +++++++++++++-------------------------- datafusion-cli/Cargo.toml | 4 ++-- 2 files changed, 17 insertions(+), 32 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index b37253d1a135d..44f11c014b42f 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -174,8 +174,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caf25cdc4a985f91df42ed9e9308e1adbcd341a31a72605c697033fcef163e3" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-arith", "arrow-array", @@ -195,8 +194,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91f2dfd1a7ec0aca967dfaa616096aec49779adc8eccec005e2f5e4111b1192a" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -210,8 +208,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d39387ca628be747394890a6e47f138ceac1aa912eab64f02519fed24b637af8" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "ahash", "arrow-buffer", @@ -227,8 +224,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e51e05228852ffe3eb391ce7178a0f97d2cf80cc6ef91d3c4a6b3cb688049ec" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "bytes", "half", @@ -238,8 +234,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09aea56ec9fa267f3f3f6cdab67d8a9974cbba90b3aa38c8fe9d0bb071bd8c1" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -259,8 +254,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c07b5232be87d115fde73e32f2ca7f1b353bff1b44ac422d3c6fc6ae38f11f0d" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -278,8 +272,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98ae0af50890b494cebd7d6b04b35e896205c1d1df7b29a6272c5d0d0249ef5" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-buffer", "arrow-schema", @@ -290,8 +283,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ed91bdeaff5a1c00d28d8f73466bcb64d32bbd7093b5a30156b4b9f4dba3eee" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -305,8 +297,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0471f51260a5309307e5d409c9dc70aede1cd9cf1d4ff0f0a1e8e1a2dd0e0d3c" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -325,8 +316,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2883d7035e0b600fb4c30ce1e50e66e53d8656aa729f2bfa4b51d359cf3ded52" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -340,8 +330,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552907e8e587a6fde4f8843fd7a27a576a260f65dab6c065741ea79f633fc5be" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "ahash", "arrow-array", @@ -354,14 +343,12 @@ dependencies = [ [[package]] name = "arrow-schema" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "539ada65246b949bd99ffa0881a9a15a4a529448af1a07a9838dd78617dafab1" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" [[package]] name = "arrow-select" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6259e566b752da6dceab91766ed8b2e67bf6270eb9ad8a6e07a33c1bede2b125" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "ahash", "arrow-array", @@ -374,8 +361,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3179ccbd18ebf04277a095ba7321b93fd1f774f18816bd5f6b3ce2f594edb6c" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "arrow-array", "arrow-buffer", @@ -2857,8 +2843,7 @@ dependencies = [ [[package]] name = "parquet" version = "53.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea02606ba6f5e856561d8d507dba8bac060aefca2a6c0f1aa1d361fed91ff3e" +source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" dependencies = [ "ahash", "arrow-array", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 784d47220c7c9..d9500f7ba04e2 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -30,7 +30,7 @@ rust-version = "1.79" readme = "README.md" [dependencies] -arrow = { version = "53.0.0" } +arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048" } async-trait = "0.1.73" aws-config = "1.5.5" aws-sdk-sso = "1.43.0" @@ -55,7 +55,7 @@ futures = "0.3" mimalloc = { version = "0.1", default-features = false } object_store = { version = "0.11.0", features = ["aws", "gcp", "http"] } parking_lot = { version = "0.12" } -parquet = { version = "53.0.0", default-features = false } +parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } regex = "1.8" rustyline = "14.0" tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] } From af26638c646e41d6a86eafe97efe6836c247f9f2 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 10 Feb 2025 11:17:48 +0800 Subject: [PATCH 035/177] missing fetch after removing SPM --- datafusion/core/src/physical_optimizer/enforce_sorting.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_sorting.rs b/datafusion/core/src/physical_optimizer/enforce_sorting.rs index cfabb4f927c02..d43b20a71ddf3 100644 --- a/datafusion/core/src/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/src/physical_optimizer/enforce_sorting.rs @@ -371,9 +371,10 @@ fn ensure_sorting( return adjust_window_sort_removal(requirements).map(Transformed::yes); } else if is_sort_preserving_merge(plan) && child_node.plan.output_partitioning().partition_count() <= 1 + && plan.fetch().is_none() { // This `SortPreservingMergeExec` is unnecessary, input already has a - // single partition. + // single partition and no fetch is required: let child_node = requirements.children.swap_remove(0); return Ok(Transformed::yes(child_node)); } From d518b51630d31252385027fb6c9d1012fc994fa0 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 10 Feb 2025 15:29:32 +0800 Subject: [PATCH 036/177] update cargo toml --- Cargo.toml | 18 +- datafusion-cli/Cargo.lock | 870 ++++++------------ datafusion-cli/Cargo.toml | 12 +- datafusion/common/Cargo.toml | 2 +- .../enforce_distribution.rs | 10 +- .../replace_with_order_preserving_variants.rs | 5 +- .../physical-plan/src/execution_plan.rs | 3 +- datafusion/sqllogictest/Cargo.toml | 9 +- 8 files changed, 320 insertions(+), 609 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7998fac8b5f07..1640d38b7381d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,22 +77,22 @@ version = "44.0.0" ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } -arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", features = [ +arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", features = [ "prettyprint", ] } -arrow-array = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false, features = [ +arrow-array = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false, features = [ "chrono-tz", ] } -arrow-buffer = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } -arrow-flight = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", features = [ +arrow-buffer = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false } +arrow-flight = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", features = [ "flight-sql-experimental", ] } -arrow-ipc = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false, features = [ "lz4", ] } -arrow-ord = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } -arrow-schema = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } -arrow-string = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } +arrow-ord = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false } +arrow-schema = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false } +arrow-string = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false } async-trait = "0.1.73" bigdecimal = "0.4.7" bytes = "1.4" @@ -134,7 +134,7 @@ itertools = "0.13" log = "^0.4" object_store = { version = "0.11.0", default-features = false } parking_lot = "0.12" -parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false, features = [ +parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index 82b6e97341fe6..5f4fee0d2f0ef 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -31,7 +31,7 @@ checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "const-random", - "getrandom", + "getrandom 0.2.15", "once_cell", "version_check", "zerocopy", @@ -123,11 +123,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "3.0.6" +version = "3.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" dependencies = [ "anstyle", + "once_cell", "windows-sys 0.59.0", ] @@ -174,14 +175,8 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c91839b07e474b3995035fd8ac33ee54f9c9ccbbb1ea33d9909c71bffdf1259d" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-arith", "arrow-array", @@ -200,14 +195,8 @@ dependencies = [ [[package]] name = "arrow-arith" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "855c57c4efd26722b044dcd3e348252560e3e0333087fb9f6479dc0bf744054f" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -220,14 +209,8 @@ dependencies = [ [[package]] name = "arrow-array" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd03279cea46569acf9295f6224fbc370c5df184b4d2ecfe97ccb131d5615a7f" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "ahash", "arrow-buffer", @@ -242,14 +225,8 @@ dependencies = [ [[package]] name = "arrow-buffer" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e4a9b9b1d6d7117f6138e13bc4dd5daa7f94e671b70e8c9c4dc37b4f5ecfc16" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "bytes", "half", @@ -258,14 +235,8 @@ dependencies = [ [[package]] name = "arrow-cast" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc70e39916e60c5b7af7a8e2719e3ae589326039e1e863675a008bee5ffe90fd" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -284,14 +255,8 @@ dependencies = [ [[package]] name = "arrow-csv" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789b2af43c1049b03a8d088ff6b2257cdcea1756cd76b174b1f2600356771b97" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -308,14 +273,8 @@ dependencies = [ [[package]] name = "arrow-data" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4e75edf21ffd53744a9b8e3ed11101f610e7ceb1a29860432824f1834a1f623" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-buffer", "arrow-schema", @@ -325,14 +284,8 @@ dependencies = [ [[package]] name = "arrow-ipc" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d186a909dece9160bf8312f5124d797884f608ef5435a36d9d608e0b2a9bcbf8" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -345,14 +298,8 @@ dependencies = [ [[package]] name = "arrow-json" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66ff2fedc1222942d0bd2fd391cb14a85baa3857be95c9373179bd616753b85" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -370,14 +317,8 @@ dependencies = [ [[package]] name = "arrow-ord" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ece7b5bc1180e6d82d1a60e1688c199829e8842e38497563c3ab6ea813e527fd" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -390,14 +331,8 @@ dependencies = [ [[package]] name = "arrow-row" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745c114c8f0e8ce211c83389270de6fbe96a9088a7b32c2a041258a443fe83ff" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "ahash", "arrow-array", @@ -409,25 +344,13 @@ dependencies = [ [[package]] name = "arrow-schema" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" - -[[package]] -name = "arrow-select" -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95513080e728e4cec37f1ff5af4f12c9688d47795d17cda80b6ec2cf74d4678" +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" [[package]] name = "arrow-select" version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e415279094ea70323c032c6e739c48ad8d80e78a09bef7117b8718ad5bf3722" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "ahash", "arrow-array", @@ -439,14 +362,8 @@ dependencies = [ [[package]] name = "arrow-string" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11d956cae7002eb8d83a27dbd34daaea1cf5b75852f0b84deb4d93a276e92bbf" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "arrow-array", "arrow-buffer", @@ -494,9 +411,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" dependencies = [ "proc-macro2", "quote", @@ -526,9 +443,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -version = "1.5.10" +version = "1.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b49afaa341e8dd8577e1a2200468f98956d6eda50bcf4a53246cc00174ba924" +checksum = "50236e4d60fe8458de90a71c0922c761e41755adf091b1b03de1cef537179915" dependencies = [ "aws-credential-types", "aws-runtime", @@ -537,7 +454,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.60.7", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -568,9 +485,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.4.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5ac934720fbb46206292d2c75b57e67acfc56fe7dfd34fb9a02334af08409ea" +checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -593,21 +510,15 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -<<<<<<< HEAD -version = "1.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded855583fa1d22e88fe39fd6062b062376e50a8211989e07cf5e38d52eb3453" -======= -version = "1.50.0" +version = "1.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ca43a4ef210894f93096039ef1d6fa4ad3edfabb3be92b80908b9f2e4b4eab" ->>>>>>> upstream/branch-44 +checksum = "16ff718c9ee45cc1ebd4774a0e086bb80a6ab752b4902edf1c9f56b86ee1f770" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -621,21 +532,15 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -<<<<<<< HEAD -version = "1.49.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9177ea1192e6601ae16c7273385690d88a7ed386a00b74a6bc894d12103cd933" -======= -version = "1.51.0" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abaf490c2e48eed0bb8e2da2fb08405647bd7f253996e0f93b981958ea0f73b0" ->>>>>>> upstream/branch-44 +checksum = "5183e088715cc135d8d396fdd3bc02f018f0da4c511f53cb8d795b6a31c55809" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -649,21 +554,15 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -<<<<<<< HEAD -version = "1.48.0" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "823ef553cf36713c97453e2ddff1eb8f62be7f4523544e2a5db64caf80100f0a" -======= -version = "1.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b68fde0d69c8bfdc1060ea7da21df3e39f6014da316783336deff0a9ec28f4bf" ->>>>>>> upstream/branch-44 +checksum = "c9f944ef032717596639cea4a2118a3a457268ef51bbb5fde9637e54c465da00" dependencies = [ "aws-credential-types", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json 0.61.1", + "aws-smithy-json", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -678,9 +577,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.6" +version = "1.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" +checksum = "0bc5bbd1e4a2648fd8c5982af03935972c24a2f9846b396de661d351ee3ce837" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -701,9 +600,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" dependencies = [ "futures-util", "pin-project-lite", @@ -712,9 +611,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.11" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8bc3e8fdc6b8d07d976e301c02fe553f72a39b7a9fea820e023268467d7ab6" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -732,18 +631,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.60.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683df9469ef09468dad3473d129960119a0d3593617542b7d52086c8486f2d6" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-json" -version = "0.61.1" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" dependencies = [ "aws-smithy-types", ] @@ -760,9 +650,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -804,9 +694,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" dependencies = [ "base64-simd", "bytes", @@ -839,9 +729,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.3" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5221b91b3e441e6675310829fd8984801b772cb1546ef6c0e54dec9f1ac13fef" +checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -910,9 +800,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" [[package]] name = "blake2" @@ -958,9 +848,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.1" +version = "4.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -968,9 +858,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "786a307d683a5bf92e6fd5fd69a7eb613751668d1d8d67d802846dfe367c62c8" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" dependencies = [ "memchr", "regex-automata", @@ -979,9 +869,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.16.0" +version = "3.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" [[package]] name = "byteorder" @@ -991,9 +881,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" +checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" [[package]] name = "bytes-utils" @@ -1038,15 +928,9 @@ dependencies = [ [[package]] name = "cc" -<<<<<<< HEAD -version = "1.1.34" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b9470d453346108f93a59222a9a1a5724db32d0a4727b7ab7ace4b4d822dc9" -======= -version = "1.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9157bbaa6b165880c27a4293a474c91cdcf265cc68cc829bf10be0964a391caf" ->>>>>>> upstream/branch-44 +checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" dependencies = [ "jobserver", "libc", @@ -1086,9 +970,9 @@ dependencies = [ [[package]] name = "chrono-tz" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd6dd8046d00723a59a2f8c5f295c515b9bb9a331ee4f8f3d4dd49e428acd3b6" +checksum = "9c6ac4f2c0bf0f44e9161aec9675e1050aa4a530663c4a9e37e108fa948bca9f" dependencies = [ "chrono", "chrono-tz-build", @@ -1107,9 +991,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.23" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" +checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" dependencies = [ "clap_builder", "clap_derive", @@ -1117,9 +1001,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.23" +version = "4.5.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" +checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" dependencies = [ "anstream", "anstyle", @@ -1129,9 +1013,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.18" +version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" dependencies = [ "heck", "proc-macro2", @@ -1162,12 +1046,11 @@ checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" [[package]] name = "comfy-table" -version = "7.1.3" +version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ - "strum", - "strum_macros", + "unicode-segmentation", "unicode-width 0.2.0", ] @@ -1186,7 +1069,7 @@ version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" dependencies = [ - "getrandom", + "getrandom 0.2.15", "once_cell", "tiny-keccak", ] @@ -1234,9 +1117,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" dependencies = [ "libc", ] @@ -1258,9 +1141,9 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" +checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" [[package]] name = "crypto-common" @@ -1325,11 +1208,7 @@ dependencies = [ [[package]] name = "datafusion" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "apache-avro", "arrow", @@ -1381,11 +1260,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow-schema", "async-trait", @@ -1398,11 +1273,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "assert_cmd", @@ -1434,11 +1305,7 @@ dependencies = [ [[package]] name = "datafusion-common" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "ahash", "apache-avro", @@ -1462,11 +1329,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "log", "tokio", @@ -1478,11 +1341,7 @@ version = "44.0.0" [[package]] name = "datafusion-execution" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "dashmap", @@ -1499,11 +1358,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "chrono", @@ -1522,11 +1377,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "datafusion-common", @@ -1535,11 +1386,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "arrow-buffer", @@ -1567,11 +1414,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "ahash", "arrow", @@ -1591,11 +1434,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "ahash", "arrow", @@ -1606,11 +1445,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "arrow-array", @@ -1644,11 +1479,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "datafusion-common", "datafusion-doc", @@ -1663,11 +1494,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1683,11 +1510,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "chrono", @@ -1704,11 +1527,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "ahash", "arrow", @@ -1731,11 +1550,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "ahash", "arrow", @@ -1747,11 +1562,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "datafusion-common", @@ -1766,11 +1577,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "ahash", "arrow", @@ -1800,11 +1607,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 dependencies = [ "arrow", "arrow-array", @@ -1897,9 +1700,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_filter" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2c92ceda6ceec50f43169f9ee8424fe2db276791afde7b2cd8bc084cb376ab" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" dependencies = [ "log", "regex", @@ -1907,9 +1710,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.5" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13fa619b91fb2381732789fc5de83b45675e882f66623b7d8cb4f643017018d" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" dependencies = [ "anstream", "anstyle", @@ -1965,9 +1768,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "24.3.25" +version = "24.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1985,9 +1788,9 @@ dependencies = [ [[package]] name = "float-cmp" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" dependencies = [ "num-traits", ] @@ -2121,10 +1924,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets 0.52.6", +] + [[package]] name = "gimli" version = "0.31.1" @@ -2133,9 +1948,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -2198,21 +2013,9 @@ dependencies = [ [[package]] name = "hashbrown" -<<<<<<< HEAD -version = "0.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" - -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -======= version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" ->>>>>>> upstream/branch-44 [[package]] name = "heck" @@ -2302,9 +2105,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.9.5" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" +checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" [[package]] name = "httpdate" @@ -2344,9 +2147,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" +checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" dependencies = [ "bytes", "futures-channel", @@ -2380,15 +2183,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http 1.2.0", - "hyper 1.5.2", + "hyper 1.6.0", "hyper-util", - "rustls 0.23.20", + "rustls 0.23.22", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -2407,7 +2210,7 @@ dependencies = [ "futures-util", "http 1.2.0", "http-body 1.0.1", - "hyper 1.5.2", + "hyper 1.6.0", "pin-project-lite", "socket2", "tokio", @@ -2579,28 +2382,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" dependencies = [ "equivalent", -<<<<<<< HEAD - "hashbrown 0.15.1", -] - -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -======= "hashbrown 0.15.2", ->>>>>>> upstream/branch-44 ] [[package]] @@ -2611,9 +2398,9 @@ checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" [[package]] name = "ipnet" -version = "2.10.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is_terminal_polyfill" @@ -2647,9 +2434,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" dependencies = [ "once_cell", "wasm-bindgen", @@ -2727,9 +2514,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libflate" @@ -2777,27 +2564,21 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "libc", ] [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" -<<<<<<< HEAD -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" -======= version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" ->>>>>>> upstream/branch-44 [[package]] name = "lock_api" @@ -2811,9 +2592,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.22" +version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" [[package]] name = "lz4_flex" @@ -2868,9 +2649,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" dependencies = [ "adler2", ] @@ -2882,7 +2663,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -2901,7 +2682,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "cfg_aliases 0.1.1", "libc", @@ -2996,18 +2777,18 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" dependencies = [ "async-trait", "base64 0.22.1", @@ -3015,7 +2796,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.5.2", + "hyper 1.6.0", "itertools", "md-5", "parking_lot", @@ -3036,15 +2817,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.2" +version = "1.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" +checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" [[package]] name = "openssl-probe" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "option-ext" @@ -3063,9 +2844,9 @@ dependencies = [ [[package]] name = "outref" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "parking_lot" @@ -3092,14 +2873,8 @@ dependencies = [ [[package]] name = "parquet" -<<<<<<< HEAD -version = "53.2.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=aa8c048#aa8c04807d85be763c2099dd4e5095e967f0ca03" -======= version = "53.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b449890367085eb65d7d3321540abc3d7babbd179ce31df0016e90719114191" ->>>>>>> upstream/branch-44 +source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" dependencies = [ "ahash", "arrow-array", @@ -3164,18 +2939,18 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -3183,9 +2958,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", "rand", @@ -3193,18 +2968,18 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -3235,9 +3010,9 @@ dependencies = [ [[package]] name = "predicates" -version = "3.1.2" +version = "3.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9086cc7640c29a356d1a29fd134380bee9d8f79a17410aa76e7ad295f42c97" +checksum = "a5d19ee57562043d37e82899fade9a22ebab7be9cef5026b07fda9cdd4293573" dependencies = [ "anstyle", "difflib", @@ -3249,15 +3024,15 @@ dependencies = [ [[package]] name = "predicates-core" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae8177bee8e75d6846599c6b9ff679ed51e882816914eec639944d7c9aa11931" +checksum = "727e462b119fe9c93fd0eb1429a5f7647394014cf3c04ab2c0350eeb09095ffa" [[package]] name = "predicates-tree" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41b740d195ed3166cd147c8047ec98db0e22ec019eb8eeb76d343b795304fb13" +checksum = "72dd2d6d381dfb73a193c7fca536518d7caee39fc8503f74e7dc0be0531b425c" dependencies = [ "predicates-core", "termtree", @@ -3274,9 +3049,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -3298,9 +3073,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" dependencies = [ "memchr", "serde", @@ -3317,9 +3092,9 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.20", + "rustls 0.23.22", "socket2", - "thiserror 2.0.7", + "thiserror 2.0.11", "tokio", "tracing", ] @@ -3331,14 +3106,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" dependencies = [ "bytes", - "getrandom", + "getrandom 0.2.15", "rand", "ring", "rustc-hash", - "rustls 0.23.20", + "rustls 0.23.22", "rustls-pki-types", "slab", - "thiserror 2.0.7", + "thiserror 2.0.11", "tinyvec", "tracing", "web-time", @@ -3346,9 +3121,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases 0.2.1", "libc", @@ -3360,9 +3135,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -3404,7 +3179,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -3433,7 +3208,7 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", ] [[package]] @@ -3442,7 +3217,7 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" dependencies = [ - "getrandom", + "getrandom 0.2.15", "libredox", "thiserror 1.0.69", ] @@ -3490,9 +3265,9 @@ checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "base64 0.22.1", "bytes", @@ -3502,8 +3277,8 @@ dependencies = [ "http 1.2.0", "http-body 1.0.1", "http-body-util", - "hyper 1.5.2", - "hyper-rustls 0.27.3", + "hyper 1.6.0", + "hyper-rustls 0.27.5", "hyper-util", "ipnet", "js-sys", @@ -3513,7 +3288,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.20", + "rustls 0.23.22", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -3524,6 +3299,7 @@ dependencies = [ "tokio", "tokio-rustls 0.26.1", "tokio-util", + "tower", "tower-service", "url", "wasm-bindgen", @@ -3541,7 +3317,7 @@ checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.15", "libc", "spin", "untrusted", @@ -3592,9 +3368,9 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fb8039b3032c191086b10f11f319a6e99e1e82889c5cc6046f515c9db1d497" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustc_version" @@ -3607,17 +3383,11 @@ dependencies = [ [[package]] name = "rustix" -<<<<<<< HEAD -version = "0.38.39" +version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "375116bee2be9ed569afe2154ea6a99dfdffd257f533f187498c2a8f5feaf4ee" -======= -version = "0.38.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" ->>>>>>> upstream/branch-44 +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "errno", "libc", "linux-raw-sys", @@ -3638,9 +3408,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" dependencies = [ "once_cell", "ring", @@ -3671,7 +3441,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.0.1", + "security-framework 3.2.0", ] [[package]] @@ -3694,9 +3464,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.1" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" +checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c" dependencies = [ "web-time", ] @@ -3724,9 +3494,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "rustyline" @@ -3734,7 +3504,7 @@ version = "14.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7803e8936da37efd9b6d4478277f4b2b9bb5cdb37a113e8d63222e58da647e63" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "cfg-if", "clipboard-win", "fd-lock", @@ -3752,9 +3522,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" [[package]] name = "same-file" @@ -3796,7 +3566,7 @@ version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation 0.9.4", "core-foundation-sys", "libc", @@ -3805,11 +3575,11 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.6.0", + "bitflags 2.8.0", "core-foundation 0.10.0", "core-foundation-sys", "libc", @@ -3818,9 +3588,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -3828,9 +3598,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" +checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" [[package]] name = "seq-macro" @@ -3840,9 +3610,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] @@ -3858,9 +3628,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.216" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -3869,9 +3639,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.138" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" dependencies = [ "itoa", "memchr", @@ -3919,9 +3689,9 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" @@ -4009,8 +3779,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -<<<<<<< HEAD -======= name = "stacker" version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4024,7 +3792,6 @@ dependencies = [ ] [[package]] ->>>>>>> upstream/branch-44 name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4063,15 +3830,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -<<<<<<< HEAD -version = "2.0.87" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" -======= -version = "2.0.90" +version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" ->>>>>>> upstream/branch-44 +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" dependencies = [ "proc-macro2", "quote", @@ -4090,20 +3851,6 @@ dependencies = [ [[package]] name = "synstructure" version = "0.13.1" -<<<<<<< HEAD -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tempfile" -version = "3.13.0" -======= ->>>>>>> upstream/branch-44 source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ @@ -4114,12 +3861,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.14.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" dependencies = [ "cfg-if", "fastrand", + "getrandom 0.3.1", "once_cell", "rustix", "windows-sys 0.59.0", @@ -4127,41 +3875,30 @@ dependencies = [ [[package]] name = "termtree" -version = "0.4.1" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" [[package]] name = "thiserror" -<<<<<<< HEAD -version = "1.0.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02dd99dc800bbb97186339685293e1cc5d9df1f8fae2d0aecd9ff1c77efea892" -======= version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" ->>>>>>> upstream/branch-44 dependencies = [ "thiserror-impl 1.0.69", ] [[package]] name = "thiserror" -version = "2.0.7" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93605438cbd668185516ab499d589afb7ee1859ea3d5fc8f6b0755e1c7443767" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ - "thiserror-impl 2.0.7", + "thiserror-impl 2.0.11", ] [[package]] name = "thiserror-impl" -<<<<<<< HEAD -version = "1.0.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7c61ec9a6f64d2793d8a45faba21efbe3ced62a886d44c36a009b2b519b4c7e" -======= version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" @@ -4173,10 +3910,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.7" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d8749b4531af2117677a5fcd12b1348a3fe2b81e36e61ffeac5c4aa3273e36" ->>>>>>> upstream/branch-44 +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", @@ -4245,9 +3981,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -4260,9 +3996,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.42.0" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -4278,9 +4014,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", @@ -4303,7 +4039,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ - "rustls 0.23.20", + "rustls 0.23.22", "tokio", ] @@ -4328,15 +4064,36 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.22" +version = "0.22.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" dependencies = [ "indexmap", "toml_datetime", "winnow", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -4418,13 +4175,9 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-ident" -version = "1.0.14" +version = "1.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -<<<<<<< HEAD -checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" -======= -checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" ->>>>>>> upstream/branch-44 +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" [[package]] name = "unicode-segmentation" @@ -4452,15 +4205,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -<<<<<<< HEAD -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d157f1b96d14500ffdc1f10ba712e780825526c03d9a49b4d0324b0d9113ada" -======= version = "2.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" ->>>>>>> upstream/branch-44 dependencies = [ "form_urlencoded", "idna", @@ -4493,11 +4240,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.11.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" +checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" dependencies = [ - "getrandom", + "getrandom 0.3.1", "serde", ] @@ -4515,9 +4262,9 @@ checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" [[package]] name = "wait-timeout" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" dependencies = [ "libc", ] @@ -4547,22 +4294,32 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" dependencies = [ "cfg-if", "once_cell", + "rustversion", "wasm-bindgen-macro", ] [[package]] name = "wasm-bindgen-backend" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" dependencies = [ "bumpalo", "log", @@ -4574,9 +4331,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.49" +version = "0.4.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" +checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" dependencies = [ "cfg-if", "js-sys", @@ -4587,9 +4344,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4597,9 +4354,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", @@ -4610,9 +4367,12 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.99" +version = "0.2.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" +checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +dependencies = [ + "unicode-ident", +] [[package]] name = "wasm-streams" @@ -4629,9 +4389,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.76" +version = "0.3.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" +checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" dependencies = [ "js-sys", "wasm-bindgen", @@ -4653,7 +4413,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4845,13 +4605,22 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.20" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" @@ -4881,15 +4650,9 @@ dependencies = [ [[package]] name = "yoke" -<<<<<<< HEAD -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" -======= version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" ->>>>>>> upstream/branch-44 dependencies = [ "serde", "stable_deref_trait", @@ -4899,15 +4662,9 @@ dependencies = [ [[package]] name = "yoke-derive" -<<<<<<< HEAD -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" -======= version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" ->>>>>>> upstream/branch-44 dependencies = [ "proc-macro2", "quote", @@ -4938,30 +4695,18 @@ dependencies = [ [[package]] name = "zerofrom" -<<<<<<< HEAD -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" -======= version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" ->>>>>>> upstream/branch-44 dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -<<<<<<< HEAD -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" -======= version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" ->>>>>>> upstream/branch-44 dependencies = [ "proc-macro2", "quote", @@ -4978,31 +4723,6 @@ checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" [[package]] name = "zerovec" version = "0.10.4" -<<<<<<< HEAD -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zstd" -version = "0.12.4" -======= ->>>>>>> upstream/branch-44 source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" dependencies = [ diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 4f666b8d43db7..16006d3d36acd 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -18,11 +18,7 @@ [package] name = "datafusion-cli" description = "Command Line Client for DataFusion query engine." -<<<<<<< HEAD -version = "43.0.0" -======= version = "44.0.0" ->>>>>>> upstream/branch-44 authors = ["Apache DataFusion "] edition = "2021" keywords = ["arrow", "datafusion", "query", "sql"] @@ -34,7 +30,7 @@ rust-version = "1.80.1" readme = "README.md" [dependencies] -arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048" } +arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c" } async-trait = "0.1.73" aws-config = "1.5.5" aws-sdk-sso = "1.43.0" @@ -43,11 +39,7 @@ aws-sdk-sts = "1.43.0" # end pin aws-sdk crates aws-credential-types = "1.2.0" clap = { version = "4.5.16", features = ["derive", "cargo"] } -<<<<<<< HEAD -datafusion = { path = "../datafusion/core", version = "43.0.0", features = [ -======= datafusion = { path = "../datafusion/core", version = "44.0.0", features = [ ->>>>>>> upstream/branch-44 "avro", "crypto_expressions", "datetime_expressions", @@ -67,7 +59,7 @@ home = "=0.5.9" mimalloc = { version = "0.1", default-features = false } object_store = { version = "0.11.0", features = ["aws", "gcp", "http"] } parking_lot = { version = "0.12" } -parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "aa8c048", default-features = false } +parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "eae176c", default-features = false } regex = "1.8" rustyline = "14.0" tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] } diff --git a/datafusion/common/Cargo.toml b/datafusion/common/Cargo.toml index b331a55a98d01..9e71c40bc4756 100644 --- a/datafusion/common/Cargo.toml +++ b/datafusion/common/Cargo.toml @@ -62,7 +62,7 @@ log = { workspace = true } object_store = { workspace = true, optional = true } parquet = { workspace = true, optional = true, default-features = true } paste = "1.0.15" -pyo3 = { version = "0.22.0", optional = true } +pyo3 = { version = "0.23.0", optional = true } recursive = { workspace = true, optional = true } sqlparser = { workspace = true } tokio = { workspace = true } diff --git a/datafusion/core/src/physical_optimizer/enforce_distribution.rs b/datafusion/core/src/physical_optimizer/enforce_distribution.rs index 0652f088ab2db..5ffb1040b1ca7 100644 --- a/datafusion/core/src/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/src/physical_optimizer/enforce_distribution.rs @@ -943,7 +943,11 @@ fn add_spm_on_top( let new_plan = if should_preserve_ordering { Arc::new( SortPreservingMergeExec::new( - LexOrdering::from_ref(input.plan.output_ordering().unwrap_or(&[])), + input + .plan + .output_ordering() + .unwrap_or(&LexOrdering::default()) + .clone(), input.plan.clone(), ) .with_fetch(fetch.take()), @@ -1376,7 +1380,9 @@ fn ensure_distribution( if fetch.is_some() { plan = Arc::new( SortPreservingMergeExec::new( - LexOrdering::from_ref(plan.output_ordering().unwrap_or(&[])), + plan.output_ordering() + .unwrap_or(&LexOrdering::default()) + .clone(), plan.clone(), ) .with_fetch(fetch.take()), diff --git a/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs b/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs index 2f128d24e7ac0..52dcc2c75a50f 100644 --- a/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs +++ b/datafusion/core/src/physical_optimizer/replace_with_order_preserving_variants.rs @@ -139,9 +139,8 @@ fn plan_with_order_preserving_variants( if let Some(ordering) = child.output_ordering() { // When the input of a `CoalescePartitionsExec` has an ordering, // replace it with a `SortPreservingMergeExec` if appropriate: - let spm = - SortPreservingMergeExec::new(LexOrdering::new(ordering), child.clone()) - .with_fetch(fetch); + let spm = SortPreservingMergeExec::new(ordering.clone(), child.clone()) + .with_fetch(fetch); sort_input.plan = Arc::new(spm) as _; sort_input.children[0].data = true; return Ok(sort_input); diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index e559d724365b8..cb60f2390b9e7 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -731,8 +731,9 @@ impl PlanProperties { /// Overwrite node id with its new value. pub fn with_node_id(mut self, node_id: usize) -> Self { self.node_id = Some(node_id); + self } - + /// Overwrite boundedness with its new value. pub fn with_boundedness(mut self, boundedness: Boundedness) -> Self { self.boundedness = boundedness; diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index 30df7b2436653..c85a3664bd7f5 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -48,17 +48,10 @@ half = { workspace = true, default-features = true } itertools = { workspace = true } log = { workspace = true } object_store = { workspace = true } -<<<<<<< HEAD -postgres-protocol = { version = "0.6.4", optional = true } -postgres-types = { version = "0.2.4", optional = true } -rust_decimal = { version = "1.27.0" } -sqllogictest = "0.26.3" -======= postgres-protocol = { version = "0.6.7", optional = true } postgres-types = { version = "0.2.8", features = ["derive", "with-chrono-0_4"], optional = true } rust_decimal = { version = "1.36.0", features = ["tokio-pg"] } -sqllogictest = "0.23.0" ->>>>>>> upstream/branch-44 +sqllogictest = "0.26.3" sqlparser = { workspace = true } tempfile = { workspace = true } thiserror = "2.0.0" From e5431f1c456b00b5bf24b4ceaeca34bae40228a2 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 10 Feb 2025 16:04:43 +0800 Subject: [PATCH 037/177] make new_group_values public --- .../src/aggregates/group_values/mod.rs | 6 +- .../physical-plan/src/aggregates/mod.rs | 2 +- datafusion/sqllogictest/bin/sqllogictests.rs | 59 +++---------------- 3 files changed, 12 insertions(+), 55 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rs index e4a7eb049e9eb..db385900bc1ae 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs @@ -86,7 +86,7 @@ mod null_builder; /// Each distinct group in a hash aggregation is identified by a unique group id /// (usize) which is assigned by instances of this trait. Group ids are /// continuous without gaps, starting from 0. -pub(crate) trait GroupValues: Send { +pub trait GroupValues: Send { /// Calculates the group id for each input row of `cols`, assigning new /// group ids as necessary. /// @@ -121,7 +121,7 @@ pub(crate) trait GroupValues: Send { /// - If group by single column, and type of this column has /// the specific [`GroupValues`] implementation, such implementation /// will be chosen. -/// +/// /// - If group by multiple columns, and all column types have the specific /// [`GroupColumn`] implementations, [`GroupValuesColumn`] will be chosen. /// @@ -129,7 +129,7 @@ pub(crate) trait GroupValues: Send { /// /// [`GroupColumn`]: crate::aggregates::group_values::multi_group_by::GroupColumn /// -pub(crate) fn new_group_values( +pub fn new_group_values( schema: SchemaRef, group_ordering: &GroupOrdering, ) -> Result> { diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 43b52dab9fe2a..47bd640d61990 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -52,7 +52,7 @@ use datafusion_physical_expr::{ use itertools::Itertools; -pub(crate) mod group_values; +pub mod group_values; mod no_grouping; pub mod order; mod row_hash; diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index 7b46109157fc6..8739a208f2397 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -<<<<<<< HEAD use clap::Parser; use datafusion_common::instant::Instant; use datafusion_common::utils::get_available_parallelism; @@ -44,18 +43,12 @@ use crate::postgres_container::{ use std::ffi::OsStr; use std::path::{Path, PathBuf}; -======= ->>>>>>> upstream/branch-44 use clap::Parser; -use datafusion_common::utils::get_available_parallelism; use datafusion_sqllogictest::{DataFusion, TestContext}; use futures::stream::StreamExt; use itertools::Itertools; use log::info; use sqllogictest::strict_column_validator; -use std::ffi::OsStr; -use std::fs; -use std::path::{Path, PathBuf}; use datafusion_common::{exec_datafusion_err, exec_err, DataFusionError, Result}; use datafusion_common_runtime::SpawnedTask; @@ -124,21 +117,20 @@ async fn run_tests() -> Result<()> { let errors: Vec<_> = futures::stream::iter(read_test_files(&options)?) .map(|test_file| { SpawnedTask::spawn(async move { - let file_path = test_file.relative_path.clone(); - let start = datafusion::common::instant::Instant::now(); - match (options.postgres_runner, options.complete) { - (false, false) => run_test_file(test_file).await?, - (false, true) => run_complete_file(test_file).await?, - (true, false) => run_test_file_with_postgres(test_file).await?, - (true, true) => run_complete_file_with_postgres(test_file).await?, + println!("Running {:?}", test_file.relative_path); + if options.complete { + run_complete_file(test_file).await?; + } else if options.postgres_runner { + run_test_file_with_postgres(test_file).await?; + } else { + run_test_file(test_file).await?; } - println!("Executed {:?}. Took {:?}", file_path, start.elapsed()); Ok(()) as Result<()> }) .join() }) // run up to num_cpus streams in parallel - .buffer_unordered(get_available_parallelism()) + .buffer_unordered(num_cpus::get()) .flat_map(|result| { // Filter out any Ok() leaving only the DataFusionErrors futures::stream::iter(match result { @@ -251,41 +243,6 @@ async fn run_complete_file(test_file: TestFile) -> Result<()> { }) } -#[cfg(feature = "postgres")] -async fn run_complete_file_with_postgres(test_file: TestFile) -> Result<()> { - use datafusion_sqllogictest::Postgres; - let TestFile { - path, - relative_path, - } = test_file; - info!( - "Using complete mode to complete with Postgres runner: {}", - path.display() - ); - setup_scratch_dir(&relative_path)?; - let mut runner = - sqllogictest::Runner::new(|| Postgres::connect(relative_path.clone())); - let col_separator = " "; - runner - .update_test_file( - path, - col_separator, - value_validator, - strict_column_validator, - ) - .await - // Can't use e directly because it isn't marked Send, so turn it into a string. - .map_err(|e| { - DataFusionError::Execution(format!("Error completing {relative_path:?}: {e}")) - }) -} - -#[cfg(not(feature = "postgres"))] -async fn run_complete_file_with_postgres(_test_file: TestFile) -> Result<()> { - use datafusion_common::plan_err; - plan_err!("Can not run with postgres as postgres feature is not enabled") -} - /// Represents a parsed test file #[derive(Debug)] struct TestFile { From c103d08a8728c00dec6f3c5fbc56b153e639f9d8 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 12 Feb 2025 17:07:55 +0800 Subject: [PATCH 038/177] cherry-pick upstream/14569 --- .../core/src/physical_optimizer/enforce_sorting.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_sorting.rs b/datafusion/core/src/physical_optimizer/enforce_sorting.rs index ab2d00ddd5ea1..b5e2c48c6c312 100644 --- a/datafusion/core/src/physical_optimizer/enforce_sorting.rs +++ b/datafusion/core/src/physical_optimizer/enforce_sorting.rs @@ -372,11 +372,15 @@ fn ensure_sorting( return adjust_window_sort_removal(requirements).map(Transformed::yes); } else if is_sort_preserving_merge(plan) && child_node.plan.output_partitioning().partition_count() <= 1 - && plan.fetch().is_none() { // This `SortPreservingMergeExec` is unnecessary, input already has a - // single partition and no fetch is required: - let child_node = requirements.children.swap_remove(0); + // single partition and no fetch is required. + let mut child_node = requirements.children.swap_remove(0); + if let Some(fetch) = plan.fetch() { + // Add the limit exec if the spm has a fetch + child_node.plan = + Arc::new(LocalLimitExec::new(Arc::clone(&child_node.plan), fetch)); + } return Ok(Transformed::yes(child_node)); } From e9fb06263db2954b1ab4cd00e15714f9f68eb5f5 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 24 Feb 2025 11:25:42 +0800 Subject: [PATCH 039/177] fix EnforceDistribution --- .../physical_optimizer/enforce_distribution.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/datafusion/core/src/physical_optimizer/enforce_distribution.rs b/datafusion/core/src/physical_optimizer/enforce_distribution.rs index 5ffb1040b1ca7..a5eee4d268c82 100644 --- a/datafusion/core/src/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/src/physical_optimizer/enforce_distribution.rs @@ -1374,24 +1374,27 @@ fn ensure_distribution( plan.with_new_children(children_plans)? }; + let mut optimized_distribution_ctx = + DistributionContext::new(Arc::clone(&plan), data.clone(), children); + // If `fetch` was not consumed, it means that there was `SortPreservingMergeExec` with fetch before // It was removed by `remove_dist_changing_operators` // and we need to add it back. if fetch.is_some() { - plan = Arc::new( + let plan = Arc::new( SortPreservingMergeExec::new( plan.output_ordering() .unwrap_or(&LexOrdering::default()) .clone(), - plan.clone(), + plan, ) .with_fetch(fetch.take()), - ) + ); + optimized_distribution_ctx = + DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); } - Ok(Transformed::yes(DistributionContext::new( - plan, data, children, - ))) + Ok(Transformed::yes(optimized_distribution_ctx)) } /// Keeps track of distribution changing operators (like `RepartitionExec`, From ee7b6581d5c53a40dcde0fd8074d77a23ea30d97 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 24 Feb 2025 14:50:40 +0800 Subject: [PATCH 040/177] Merge remote-tracking branch 'upstream/branch-45'(with our fixes) --- datafusion-cli/Cargo.lock | 449 +++++------------- .../core/src/datasource/physical_plan/csv.rs | 3 +- .../src/enforce_distribution.rs | 297 +++++++----- .../physical-plan/src/coalesce_partitions.rs | 3 +- datafusion/physical-plan/src/filter.rs | 2 +- datafusion/physical-plan/src/insert.rs | 1 - .../physical-plan/src/joins/cross_join.rs | 3 +- .../physical-plan/src/joins/hash_join.rs | 3 +- .../src/joins/sort_merge_join.rs | 3 +- datafusion/physical-plan/src/memory.rs | 3 +- datafusion/physical-plan/src/projection.rs | 3 +- .../physical-plan/src/repartition/mod.rs | 3 +- datafusion/physical-plan/src/sorts/sort.rs | 3 +- .../src/sorts/sort_preserving_merge.rs | 3 +- datafusion/physical-plan/src/union.rs | 3 +- 15 files changed, 308 insertions(+), 474 deletions(-) diff --git a/datafusion-cli/Cargo.lock b/datafusion-cli/Cargo.lock index ebb1a398cccc3..4ec28878b09e2 100644 --- a/datafusion-cli/Cargo.lock +++ b/datafusion-cli/Cargo.lock @@ -175,14 +175,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6422e12ac345a0678d7a17e316238e3a40547ae7f92052b77bd86d5e0239f3fc" ->>>>>>> upstream/branch-45 +checksum = "755b6da235ac356a869393c23668c663720b8749dd6f15e52b6c214b4b964cc7" dependencies = [ "arrow-arith", "arrow-array", @@ -201,14 +196,9 @@ dependencies = [ [[package]] name = "arrow-arith" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cf34bb1f48c41d3475927bcc7be498665b8e80b379b88f62a840337f8b8248" ->>>>>>> upstream/branch-45 +checksum = "64656a1e0b13ca766f8440752e9a93e11014eec7b67909986f83ed0ab1fe37b8" dependencies = [ "arrow-array", "arrow-buffer", @@ -220,14 +210,9 @@ dependencies = [ [[package]] name = "arrow-array" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb4a06d507f54b70a277be22a127c8ffe0cec6cd98c0ad8a48e77779bbda8223" ->>>>>>> upstream/branch-45 +checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a" dependencies = [ "ahash", "arrow-buffer", @@ -242,14 +227,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69d326d5ad1cb82dcefa9ede3fee8fdca98f9982756b16f9cb142f4aa6edc89" ->>>>>>> upstream/branch-45 +checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a" dependencies = [ "bytes", "half", @@ -258,14 +238,9 @@ dependencies = [ [[package]] name = "arrow-cast" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626e65bd42636a84a238bed49d09c8777e3d825bf81f5087a70111c2831d9870" ->>>>>>> upstream/branch-45 +checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee" dependencies = [ "arrow-array", "arrow-buffer", @@ -284,14 +259,9 @@ dependencies = [ [[package]] name = "arrow-csv" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c8f959f7a1389b1dbd883cdcd37c3ed12475329c111912f7f69dad8195d8c6" ->>>>>>> upstream/branch-45 +checksum = "90f12542b8164398fc9ec595ff783c4cf6044daa89622c5a7201be920e4c0d4c" dependencies = [ "arrow-array", "arrow-cast", @@ -305,14 +275,9 @@ dependencies = [ [[package]] name = "arrow-data" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1858e7c7d01c44cf71c21a85534fd1a54501e8d60d1195d0d6fbcc00f4b10754" ->>>>>>> upstream/branch-45 +checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83" dependencies = [ "arrow-buffer", "arrow-schema", @@ -322,14 +287,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6bb3f727f049884c7603f0364bc9315363f356b59e9f605ea76541847e06a1e" ->>>>>>> upstream/branch-45 +checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,14 +301,9 @@ dependencies = [ [[package]] name = "arrow-json" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35de94f165ed8830aede72c35f238763794f0d49c69d30c44d49c9834267ff8c" ->>>>>>> upstream/branch-45 +checksum = "9551d9400532f23a370cabbea1dc5a53c49230397d41f96c4c8eedf306199305" dependencies = [ "arrow-array", "arrow-buffer", @@ -366,14 +321,9 @@ dependencies = [ [[package]] name = "arrow-ord" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8aa06e5f267dc53efbacb933485c79b6fc1685d3ffbe870a16ce4e696fb429da" ->>>>>>> upstream/branch-45 +checksum = "6c07223476f8219d1ace8cd8d85fa18c4ebd8d945013f25ef5c72e85085ca4ee" dependencies = [ "arrow-array", "arrow-buffer", @@ -384,14 +334,9 @@ dependencies = [ [[package]] name = "arrow-row" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f1144bb456a2f9d82677bd3abcea019217e572fc8f07de5a7bac4b2c56eb2c" ->>>>>>> upstream/branch-45 +checksum = "91b194b38bfd89feabc23e798238989c6648b2506ad639be42ec8eb1658d82c4" dependencies = [ "arrow-array", "arrow-buffer", @@ -402,25 +347,15 @@ dependencies = [ [[package]] name = "arrow-schema" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" - -[[package]] -name = "arrow-select" -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "105f01ec0090259e9a33a9263ec18ff223ab91a0ea9fbc18042f7e38005142f6" +checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735" [[package]] name = "arrow-select" -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f690752fdbd2dee278b5f1636fefad8f2f7134c85e20fd59c4199e15a39a6807" ->>>>>>> upstream/branch-45 +checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539" dependencies = [ "ahash", "arrow-array", @@ -432,14 +367,9 @@ dependencies = [ [[package]] name = "arrow-string" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fff9cd745a7039b66c47ecaf5954460f9fa12eed628f65170117ea93e64ee0" ->>>>>>> upstream/branch-45 +checksum = "d44c8eed43be4ead49128370f7131f054839d3d6003e52aebf64322470b8fbd0" dependencies = [ "arrow-array", "arrow-buffer", @@ -487,15 +417,9 @@ dependencies = [ [[package]] name = "async-trait" -<<<<<<< HEAD version = "0.1.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" -======= -version = "0.1.85" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" ->>>>>>> upstream/branch-45 dependencies = [ "proc-macro2", "quote", @@ -525,15 +449,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -<<<<<<< HEAD version = "1.5.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "50236e4d60fe8458de90a71c0922c761e41755adf091b1b03de1cef537179915" -======= -version = "1.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc47e70fc35d054c8fcd296d47a61711f043ac80534a10b4f741904f81e73a90" ->>>>>>> upstream/branch-45 dependencies = [ "aws-credential-types", "aws-runtime", @@ -573,15 +491,9 @@ dependencies = [ [[package]] name = "aws-runtime" -<<<<<<< HEAD version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad" -======= -version = "1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee7643696e7fdd74c10f9eb42848a87fe469d35eae9c3323f80aa98f350baac" ->>>>>>> upstream/branch-45 dependencies = [ "aws-credential-types", "aws-sigv4", @@ -604,15 +516,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -<<<<<<< HEAD -version = "1.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16ff718c9ee45cc1ebd4774a0e086bb80a6ab752b4902edf1c9f56b86ee1f770" -======= -version = "1.57.0" +version = "1.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c54bab121fe1881a74c338c5f723d1592bf3b53167f80268a1274f404e1acc38" ->>>>>>> upstream/branch-45 +checksum = "00a35fc7e74f5be45839eb753568535c074a592185dd0a2d406685018d581c43" dependencies = [ "aws-credential-types", "aws-runtime", @@ -632,15 +538,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -<<<<<<< HEAD -version = "1.59.0" +version = "1.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5183e088715cc135d8d396fdd3bc02f018f0da4c511f53cb8d795b6a31c55809" -======= -version = "1.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c8234fd024f7ac61c4e44ea008029bde934250f371efe7d4a39708397b1080c" ->>>>>>> upstream/branch-45 +checksum = "f8fa655b4f313124ce272cbc38c5fef13793c832279cec750103e5e6b71a54b8" dependencies = [ "aws-credential-types", "aws-runtime", @@ -660,15 +560,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -<<<<<<< HEAD -version = "1.59.0" +version = "1.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9f944ef032717596639cea4a2118a3a457268ef51bbb5fde9637e54c465da00" -======= -version = "1.58.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba60e1d519d6f23a9df712c04fdeadd7872ac911c84b2f62a8bda92e129b7962" ->>>>>>> upstream/branch-45 +checksum = "dc1cfe5e16b90421ea031f4c6348b534ef442e76f6bf4a1b2b592c12cc2c6af9" dependencies = [ "aws-credential-types", "aws-runtime", @@ -689,15 +583,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -<<<<<<< HEAD -version = "1.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bc5bbd1e4a2648fd8c5982af03935972c24a2f9846b396de661d351ee3ce837" -======= -version = "1.2.7" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "690118821e46967b3c4501d67d7d52dd75106a9c54cf36cefa1985cedbe94e05" ->>>>>>> upstream/branch-45 +checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -768,15 +656,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -<<<<<<< HEAD version = "1.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92" -======= -version = "1.7.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "865f7050bbc7107a6c98a397a9fcd9413690c27fa718446967cf03b2d3ac517e" ->>>>>>> upstream/branch-45 dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -818,15 +700,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -<<<<<<< HEAD version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" -======= -version = "1.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28f6feb647fb5e0d5b50f0472c19a7db9462b74e2fec01bb0b44eedcc834e97" ->>>>>>> upstream/branch-45 dependencies = [ "base64-simd", "bytes", @@ -859,15 +735,9 @@ dependencies = [ [[package]] name = "aws-types" -<<<<<<< HEAD version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" -======= -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0df5a18c4f951c645300d365fec53a61418bcf4650f604f85fe2a665bfaa0c2" ->>>>>>> upstream/branch-45 dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -951,15 +821,16 @@ dependencies = [ [[package]] name = "blake3" -version = "1.5.5" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8ee0c1824c4dea5b5f81736aff91bae041d2c07ee1192bec91054e10e3e601e" +checksum = "1230237285e3e10cde447185e8975408ae24deaa67205ce684805c25bc0c7937" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", + "memmap2", ] [[package]] @@ -1043,19 +914,18 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bafdbf26611df8c14810e268ddceda071c297570a5fb360ceddf617fe417ef58" +checksum = "75b89e7c29231c673a61a46e722602bcd138298f6b9e81e71119693534585f5c" dependencies = [ "bzip2-sys", - "libc", ] [[package]] name = "bzip2-sys" -version = "0.1.11+1.0.8" +version = "0.1.12+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +checksum = "72ebc2f1a417f01e1da30ef264ee86ae31d2dcd2d603ea283d3c244a883ca2a9" dependencies = [ "cc", "libc", @@ -1064,15 +934,9 @@ dependencies = [ [[package]] name = "cc" -<<<<<<< HEAD -version = "1.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7777341816418c02e033934a09f20dc0ccaf65a5201ef8a450ae0105a573fda" -======= -version = "1.2.10" +version = "1.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13208fcbb66eaeffe09b99fffbe1af420f00a7b35aa99ad683dfc1aa76145229" ->>>>>>> upstream/branch-45 +checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" dependencies = [ "jobserver", "libc", @@ -1127,15 +991,9 @@ dependencies = [ [[package]] name = "clap" -<<<<<<< HEAD -version = "4.5.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e77c3243bd94243c03672cb5154667347c457ca271254724f9f393aee1c05ff" -======= -version = "4.5.27" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "769b0145982b4b48713e01ec42d61614425f27b7058bda7180a3a41f30104796" ->>>>>>> upstream/branch-45 +checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" dependencies = [ "clap_builder", "clap_derive", @@ -1143,9 +1001,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.27" +version = "4.5.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b26884eb4b57140e4d2d93652abfa49498b938b3c9179f9fc487b0acc3edad7" +checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" dependencies = [ "anstream", "anstyle", @@ -1155,15 +1013,9 @@ dependencies = [ [[package]] name = "clap_derive" -<<<<<<< HEAD version = "4.5.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" -======= -version = "4.5.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" ->>>>>>> upstream/branch-45 dependencies = [ "heck", "proc-macro2", @@ -1198,14 +1050,8 @@ version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ -<<<<<<< HEAD "unicode-segmentation", - "unicode-width 0.2.0", -======= - "strum", - "strum_macros", "unicode-width", ->>>>>>> upstream/branch-45 ] [[package]] @@ -1323,9 +1169,9 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" dependencies = [ "memchr", ] @@ -1372,7 +1218,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2 0.5.0", + "bzip2 0.5.1", "chrono", "datafusion-catalog", "datafusion-common", @@ -1860,9 +1706,9 @@ checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" [[package]] name = "either" -version = "1.13.0" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" [[package]] name = "endian-type" @@ -1895,9 +1741,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" @@ -2109,11 +1955,7 @@ dependencies = [ "cfg-if", "libc", "wasi 0.13.3+wasi-0.2.2", -<<<<<<< HEAD - "windows-targets 0.52.6", -======= "windows-targets", ->>>>>>> upstream/branch-45 ] [[package]] @@ -2149,9 +1991,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccae279728d634d083c00f6099cb58f01cc99c145b84b8be2f6c74618d79922e" +checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" dependencies = [ "atomic-waker", "bytes", @@ -2330,7 +2172,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.7", + "h2 0.4.8", "http 1.2.0", "http-body 1.0.1", "httparse", @@ -2367,11 +2209,7 @@ dependencies = [ "http 1.2.0", "hyper 1.6.0", "hyper-util", -<<<<<<< HEAD - "rustls 0.23.22", -======= - "rustls 0.23.21", ->>>>>>> upstream/branch-45 + "rustls 0.23.23", "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", @@ -2703,9 +2541,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.169" +version = "0.2.170" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" +checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" [[package]] name = "libflate" @@ -2781,9 +2619,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.25" +version = "0.4.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +checksum = "30bde2b3dc3671ae49d8e2e9f044c7c005836e7a023ee57cffa25ab82764bb9e" [[package]] name = "lz4_flex" @@ -2821,6 +2659,15 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", +] + [[package]] name = "mimalloc" version = "0.1.43" @@ -2838,9 +2685,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8402cab7aefae129c6977bb0ff1b8fd9a04eb5b51efc50a70bea51cda0c7924" +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", ] @@ -2986,11 +2833,7 @@ dependencies = [ "futures", "humantime", "hyper 1.6.0", -<<<<<<< HEAD - "itertools", -======= "itertools 0.13.0", ->>>>>>> upstream/branch-45 "md-5", "parking_lot", "percent-encoding", @@ -3066,14 +2909,9 @@ dependencies = [ [[package]] name = "parquet" -<<<<<<< HEAD -version = "53.3.0" -source = "git+https://github.com/influxdata/arrow-rs?rev=eae176c#eae176c21b1ef915227294e8a8a201b6f266031a" -======= -version = "54.1.0" +version = "54.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a01a0efa30bbd601ae85b375c728efdb211ade54390281628a7b16708beb235" ->>>>>>> upstream/branch-45 +checksum = "761c44d824fe83106e0600d2510c07bf4159a4985bf0569b513ea4288dc1b4fb" dependencies = [ "ahash", "arrow-array", @@ -3258,9 +3096,9 @@ dependencies = [ [[package]] name = "psm" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" dependencies = [ "cc", ] @@ -3292,11 +3130,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", -<<<<<<< HEAD - "rustls 0.23.22", -======= - "rustls 0.23.21", ->>>>>>> upstream/branch-45 + "rustls 0.23.23", "socket2", "thiserror 2.0.11", "tokio", @@ -3314,11 +3148,7 @@ dependencies = [ "rand", "ring", "rustc-hash", -<<<<<<< HEAD - "rustls 0.23.22", -======= - "rustls 0.23.21", ->>>>>>> upstream/branch-45 + "rustls 0.23.23", "rustls-pki-types", "slab", "thiserror 2.0.11", @@ -3329,9 +3159,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" +checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" dependencies = [ "cfg_aliases", "libc", @@ -3412,9 +3242,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" dependencies = [ "bitflags 2.8.0", ] @@ -3481,7 +3311,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "h2 0.4.7", + "h2 0.4.8", "http 1.2.0", "http-body 1.0.1", "http-body-util", @@ -3496,11 +3326,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", -<<<<<<< HEAD - "rustls 0.23.22", -======= - "rustls 0.23.21", ->>>>>>> upstream/branch-45 + "rustls 0.23.23", "rustls-native-certs 0.8.1", "rustls-pemfile 2.2.0", "rustls-pki-types", @@ -3523,15 +3349,14 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.8" +version = "0.17.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" dependencies = [ "cc", "cfg-if", "getrandom 0.2.15", "libc", - "spin", "untrusted", "windows-sys 0.52.0", ] @@ -3620,15 +3445,9 @@ dependencies = [ [[package]] name = "rustls" -<<<<<<< HEAD -version = "0.23.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb9263ab4eb695e42321db096e3b8fbd715a59b154d5c88d82db2175b681ba7" -======= -version = "0.23.21" +version = "0.23.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" ->>>>>>> upstream/branch-45 +checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" dependencies = [ "once_cell", "ring", @@ -3828,9 +3647,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" dependencies = [ "serde_derive", ] @@ -3846,9 +3665,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.217" +version = "1.0.218" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" dependencies = [ "proc-macro2", "quote", @@ -3857,9 +3676,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.138" +version = "1.0.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d434192e7da787e94a6ea7e9670b26a036d0ca41e0b7efb2676dd32bae872949" +checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" dependencies = [ "itoa", "memchr", @@ -3906,11 +3725,6 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD -name = "siphasher" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -======= name = "simdutf8" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3920,7 +3734,6 @@ checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" name = "siphasher" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" ->>>>>>> upstream/branch-45 checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] @@ -3934,9 +3747,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.2" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" [[package]] name = "snafu" @@ -3975,12 +3788,6 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "spin" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - [[package]] name = "sqlparser" version = "0.53.0" @@ -4010,9 +3817,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +checksum = "d9156ebd5870ef293bfb43f91c7a74528d363ec0d424afe24160ed5a4343d08a" dependencies = [ "cc", "cfg-if", @@ -4060,15 +3867,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -<<<<<<< HEAD version = "2.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" -======= -version = "2.0.96" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" ->>>>>>> upstream/branch-45 dependencies = [ "proc-macro2", "quote", @@ -4097,9 +3898,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.16.0" +version = "3.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c246215d7d24f48ae091a2902398798e05d978b24315d6efbc00ede9a8bb91" +checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" dependencies = [ "cfg-if", "fastrand", @@ -4275,11 +4076,7 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ -<<<<<<< HEAD - "rustls 0.23.22", -======= - "rustls 0.23.21", ->>>>>>> upstream/branch-45 + "rustls 0.23.23", "tokio", ] @@ -4304,9 +4101,9 @@ checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" [[package]] name = "toml_edit" -version = "0.22.23" +version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a8b472d1a3d7c18e2d61a489aee3453fd9031c33e4f55bd533f4a7adca1bee" +checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ "indexmap", "toml_datetime", @@ -4409,15 +4206,15 @@ dependencies = [ [[package]] name = "typenum" -version = "1.17.0" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "unicode-ident" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" +checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" [[package]] name = "unicode-segmentation" @@ -4474,19 +4271,11 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -<<<<<<< HEAD -version = "1.13.1" +version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced87ca4be083373936a67f8de945faa23b6b42384bd5b64434850802c6dccd0" +checksum = "93d59ca99a559661b96bf898d8fce28ed87935fd2bea9f05983c1464dd6c71b1" dependencies = [ "getrandom 0.3.1", -======= -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3758f5e68192bb96cc8f9b7e2c2cfdabb435499a28499a42f8f984092adad4b" -dependencies = [ - "getrandom 0.2.15", ->>>>>>> upstream/branch-45 "serde", ] @@ -4781,15 +4570,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -<<<<<<< HEAD -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e376c75f4f43f44db463cf729e0d3acbf954d13e22c51e26e4c264b4ab545f" -======= -version = "0.6.26" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e90edd2ac1aa278a5c4599b1d89cf03074b610800f866d4026dc199d7929a28" ->>>>>>> upstream/branch-45 +checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" dependencies = [ "memchr", ] @@ -4926,9 +4709,9 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ "zstd-safe", ] diff --git a/datafusion/core/src/datasource/physical_plan/csv.rs b/datafusion/core/src/datasource/physical_plan/csv.rs index 0b2955c170edb..d0334dbc26ea8 100644 --- a/datafusion/core/src/datasource/physical_plan/csv.rs +++ b/datafusion/core/src/datasource/physical_plan/csv.rs @@ -502,7 +502,8 @@ impl ExecutionPlan for CsvExec { metrics: self.metrics.clone(), file_compression_type: self.file_compression_type, cache: new_cache, - })))} + }))) + } fn try_swapping_with_projection( &self, projection: &ProjectionExec, diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 176c1e69aba8c..2f18ee8ae3c98 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -937,7 +937,10 @@ fn add_hash_on_top( /// /// Updated node with an execution plan, where desired single /// distribution is satisfied by adding [`SortPreservingMergeExec`]. -fn add_spm_on_top(input: DistributionContext) -> DistributionContext { +fn add_spm_on_top( + input: DistributionContext, + fetch: &mut Option, +) -> DistributionContext { // Add SortPreservingMerge only when partition count is larger than 1. if input.plan.output_partitioning().partition_count() > 1 { // When there is an existing ordering, we preserve ordering @@ -949,14 +952,17 @@ fn add_spm_on_top(input: DistributionContext) -> DistributionContext { let should_preserve_ordering = input.plan.output_ordering().is_some(); let new_plan = if should_preserve_ordering { - Arc::new(SortPreservingMergeExec::new( - input - .plan - .output_ordering() - .unwrap_or(&LexOrdering::default()) - .clone(), - Arc::clone(&input.plan), - )) as _ + Arc::new( + SortPreservingMergeExec::new( + input + .plan + .output_ordering() + .unwrap_or(&LexOrdering::default()) + .clone(), + input.plan.clone(), + ) + .with_fetch(fetch.take()), + ) as _ } else { Arc::new(CoalescePartitionsExec::new(Arc::clone(&input.plan))) as _ }; @@ -986,18 +992,28 @@ fn add_spm_on_top(input: DistributionContext) -> DistributionContext { /// ``` fn remove_dist_changing_operators( mut distribution_context: DistributionContext, -) -> Result { +) -> Result<(DistributionContext, Option)> { + let mut fetch = None; while is_repartition(&distribution_context.plan) || is_coalesce_partitions(&distribution_context.plan) || is_sort_preserving_merge(&distribution_context.plan) { + if is_sort_preserving_merge(&distribution_context.plan) { + if let Some(child_fetch) = distribution_context.plan.fetch() { + if fetch.is_none() { + fetch = Some(child_fetch); + } else { + fetch = Some(fetch.unwrap().min(child_fetch)); + } + } + } // All of above operators have a single child. First child is only child. // Remove any distribution changing operators at the beginning: distribution_context = distribution_context.children.swap_remove(0); // Note that they will be re-inserted later on if necessary or helpful. } - Ok(distribution_context) + Ok((distribution_context, fetch)) } /// Updates the [`DistributionContext`] if preserving ordering while changing partitioning is not helpful or desirable. @@ -1020,23 +1036,25 @@ fn remove_dist_changing_operators( /// ``` fn replace_order_preserving_variants( mut context: DistributionContext, -) -> Result { - context.children = context - .children - .into_iter() - .map(|child| { - if child.data { - replace_order_preserving_variants(child) - } else { - Ok(child) - } - }) - .collect::>>()?; - +) -> Result<(DistributionContext, Option)> { + let mut children = vec![]; + let mut fetch = None; + for child in context.children.into_iter() { + if child.data { + let (child, inner_fetch) = replace_order_preserving_variants(child)?; + children.push(child); + fetch = inner_fetch; + } else { + children.push(child); + } + } + context.children = children; if is_sort_preserving_merge(&context.plan) { - let child_plan = Arc::clone(&context.children[0].plan); + // Keep the fetch value of the SortPreservingMerge operator, maybe it will be used later. + let fetch = context.plan.fetch(); + let child_plan = context.children[0].plan.clone(); context.plan = Arc::new(CoalescePartitionsExec::new(child_plan)); - return Ok(context); + return Ok((context, fetch)); } else if let Some(repartition) = context.plan.as_any().downcast_ref::() { @@ -1045,11 +1063,11 @@ fn replace_order_preserving_variants( Arc::clone(&context.children[0].plan), repartition.partitioning().clone(), )?); - return Ok(context); + return Ok((context, None)); } } - context.update_plan_from_children() + Ok((context.update_plan_from_children()?, fetch)) } /// A struct to keep track of repartition requirements for each child node. @@ -1186,11 +1204,14 @@ pub fn ensure_distribution( unbounded_and_pipeline_friendly || config.optimizer.prefer_existing_sort; // Remove unnecessary repartition from the physical plan if any - let DistributionContext { - mut plan, - data, - children, - } = remove_dist_changing_operators(dist_context)?; + let ( + DistributionContext { + mut plan, + data, + children, + }, + mut fetch, + ) = remove_dist_changing_operators(dist_context)?; if let Some(exec) = plan.as_any().downcast_ref::() { if let Some(updated_window) = get_best_fitting_window( @@ -1223,107 +1244,109 @@ pub fn ensure_distribution( plan.maintains_input_order(), repartition_status_flags.into_iter() ) - .map( - |( - mut child, - required_input_ordering, - maintains, - RepartitionRequirementStatus { - requirement, - roundrobin_beneficial, - roundrobin_beneficial_stats, - hash_necessary, - }, - )| { - let add_roundrobin = enable_round_robin - // Operator benefits from partitioning (e.g. filter): - && roundrobin_beneficial - && roundrobin_beneficial_stats - // Unless partitioning increases the partition count, it is not beneficial: - && child.plan.output_partitioning().partition_count() < target_partitions; - - // When `repartition_file_scans` is set, attempt to increase - // parallelism at the source. - if repartition_file_scans && roundrobin_beneficial_stats { - if let Some(new_child) = - child.plan.repartitioned(target_partitions, config)? - { - child.plan = new_child; + .map( + |( + mut child, + required_input_ordering, + maintains, + RepartitionRequirementStatus { + requirement, + roundrobin_beneficial, + roundrobin_beneficial_stats, + hash_necessary, + }, + )| { + let add_roundrobin = enable_round_robin + // Operator benefits from partitioning (e.g. filter): + && roundrobin_beneficial + && roundrobin_beneficial_stats + // Unless partitioning increases the partition count, it is not beneficial: + && child.plan.output_partitioning().partition_count() < target_partitions; + + // When `repartition_file_scans` is set, attempt to increase + // parallelism at the source. + if repartition_file_scans && roundrobin_beneficial_stats { + if let Some(new_child) = + child.plan.repartitioned(target_partitions, config)? + { + child.plan = new_child; + } } - } - // Satisfy the distribution requirement if it is unmet. - match &requirement { - Distribution::SinglePartition => { - child = add_spm_on_top(child); - } - Distribution::HashPartitioned(exprs) => { - if add_roundrobin { - // Add round-robin repartitioning on top of the operator - // to increase parallelism. - child = add_roundrobin_on_top(child, target_partitions)?; - } - // When inserting hash is necessary to satisfy hash requirement, insert hash repartition. - if hash_necessary { - child = - add_hash_on_top(child, exprs.to_vec(), target_partitions)?; + // Satisfy the distribution requirement if it is unmet. + match &requirement { + Distribution::SinglePartition => { + child = add_spm_on_top(child, &mut fetch); } - } - Distribution::UnspecifiedDistribution => { - if add_roundrobin { - // Add round-robin repartitioning on top of the operator - // to increase parallelism. - child = add_roundrobin_on_top(child, target_partitions)?; + Distribution::HashPartitioned(exprs) => { + if add_roundrobin { + // Add round-robin repartitioning on top of the operator + // to increase parallelism. + child = add_roundrobin_on_top(child, target_partitions)?; + } + // When inserting hash is necessary to satisfy hash requirement, insert hash repartition. + if hash_necessary { + child = + add_hash_on_top(child, exprs.to_vec(), target_partitions)?; + } } - } - }; - - // There is an ordering requirement of the operator: - if let Some(required_input_ordering) = required_input_ordering { - // Either: - // - Ordering requirement cannot be satisfied by preserving ordering through repartitions, or - // - using order preserving variant is not desirable. - let ordering_satisfied = child - .plan - .equivalence_properties() - .ordering_satisfy_requirement(&required_input_ordering); - if (!ordering_satisfied || !order_preserving_variants_desirable) - && child.data - { - child = replace_order_preserving_variants(child)?; - // If ordering requirements were satisfied before repartitioning, - // make sure ordering requirements are still satisfied after. - if ordering_satisfied { - // Make sure to satisfy ordering requirement: - child = add_sort_above_with_check( - child, - required_input_ordering.clone(), - None, - ); + Distribution::UnspecifiedDistribution => { + if add_roundrobin { + // Add round-robin repartitioning on top of the operator + // to increase parallelism. + child = add_roundrobin_on_top(child, target_partitions)?; + } } - } - // Stop tracking distribution changing operators - child.data = false; - } else { - // no ordering requirement - match requirement { - // Operator requires specific distribution. - Distribution::SinglePartition | Distribution::HashPartitioned(_) => { - // Since there is no ordering requirement, preserving ordering is pointless - child = replace_order_preserving_variants(child)?; + }; + + // There is an ordering requirement of the operator: + if let Some(required_input_ordering) = required_input_ordering { + // Either: + // - Ordering requirement cannot be satisfied by preserving ordering through repartitions, or + // - using order preserving variant is not desirable. + let ordering_satisfied = child + .plan + .equivalence_properties() + .ordering_satisfy_requirement(&required_input_ordering); + if (!ordering_satisfied || !order_preserving_variants_desirable) + && child.data + { + let (replaced_child, fetch) = + replace_order_preserving_variants(child)?; + child = replaced_child; + // If ordering requirements were satisfied before repartitioning, + // make sure ordering requirements are still satisfied after. + if ordering_satisfied { + // Make sure to satisfy ordering requirement: + child = add_sort_above_with_check( + child, + required_input_ordering.clone(), + fetch, + ); + } } - Distribution::UnspecifiedDistribution => { - // Since ordering is lost, trying to preserve ordering is pointless - if !maintains || plan.as_any().is::() { - child = replace_order_preserving_variants(child)?; + // Stop tracking distribution changing operators + child.data = false; + } else { + // no ordering requirement + match requirement { + // Operator requires specific distribution. + Distribution::SinglePartition | Distribution::HashPartitioned(_) => { + // Since there is no ordering requirement, preserving ordering is pointless + child = replace_order_preserving_variants(child)?.0; + } + Distribution::UnspecifiedDistribution => { + // Since ordering is lost, trying to preserve ordering is pointless + if !maintains || plan.as_any().is::() { + child = replace_order_preserving_variants(child)?.0; + } } } } - } - Ok(child) - }, - ) - .collect::>>()?; + Ok(child) + }, + ) + .collect::>>()?; let children_plans = children .iter() @@ -1362,9 +1385,27 @@ pub fn ensure_distribution( plan.with_new_children(children_plans)? }; - Ok(Transformed::yes(DistributionContext::new( - plan, data, children, - ))) + let mut optimized_distribution_ctx = + DistributionContext::new(Arc::clone(&plan), data.clone(), children); + + // If `fetch` was not consumed, it means that there was `SortPreservingMergeExec` with fetch before + // It was removed by `remove_dist_changing_operators` + // and we need to add it back. + if fetch.is_some() { + let plan = Arc::new( + SortPreservingMergeExec::new( + plan.output_ordering() + .unwrap_or(&LexOrdering::default()) + .clone(), + plan, + ) + .with_fetch(fetch.take()), + ); + optimized_distribution_ctx = + DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); + } + + Ok(Transformed::yes(optimized_distribution_ctx)) } /// Keeps track of distribution changing operators (like `RepartitionExec`, diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index d8d82a2d0187b..785834a2bd100 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -191,7 +191,8 @@ impl ExecutionPlan for CoalescePartitionsExec { let mut new_plan = CoalescePartitionsExec::new(self.input.clone()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan)))} + Ok(Some(Arc::new(new_plan))) + } /// Tries to swap `projection` with its input, which is known to be a /// [`CoalescePartitionsExec`]. If possible, performs the swap and returns diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 2011e43a7cde2..69afd77d4e3fb 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -415,7 +415,7 @@ impl ExecutionPlan for FilterExec { new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } - + /// Tries to swap `projection` with its input (`filter`). If possible, performs /// the swap and returns [`FilterExec`] as the top plan. Otherwise, returns `None`. fn try_swapping_with_projection( diff --git a/datafusion/physical-plan/src/insert.rs b/datafusion/physical-plan/src/insert.rs index 449b7337e4bd2..3e149b6b0835f 100644 --- a/datafusion/physical-plan/src/insert.rs +++ b/datafusion/physical-plan/src/insert.rs @@ -257,7 +257,6 @@ impl ExecutionPlan for DataSinkExec { let mut new_plan = DataSinkExec::new( self.input.clone(), self.sink.clone(), - self.sink_schema.clone(), self.sort_order.clone(), ); let new_props = new_plan.cache.clone().with_node_id(_node_id); diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 3235b3e9c8786..3196f0510c3a9 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -348,7 +348,8 @@ impl ExecutionPlan for CrossJoinExec { let mut new_plan = CrossJoinExec::new(self.left.clone(), self.right.clone()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } /// Tries to swap the projection with its input [`CrossJoinExec`]. If it can be done, /// it returns the new swapped version having the [`CrossJoinExec`] as the top plan. /// Otherwise, it returns None. diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 7749d0efa88b8..325590b339244 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -885,7 +885,8 @@ impl ExecutionPlan for HashJoinExec { )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } /// Tries to push `projection` down through `hash_join`. If possible, performs the /// pushdown and returns a new [`HashJoinExec`] as the top plan which has projections /// as its children. Otherwise, returns `None`. diff --git a/datafusion/physical-plan/src/joins/sort_merge_join.rs b/datafusion/physical-plan/src/joins/sort_merge_join.rs index d82ccb5181621..94ed9f2c62256 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join.rs @@ -524,7 +524,8 @@ impl ExecutionPlan for SortMergeJoinExec { )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } /// Tries to swap the projection with its input [`SortMergeJoinExec`]. If it can be done, /// it returns the new swapped version having the [`SortMergeJoinExec`] as the top plan. /// Otherwise, it returns None. diff --git a/datafusion/physical-plan/src/memory.rs b/datafusion/physical-plan/src/memory.rs index e18c3e4e0521d..262d1f7737a89 100644 --- a/datafusion/physical-plan/src/memory.rs +++ b/datafusion/physical-plan/src/memory.rs @@ -185,7 +185,8 @@ impl ExecutionPlan for MemoryExec { }; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan)))} + Ok(Some(Arc::new(new_plan))) + } fn try_swapping_with_projection( &self, projection: &ProjectionExec, diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index 5a5a1d33a81a5..9e85983b43787 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -253,7 +253,8 @@ impl ExecutionPlan for ProjectionExec { ProjectionExec::try_new(self.expr.clone(), self.input.clone())?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } fn try_swapping_with_projection( &self, diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 1a2646bb3fac0..4e90fbf696347 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -686,7 +686,8 @@ impl ExecutionPlan for RepartitionExec { }; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } fn try_swapping_with_projection( &self, diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index cdc9fc0dcc355..38ad037ea6d0d 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1037,7 +1037,8 @@ impl ExecutionPlan for SortExec { preserve_partitioning: self.preserve_partitioning, cache: self.cache.clone().with_node_id(_node_id), }; - Ok(Some(Arc::new(new_plan)))} + Ok(Some(Arc::new(new_plan))) + } /// Tries to swap the projection with its input [`SortExec`]. If it can be done, /// it returns the new swapped version having the [`SortExec`] as the top plan. diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index 5176d4adff537..c60949bd37b08 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -346,7 +346,8 @@ impl ExecutionPlan for SortPreservingMergeExec { .with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } /// Tries to swap the projection with its input [`SortPreservingMergeExec`]. /// If this is possible, it returns the new [`SortPreservingMergeExec`] whose /// child is a projection. Otherwise, it returns None. diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 84d2e3f054309..8464c02197fa3 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -280,7 +280,8 @@ impl ExecutionPlan for UnionExec { let mut new_plan = UnionExec::new(self.inputs.clone()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } /// Tries to push `projection` down through `union`. If possible, performs the /// pushdown and returns a new [`UnionExec`] as the top plan which has projections /// as its children. Otherwise, returns `None`. From 3766da9e4e7ebd8fbeb507a1deed5df0c99cdaac Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 24 Feb 2025 15:12:02 +0800 Subject: [PATCH 041/177] downgrade tonic --- Cargo.toml | 16 ++++++++-------- datafusion-cli/Cargo.toml | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 755ef751629fc..5b1e275d25eb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,21 +77,21 @@ version = "45.0.0" ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } -arrow = { version = "54.1.0", features = [ +arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", features = [ "prettyprint", ] } -arrow-array = { version = "54.1.0", default-features = false, features = [ +arrow-array = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false, features = [ "chrono-tz", ] } -arrow-buffer = { version = "54.1.0", default-features = false } -arrow-flight = { version = "54.1.0", features = [ +arrow-buffer = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } +arrow-flight = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "54.1.0", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "54.1.0", default-features = false } -arrow-schema = { version = "54.1.0", default-features = false } +arrow-ord = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } +arrow-schema = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } async-trait = "0.1.73" bigdecimal = "0.4.7" bytes = "1.4" @@ -133,7 +133,7 @@ itertools = "0.14" log = "^0.4" object_store = { version = "0.11.0", default-features = false } parking_lot = "0.12" -parquet = { version = "54.1.0", default-features = false, features = [ +parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 7daa32562173b..5d92c844df1df 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -29,7 +29,7 @@ rust-version = "1.81.0" readme = "README.md" [dependencies] -arrow = { version = "54.1.0" } +arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0" } async-trait = "0.1.0" aws-config = "1.5.0" aws-credential-types = "1.2.0" @@ -57,7 +57,7 @@ home = "=0.5.11" mimalloc = { version = "0.1", default-features = false } object_store = { version = "0.11.0", features = ["aws", "gcp", "http"] } parking_lot = { version = "0.12" } -parquet = { version = "54.1.0", default-features = false } +parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } regex = "1.8" rustyline = "15.0" tokio = { version = "1.24", features = ["macros", "rt", "rt-multi-thread", "sync", "parking_lot", "signal"] } From 2b5cec270fb29c04ac47785e517add3da3d574b0 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 24 Feb 2025 17:15:44 +0800 Subject: [PATCH 042/177] cherry-pick upstream/14569 --- datafusion/physical-optimizer/src/enforce_sorting/mod.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs index 2d23894d6b5e7..a4ae5501fc9c8 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs @@ -375,8 +375,13 @@ pub fn ensure_sorting( && child_node.plan.output_partitioning().partition_count() <= 1 { // This `SortPreservingMergeExec` is unnecessary, input already has a - // single partition. - let child_node = requirements.children.swap_remove(0); + // single partition and no fetch is required. + let mut child_node = requirements.children.swap_remove(0); + if let Some(fetch) = plan.fetch() { + // Add the limit exec if the spm has a fetch + child_node.plan = + Arc::new(LocalLimitExec::new(Arc::clone(&child_node.plan), fetch)); + } return Ok(Transformed::yes(child_node)); } From 08b3ce0a1656be99547624d1e6e171959cdc66d5 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 28 Feb 2025 15:48:51 +0800 Subject: [PATCH 043/177] public more parquet components --- .../core/src/datasource/physical_plan/parquet/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs index 8162bdee8194e..3c6f4a342d5b9 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs @@ -32,10 +32,7 @@ use std::sync::Arc; use crate::datasource::listing::PartitionedFile; use crate::datasource::physical_plan::file_stream::FileStream; -use crate::datasource::physical_plan::{ - parquet::page_filter::PagePruningAccessPlanFilter, DisplayAs, FileGroupPartitioner, - FileScanConfig, -}; +use crate::datasource::physical_plan::{DisplayAs, FileGroupPartitioner, FileScanConfig}; use crate::datasource::schema_adapter::{ DefaultSchemaAdapterFactory, SchemaAdapterFactory, }; @@ -58,8 +55,11 @@ use datafusion_physical_optimizer::pruning::PruningPredicate; use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; pub use metrics::ParquetFileMetrics; use opener::ParquetOpener; +pub use page_filter::PagePruningAccessPlanFilter; pub use reader::{DefaultParquetFileReaderFactory, ParquetFileReaderFactory}; +pub use row_filter::build_row_filter; pub use row_filter::can_expr_be_pushed_down_with_schemas; +pub use row_group_filter::RowGroupAccessPlanFilter; pub use writer::plan_to_parquet; use itertools::Itertools; From 8b3cd7b7b51fc1424fe4939af1b0987c68108c1c Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 2 Mar 2025 08:23:36 -0500 Subject: [PATCH 044/177] Do not swap with projection when file is partitioned (#14956) (#14964) * Do not swap with projection when file is partitioned * Narrow the case when not swapping * Add test Co-authored-by: Dmitrii Blaginin --- .../physical_optimizer/projection_pushdown.rs | 51 +++++++++++++++++++ datafusion/datasource/src/file_scan_config.rs | 43 ++++++++++------ 2 files changed, 78 insertions(+), 16 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs index 836758b21318e..77c32f5623556 100644 --- a/datafusion/core/tests/physical_optimizer/projection_pushdown.rs +++ b/datafusion/core/tests/physical_optimizer/projection_pushdown.rs @@ -1382,3 +1382,54 @@ fn test_union_after_projection() -> Result<()> { Ok(()) } + +#[test] +fn test_partition_col_projection_pushdown() -> Result<()> { + let file_schema = Arc::new(Schema::new(vec![ + Field::new("int_col", DataType::Int32, true), + Field::new("string_col", DataType::Utf8, true), + ])); + + let partitioned_schema = Arc::new(Schema::new(vec![ + Field::new("int_col", DataType::Int32, true), + Field::new("string_col", DataType::Utf8, true), + Field::new("partition_col", DataType::Utf8, true), + ])); + + let source = FileScanConfig::new( + ObjectStoreUrl::parse("test:///").unwrap(), + file_schema.clone(), + Arc::new(CsvSource::default()), + ) + .with_file(PartitionedFile::new("x".to_string(), 100)) + .with_table_partition_cols(vec![Field::new("partition_col", DataType::Utf8, true)]) + .with_projection(Some(vec![0, 1, 2])) + .build(); + + let projection = Arc::new(ProjectionExec::try_new( + vec![ + ( + col("string_col", partitioned_schema.as_ref())?, + "string_col".to_string(), + ), + ( + col("partition_col", partitioned_schema.as_ref())?, + "partition_col".to_string(), + ), + ( + col("int_col", partitioned_schema.as_ref())?, + "int_col".to_string(), + ), + ], + source, + )?); + + let after_optimize = + ProjectionPushdown::new().optimize(projection, &ConfigOptions::new())?; + + let expected = ["ProjectionExec: expr=[string_col@1 as string_col, partition_col@2 as partition_col, int_col@0 as int_col]", + " DataSourceExec: file_groups={1 group: [[x]]}, projection=[int_col, string_col, partition_col], file_type=csv, has_header=false"]; + assert_eq!(get_plan_string(&after_optimize), expected); + + Ok(()) +} diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs index 79279b5c82311..bee74e042f220 100644 --- a/datafusion/datasource/src/file_scan_config.rs +++ b/datafusion/datasource/src/file_scan_config.rs @@ -266,22 +266,33 @@ impl DataSource for FileScanConfig { ) -> Result>> { // If there is any non-column or alias-carrier expression, Projection should not be removed. // This process can be moved into CsvExec, but it would be an overlap of their responsibility. - Ok(all_alias_free_columns(projection.expr()).then(|| { - let file_scan = self.clone(); - let source = Arc::clone(&file_scan.file_source); - let new_projections = new_projections_for_columns( - projection, - &file_scan - .projection - .clone() - .unwrap_or((0..self.file_schema.fields().len()).collect()), - ); - file_scan - // Assign projected statistics to source - .with_projection(Some(new_projections)) - .with_source(source) - .build() as _ - })) + + let partitioned_columns_in_proj = projection.expr().iter().any(|(expr, _)| { + expr.as_any() + .downcast_ref::() + .map(|expr| expr.index() >= self.file_schema.fields().len()) + .unwrap_or(false) + }); + + Ok( + (all_alias_free_columns(projection.expr()) && !partitioned_columns_in_proj) + .then(|| { + let file_scan = self.clone(); + let source = Arc::clone(&file_scan.file_source); + let new_projections = new_projections_for_columns( + projection, + &file_scan + .projection + .clone() + .unwrap_or((0..self.file_schema.fields().len()).collect()), + ); + file_scan + // Assign projected statistics to source + .with_projection(Some(new_projections)) + .with_source(source) + .build() as _ + }), + ) } } From 76d833ac215053e102424617f754946ea198388f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 2 Mar 2025 08:23:39 -0500 Subject: [PATCH 045/177] Improve documentation for `DataSourceExec`, `FileScanConfig`, `DataSource` etc (#14941) (#14965) --- .../core/src/datasource/listing/table.rs | 13 ++++++++--- datafusion/core/src/lib.rs | 10 ++++---- datafusion/datasource/src/file.rs | 4 ++-- datafusion/datasource/src/source.rs | 23 +++++++++++++++++-- 4 files changed, 38 insertions(+), 12 deletions(-) diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 41e939d60b086..a983f0696e83b 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -616,6 +616,7 @@ impl ListingOptions { /// using an [`ObjectStore`] instance, for example from local files or objects /// from AWS S3. /// +/// # Reading Directories /// For example, given the `table1` directory (or object store prefix) /// /// ```text @@ -651,13 +652,19 @@ impl ListingOptions { /// If the query has a predicate like `WHERE date = '2024-06-01'` /// only the corresponding directory will be read. /// -/// `ListingTable` also supports filter and projection pushdown for formats that +/// `ListingTable` also supports limit, filter and projection pushdown for formats that /// support it as such as Parquet. /// +/// # Implementation +/// +/// `ListingTable` Uses [`DataSourceExec`] to execute the data. See that struct +/// for more details. +/// +/// [`DataSourceExec`]: crate::datasource::source::DataSourceExec +/// /// # Example /// -/// Here is an example of reading a directory of parquet files using a -/// [`ListingTable`]: +/// To read a directory of parquet files using a [`ListingTable`]: /// /// ```no_run /// # use datafusion::prelude::SessionContext; diff --git a/datafusion/core/src/lib.rs b/datafusion/core/src/lib.rs index 9a0d0157c1ae2..dfd171082f021 100644 --- a/datafusion/core/src/lib.rs +++ b/datafusion/core/src/lib.rs @@ -298,10 +298,10 @@ //! (built in or user provided) ExecutionPlan //! ``` //! -//! DataFusion includes several built in data sources for common use -//! cases, and can be extended by implementing the [`TableProvider`] -//! trait. A [`TableProvider`] provides information for planning and -//! an [`ExecutionPlan`]s for execution. +//! A [`TableProvider`] provides information for planning and +//! an [`ExecutionPlan`]s for execution. DataFusion includes [`ListingTable`] +//! which supports reading several common file formats, and you can support any +//! new file format by implementing the [`TableProvider`] trait. See also: //! //! 1. [`ListingTable`]: Reads data from Parquet, JSON, CSV, or AVRO //! files. Supports single files or multiple files with HIVE style @@ -314,7 +314,7 @@ //! //! [`ListingTable`]: crate::datasource::listing::ListingTable //! [`MemTable`]: crate::datasource::memory::MemTable -//! [`StreamingTable`]: datafusion_catalog::streaming::StreamingTable +//! [`StreamingTable`]: crate::catalog::streaming::StreamingTable //! //! ## Plan Representations //! diff --git a/datafusion/datasource/src/file.rs b/datafusion/datasource/src/file.rs index 8d8cbbc67b9aa..0066f39801a1b 100644 --- a/datafusion/datasource/src/file.rs +++ b/datafusion/datasource/src/file.rs @@ -33,9 +33,9 @@ use datafusion_physical_plan::DisplayFormatType; use object_store::ObjectStore; -/// Common behaviors that every file format needs to implement. +/// Common file format behaviors needs to implement. /// -/// See initialization examples on `ParquetSource`, `CsvSource` +/// See implementation examples such as `ParquetSource`, `CsvSource` pub trait FileSource: Send + Sync { /// Creates a `dyn FileOpener` based on given parameters fn create_file_opener( diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index b3089a6e59fef..07cee7fba00ee 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -15,6 +15,8 @@ // specific language governing permissions and limitations // under the License. +//! [`DataSource`] and [`DataSourceExec`] + use std::any::Any; use std::fmt; use std::fmt::{Debug, Formatter}; @@ -34,9 +36,15 @@ use datafusion_physical_expr::{EquivalenceProperties, Partitioning}; use datafusion_physical_expr_common::sort_expr::LexOrdering; /// Common behaviors in Data Sources for both from Files and Memory. -/// See `DataSourceExec` for physical plan implementation /// +/// # See Also +/// * [`DataSourceExec`] for physical plan implementation +/// * [`FileSource`] for file format implementations (Parquet, Json, etc) +/// +/// # Notes /// Requires `Debug` to assist debugging +/// +/// [`FileSource`]: crate::file::FileSource pub trait DataSource: Send + Sync + Debug { fn open( &self, @@ -71,10 +79,21 @@ pub trait DataSource: Send + Sync + Debug { ) -> datafusion_common::Result>>; } -/// Unified data source for file formats like JSON, CSV, AVRO, ARROW, PARQUET +/// [`ExecutionPlan`] handles different file formats like JSON, CSV, AVRO, ARROW, PARQUET +/// +/// `DataSourceExec` implements common functionality such as applying projections, +/// and caching plan properties. +/// +/// The [`DataSource`] trait describes where to find the data for this data +/// source (for example what files or what in memory partitions). Format +/// specifics are implemented with the [`FileSource`] trait. +/// +/// [`FileSource`]: crate::file::FileSource #[derive(Clone, Debug)] pub struct DataSourceExec { + /// The source of the data -- for example, `FileScanConfig` or `MemorySourceConfig` data_source: Arc, + /// Cached plan properties such as sort order cache: PlanProperties, } From b494e975e0fb4ba2c004a95dced4f9291db89d71 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Mon, 3 Mar 2025 18:59:46 +0800 Subject: [PATCH 046/177] Deprecate `Expr::Wildcard` (#14959) (#14976) * Deprecate `Expr::Wildcard` * Update * Update --------- Co-authored-by: Heran Lin Co-authored-by: Andrew Lamb --- datafusion/catalog-listing/src/helpers.rs | 2 ++ datafusion/expr/src/expr.rs | 11 +++++++++++ datafusion/expr/src/expr_fn.rs | 4 ++++ datafusion/expr/src/expr_rewriter/mod.rs | 1 + datafusion/expr/src/expr_schema.rs | 2 ++ datafusion/expr/src/logical_plan/builder.rs | 1 + datafusion/expr/src/logical_plan/plan.rs | 2 ++ datafusion/expr/src/tree_node.rs | 4 ++++ datafusion/expr/src/utils.rs | 5 +++++ datafusion/functions-aggregate/src/planner.rs | 2 ++ datafusion/functions-window/src/planner.rs | 2 ++ .../optimizer/src/analyzer/expand_wildcard_rule.rs | 1 + datafusion/optimizer/src/analyzer/type_coercion.rs | 3 +++ datafusion/optimizer/src/common_subexpr_eliminate.rs | 2 ++ datafusion/optimizer/src/push_down_filter.rs | 2 ++ .../src/simplify_expressions/expr_simplifier.rs | 2 ++ datafusion/proto/src/logical_plan/from_proto.rs | 1 + datafusion/proto/src/logical_plan/to_proto.rs | 1 + datafusion/sql/src/expr/mod.rs | 2 ++ datafusion/sql/src/unparser/expr.rs | 3 +++ datafusion/sql/src/utils.rs | 2 ++ datafusion/substrait/src/logical_plan/producer.rs | 1 + 22 files changed, 56 insertions(+) diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rs index cf475263535a3..3023dc09184ec 100644 --- a/datafusion/catalog-listing/src/helpers.rs +++ b/datafusion/catalog-listing/src/helpers.rs @@ -103,6 +103,8 @@ pub fn expr_applicable_for_cols(col_names: &[&str], expr: &Expr) -> bool { // - AGGREGATE and WINDOW should not end up in filter conditions, except maybe in some edge cases // - Can `Wildcard` be considered as a `Literal`? // - ScalarVariable could be `applicable`, but that would require access to the context + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::AggregateFunction { .. } | Expr::WindowFunction { .. } | Expr::Wildcard { .. } diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index f8baf9c94b3cf..3323ea1614fd9 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -311,6 +311,10 @@ pub enum Expr { /// /// This expr has to be resolved to a list of columns before translating logical /// plan into physical plan. + #[deprecated( + since = "46.0.0", + note = "A wildcard needs to be resolved to concrete expressions when constructing the logical plan. See https://github.com/apache/datafusion/issues/7765" + )] Wildcard { qualifier: Option, options: Box, @@ -1175,6 +1179,7 @@ impl Expr { Expr::ScalarVariable(..) => "ScalarVariable", Expr::TryCast { .. } => "TryCast", Expr::WindowFunction { .. } => "WindowFunction", + #[expect(deprecated)] Expr::Wildcard { .. } => "Wildcard", Expr::Unnest { .. } => "Unnest", } @@ -1648,6 +1653,8 @@ impl Expr { // Use explicit pattern match instead of a default // implementation, so that in the future if someone adds // new Expr types, they will check here as well + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::AggregateFunction(..) | Expr::Alias(..) | Expr::Between(..) @@ -2229,6 +2236,7 @@ impl HashNode for Expr { Expr::ScalarSubquery(subquery) => { subquery.hash(state); } + #[expect(deprecated)] Expr::Wildcard { qualifier, options } => { qualifier.hash(state); options.hash(state); @@ -2288,6 +2296,8 @@ impl Display for SchemaDisplay<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self.0 { // The same as Display + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::Column(_) | Expr::Literal(_) | Expr::ScalarVariable(..) @@ -2758,6 +2768,7 @@ impl Display for Expr { write!(f, "{expr} IN ([{}])", expr_vec_fmt!(list)) } } + #[expect(deprecated)] Expr::Wildcard { qualifier, options } => match qualifier { Some(qualifier) => write!(f, "{qualifier}.*{options}"), None => write!(f, "*{options}"), diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index f47de4a8178fb..a8e7fd76d037c 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -121,6 +121,7 @@ pub fn placeholder(id: impl Into) -> Expr { /// assert_eq!(p.to_string(), "*") /// ``` pub fn wildcard() -> Expr { + #[expect(deprecated)] Expr::Wildcard { qualifier: None, options: Box::new(WildcardOptions::default()), @@ -129,6 +130,7 @@ pub fn wildcard() -> Expr { /// Create an '*' [`Expr::Wildcard`] expression with the wildcard options pub fn wildcard_with_options(options: WildcardOptions) -> Expr { + #[expect(deprecated)] Expr::Wildcard { qualifier: None, options: Box::new(options), @@ -146,6 +148,7 @@ pub fn wildcard_with_options(options: WildcardOptions) -> Expr { /// assert_eq!(p.to_string(), "t.*") /// ``` pub fn qualified_wildcard(qualifier: impl Into) -> Expr { + #[expect(deprecated)] Expr::Wildcard { qualifier: Some(qualifier.into()), options: Box::new(WildcardOptions::default()), @@ -157,6 +160,7 @@ pub fn qualified_wildcard_with_options( qualifier: impl Into, options: WildcardOptions, ) -> Expr { + #[expect(deprecated)] Expr::Wildcard { qualifier: Some(qualifier.into()), options: Box::new(options), diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rs index c6739c1f48a41..90dcbce46b017 100644 --- a/datafusion/expr/src/expr_rewriter/mod.rs +++ b/datafusion/expr/src/expr_rewriter/mod.rs @@ -253,6 +253,7 @@ fn coerce_exprs_for_schema( Expr::Alias(Alias { expr, name, .. }) => { Ok(expr.cast_to(new_type, src_schema)?.alias(name)) } + #[expect(deprecated)] Expr::Wildcard { .. } => Ok(expr), _ => expr.cast_to(new_type, src_schema), } diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index ce1dd2f34c059..0a14cb5c60a0f 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -215,6 +215,7 @@ impl ExprSchemable for Expr { Ok(DataType::Null) } } + #[expect(deprecated)] Expr::Wildcard { .. } => Ok(DataType::Null), Expr::GroupingSet(_) => { // Grouping sets do not really have a type and do not appear in projections @@ -329,6 +330,7 @@ impl ExprSchemable for Expr { | Expr::SimilarTo(Like { expr, pattern, .. }) => { Ok(expr.nullable(input_schema)? || pattern.nullable(input_schema)?) } + #[expect(deprecated)] Expr::Wildcard { .. } => Ok(false), Expr::GroupingSet(_) => { // Grouping sets do not really have the concept of nullable and do not appear diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index 2bb15da21863e..f60bb2f007714 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -1675,6 +1675,7 @@ fn project_with_validation( for (e, validate) in expr { let e = e.into(); match e { + #[expect(deprecated)] Expr::Wildcard { .. } => projected_expr.push(e), _ => { if validate { diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index c6fd95595233e..72b82fc219eb6 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -2141,6 +2141,7 @@ impl Projection { input: Arc, schema: DFSchemaRef, ) -> Result { + #[expect(deprecated)] if !expr.iter().any(|e| matches!(e, Expr::Wildcard { .. })) && expr.len() != schema.fields().len() { @@ -3451,6 +3452,7 @@ fn calc_func_dependencies_for_project( let proj_indices = exprs .iter() .map(|expr| match expr { + #[expect(deprecated)] Expr::Wildcard { qualifier, options } => { let wildcard_fields = exprlist_to_fields( vec![&Expr::Wildcard { diff --git a/datafusion/expr/src/tree_node.rs b/datafusion/expr/src/tree_node.rs index 50af620603469..49cc79c60a271 100644 --- a/datafusion/expr/src/tree_node.rs +++ b/datafusion/expr/src/tree_node.rs @@ -67,6 +67,8 @@ impl TreeNode for Expr { Expr::GroupingSet(GroupingSet::GroupingSets(lists_of_exprs)) => { lists_of_exprs.apply_elements(f) } + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::Column(_) // Treat OuterReferenceColumn as a leaf expression | Expr::OuterReferenceColumn(_, _) @@ -113,6 +115,8 @@ impl TreeNode for Expr { mut f: F, ) -> Result> { Ok(match self { + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::Column(_) | Expr::Wildcard { .. } | Expr::Placeholder(Placeholder { .. }) diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index 56c1e64554a90..3846566e27fec 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -282,6 +282,8 @@ pub fn expr_to_columns(expr: &Expr, accum: &mut HashSet) -> Result<()> { // Use explicit pattern match instead of a default // implementation, so that in the future if someone adds // new Expr types, they will check here as well + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::Unnest(_) | Expr::ScalarVariable(_, _) | Expr::Alias(_) @@ -709,6 +711,7 @@ pub fn exprlist_to_fields<'a>( let result = exprs .into_iter() .map(|e| match e { + #[expect(deprecated)] Expr::Wildcard { qualifier, options } => match qualifier { None => { let mut excluded = exclude_using_columns(plan)?; @@ -801,6 +804,7 @@ pub fn exprlist_len( exprs .iter() .map(|e| match e { + #[expect(deprecated)] Expr::Wildcard { qualifier: None, options, @@ -818,6 +822,7 @@ pub fn exprlist_len( .len(), ) } + #[expect(deprecated)] Expr::Wildcard { qualifier: Some(qualifier), options, diff --git a/datafusion/functions-aggregate/src/planner.rs b/datafusion/functions-aggregate/src/planner.rs index 7c88a8b82624c..c8cb841189954 100644 --- a/datafusion/functions-aggregate/src/planner.rs +++ b/datafusion/functions-aggregate/src/planner.rs @@ -82,6 +82,8 @@ impl ExprPlanner for AggregateFunctionPlanner { // handle count() and count(*) case // convert to count(1) as "count()" // or count(1) as "count(*)" + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] if raw_expr.func.name() == "count" && (raw_expr.args.len() == 1 && matches!(raw_expr.args[0], Expr::Wildcard { .. }) diff --git a/datafusion/functions-window/src/planner.rs b/datafusion/functions-window/src/planner.rs index ffaccd9369bc1..1ddd8b27c4205 100644 --- a/datafusion/functions-window/src/planner.rs +++ b/datafusion/functions-window/src/planner.rs @@ -79,6 +79,8 @@ impl ExprPlanner for WindowFunctionPlanner { null_treatment, }; + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] if raw_expr.func_def.name() == "count" && (raw_expr.args.len() == 1 && matches!(raw_expr.args[0], Expr::Wildcard { .. }) diff --git a/datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs b/datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs index 7df4e970ada22..8015ebfc75348 100644 --- a/datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs +++ b/datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs @@ -89,6 +89,7 @@ fn expand_exprlist(input: &LogicalPlan, expr: Vec) -> Result> { let input = find_base_plan(input); for e in expr { match e { + #[expect(deprecated)] Expr::Wildcard { qualifier, options } => { if let Some(qualifier) = qualifier { let expanded = expand_qualified_wildcard( diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index d1d491cc7a643..538ef98ac7bed 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -565,6 +565,8 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { .build()?, )) } + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::Alias(_) | Expr::Column(_) | Expr::ScalarVariable(_, _) @@ -1021,6 +1023,7 @@ fn project_with_column_index( spans: _, }) if name != schema.field(i).name() => Ok(e.alias(schema.field(i).name())), Expr::Alias { .. } | Expr::Column { .. } => Ok(e), + #[expect(deprecated)] Expr::Wildcard { .. } => { plan_err!("Wildcard should be expanded before type coercion") } diff --git a/datafusion/optimizer/src/common_subexpr_eliminate.rs b/datafusion/optimizer/src/common_subexpr_eliminate.rs index bfa53a5ce8524..5dc1a7e5ac5b3 100644 --- a/datafusion/optimizer/src/common_subexpr_eliminate.rs +++ b/datafusion/optimizer/src/common_subexpr_eliminate.rs @@ -678,6 +678,8 @@ impl CSEController for ExprCSEController<'_> { } fn is_ignored(&self, node: &Expr) -> bool { + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] let is_normal_minus_aggregates = matches!( node, Expr::Literal(..) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index c38dd35abd360..6b408521c5cf9 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -285,6 +285,8 @@ fn can_evaluate_as_join_condition(predicate: &Expr) -> Result { | Expr::TryCast(_) | Expr::InList { .. } | Expr::ScalarFunction(_) => Ok(TreeNodeRecursion::Continue), + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::AggregateFunction(_) | Expr::WindowFunction(_) | Expr::Wildcard { .. } diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index e43e2e704080e..e20580df4ce05 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -582,6 +582,8 @@ impl<'a> ConstEvaluator<'a> { // added they can be checked for their ability to be evaluated // at plan time match expr { + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] Expr::AggregateFunction { .. } | Expr::ScalarVariable(_, _) | Expr::Column(_) diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index e04a89a03dae5..cac2f9db1645b 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -527,6 +527,7 @@ pub fn parse_expr( ))), ExprType::Wildcard(protobuf::Wildcard { qualifier }) => { let qualifier = qualifier.to_owned().map(|x| x.try_into()).transpose()?; + #[expect(deprecated)] Ok(Expr::Wildcard { qualifier, options: Box::new(WildcardOptions::default()), diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 5785bc0c49661..5bb0cdb20c9c8 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -560,6 +560,7 @@ pub fn serialize_expr( expr_type: Some(ExprType::InList(expr)), } } + #[expect(deprecated)] Expr::Wildcard { qualifier, .. } => protobuf::LogicalExprNode { expr_type: Some(ExprType::Wildcard(protobuf::Wildcard { qualifier: qualifier.to_owned().map(|x| x.into()), diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index fa2619111e7e9..c5bcf5a2fae9e 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -593,10 +593,12 @@ impl SqlToRel<'_, S> { } not_impl_err!("AnyOp not supported by ExprPlanner: {binary_expr:?}") } + #[expect(deprecated)] SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard { qualifier: None, options: Box::new(WildcardOptions::default()), }), + #[expect(deprecated)] SQLExpr::QualifiedWildcard(object_name, _token) => Ok(Expr::Wildcard { qualifier: Some(self.object_name_to_table_reference(object_name)?), options: Box::new(WildcardOptions::default()), diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index d051cb78a8d5f..a48c077afd9be 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -429,6 +429,7 @@ impl Unparser<'_> { }) } // TODO: unparsing wildcard addition options + #[expect(deprecated)] Expr::Wildcard { qualifier, .. } => { let attached_token = AttachedToken::empty(); if let Some(qualifier) = qualifier { @@ -729,6 +730,7 @@ impl Unparser<'_> { ) -> Result> { args.iter() .map(|e| { + #[expect(deprecated)] if matches!( e, Expr::Wildcard { @@ -1715,6 +1717,7 @@ mod tests { #[test] fn expr_to_sql_ok() -> Result<()> { let dummy_schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]); + #[expect(deprecated)] let dummy_logical_plan = table_scan(Some("t"), &dummy_schema, None)? .project(vec![Expr::Wildcard { qualifier: None, diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index 3f093afaf26ae..4a248de101dc8 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -632,6 +632,8 @@ pub(crate) fn rewrite_recursive_unnest_bottom_up( } = original_expr.clone().rewrite(&mut rewriter)?; if !transformed { + // TODO: remove the next line after `Expr::Wildcard` is removed + #[expect(deprecated)] if matches!(&transformed_expr, Expr::Column(_)) || matches!(&transformed_expr, Expr::Wildcard { .. }) { diff --git a/datafusion/substrait/src/logical_plan/producer.rs b/datafusion/substrait/src/logical_plan/producer.rs index 9dbb246453be5..e4df9703b20ca 100644 --- a/datafusion/substrait/src/logical_plan/producer.rs +++ b/datafusion/substrait/src/logical_plan/producer.rs @@ -1366,6 +1366,7 @@ pub fn to_substrait_rex( Expr::ScalarSubquery(expr) => { not_impl_err!("Cannot convert {expr:?} to Substrait") } + #[expect(deprecated)] Expr::Wildcard { .. } => not_impl_err!("Cannot convert {expr:?} to Substrait"), Expr::GroupingSet(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"), Expr::Placeholder(expr) => not_impl_err!("Cannot convert {expr:?} to Substrait"), From 65c85606cd93b34dad578c5812bbe0ae4c57df4c Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Mon, 3 Mar 2025 19:41:22 +0800 Subject: [PATCH 047/177] [branch-46] Update changelog for backports to 46.0.0 (#14977) --- dev/changelog/46.0.0.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dev/changelog/46.0.0.md b/dev/changelog/46.0.0.md index 1ef5260a1df18..96b1e9ec5be02 100644 --- a/dev/changelog/46.0.0.md +++ b/dev/changelog/46.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 46.0.0 Changelog -This release consists of 283 commits from 80 contributors. See credits at the end of this changelog for more information. +This release consists of 288 commits from 79 contributors. See credits at the end of this changelog for more information. **Breaking changes:** @@ -102,6 +102,8 @@ This release consists of 283 commits from 80 contributors. See credits at the en - Examples: boundary analysis example for `AND/OR` conjunctions [#14735](https://github.com/apache/datafusion/pull/14735) (clflushopt) - Allow setting the recursion limit for sql parsing [#14756](https://github.com/apache/datafusion/pull/14756) (cetra3) - Document SQL literal syntax and escaping [#14934](https://github.com/apache/datafusion/pull/14934) (alamb) +- Prepare for 46.0.0 release: Version and Changelog [#14903](https://github.com/apache/datafusion/pull/14903) (xudong963) +- MINOR fix(docs): set the proper link for dev-env setup in contrib guide [#14960](https://github.com/apache/datafusion/pull/14960) (clflushopt) **Other:** @@ -319,25 +321,33 @@ This release consists of 283 commits from 80 contributors. See credits at the en - refactor(properties): Split properties.rs into smaller modules [#14925](https://github.com/apache/datafusion/pull/14925) (Standing-Man) - Fix failing extended `sqlite`test on main / update `datafusion-testing` pin [#14940](https://github.com/apache/datafusion/pull/14940) (alamb) - Revert Datafusion-cli: Redesign the datafusion-cli execution and print, make it totally streaming printing without memory overhead [#14948](https://github.com/apache/datafusion/pull/14948) (alamb) +- Remove invalid bug reproducer. [#14950](https://github.com/apache/datafusion/pull/14950) (wiedld) +- Improve documentation for `DataSourceExec`, `FileScanConfig`, `DataSource` etc [#14941](https://github.com/apache/datafusion/pull/14941) (alamb) +- Do not swap with projection when file is partitioned [#14956](https://github.com/apache/datafusion/pull/14956) (blaginin) +- Minor: Add more projection pushdown tests, clarify comments [#14963](https://github.com/apache/datafusion/pull/14963) (alamb) +- Update labeler components [#14942](https://github.com/apache/datafusion/pull/14942) (alamb) +- Deprecate `Expr::Wildcard` [#14959](https://github.com/apache/datafusion/pull/14959) (linhr) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` - 35 Andrew Lamb + 38 Andrew Lamb + 35 dependabot[bot] 14 Piotr Findeisen 10 Matthijs Brobbel 10 logan-keede 9 Bruce Ritchie + 9 xudong.w 8 Jay Zhan 8 Qi Zhu - 8 xudong.w 6 Adam Gutglick 6 Joseph Koshakow 5 Ian Lai 5 Lordworms 5 Simon Vandel Sillesen + 5 wiedld 5 zjregee 4 Dmitrii Blaginin 4 Kristin Cowalcijk @@ -345,7 +355,6 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co 4 Yongting You 4 irenjj 4 oznur-synnada - 4 wiedld 3 Andy Yen 3 Jax Liu 3 Oleks V From ec4862fa2d870fcab973fd1589ef99b6bf8d560f Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 3 Mar 2025 07:19:27 -0500 Subject: [PATCH 048/177] Add note about upgrade guide into the release notes (#14979) --- dev/changelog/46.0.0.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dev/changelog/46.0.0.md b/dev/changelog/46.0.0.md index 96b1e9ec5be02..3734161e032f4 100644 --- a/dev/changelog/46.0.0.md +++ b/dev/changelog/46.0.0.md @@ -21,6 +21,10 @@ under the License. This release consists of 288 commits from 79 contributors. See credits at the end of this changelog for more information. +Please see the [Upgrade Guide] for help updating to DataFusion `46.0.0` + +[upgrade guide]: https://datafusion.apache.org/library-user-guide/upgrading.html#datafusion-46-0-0 + **Breaking changes:** - bug: Fix NULL handling in array_slice, introduce `NullHandling` enum to `Signature` [#14289](https://github.com/apache/datafusion/pull/14289) (jkosh44) From d5ca8307940c1a6345419a2c8d91ef87704659be Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 4 Mar 2025 01:54:44 -0500 Subject: [PATCH 049/177] Fix verification script and extended tests due to `rustup` changes (#14990) * Fix rustup toolchain errors * Use standard builder setup in extended tests * fix yaml * sprinkle sudo * Revert "sprinkle sudo" This reverts commit 0ed0e0ab5ad7f904cb6fb93c57fd063e5911ffc6. * no contianer * use rust container * fix extended jobs * fix * Update .github/workflows/extended.yml --- .github/workflows/extended.yml | 6 ++++-- dev/release/verify-release-candidate.sh | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index 3f882d7a3a821..1ad1c36e1aa7a 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -36,6 +36,7 @@ jobs: linux-build-lib: name: linux build test runs-on: ubuntu-latest + # note: do not use amd/rust container to preserve disk space steps: - uses: actions/checkout@v4 with: @@ -45,7 +46,7 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source $HOME/.cargo/env - rustup default stable + rustup toolchain install - name: Install Protobuf Compiler run: sudo apt-get install -y protobuf-compiler - name: Prepare cargo build @@ -58,6 +59,7 @@ jobs: name: cargo test 'extended_tests' (amd64) needs: linux-build-lib runs-on: ubuntu-latest + # note: do not use amd/rust container to preserve disk space steps: - uses: actions/checkout@v4 with: @@ -69,7 +71,7 @@ jobs: run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source $HOME/.cargo/env - rustup default stable + rustup toolchain install - name: Install Protobuf Compiler run: sudo apt-get install -y protobuf-compiler # For debugging, test binaries can be large. diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh index 2c0bd216b3ac9..a053569dcb249 100755 --- a/dev/release/verify-release-candidate.sh +++ b/dev/release/verify-release-candidate.sh @@ -117,8 +117,11 @@ test_source_distribution() { # build and test rust + # install the needed version of rust defined in rust-toolchain.toml + rustup toolchain install + # raises on any formatting errors - rustup component add rustfmt --toolchain stable + rustup component add rustfmt cargo fmt --all -- --check # Clone testing repositories into the expected location From 1c92803e506e67c658c35da714116ff1c408776e Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 13 Mar 2025 14:27:17 +0800 Subject: [PATCH 050/177] upgrade tonic --- Cargo.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5b1e275d25eb9..55855d09d50ed 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,21 +77,21 @@ version = "45.0.0" ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } -arrow = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", features = [ +arrow = { version = "54.1.0", features = [ "prettyprint", ] } -arrow-array = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false, features = [ +arrow-array = { version = "54.1.0", default-features = false, features = [ "chrono-tz", ] } -arrow-buffer = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } -arrow-flight = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", features = [ +arrow-buffer = { version = "54.1.0", default-features = false } +arrow-flight = { version = "54.1.0", features = [ "flight-sql-experimental", ] } -arrow-ipc = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false, features = [ +arrow-ipc = { version = "54.1.0", default-features = false, features = [ "lz4", ] } -arrow-ord = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } -arrow-schema = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false } +arrow-ord = { version = "54.1.0", default-features = false } +arrow-schema = { version = "54.1.0", default-features = false } async-trait = "0.1.73" bigdecimal = "0.4.7" bytes = "1.4" @@ -133,15 +133,15 @@ itertools = "0.14" log = "^0.4" object_store = { version = "0.11.0", default-features = false } parking_lot = "0.12" -parquet = { git = "https://github.com/influxdata/arrow-rs", rev = "36685f0", default-features = false, features = [ +parquet = { version = "54.1.0", default-features = false, features = [ "arrow", "async", "object_store", ] } pbjson = { version = "0.7.0" } # Should match arrow-flight's version of prost. -prost = "0.12.3" -prost-derive = "0.12.3" +prost = "0.13.1" +prost-derive = "0.13.1" rand = "0.8" recursive = "0.1.1" regex = "1.8" From 112e9ebb9f8b918e82136bcb9f7c3b53d27566a2 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Mar 2025 10:42:37 -0400 Subject: [PATCH 051/177] Update ring to v0.17.13 (#15063) (#15228) --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8eaaad283b3bc..7c81dd99c87de 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4937,9 +4937,9 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.9" +version = "0.17.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e75ec5e92c4d8aede845126adc388046234541629e76029599ed35a003c7ed24" +checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" dependencies = [ "cc", "cfg-if", From 0877c9930cb52e7e83307c347852d3f77779bbfe Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Mar 2025 14:58:50 -0400 Subject: [PATCH 052/177] Fix broken `serde` feature (#15124) (#15227) * Fix broked `serde` feature * Test `serde` feature * consolidate serde test into core_integration, update run --------- Co-authored-by: Vadim Piven --- .github/workflows/rust.yml | 2 +- Cargo.lock | 1 + datafusion/core/Cargo.toml | 7 ++++- datafusion/core/tests/core_integration.rs | 5 ++++ datafusion/core/tests/serde/mod.rs | 34 +++++++++++++++++++++++ 5 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 datafusion/core/tests/serde/mod.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 99aaa7d6f2907..f9a8a456fbc7a 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -183,7 +183,7 @@ jobs: with: rust-version: stable - name: Run tests (excluding doctests) - run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests + run: cargo test --profile ci --exclude datafusion-examples --exclude ffi_example_table_provider --exclude datafusion-benchmarks --workspace --lib --tests --bins --features serde,avro,json,backtrace,integration-tests - name: Verify Working Directory Clean run: git diff --exit-code diff --git a/Cargo.lock b/Cargo.lock index 7c81dd99c87de..d1f63ef001a73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -453,6 +453,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85934a9d0261e0fa5d4e2a5295107d743b543a6e0484a835d4b8db2da15306f9" dependencies = [ "bitflags 2.8.0", + "serde", ] [[package]] diff --git a/datafusion/core/Cargo.toml b/datafusion/core/Cargo.toml index 438e2600a66d7..27e261739357b 100644 --- a/datafusion/core/Cargo.toml +++ b/datafusion/core/Cargo.toml @@ -73,7 +73,12 @@ recursive_protection = [ "datafusion-physical-optimizer/recursive_protection", "datafusion-sql/recursive_protection", ] -serde = ["dep:serde"] +serde = [ + "dep:serde", + # Enable `#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]` + # statements in `arrow-schema` crate + "arrow-schema/serde", +] string_expressions = ["datafusion-functions/string_expressions"] unicode_expressions = [ "datafusion-sql/unicode_expressions", diff --git a/datafusion/core/tests/core_integration.rs b/datafusion/core/tests/core_integration.rs index 66b4103160e7b..9bcb9e41f86a9 100644 --- a/datafusion/core/tests/core_integration.rs +++ b/datafusion/core/tests/core_integration.rs @@ -42,8 +42,13 @@ mod custom_sources_cases; /// Run all tests that are found in the `optimizer` directory mod optimizer; +/// Run all tests that are found in the `physical_optimizer` directory mod physical_optimizer; +/// Run all tests that are found in the `serde` directory +mod serde; + +/// Run all tests that are found in the `catalog` directory mod catalog; #[cfg(test)] diff --git a/datafusion/core/tests/serde/mod.rs b/datafusion/core/tests/serde/mod.rs new file mode 100644 index 0000000000000..05dde7a541867 --- /dev/null +++ b/datafusion/core/tests/serde/mod.rs @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +/// Ensure `serde` feature from `arrow-schema` crate is re-exported. +#[test] +#[cfg(feature = "serde")] +fn ensure_serde_support() { + use datafusion::arrow::datatypes::DataType; + + #[derive(Debug, PartialEq, serde::Serialize, serde::Deserialize)] + struct WrappingStruct(DataType); + + let boolean = WrappingStruct(DataType::Boolean); + + let serialized = serde_json::to_string(&boolean).unwrap(); + assert_eq!("\"Boolean\"", serialized); + + let deserialized = serde_json::from_str(&serialized).unwrap(); + assert_eq!(boolean, deserialized); +} From 048a1253a57a5263b0e9c6babf04449e7bb91425 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 14 Mar 2025 14:59:02 -0400 Subject: [PATCH 053/177] [branch-46] Fix wasm32 build on version 46 (#15229) --- .github/workflows/rust.yml | 4 ++++ datafusion/core/src/datasource/file_format/parquet.rs | 2 -- datafusion/wasmtest/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index f9a8a456fbc7a..f1516e247d50c 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -259,6 +259,10 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable + - name: Install dependencies + run: | + apt-get update -qq + apt-get install -y -qq clang - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - name: Build with wasm-pack diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs index 4a24871aeef77..32713923b18b8 100644 --- a/datafusion/core/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -469,7 +469,6 @@ impl FileFormat for ParquetFormat { } /// Coerces the file schema if the table schema uses a view type. -#[cfg(not(target_arch = "wasm32"))] pub fn coerce_file_schema_to_view_type( table_schema: &Schema, file_schema: &Schema, @@ -519,7 +518,6 @@ pub fn coerce_file_schema_to_view_type( /// If the table schema uses a string type, coerce the file schema to use a string type. /// /// See [ParquetFormat::binary_as_string] for details -#[cfg(not(target_arch = "wasm32"))] pub fn coerce_file_schema_to_string_type( table_schema: &Schema, file_schema: &Schema, diff --git a/datafusion/wasmtest/Cargo.toml b/datafusion/wasmtest/Cargo.toml index 30d5bcaedcb73..94515c6754a7a 100644 --- a/datafusion/wasmtest/Cargo.toml +++ b/datafusion/wasmtest/Cargo.toml @@ -45,7 +45,7 @@ chrono = { version = "0.4", features = ["wasmbind"] } # all the `std::fmt` and `std::panicking` infrastructure, so isn't great for # code size when deploying. console_error_panic_hook = { version = "0.1.1", optional = true } -datafusion = { workspace = true } +datafusion = { workspace = true, features = ["parquet"] } datafusion-common = { workspace = true, default-features = true } datafusion-execution = { workspace = true } datafusion-expr = { workspace = true } From 68f29038953d91191a46656968488f1e7bb4c327 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Sat, 15 Mar 2025 15:09:37 +0800 Subject: [PATCH 054/177] Update version to 46.0.1, add CHANGELOG (#15243) --- Cargo.lock | 66 +++++++++++++++---------------- Cargo.toml | 56 +++++++++++++------------- dev/changelog/46.0.1.md | 38 ++++++++++++++++++ docs/source/user-guide/configs.md | 2 +- 4 files changed, 100 insertions(+), 62 deletions(-) create mode 100644 dev/changelog/46.0.1.md diff --git a/Cargo.lock b/Cargo.lock index d1f63ef001a73..1dc438fb8db2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1718,7 +1718,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "46.0.0" +version = "46.0.1" dependencies = [ "apache-avro", "arrow", @@ -1784,7 +1784,7 @@ dependencies = [ [[package]] name = "datafusion-benchmarks" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "datafusion", @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "async-trait", @@ -1827,7 +1827,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "async-trait", @@ -1848,7 +1848,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "assert_cmd", @@ -1875,7 +1875,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "46.0.0" +version = "46.0.1" dependencies = [ "ahash 0.8.11", "apache-avro", @@ -1901,7 +1901,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "46.0.0" +version = "46.0.1" dependencies = [ "log", "tokio", @@ -1909,7 +1909,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "async-compression", @@ -1942,11 +1942,11 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "46.0.0" +version = "46.0.1" [[package]] name = "datafusion-examples" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "arrow-flight", @@ -1973,7 +1973,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "chrono", @@ -1991,7 +1991,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "chrono", @@ -2012,7 +2012,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2023,7 +2023,7 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "46.0.0" +version = "46.0.1" dependencies = [ "abi_stable", "arrow", @@ -2041,7 +2041,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "arrow-buffer", @@ -2070,7 +2070,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "46.0.0" +version = "46.0.1" dependencies = [ "ahash 0.8.11", "arrow", @@ -2091,7 +2091,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "46.0.0" +version = "46.0.1" dependencies = [ "ahash 0.8.11", "arrow", @@ -2104,7 +2104,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "arrow-ord", @@ -2125,7 +2125,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "async-trait", @@ -2139,7 +2139,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2155,7 +2155,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "46.0.0" +version = "46.0.1" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2163,7 +2163,7 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "46.0.0" +version = "46.0.1" dependencies = [ "datafusion-expr", "quote", @@ -2172,7 +2172,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "async-trait", @@ -2196,7 +2196,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "46.0.0" +version = "46.0.1" dependencies = [ "ahash 0.8.11", "arrow", @@ -2220,7 +2220,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "46.0.0" +version = "46.0.1" dependencies = [ "ahash 0.8.11", "arrow", @@ -2232,7 +2232,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2250,7 +2250,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "46.0.0" +version = "46.0.1" dependencies = [ "ahash 0.8.11", "arrow", @@ -2284,7 +2284,7 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "chrono", @@ -2307,7 +2307,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2320,7 +2320,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "bigdecimal", @@ -2343,7 +2343,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "46.0.0" +version = "46.0.1" dependencies = [ "arrow", "async-trait", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "46.0.0" +version = "46.0.1" dependencies = [ "async-recursion", "async-trait", @@ -2393,7 +2393,7 @@ dependencies = [ [[package]] name = "datafusion-wasmtest" -version = "46.0.0" +version = "46.0.1" dependencies = [ "chrono", "console_error_panic_hook", diff --git a/Cargo.toml b/Cargo.toml index 627a50d38d78a..73e82580ce3db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -69,7 +69,7 @@ repository = "https://github.com/apache/datafusion" # Define Minimum Supported Rust Version (MSRV) rust-version = "1.82.0" # Define DataFusion version -version = "46.0.0" +version = "46.0.1" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can @@ -100,33 +100,33 @@ chrono = { version = "0.4.38", default-features = false } criterion = "0.5.1" ctor = "0.2.9" dashmap = "6.0.1" -datafusion = { path = "datafusion/core", version = "46.0.0", default-features = false } -datafusion-catalog = { path = "datafusion/catalog", version = "46.0.0" } -datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "46.0.0" } -datafusion-common = { path = "datafusion/common", version = "46.0.0", default-features = false } -datafusion-common-runtime = { path = "datafusion/common-runtime", version = "46.0.0" } -datafusion-datasource = { path = "datafusion/datasource", version = "46.0.0", default-features = false } -datafusion-doc = { path = "datafusion/doc", version = "46.0.0" } -datafusion-execution = { path = "datafusion/execution", version = "46.0.0" } -datafusion-expr = { path = "datafusion/expr", version = "46.0.0" } -datafusion-expr-common = { path = "datafusion/expr-common", version = "46.0.0" } -datafusion-ffi = { path = "datafusion/ffi", version = "46.0.0" } -datafusion-functions = { path = "datafusion/functions", version = "46.0.0" } -datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "46.0.0" } -datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "46.0.0" } -datafusion-functions-nested = { path = "datafusion/functions-nested", version = "46.0.0" } -datafusion-functions-table = { path = "datafusion/functions-table", version = "46.0.0" } -datafusion-functions-window = { path = "datafusion/functions-window", version = "46.0.0" } -datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "46.0.0" } -datafusion-macros = { path = "datafusion/macros", version = "46.0.0" } -datafusion-optimizer = { path = "datafusion/optimizer", version = "46.0.0", default-features = false } -datafusion-physical-expr = { path = "datafusion/physical-expr", version = "46.0.0", default-features = false } -datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "46.0.0", default-features = false } -datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "46.0.0" } -datafusion-physical-plan = { path = "datafusion/physical-plan", version = "46.0.0" } -datafusion-proto = { path = "datafusion/proto", version = "46.0.0" } -datafusion-proto-common = { path = "datafusion/proto-common", version = "46.0.0" } -datafusion-sql = { path = "datafusion/sql", version = "46.0.0" } +datafusion = { path = "datafusion/core", version = "46.0.1", default-features = false } +datafusion-catalog = { path = "datafusion/catalog", version = "46.0.1" } +datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "46.0.1" } +datafusion-common = { path = "datafusion/common", version = "46.0.1", default-features = false } +datafusion-common-runtime = { path = "datafusion/common-runtime", version = "46.0.1" } +datafusion-datasource = { path = "datafusion/datasource", version = "46.0.1", default-features = false } +datafusion-doc = { path = "datafusion/doc", version = "46.0.1" } +datafusion-execution = { path = "datafusion/execution", version = "46.0.1" } +datafusion-expr = { path = "datafusion/expr", version = "46.0.1" } +datafusion-expr-common = { path = "datafusion/expr-common", version = "46.0.1" } +datafusion-ffi = { path = "datafusion/ffi", version = "46.0.1" } +datafusion-functions = { path = "datafusion/functions", version = "46.0.1" } +datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "46.0.1" } +datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "46.0.1" } +datafusion-functions-nested = { path = "datafusion/functions-nested", version = "46.0.1" } +datafusion-functions-table = { path = "datafusion/functions-table", version = "46.0.1" } +datafusion-functions-window = { path = "datafusion/functions-window", version = "46.0.1" } +datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "46.0.1" } +datafusion-macros = { path = "datafusion/macros", version = "46.0.1" } +datafusion-optimizer = { path = "datafusion/optimizer", version = "46.0.1", default-features = false } +datafusion-physical-expr = { path = "datafusion/physical-expr", version = "46.0.1", default-features = false } +datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "46.0.1", default-features = false } +datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "46.0.1" } +datafusion-physical-plan = { path = "datafusion/physical-plan", version = "46.0.1" } +datafusion-proto = { path = "datafusion/proto", version = "46.0.1" } +datafusion-proto-common = { path = "datafusion/proto-common", version = "46.0.1" } +datafusion-sql = { path = "datafusion/sql", version = "46.0.1" } doc-comment = "0.3" env_logger = "0.11" futures = "0.3" diff --git a/dev/changelog/46.0.1.md b/dev/changelog/46.0.1.md new file mode 100644 index 0000000000000..17308bea87ac5 --- /dev/null +++ b/dev/changelog/46.0.1.md @@ -0,0 +1,38 @@ + + +# Apache DataFusion 46.0.1 Changelog + +This release consists of 3 commits from 1 contributors. See credits at the end of this changelog for more information. + +**Other:** + +- [branch-46] Update ring to v0.17.13 (#15063) [#15228](https://github.com/apache/datafusion/pull/15228) (alamb) +- [branch-46] Fix broken `serde` feature (#15124) [#15227](https://github.com/apache/datafusion/pull/15227) (alamb) +- [branch-46] Fix wasm32 build on version 46 [#15229](https://github.com/apache/datafusion/pull/15229) (alamb) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 3 Andrew Lamb +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 8c4aad51077c4..3f5fc53f1cc6e 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -68,7 +68,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 46.0.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 46.0.1 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | From 2e5b5e277bcc096ddcce7b9738d624b9498f23d5 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 20 Mar 2025 14:26:54 +0800 Subject: [PATCH 055/177] fix with_node_id and clippy --- Cargo.lock | 1455 ++++++++--------- .../datasource/physical_plan/arrow_file.rs | 16 - .../core/src/datasource/physical_plan/avro.rs | 16 - .../core/src/datasource/physical_plan/json.rs | 15 - .../datasource/physical_plan/parquet/mod.rs | 8 +- datafusion/core/tests/fuzz_cases/sort_fuzz.rs | 2 +- datafusion/datasource/src/source.rs | 10 + .../src/enforce_distribution.rs | 6 +- datafusion/physical-plan/src/analyze.rs | 4 +- .../physical-plan/src/coalesce_batches.rs | 2 +- .../physical-plan/src/coalesce_partitions.rs | 2 +- datafusion/physical-plan/src/filter.rs | 2 +- datafusion/physical-plan/src/insert.rs | 4 +- .../physical-plan/src/joins/cross_join.rs | 3 +- .../physical-plan/src/joins/hash_join.rs | 6 +- .../src/joins/sort_merge_join.rs | 4 +- .../src/joins/symmetric_hash_join.rs | 4 +- datafusion/physical-plan/src/limit.rs | 2 +- datafusion/physical-plan/src/node_id.rs | 12 +- .../physical-plan/src/placeholder_row.rs | 2 +- datafusion/physical-plan/src/projection.rs | 2 +- .../physical-plan/src/recursive_query.rs | 4 +- datafusion/physical-plan/src/sorts/sort.rs | 2 +- .../src/sorts/sort_preserving_merge.rs | 2 +- datafusion/physical-plan/src/unnest.rs | 4 +- .../src/windows/window_agg_exec.rs | 4 +- datafusion/physical-plan/src/work_table.rs | 3 +- datafusion/sqllogictest/Cargo.toml | 2 +- 28 files changed, 746 insertions(+), 852 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3b5430f3d78ad..712f856f58406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ dependencies = [ "core_extensions", "crossbeam-channel", "generational-arena", - "libloading", + "libloading 0.7.4", "lock_api", "parking_lot", "paste", @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.95" +version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" +checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "apache-avro" @@ -246,15 +246,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755b6da235ac356a869393c23668c663720b8749dd6f15e52b6c214b4b964cc7" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc208515aa0151028e464cc94a692156e945ce5126abd3537bb7fd6ba2143ed1" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-arith", "arrow-array", @@ -276,15 +270,9 @@ dependencies = [ [[package]] name = "arrow-arith" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64656a1e0b13ca766f8440752e9a93e11014eec7b67909986f83ed0ab1fe37b8" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e07e726e2b3f7816a85c6a45b6ec118eeeabf0b2a8c208122ad949437181f49a" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -296,15 +284,9 @@ dependencies = [ [[package]] name = "arrow-array" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57a4a6d2896083cfbdf84a71a863b22460d0708f8206a8373c52e326cc72ea1a" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2262eba4f16c78496adfd559a29fe4b24df6088efc9985a873d58e92be022d5" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "ahash 0.8.11", "arrow-buffer", @@ -319,15 +301,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef870583ce5e4f3b123c181706f2002fb134960f9a911900f64ba4830c7a43a" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e899dade2c3b7f5642eb8366cfd898958bcca099cde6dfea543c7e8d3ad88d4" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "bytes", "half", @@ -336,15 +312,9 @@ dependencies = [ [[package]] name = "arrow-cast" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac7eba5a987f8b4a7d9629206ba48e19a1991762795bbe5d08497b7736017ee" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4103d88c5b441525ed4ac23153be7458494c2b0c9a11115848fdb9b81f6f886a" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -363,15 +333,9 @@ dependencies = [ [[package]] name = "arrow-csv" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f12542b8164398fc9ec595ff783c4cf6044daa89622c5a7201be920e4c0d4c" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43d3cb0914486a3cae19a5cad2598e44e225d53157926d0ada03c20521191a65" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-cast", @@ -385,15 +349,9 @@ dependencies = [ [[package]] name = "arrow-data" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b095e8a4f3c309544935d53e04c3bfe4eea4e71c3de6fe0416d1f08bb4441a83" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a329fb064477c9ec5f0870d2f5130966f91055c7c5bce2b3a084f116bc28c3b" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-buffer", "arrow-schema", @@ -402,12 +360,6 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "arrow-ipc" -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65c63da4afedde2b25ef69825cd4663ca76f78f79ffe2d057695742099130ff6" -======= name = "arrow-flight" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -439,7 +391,6 @@ name = "arrow-ipc" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ddecdeab02491b1ce88885986e25002a3da34dd349f682c7cfe67bab7cc17b86" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -451,15 +402,9 @@ dependencies = [ [[package]] name = "arrow-json" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9551d9400532f23a370cabbea1dc5a53c49230397d41f96c4c8eedf306199305" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d03b9340013413eb84868682ace00a1098c81a5ebc96d279f7ebf9a4cac3c0fd" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -468,7 +413,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.7.1", + "indexmap 2.8.0", "lexical-core", "num", "serde", @@ -477,15 +422,9 @@ dependencies = [ [[package]] name = "arrow-ord" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c07223476f8219d1ace8cd8d85fa18c4ebd8d945013f25ef5c72e85085ca4ee" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f841bfcc1997ef6ac48ee0305c4dfceb1f7c786fe31e67c1186edf775e1f1160" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -496,15 +435,9 @@ dependencies = [ [[package]] name = "arrow-row" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91b194b38bfd89feabc23e798238989c6648b2506ad639be42ec8eb1658d82c4" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1eeb55b0a0a83851aa01f2ca5ee5648f607e8506ba6802577afdda9d75cdedcd" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -515,22 +448,11 @@ dependencies = [ [[package]] name = "arrow-schema" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f40f6be8f78af1ab610db7d9b236e21d587b7168e368a36275d2e5670096735" - -[[package]] -name = "arrow-select" -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac265273864a820c4a179fc67182ccc41ea9151b97024e1be956f0f2369c2539" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85934a9d0261e0fa5d4e2a5295107d743b543a6e0484a835d4b8db2da15306f9" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "serde", ] @@ -539,7 +461,6 @@ name = "arrow-select" version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e2932aece2d0c869dd2125feb9bd1709ef5c445daa3838ac4112dcfa0fda52c" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "ahash 0.8.11", "arrow-array", @@ -551,15 +472,9 @@ dependencies = [ [[package]] name = "arrow-string" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d44c8eed43be4ead49128370f7131f054839d3d6003e52aebf64322470b8fbd0" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "912e38bd6a7a7714c1d9b61df80315685553b7455e8a6045c27531d8ecd5b458" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "arrow-array", "arrow-buffer", @@ -602,11 +517,11 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.18" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" +checksum = "06575e6a9673580f52661c92107baabffbf41e2141373441cbcdc47cb733003c" dependencies = [ - "bzip2 0.4.4", + "bzip2 0.5.2", "flate2", "futures-core", "memchr", @@ -618,12 +533,6 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "async-trait" -version = "0.1.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" -======= name = "async-ffi" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -637,11 +546,10 @@ name = "async-recursion" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -663,18 +571,18 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "async-trait" -version = "0.1.86" +version = "0.1.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644dd749086bf3771a2fbc5f256fdb982d53f011c7d5d560304eafeecebce79d" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -700,15 +608,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "1.5.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50236e4d60fe8458de90a71c0922c761e41755adf091b1b03de1cef537179915" -======= -version = "1.5.17" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490aa7465ee685b2ced076bb87ef654a47724a7844e2c7d3af4e749ce5b875dd" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "6a84fe2c5e9965fba0fbc2001db252f1d57527d82a905cca85127df227bca748" dependencies = [ "aws-credential-types", "aws-runtime", @@ -725,7 +627,7 @@ dependencies = [ "bytes", "fastrand", "hex", - "http 0.2.12", + "http 1.3.1", "ring", "time", "tokio", @@ -736,9 +638,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" +checksum = "4471bef4c22a06d2c7a1b6492493d3fdf24a805323109d6874f9c94d5906ac14" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -746,11 +648,34 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-lc-rs" +version = "1.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" +dependencies = [ + "bindgen", + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-runtime" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad" +checksum = "0aff45ffe35196e593ea3b9dd65b320e51e2dda95aff4390bc459e461d09c6ad" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -773,15 +698,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "1.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00a35fc7e74f5be45839eb753568535c074a592185dd0a2d406685018d581c43" -======= -version = "1.60.0" +version = "1.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60186fab60b24376d3e33b9ff0a43485f99efd470e3b75a9160c849741d63d56" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "1d5330ad4e8a1ff49e9f26b738611caa72b105c41d41733801d1a36e8f9de936" dependencies = [ "aws-credential-types", "aws-runtime", @@ -801,15 +720,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "1.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fa655b4f313124ce272cbc38c5fef13793c832279cec750103e5e6b71a54b8" -======= -version = "1.61.0" +version = "1.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7033130ce1ee13e6018905b7b976c915963755aef299c1521897679d6cd4f8ef" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "7956b1a85d49082347a7d17daa2e32df191f3e23c03d47294b99f95413026a78" dependencies = [ "aws-credential-types", "aws-runtime", @@ -829,15 +742,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "1.60.0" +version = "1.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1cfe5e16b90421ea031f4c6348b534ef442e76f6bf4a1b2b592c12cc2c6af9" -======= -version = "1.61.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5c1cac7677179d622b4448b0d31bcb359185295dc6fca891920cfb17e2b5156" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "065c533fbe6f84962af33fcf02b0350b7c1f79285baab5924615d2be3b232855" dependencies = [ "aws-credential-types", "aws-runtime", @@ -858,9 +765,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.9" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051" +checksum = "69d03c3c05ff80d54ff860fe38c726f6f494c639ae975203a101335f223386db" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -871,7 +778,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.2.0", + "http 1.3.1", "once_cell", "percent-encoding", "sha2", @@ -881,9 +788,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.4" +version = "1.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" +checksum = "1e190749ea56f8c42bf15dd76c65e14f8f765233e6df9b0506d9d934ebef867c" dependencies = [ "futures-util", "pin-project-lite", @@ -892,9 +799,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.12" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" +checksum = "c5949124d11e538ca21142d1fba61ab0a2a2c1bc3ed323cdb3e4b878bfb83166" dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", @@ -902,6 +809,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", + "http 1.3.1", "http-body 0.4.6", "once_cell", "percent-encoding", @@ -910,11 +818,34 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-http-client" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0497ef5d53065b7cd6a35e9c1654bd1fefeae5c52900d91d1b188b0af0f29324" +dependencies = [ + "aws-smithy-async", + "aws-smithy-runtime-api", + "aws-smithy-types", + "h2", + "http 1.3.1", + "hyper", + "hyper-rustls", + "hyper-util", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tower 0.5.2", + "tracing", +] + [[package]] name = "aws-smithy-json" -version = "0.61.2" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" +checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" dependencies = [ "aws-smithy-types", ] @@ -931,42 +862,39 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.8" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92" +checksum = "f6328865e36c6fd970094ead6b05efd047d3a80ec5fc3be5e743910da9f2ebf8" dependencies = [ "aws-smithy-async", "aws-smithy-http", + "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "fastrand", - "h2 0.3.26", "http 0.2.12", + "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", - "httparse", - "hyper 0.14.32", - "hyper-rustls 0.24.2", "once_cell", "pin-project-lite", "pin-utils", - "rustls 0.21.12", "tokio", "tracing", ] [[package]] name = "aws-smithy-runtime-api" -version = "1.7.3" +version = "1.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" +checksum = "3da37cf5d57011cb1753456518ec76e31691f1f474b73934a284eb2a1c76510f" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", "http 0.2.12", - "http 1.2.0", + "http 1.3.1", "pin-project-lite", "tokio", "tracing", @@ -975,15 +903,15 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.13" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" +checksum = "836155caafba616c0ff9b07944324785de2ab016141c3550bd1c07882f8cee8f" dependencies = [ "base64-simd", "bytes", "bytes-utils", "http 0.2.12", - "http 1.2.0", + "http 1.3.1", "http-body 0.4.6", "http-body 1.0.1", "http-body-util", @@ -1007,9 +935,9 @@ dependencies = [ [[package]] name = "aws-types" -version = "1.3.5" +version = "1.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" +checksum = "3873f8deed8927ce8d04487630dc9ff73193bab64742a61d050e57a68dec4125" dependencies = [ "aws-credential-types", "aws-smithy-async", @@ -1029,7 +957,7 @@ dependencies = [ "axum-core", "bytes", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "itoa", @@ -1055,7 +983,7 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", "http-body-util", "mime", @@ -1117,6 +1045,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.9.0", + "cexpr", + "clang-sys", + "itertools 0.12.1", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.100", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1125,9 +1076,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" [[package]] name = "bitvec" @@ -1152,16 +1103,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1230237285e3e10cde447185e8975408ae24deaa67205ce684805c25bc0c7937" +checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" dependencies = [ "arrayref", "arrayvec", "cc", "cfg-if", "constant_time_eq", - "memmap2", ] [[package]] @@ -1186,25 +1136,25 @@ dependencies = [ "futures-util", "hex", "home", - "http 1.2.0", + "http 1.3.1", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-named-pipe", - "hyper-rustls 0.27.5", + "hyper-rustls", "hyper-util", "hyperlocal", "log", "pin-project-lite", - "rustls 0.23.23", - "rustls-native-certs 0.8.1", - "rustls-pemfile 2.2.0", + "rustls", + "rustls-native-certs", + "rustls-pemfile", "rustls-pki-types", "serde", "serde_derive", "serde_json", "serde_repr", "serde_urlencoded", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tokio-util", "tower-service", @@ -1225,9 +1175,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5430e3be710b68d984d1391c854eb431a9d548640711faa54eecb1df93db91cc" +checksum = "b2b74d67a0fc0af8e9823b79fd1c43a0900e5a8f0e0f4cc9210796bf3a820126" dependencies = [ "borsh-derive", "cfg_aliases", @@ -1235,15 +1185,15 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8b668d39970baad5356d7c83a86fee3a539e6f93bf6764c97368243e17a0487" +checksum = "2d37ed1b2c9b78421218a0b4f6d8349132d6ec2cfeba1cfb0118b0a8e268df9e" dependencies = [ "once_cell", "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1314,9 +1264,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f61dac84819c6588b558454b194026eb1f09c293b9036ae9b159e74e73ab6cf9" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bytes-utils" @@ -1340,31 +1290,24 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b89e7c29231c673a61a46e722602bcd138298f6b9e81e71119693534585f5c" +checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" dependencies = [ "bzip2-sys", ] [[package]] name = "bzip2-sys" -version = "0.1.12+1.0.8" +version = "0.1.13+1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ebc2f1a417f01e1da30ef264ee86ae31d2dcd2d603ea283d3c244a883ca2a9" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" dependencies = [ "cc", - "libc", "pkg-config", ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "cc" -version = "1.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c736e259eea577f443d5c86c304f9f4ae0295c43f3ba05c21f1d66b5f06001af" -======= name = "cast" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1372,16 +1315,24 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.14" +version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ "jobserver", "libc", "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -1431,11 +1382,6 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "clap" -version = "4.5.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -======= name = "ciborium" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1462,6 +1408,17 @@ dependencies = [ "half", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading 0.8.6", +] + [[package]] name = "clap" version = "2.34.0" @@ -1475,10 +1432,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.30" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" ->>>>>>> upstream/branch-46:Cargo.lock -checksum = "92b7b18d71fad5313a1e320fa9897994228ce274b60faa4d694fe0ea89cd9e6d" +checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" dependencies = [ "clap_builder", "clap_derive", @@ -1486,9 +1442,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.30" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35db2071778a7344791a4fb4f95308b5673d219dee3ae348b86642574ecc90c" +checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" dependencies = [ "anstream", "anstyle", @@ -1498,14 +1454,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.28" +version = "4.5.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1545,17 +1501,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ "unicode-segmentation", -<<<<<<< HEAD:datafusion-cli/Cargo.lock - "unicode-width", -======= "unicode-width 0.2.0", ] [[package]] name = "console" -version = "0.15.10" +version = "0.15.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea3c6ecd8059b57859df5c69830340ed3c41d30e3da0c1cbed90a96ac853041b" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" dependencies = [ "encode_unicode", "libc", @@ -1572,7 +1525,6 @@ checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ "cfg-if", "wasm-bindgen", ->>>>>>> upstream/branch-46:Cargo.lock ] [[package]] @@ -1607,16 +1559,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation" version = "0.10.0" @@ -1684,7 +1626,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.30", + "clap 4.5.32", "criterion-plot", "futures", "is-terminal", @@ -1791,7 +1733,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a2785755761f3ddc1492979ce1e48d2c00d09311c39e4466429188f3dd6501" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1815,7 +1757,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1826,7 +1768,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -1859,7 +1801,7 @@ dependencies = [ "arrow-schema", "async-trait", "bytes", - "bzip2 0.5.1", + "bzip2 0.5.2", "chrono", "criterion", "ctor", @@ -1988,7 +1930,7 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", - "clap 4.5.30", + "clap 4.5.32", "ctor", "datafusion", "dirs", @@ -2018,7 +1960,7 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", - "indexmap 2.7.1", + "indexmap 2.8.0", "libc", "log", "object_store", @@ -2048,7 +1990,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2 0.5.1", + "bzip2 0.5.2", "chrono", "datafusion-catalog", "datafusion-common", @@ -2136,7 +2078,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "env_logger", - "indexmap 2.7.1", + "indexmap 2.8.0", "paste", "recursive", "serde_json", @@ -2149,7 +2091,7 @@ version = "46.0.1" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.7.1", + "indexmap 2.8.0", "itertools 0.14.0", "paste", ] @@ -2300,7 +2242,7 @@ version = "46.0.1" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2319,7 +2261,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-sql", "env_logger", - "indexmap 2.7.1", + "indexmap 2.8.0", "itertools 0.14.0", "log", "recursive", @@ -2342,7 +2284,7 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", - "indexmap 2.7.1", + "indexmap 2.8.0", "itertools 0.14.0", "log", "paste", @@ -2404,7 +2346,7 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", - "indexmap 2.7.1", + "indexmap 2.8.0", "itertools 0.14.0", "log", "parking_lot", @@ -2465,7 +2407,7 @@ dependencies = [ "datafusion-functions-nested", "datafusion-functions-window", "env_logger", - "indexmap 2.7.1", + "indexmap 2.8.0", "log", "paste", "recursive", @@ -2483,7 +2425,7 @@ dependencies = [ "bigdecimal", "bytes", "chrono", - "clap 4.5.30", + "clap 4.5.32", "datafusion", "env_logger", "futures", @@ -2500,7 +2442,7 @@ dependencies = [ "tempfile", "testcontainers", "testcontainers-modules", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tokio-postgres", ] @@ -2545,9 +2487,9 @@ dependencies = [ [[package]] name = "deranged" -version = "0.3.11" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" dependencies = [ "powerfmt", "serde", @@ -2599,7 +2541,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2619,11 +2561,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feeef44e73baff3a26d371801df019877a9866a8c493d315ab00177843314f35" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" [[package]] name = "educe" @@ -2634,14 +2582,14 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "either" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7914353092ddf589ad78f25c5c1c21b7f80b0ff8621e7c814c3485b5306da9d" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" [[package]] name = "encode_unicode" @@ -2672,7 +2620,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -2687,14 +2635,14 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" dependencies = [ "anstream", "anstyle", "env_filter", - "humantime", + "jiff", "log", ] @@ -2751,13 +2699,13 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fd-lock" -version = "4.0.2" +version = "4.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5768da2206272c81ef0b5e951a41862938a6070da63bcea197899942d3b947" +checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix", - "windows-sys 0.52.0", + "rustix 1.0.3", + "windows-sys 0.59.0", ] [[package]] @@ -2845,9 +2793,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" [[package]] name = "form_urlencoded" @@ -2867,6 +2815,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "2.0.0" @@ -2929,7 +2883,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3018,14 +2972,16 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" dependencies = [ "cfg-if", + "js-sys", "libc", - "wasi 0.13.3+wasi-0.2.2", - "windows-targets 0.52.6", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -3040,25 +2996,6 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" -[[package]] -name = "h2" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap 2.7.1", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "h2" version = "0.4.8" @@ -3070,8 +3007,8 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http 1.2.0", - "indexmap 2.7.1", + "http 1.3.1", + "indexmap 2.8.0", "slab", "tokio", "tokio-util", @@ -3080,9 +3017,9 @@ dependencies = [ [[package]] name = "half" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" dependencies = [ "cfg-if", "crunchy", @@ -3136,9 +3073,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +checksum = "fbd780fe5cc30f81464441920d82ac8740e2e46b29a6fad543ddd075229ce37e" [[package]] name = "hex" @@ -3177,9 +3114,9 @@ dependencies = [ [[package]] name = "http" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -3204,27 +3141,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http 1.2.0", + "http 1.3.1", ] [[package]] name = "http-body-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", - "futures-util", - "http 1.2.0", + "futures-core", + "http 1.3.1", "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d708df4e7140240a16cd6ab0ab65c972d7433ab77819ea693fde9c43811e2a" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "httpdate" @@ -3234,33 +3171,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "hyper" -version = "0.14.32" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", - "want", -] +checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" [[package]] name = "hyper" @@ -3271,8 +3184,8 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.8", - "http 1.2.0", + "h2", + "http 1.3.1", "http-body 1.0.1", "httparse", "httpdate", @@ -3290,7 +3203,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper 1.6.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -3298,22 +3211,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "hyper-rustls" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" -dependencies = [ - "futures-util", - "http 0.2.12", - "hyper 0.14.32", - "log", - "rustls 0.21.12", - "rustls-native-certs 0.6.3", - "tokio", - "tokio-rustls 0.24.1", -] - [[package]] name = "hyper-rustls" version = "0.27.5" @@ -3321,14 +3218,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", - "http 1.2.0", - "hyper 1.6.0", + "http 1.3.1", + "hyper", "hyper-util", - "rustls 0.23.23", - "rustls-native-certs 0.8.1", + "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls", "tower-service", ] @@ -3338,7 +3235,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.6.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -3354,9 +3251,9 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http 1.2.0", + "http 1.3.1", "http-body 1.0.1", - "hyper 1.6.0", + "hyper", "pin-project-lite", "socket2", "tokio", @@ -3372,7 +3269,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -3517,7 +3414,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -3560,9 +3457,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.7.1" +version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9c992b02b5b4c94ea26e32fe5bccb7aa7d9f390ab5c1221ff895bc7ea8b652" +checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -3584,9 +3481,9 @@ dependencies = [ [[package]] name = "indoc" -version = "2.0.5" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" +checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "integer-encoding" @@ -3602,9 +3499,9 @@ checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" [[package]] name = "is-terminal" -version = "0.4.15" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e19b23d53f35ce9f56aebc7d1bb4e6ac1e9c0db7ac85c8d1760c04379edced37" +checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", @@ -3626,6 +3523,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -3646,9 +3552,33 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.14" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "jiff" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" +checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] [[package]] name = "jobserver" @@ -3675,6 +3605,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lexical-core" version = "1.0.5" @@ -3741,9 +3677,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.170" +version = "0.2.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "875b3680cb2f8f71bdcf9a30f38d48282f5d3c95cbf9b3fa57269bb5d5c06828" +checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" [[package]] name = "libflate" @@ -3779,6 +3715,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "libloading" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +dependencies = [ + "cfg-if", + "windows-targets 0.52.6", +] + [[package]] name = "libm" version = "0.2.11" @@ -3787,9 +3733,9 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libmimalloc-sys" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44" +checksum = "07d0e07885d6a754b9c7993f2625187ad694ee985d60f23355ff0e7077261502" dependencies = [ "cc", "libc", @@ -3801,9 +3747,9 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "libc", - "redox_syscall 0.5.8", + "redox_syscall 0.5.10", ] [[package]] @@ -3814,7 +3760,7 @@ checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" dependencies = [ "anstream", "anstyle", - "clap 4.5.30", + "clap 4.5.32", "escape8259", ] @@ -3824,11 +3770,17 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" +[[package]] +name = "linux-raw-sys" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" + [[package]] name = "litemap" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" [[package]] name = "lock_api" @@ -3889,17 +3841,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] -name = "memmap2" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" -dependencies = [ - "libc", -] - -[[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -======= name = "memoffset" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3909,11 +3850,10 @@ dependencies = [ ] [[package]] ->>>>>>> upstream/branch-46:Cargo.lock name = "mimalloc" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633" +checksum = "99585191385958383e13f6b822e6b6d8d9cf928e7d286ceb092da92b43c87bc1" dependencies = [ "libmimalloc-sys", ] @@ -3925,12 +3865,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "miniz_oxide" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" -======= name = "minicov" version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3940,12 +3874,17 @@ dependencies = [ "walkdir", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3b1c9bd4fe1f0f8b387f6eb9eb3b4a1aa26185e5750efb9140301703f62cd1b" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" dependencies = [ "adler2", ] @@ -3982,12 +3921,22 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cfg-if", "cfg_aliases", "libc", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -4111,7 +4060,7 @@ dependencies = [ "chrono", "futures", "humantime", - "hyper 1.6.0", + "hyper", "itertools 0.13.0", "md-5", "parking_lot", @@ -4120,7 +4069,7 @@ dependencies = [ "rand 0.8.5", "reqwest", "ring", - "rustls-pemfile 2.2.0", + "rustls-pemfile", "serde", "serde_json", "snafu", @@ -4132,18 +4081,15 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.20.3" +version = "1.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "945462a4b81e43c4e3ba96bd7b49d834c6f61198356aa858733bc4acf3cbe62e" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -======= +checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" [[package]] name = "oorandom" -version = "11.1.4" +version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b410bbe7e14ab526a0e86877eb47c6996a2bd7746f027ba551028c925390e4e9" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl-probe" @@ -4174,9 +4120,9 @@ checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" [[package]] name = "owo-colors" -version = "4.1.0" +version = "4.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +checksum = "1036865bb9422d3300cf723f657c2851d0e9ab12567854b1f4eba3d77decf564" [[package]] name = "parking_lot" @@ -4196,22 +4142,16 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.8", + "redox_syscall 0.5.10", "smallvec", "windows-targets 0.52.6", ] [[package]] name = "parquet" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "54.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "761c44d824fe83106e0600d2510c07bf4159a4985bf0569b513ea4288dc1b4fb" -======= version = "54.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f88838dca3b84d41444a0341b19f347e8098a3898b0f21536654b8b799e11abd" ->>>>>>> upstream/branch-46:Cargo.lock dependencies = [ "ahash 0.8.11", "arrow-array", @@ -4266,7 +4206,7 @@ dependencies = [ "regex", "regex-syntax", "structmeta", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4334,7 +4274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", - "indexmap 2.7.1", + "indexmap 2.8.0", ] [[package]] @@ -4377,22 +4317,22 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe2e71e1471fe07709406bf725f710b02927c9c54b2b5b2ec0e8087d97c327d" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6e859e6e5bd50440ab63c47e3ebabc90f26251f7c73c3d3e837b74a1cc3fa67" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4409,9 +4349,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.31" +version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "plotters" @@ -4443,9 +4383,18 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.10.0" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] [[package]] name = "postgres-derive" @@ -4456,7 +4405,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4498,11 +4447,11 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.7.35", + "zerocopy 0.8.23", ] [[package]] @@ -4537,19 +4486,19 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.29" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6924ced06e1f7dfe3fa48d57b9f74f55d8915f5036121bef647ef4b204895fac" +checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" dependencies = [ "proc-macro2", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "proc-macro-crate" -version = "3.2.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" dependencies = [ "toml_edit", ] @@ -4580,19 +4529,14 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.93" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" dependencies = [ "unicode-ident", ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "psm" -version = "0.1.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -======= name = "prost" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4618,7 +4562,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.98", + "syn 2.0.100", "tempfile", ] @@ -4632,7 +4576,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4657,7 +4601,6 @@ dependencies = [ name = "psm" version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" ->>>>>>> upstream/branch-46:Cargo.lock checksum = "f58e5423e24c18cc840e1c98370b3993c6649cd1678b4d24318bcf0a083cbe88" dependencies = [ "cc", @@ -4685,9 +4628,9 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fe09249128b3173d092de9523eaa75136bf7ba85e0d69eca241c7939c933cc" +checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" dependencies = [ "cfg-if", "indoc", @@ -4703,9 +4646,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd3927b5a78757a0d71aa9dff669f903b1eb64b54142a9bd9f757f8fde65fd7" +checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" dependencies = [ "once_cell", "target-lexicon", @@ -4713,9 +4656,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab6bb2102bd8f991e7749f130a70d05dd557613e39ed2deeee8e9ca0c4d548d" +checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" dependencies = [ "libc", "pyo3-build-config", @@ -4723,27 +4666,27 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91871864b353fd5ffcb3f91f2f703a22a9797c91b9ab497b1acac7b07ae509c7" +checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "pyo3-macros-backend" -version = "0.23.4" +version = "0.23.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43abc3b80bc20f3facd86cd3c60beed58c3e2aa26213f3cda368de39c60a27e4" +checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" dependencies = [ "heck 0.5.0", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4764,37 +4707,39 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +checksum = "c3bd15a6f2967aef83887dcb9fec0014580467e33720d073560cf015a5683012" dependencies = [ "bytes", + "cfg_aliases", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls 0.23.23", + "rustc-hash 2.1.1", + "rustls", "socket2", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tracing", + "web-time", ] [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", - "getrandom 0.2.15", - "rand 0.8.5", + "getrandom 0.3.2", + "rand 0.9.0", "ring", - "rustc-hash", - "rustls 0.23.23", + "rustc-hash 2.1.1", + "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.11", + "thiserror 2.0.12", "tinyvec", "tracing", "web-time", @@ -4816,13 +4761,19 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.38" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" + [[package]] name = "radium" version = "0.7.0" @@ -4857,8 +4808,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ "rand_chacha 0.9.0", - "rand_core 0.9.1", - "zerocopy 0.8.18", + "rand_core 0.9.3", + "zerocopy 0.8.23", ] [[package]] @@ -4878,7 +4829,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.1", + "rand_core 0.9.3", ] [[package]] @@ -4892,12 +4843,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.9.1" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88e0da7a2c97baa202165137c158d0a2e824ac465d13d81046727b34cb247d3" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.1", - "zerocopy 0.8.18", + "getrandom 0.3.2", ] [[package]] @@ -4947,7 +4897,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -4961,11 +4911,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82b568323e98e49e2a0899dcee453dd679fae22d69adf9b11dd508d1549b7e2f" +checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -4976,7 +4926,7 @@ checksum = "dd6f9d3d47bdd2ad6945c5015a226ec6155d0bcdfd8f7cd29f86b71f8de99d2b" dependencies = [ "getrandom 0.2.15", "libredox", - "thiserror 2.0.11", + "thiserror 2.0.12", ] [[package]] @@ -5050,20 +5000,20 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.12" +version = "0.12.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" +checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" dependencies = [ "base64 0.22.1", "bytes", "futures-core", "futures-util", - "h2 0.4.8", - "http 1.2.0", + "h2", + "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.6.0", - "hyper-rustls 0.27.5", + "hyper", + "hyper-rustls", "hyper-util", "ipnet", "js-sys", @@ -5073,16 +5023,16 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.23", - "rustls-native-certs 0.8.1", - "rustls-pemfile 2.2.0", + "rustls", + "rustls-native-certs", + "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-rustls 0.26.1", + "tokio-rustls", "tokio-util", "tower 0.5.2", "tower-service", @@ -5096,15 +5046,9 @@ dependencies = [ [[package]] name = "ring" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "0.17.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da5349ae27d3887ca812fb375b45a4fbb36d8d12d2df394968cd86e35683fe73" -======= -version = "0.17.13" +version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", @@ -5175,7 +5119,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.98", + "syn 2.0.100", "unicode-ident", ] @@ -5187,14 +5131,14 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", "rand 0.8.5", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "rust_decimal" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b082d80e3e3cc52b2ed634388d436fe1f4de6af5786cc2de9ba9737527bdf555" +checksum = "5c24af6e7ac43c88a8a458d1139d0246fdce2f6cd2f1ac6cb51eb88b29c978af" dependencies = [ "arrayvec", "borsh", @@ -5213,6 +5157,12 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -5234,51 +5184,41 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.4.15", "windows-sys 0.59.0", ] [[package]] -name = "rustls" -version = "0.21.12" +name = "rustix" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" dependencies = [ - "log", - "ring", - "rustls-webpki 0.101.7", - "sct", + "bitflags 2.9.0", + "errno", + "libc", + "linux-raw-sys 0.9.3", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" dependencies = [ + "aws-lc-rs", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.102.8", + "rustls-webpki", "subtle", "zeroize", ] -[[package]] -name = "rustls-native-certs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" -dependencies = [ - "openssl-probe", - "rustls-pemfile 1.0.4", - "schannel", - "security-framework 2.11.1", -] - [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -5288,16 +5228,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.2.0", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64 0.21.7", + "security-framework", ] [[package]] @@ -5320,20 +5251,11 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring", - "untrusted", -] - -[[package]] -name = "rustls-webpki" -version = "0.102.8" +version = "0.103.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" dependencies = [ + "aws-lc-rs", "ring", "rustls-pki-types", "untrusted", @@ -5341,9 +5263,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" +checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" [[package]] name = "rustyline" @@ -5351,7 +5273,7 @@ version = "15.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "cfg-if", "clipboard-win", "fd-lock", @@ -5369,9 +5291,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea1a2d0a644769cc99faa24c3ad26b379b786fe7c36fd3c546254801650e6dd" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" [[package]] name = "same-file" @@ -5393,9 +5315,9 @@ dependencies = [ [[package]] name = "schemars" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09c024468a378b7e36765cd36702b7a90cc3cba11654f6685c8f233408e89e92" +checksum = "3fbf2ae1b8bc8e02df939598064d22402220cd5bbcca1c76f7d6a310974d5615" dependencies = [ "dyn-clone", "schemars_derive", @@ -5405,14 +5327,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "0.8.21" +version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1eee588578aff73f856ab961cd2f79e36bc45d7ded33a7562adba4667aecc0e" +checksum = "32e265784ad618884abaea0600a9adf15393368d840e0222d101a072f3f7534d" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5421,43 +5343,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "seahash" version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.8.0", - "core-foundation 0.9.4", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - [[package]] name = "security-framework" version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", - "core-foundation 0.10.0", + "bitflags 2.9.0", + "core-foundation", "core-foundation-sys", "libc", "security-framework-sys", @@ -5475,46 +5374,46 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.25" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" dependencies = [ "serde", ] [[package]] name = "seq-macro" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8dfc9d19bdbf6d17e22319da49161d5d0108e4188e8b680aef6299eed22df60" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.15" +version = "0.11.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" dependencies = [ "serde", ] [[package]] name = "serde_derive" -version = "1.0.218" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09503e191f4e797cb8aac08e9a4a4695c5edf6a2e70e376d961ddd5c969f82b" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5525,14 +5424,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "serde_json" -version = "1.0.139" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44f86c3acccc9c65b153fe1b85a3be07fe5515274ec9f0653b4a0875731c72a6" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -5542,13 +5441,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5560,7 +5459,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5585,7 +5484,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.7.1", + "indexmap 2.8.0", "serde", "serde_derive", "serde_json", @@ -5602,7 +5501,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5611,7 +5510,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.7.1", + "indexmap 2.8.0", "itoa", "ryu", "serde", @@ -5651,15 +5550,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -======= name = "similar" version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" [[package]] ->>>>>>> upstream/branch-46:Cargo.lock name = "siphasher" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -5698,7 +5594,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5736,12 +5632,10 @@ dependencies = [ ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -======= name = "sqllogictest" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f1c93848602f92e5925690d4805ccbc1ccdb61bee7d4ae79ad6862b542a539c" +checksum = "17b2f0b80fc250ed3fdd82fc88c0ada5ad62ee1ed5314ac5474acfa52082f518" dependencies = [ "async-trait", "educe", @@ -5758,12 +5652,11 @@ dependencies = [ "similar", "subst", "tempfile", - "thiserror 2.0.11", + "thiserror 2.0.12", "tracing", ] [[package]] ->>>>>>> upstream/branch-46:Cargo.lock name = "sqlparser" version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -5782,7 +5675,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5793,15 +5686,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9156ebd5870ef293bfb43f91c7a74528d363ec0d424afe24160ed5a4343d08a" -======= -version = "0.1.18" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "601f9201feb9b09c00266478bf459952b9ef9a6b94edb2f21eba14ab681a60a9" dependencies = [ "cc", "cfg-if", @@ -5842,7 +5729,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5853,7 +5740,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5905,7 +5792,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5918,7 +5805,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -5952,7 +5839,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.98", + "syn 2.0.100", "typify", "walkdir", ] @@ -5965,10 +5852,6 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "2.0.98" -source = "registry+https://github.com/rust-lang/crates.io-index" -======= version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" @@ -5980,10 +5863,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.98" +version = "2.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" ->>>>>>> upstream/branch-46:Cargo.lock -checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +checksum = "b09a44accad81e1ba1cd74a32461ba89dee89095ba17b32f5d03683b1b1fc2a0" dependencies = [ "proc-macro2", "quote", @@ -6007,15 +5889,10 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] -<<<<<<< HEAD:datafusion-cli/Cargo.lock -name = "tempfile" -version = "3.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -======= name = "sysinfo" version = "0.33.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6043,16 +5920,14 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.17.1" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" ->>>>>>> upstream/branch-46:Cargo.lock -checksum = "22e5a0acb1f3f55f65cc4a866c361b2fb2a0ff6366785ae6fbb5f85df07ba230" +checksum = "7437ac7763b9b123ccf33c338a5cc1bac6f69b45a136c19bdd8a65e3916435bf" dependencies = [ - "cfg-if", "fastrand", - "getrandom 0.3.1", + "getrandom 0.3.2", "once_cell", - "rustix", + "rustix 1.0.3", "windows-sys 0.59.0", ] @@ -6094,7 +5969,7 @@ dependencies = [ "serde", "serde_json", "serde_with", - "thiserror 2.0.11", + "thiserror 2.0.12", "tokio", "tokio-stream", "tokio-tar", @@ -6131,11 +6006,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" dependencies = [ - "thiserror-impl 2.0.11", + "thiserror-impl 2.0.12", ] [[package]] @@ -6146,18 +6021,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "thiserror-impl" -version = "2.0.11" +version = "2.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -6173,9 +6048,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.37" +version = "0.3.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618" dependencies = [ "deranged", "itoa", @@ -6188,15 +6063,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" +checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.19" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04" dependencies = [ "num-conv", "time-core", @@ -6233,9 +6108,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -6248,9 +6123,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.43.0" +version = "1.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" +checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" dependencies = [ "backtrace", "bytes", @@ -6272,7 +6147,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -6303,21 +6178,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" dependencies = [ - "rustls 0.21.12", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" -dependencies = [ - "rustls 0.23.23", + "rustls", "tokio", ] @@ -6349,9 +6214,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.13" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" +checksum = "6b9590b93e6fcc1739458317cccd391ad3955e2bde8913edf6f95f9e65a8f034" dependencies = [ "bytes", "futures-core", @@ -6372,7 +6237,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ - "indexmap 2.7.1", + "indexmap 2.8.0", "toml_datetime", "winnow", ] @@ -6388,11 +6253,11 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", - "h2 0.4.8", - "http 1.2.0", + "h2", + "http 1.3.1", "http-body 1.0.1", "http-body-util", - "hyper 1.6.0", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", @@ -6473,7 +6338,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -6539,7 +6404,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -6547,8 +6412,6 @@ name = "typenum" version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -======= [[package]] name = "typify" @@ -6575,8 +6438,8 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.98", - "thiserror 2.0.11", + "syn 2.0.100", + "thiserror 2.0.12", "unicode-ident", ] @@ -6593,7 +6456,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.98", + "syn 2.0.100", "typify-impl", ] @@ -6602,13 +6465,12 @@ name = "unicode-bidi" version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" ->>>>>>> upstream/branch-46:Cargo.lock [[package]] name = "unicode-ident" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00e2473a93778eb0bad35909dff6a10d28e63f792f16ed15e404fca9d5eeedbe" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unicode-normalization" @@ -6645,9 +6507,9 @@ checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" [[package]] name = "unindent" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" [[package]] name = "unsafe-libyaml" @@ -6699,20 +6561,12 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d59ca99a559661b96bf898d8fce28ed87935fd2bea9f05983c1464dd6c71b1" -dependencies = [ - "getrandom 0.3.1", -======= -version = "1.15.1" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.2", "js-sys", ->>>>>>> upstream/branch-46:Cargo.lock "serde", "wasm-bindgen", ] @@ -6765,9 +6619,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi" -version = "0.13.3+wasi-0.2.2" +version = "0.14.2+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" dependencies = [ "wit-bindgen-rt", ] @@ -6800,7 +6654,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-shared", ] @@ -6835,7 +6689,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6870,7 +6724,7 @@ checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -6906,13 +6760,25 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" dependencies = [ - "redox_syscall 0.5.8", + "redox_syscall 0.5.10", "wasite", "web-sys", ] @@ -6987,7 +6853,7 @@ checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] @@ -6998,18 +6864,24 @@ checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] +[[package]] +name = "windows-link" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" + [[package]] name = "windows-registry" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" +checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ - "windows-result 0.2.0", + "windows-result 0.3.2", "windows-strings", - "windows-targets 0.52.6", + "windows-targets 0.53.0", ] [[package]] @@ -7023,21 +6895,20 @@ dependencies = [ [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" dependencies = [ - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "windows-strings" -version = "0.1.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" +checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -7091,13 +6962,29 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +dependencies = [ + "windows_aarch64_gnullvm 0.53.0", + "windows_aarch64_msvc 0.53.0", + "windows_i686_gnu 0.53.0", + "windows_i686_gnullvm 0.53.0", + "windows_i686_msvc 0.53.0", + "windows_x86_64_gnu 0.53.0", + "windows_x86_64_gnullvm 0.53.0", + "windows_x86_64_msvc 0.53.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -7110,6 +6997,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -7122,6 +7015,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -7134,12 +7033,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -7152,6 +7063,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -7164,6 +7081,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -7176,6 +7099,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -7189,27 +7118,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winnow" -<<<<<<< HEAD:datafusion-cli/Cargo.lock -version = "0.7.3" +name = "windows_x86_64_msvc" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7f4ea97f6f78012141bcdb6a216b2609f0979ada50b20ca5b52dde2eac2bb1" -======= -version = "0.7.2" +checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" + +[[package]] +name = "winnow" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59690dea168f2198d1a3b0cac23b8063efcd11012f10ae4698f284808c8ef603" ->>>>>>> upstream/branch-46:Cargo.lock +checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" dependencies = [ "memchr", ] [[package]] name = "wit-bindgen-rt" -version = "0.33.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -7235,13 +7164,12 @@ dependencies = [ [[package]] name = "xattr" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e105d177a3871454f754b33bb0ee637ecaaac997446375fd3e5d43a2ed00c909" +checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", - "linux-raw-sys", - "rustix", + "rustix 1.0.3", ] [[package]] @@ -7279,7 +7207,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "synstructure", ] @@ -7289,17 +7217,16 @@ version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ - "byteorder", "zerocopy-derive 0.7.35", ] [[package]] name = "zerocopy" -version = "0.8.18" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" +checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" dependencies = [ - "zerocopy-derive 0.8.18", + "zerocopy-derive 0.8.23", ] [[package]] @@ -7310,38 +7237,38 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "zerocopy-derive" -version = "0.8.18" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] name = "zerofrom" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" dependencies = [ "zerofrom-derive", ] [[package]] name = "zerofrom-derive" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", "synstructure", ] @@ -7370,7 +7297,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.98", + "syn 2.0.100", ] [[package]] diff --git a/datafusion/core/src/datasource/physical_plan/arrow_file.rs b/datafusion/core/src/datasource/physical_plan/arrow_file.rs index ac5027de56383..a0e1135e2cac6 100644 --- a/datafusion/core/src/datasource/physical_plan/arrow_file.rs +++ b/datafusion/core/src/datasource/physical_plan/arrow_file.rs @@ -198,22 +198,6 @@ impl ExecutionPlan for ArrowExec { fn with_fetch(&self, limit: Option) -> Option> { self.inner.with_fetch(limit) } - - fn with_node_id( - self: Arc, - _node_id: usize, - ) -> Result>> { - let new_cache = self.cache.clone().with_node_id(_node_id); - - Ok(Some(Arc::new(Self { - base_config: self.base_config.clone(), - projected_statistics: self.projected_statistics.clone(), - projected_schema: self.projected_schema.clone(), - projected_output_ordering: self.projected_output_ordering.clone(), - metrics: self.metrics.clone(), - cache: new_cache, - }))) - } } /// Arrow configuration struct that is given to DataSourceExec diff --git a/datafusion/core/src/datasource/physical_plan/avro.rs b/datafusion/core/src/datasource/physical_plan/avro.rs index 32228291636e4..08c22183302b8 100644 --- a/datafusion/core/src/datasource/physical_plan/avro.rs +++ b/datafusion/core/src/datasource/physical_plan/avro.rs @@ -267,22 +267,6 @@ impl FileSource for AvroSource { ) -> Result> { Ok(None) } - - fn with_node_id( - self: Arc, - _node_id: usize, - ) -> Result>> { - let new_cache = self.cache.clone().with_node_id(_node_id); - - Ok(Some(Arc::new(Self { - base_config: self.base_config.clone(), - projected_statistics: self.projected_statistics.clone(), - projected_schema: self.projected_schema.clone(), - projected_output_ordering: self.projected_output_ordering.clone(), - metrics: self.metrics.clone(), - cache: new_cache, - }))) - } } #[cfg(feature = "avro")] diff --git a/datafusion/core/src/datasource/physical_plan/json.rs b/datafusion/core/src/datasource/physical_plan/json.rs index 121b82a82fdbe..c9a22add2afcd 100644 --- a/datafusion/core/src/datasource/physical_plan/json.rs +++ b/datafusion/core/src/datasource/physical_plan/json.rs @@ -219,21 +219,6 @@ impl ExecutionPlan for NdJsonExec { fn with_fetch(&self, limit: Option) -> Option> { self.inner.with_fetch(limit) } - - fn with_node_id( - self: Arc, - _node_id: usize, - ) -> Result>> { - let new_cache = self.cache.clone().with_node_id(_node_id); - - Ok(Some(Arc::new(Self { - base_config: self.base_config.clone(), - projected_statistics: self.projected_statistics.clone(), - metrics: self.metrics.clone(), - file_compression_type: self.file_compression_type, - cache: new_cache, - }))) - } } /// A [`FileOpener`] that opens a JSON file and yields a [`FileOpenFuture`] diff --git a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs index adf47b4f4d0cd..34b5e9d255200 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet/mod.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet/mod.rs @@ -537,18 +537,12 @@ impl ExecutionPlan for ParquetExec { self: Arc, _node_id: usize, ) -> Result>> { - let new_cache = self.cache.clone().with_node_id(_node_id); - let new_plan = Self { + inner: self.inner.clone(), base_config: self.base_config.clone(), - projected_statistics: self.projected_statistics.clone(), - metrics: self.metrics.clone(), predicate: self.predicate.clone(), pruning_predicate: self.pruning_predicate.clone(), - page_pruning_predicate: self.page_pruning_predicate.clone(), - metadata_size_hint: self.metadata_size_hint, parquet_file_reader_factory: self.parquet_file_reader_factory.clone(), - cache: new_cache, table_parquet_options: self.table_parquet_options.clone(), schema_adapter_factory: self.schema_adapter_factory.clone(), }; diff --git a/datafusion/core/tests/fuzz_cases/sort_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_fuzz.rs index 0b0f0aa2f105a..6d6ac7773785a 100644 --- a/datafusion/core/tests/fuzz_cases/sort_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/sort_fuzz.rs @@ -224,7 +224,7 @@ impl SortTest { /// Sort the input using SortExec and ensure the results are /// correct according to `Vec::sort` both with and without spilling async fn run(&self) -> (Vec>, Vec) { - let input = self.input.clone(); + let input = Arc::clone(self.input()); let first_batch = input .iter() .flat_map(|p| p.iter()) diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index 07cee7fba00ee..204a9414bd15d 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -185,6 +185,16 @@ impl ExecutionPlan for DataSourceExec { ) -> datafusion_common::Result>> { self.data_source.try_swapping_with_projection(projection) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> datafusion_common::Result>> { + let mut new_plan = DataSourceExec::new(self.data_source.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } impl DataSourceExec { diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 452ae82bbaeda..8060b9639892a 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -959,7 +959,7 @@ fn add_spm_on_top( .output_ordering() .unwrap_or(&LexOrdering::default()) .clone(), - input.plan.clone(), + Arc::clone(&input.plan), ) .with_fetch(fetch.take()), ) as _ @@ -1052,7 +1052,7 @@ fn replace_order_preserving_variants( if is_sort_preserving_merge(&context.plan) { // Keep the fetch value of the SortPreservingMerge operator, maybe it will be used later. let fetch = context.plan.fetch(); - let child_plan = context.children[0].plan.clone(); + let child_plan = Arc::clone(&context.children[0].plan); context.plan = Arc::new(CoalescePartitionsExec::new(child_plan)); return Ok((context, fetch)); } else if let Some(repartition) = @@ -1386,7 +1386,7 @@ pub fn ensure_distribution( }; let mut optimized_distribution_ctx = - DistributionContext::new(Arc::clone(&plan), data.clone(), children); + DistributionContext::new(Arc::clone(&plan), data, children); // If `fetch` was not consumed, it means that there was `SortPreservingMergeExec` with fetch before // It was removed by `remove_dist_changing_operators` diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs index e435d79b1416c..df485f2ff8331 100644 --- a/datafusion/physical-plan/src/analyze.rs +++ b/datafusion/physical-plan/src/analyze.rs @@ -214,8 +214,8 @@ impl ExecutionPlan for AnalyzeExec { let mut new_plan = AnalyzeExec::new( self.verbose, self.show_statistics, - self.input.clone(), - self.schema.clone(), + Arc::clone(self.input()), + Arc::clone(&self.schema), ); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs index 1dcf1bec82c6c..01d01fa596d34 100644 --- a/datafusion/physical-plan/src/coalesce_batches.rs +++ b/datafusion/physical-plan/src/coalesce_batches.rs @@ -210,7 +210,7 @@ impl ExecutionPlan for CoalesceBatchesExec { _node_id: usize, ) -> Result>> { let mut new_plan = - CoalesceBatchesExec::new(self.input.clone(), self.target_batch_size) + CoalesceBatchesExec::new(Arc::clone(self.input()), self.target_batch_size) .with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index 4a26a9895f1dd..a8be6faeb9ac6 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -198,7 +198,7 @@ impl ExecutionPlan for CoalescePartitionsExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = CoalescePartitionsExec::new(self.input.clone()); + let mut new_plan = CoalescePartitionsExec::new(Arc::clone(self.input())); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 4be52b934a144..80d8ccd9633b0 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -409,7 +409,7 @@ impl ExecutionPlan for FilterExec { _node_id: usize, ) -> Result>> { let mut new_plan = - FilterExec::try_new(self.predicate.clone(), self.input.clone())? + FilterExec::try_new(Arc::clone(&self.predicate), Arc::clone(self.input()))? .with_projection(self.projection.clone())?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; diff --git a/datafusion/physical-plan/src/insert.rs b/datafusion/physical-plan/src/insert.rs index f95fd495e1001..02b628a8f20a0 100644 --- a/datafusion/physical-plan/src/insert.rs +++ b/datafusion/physical-plan/src/insert.rs @@ -253,8 +253,8 @@ impl ExecutionPlan for DataSinkExec { _node_id: usize, ) -> Result>> { let mut new_plan = DataSinkExec::new( - self.input.clone(), - self.sink.clone(), + Arc::clone(self.input()), + Arc::clone(&self.sink), self.sort_order.clone(), ); let new_props = new_plan.cache.clone().with_node_id(_node_id); diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 8cff06567f802..1382aedcfab2c 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -344,7 +344,8 @@ impl ExecutionPlan for CrossJoinExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = CrossJoinExec::new(self.left.clone(), self.right.clone()); + let mut new_plan = + CrossJoinExec::new(Arc::clone(&self.left), Arc::clone(&self.right)); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 062ad2dc4d5d0..54df87706e8fc 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -874,13 +874,13 @@ impl ExecutionPlan for HashJoinExec { _node_id: usize, ) -> Result>> { let mut new_plan = HashJoinExec::try_new( - self.left.clone(), - self.right.clone(), + Arc::clone(&self.left), + Arc::clone(&self.right), self.on.clone(), self.filter.clone(), self.join_type(), self.projection.clone(), - self.partition_mode().clone(), + *self.partition_mode(), self.null_equals_null, )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); diff --git a/datafusion/physical-plan/src/joins/sort_merge_join.rs b/datafusion/physical-plan/src/joins/sort_merge_join.rs index 8759a5e4f38ca..3ed0210e42dd2 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join.rs @@ -513,8 +513,8 @@ impl ExecutionPlan for SortMergeJoinExec { _node_id: usize, ) -> Result>> { let mut new_plan = SortMergeJoinExec::try_new( - self.left.clone(), - self.right.clone(), + Arc::clone(&self.left), + Arc::clone(&self.right), self.on.clone(), self.filter.clone(), self.join_type(), diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index f6a8588b23df7..adfa2c0ef362e 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -464,8 +464,8 @@ impl ExecutionPlan for SymmetricHashJoinExec { _node_id: usize, ) -> Result>> { let mut new_plan = SymmetricHashJoinExec::try_new( - self.left.clone(), - self.right.clone(), + Arc::clone(&self.left), + Arc::clone(&self.right), self.on.clone(), self.filter.clone(), self.join_type(), diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index 505fe1222f2b6..3eaf5c66f1db1 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -209,7 +209,7 @@ impl ExecutionPlan for GlobalLimitExec { _node_id: usize, ) -> Result>> { let mut new_plan = - GlobalLimitExec::new(self.input.clone(), self.skip, self.fetch); + GlobalLimitExec::new(Arc::clone(self.input()), self.skip, self.fetch); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/node_id.rs b/datafusion/physical-plan/src/node_id.rs index a03c747f7c15d..2a246db0a77b5 100644 --- a/datafusion/physical-plan/src/node_id.rs +++ b/datafusion/physical-plan/src/node_id.rs @@ -34,12 +34,20 @@ impl NodeIdAnnotator { &mut self, plan: Arc, ) -> Result, DataFusionError> { - let plan_with_id = plan.clone().with_node_id(self.next_id)?.unwrap_or(plan); + let plan_with_id = Arc::clone(&plan) + .with_node_id(self.next_id)? + .unwrap_or(plan); self.next_id += 1; Ok(plan_with_id) } } +impl Default for NodeIdAnnotator { + fn default() -> Self { + Self::new() + } +} + pub fn annotate_node_id_for_execution_plan( plan: &Arc, annotator: &mut NodeIdAnnotator, @@ -50,7 +58,7 @@ pub fn annotate_node_id_for_execution_plan( annotate_node_id_for_execution_plan(child, annotator)?; new_children.push(new_child); } - let new_plan = plan.clone().with_new_children(new_children)?; + let new_plan = Arc::clone(plan).with_new_children(new_children)?; let new_plan_with_id = annotator.annotate_execution_plan_with_node_id(new_plan)?; Ok(new_plan_with_id) } diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs index 9a03540f3bc57..f8706e9515168 100644 --- a/datafusion/physical-plan/src/placeholder_row.rs +++ b/datafusion/physical-plan/src/placeholder_row.rs @@ -178,7 +178,7 @@ impl ExecutionPlan for PlaceholderRowExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = PlaceholderRowExec::new(self.schema.clone()); + let mut new_plan = PlaceholderRowExec::new(Arc::clone(&self.schema)); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index 3902778f6a9de..e7ec2c1e4ed71 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -250,7 +250,7 @@ impl ExecutionPlan for ProjectionExec { _node_id: usize, ) -> Result>> { let mut new_plan = - ProjectionExec::try_new(self.expr.clone(), self.input.clone())?; + ProjectionExec::try_new(self.expr.clone(), Arc::clone(self.input()))?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs index 77a36db1ad3d0..ab66c7356e6b9 100644 --- a/datafusion/physical-plan/src/recursive_query.rs +++ b/datafusion/physical-plan/src/recursive_query.rs @@ -214,8 +214,8 @@ impl ExecutionPlan for RecursiveQueryExec { ) -> Result>> { let mut new_plan = RecursiveQueryExec::try_new( self.name.clone(), - self.static_term.clone(), - self.recursive_term.clone(), + Arc::clone(&self.static_term), + Arc::clone(&self.recursive_term), self.is_distinct, )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 6b3cbc07f4fdb..247dbedffa9e7 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1146,7 +1146,7 @@ impl ExecutionPlan for SortExec { _node_id: usize, ) -> Result>> { let new_plan = SortExec { - input: self.input.clone(), + input: Arc::clone(self.input()), expr: self.expr.clone(), fetch: self.fetch, metrics_set: self.metrics_set.clone(), diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index 36eff3b7a7bec..ba7523aa68324 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -342,7 +342,7 @@ impl ExecutionPlan for SortPreservingMergeExec { _node_id: usize, ) -> Result>> { let mut new_plan = - SortPreservingMergeExec::new(self.expr.clone(), self.input.clone()) + SortPreservingMergeExec::new(self.expr.clone(), Arc::clone(self.input())) .with_fetch(self.fetch()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index a0da7a3ebbe57..bdc21055d2980 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -202,10 +202,10 @@ impl ExecutionPlan for UnnestExec { _node_id: usize, ) -> Result>> { let mut new_plan = UnnestExec::new( - self.input.clone(), + Arc::clone(self.input()), self.list_column_indices.clone(), self.struct_column_indices.clone(), - self.schema.clone(), + Arc::clone(&self.schema), self.options.clone(), ); let new_props = new_plan.cache.clone().with_node_id(_node_id); diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs index 18f2ece325391..4adc561994b68 100644 --- a/datafusion/physical-plan/src/windows/window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs @@ -286,8 +286,8 @@ impl ExecutionPlan for WindowAggExec { ) -> Result>> { let mut new_plan = WindowAggExec::try_new( self.window_expr.clone(), - self.input.clone(), - self.partition_keys.clone(), + Arc::clone(self.input()), + self.can_repartition, )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs index 86a159258ed38..df01a836076c6 100644 --- a/datafusion/physical-plan/src/work_table.rs +++ b/datafusion/physical-plan/src/work_table.rs @@ -229,7 +229,8 @@ impl ExecutionPlan for WorkTableExec { self: Arc, _node_id: usize, ) -> Result>> { - let mut new_plan = WorkTableExec::new(self.name.clone(), self.schema.clone()); + let mut new_plan = + WorkTableExec::new(self.name.clone(), Arc::clone(&self.schema)); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index 504e5164805a3..65402df014850 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -55,7 +55,7 @@ postgres-types = { version = "0.2.8", features = ["derive", "with-chrono-0_4"], rust_decimal = { version = "1.36.0", features = ["tokio-pg"] } # When updating the following dependency verify that sqlite test file regeneration works correctly # by running the regenerate_sqlite_files.sh script. -sqllogictest = "0.27.2" +sqllogictest = "0.28.0" sqlparser = { workspace = true } tempfile = { workspace = true } testcontainers = { version = "0.23", features = ["default"], optional = true } From 3be582f1d9847f6865689384e04c27da38e94878 Mon Sep 17 00:00:00 2001 From: Matt Friede <7852262+Friede80@users.noreply.github.com> Date: Sun, 16 Mar 2025 06:32:33 -0400 Subject: [PATCH 056/177] Fix invalid schema for unions in ViewTables (#15135) * Add test for coerce_union_schema * coerce_union to use its own schema instead of that of the first plan * Generate unique field names for union schema instead of using table qualifiers * Review feedback: avoid cloning schema * start from union schema when coercing * cargo fmt * dont use wildcard in test * Dont strip qualifiers for sorts over unions --- datafusion/expr/src/logical_plan/builder.rs | 1 + datafusion/expr/src/logical_plan/plan.rs | 20 ++++-- .../optimizer/src/analyzer/type_coercion.rs | 62 +++++++++++++++--- .../optimizer/src/propagate_empty_relation.rs | 4 +- datafusion/sqllogictest/test_files/limit.slt | 2 +- datafusion/sqllogictest/test_files/order.slt | 43 ++++++------ .../sqllogictest/test_files/type_coercion.slt | 2 +- datafusion/sqllogictest/test_files/union.slt | 65 +++++++++++++++++-- .../sqllogictest/test_files/union_by_name.slt | 12 ++-- 9 files changed, 161 insertions(+), 50 deletions(-) diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index f60bb2f007714..2ade93a25f177 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -776,6 +776,7 @@ impl LogicalPlanBuilder { &missing_cols, is_distinct, )?; + let sort_plan = LogicalPlan::Sort(Sort { expr: normalize_sorts(sorts, &plan)?, input: Arc::new(plan), diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 72b82fc219eb6..8e46811a20a35 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -2814,6 +2814,7 @@ impl Union { } } + let mut name_counts: HashMap = HashMap::new(); let union_fields = (0..fields_count) .map(|i| { let fields = inputs @@ -2821,7 +2822,8 @@ impl Union { .map(|input| input.schema().field(i)) .collect::>(); let first_field = fields[0]; - let name = first_field.name(); + let base_name = first_field.name().to_string(); + let data_type = if loose_types { // TODO apply type coercion here, or document why it's better to defer // temporarily use the data type from the left input and later rely on the analyzer to @@ -2844,13 +2846,21 @@ impl Union { )? }; let nullable = fields.iter().any(|field| field.is_nullable()); - let mut field = Field::new(name, data_type.clone(), nullable); + + // Generate unique field name + let name = if let Some(count) = name_counts.get_mut(&base_name) { + *count += 1; + format!("{}_{}", base_name, count) + } else { + name_counts.insert(base_name.clone(), 0); + base_name + }; + + let mut field = Field::new(&name, data_type.clone(), nullable); let field_metadata = intersect_maps(fields.iter().map(|field| field.metadata())); field.set_metadata(field_metadata); - // TODO reusing table reference from the first schema is probably wrong - let table_reference = first_schema.qualified_field(i).0.cloned(); - Ok((table_reference, Arc::new(field))) + Ok((None, Arc::new(field))) }) .collect::>()?; let union_schema_metadata = diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index 538ef98ac7bed..8248a0cc793a0 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -214,7 +214,10 @@ impl<'a> TypeCoercionRewriter<'a> { /// Coerce the union’s inputs to a common schema compatible with all inputs. /// This occurs after wildcard expansion and the coercion of the input expressions. pub fn coerce_union(union_plan: Union) -> Result { - let union_schema = Arc::new(coerce_union_schema(&union_plan.inputs)?); + let union_schema = Arc::new(coerce_union_schema_with_schema( + &union_plan.inputs, + &union_plan.schema, + )?); let new_inputs = union_plan .inputs .into_iter() @@ -930,7 +933,12 @@ fn coerce_case_expression(case: Case, schema: &DFSchema) -> Result { /// This method presumes that the wildcard expansion is unneeded, or has already /// been applied. pub fn coerce_union_schema(inputs: &[Arc]) -> Result { - let base_schema = inputs[0].schema(); + coerce_union_schema_with_schema(&inputs[1..], inputs[0].schema()) +} +fn coerce_union_schema_with_schema( + inputs: &[Arc], + base_schema: &DFSchemaRef, +) -> Result { let mut union_datatypes = base_schema .fields() .iter() @@ -949,7 +957,7 @@ pub fn coerce_union_schema(inputs: &[Arc]) -> Result { let mut metadata = base_schema.metadata().clone(); - for (i, plan) in inputs.iter().enumerate().skip(1) { + for (i, plan) in inputs.iter().enumerate() { let plan_schema = plan.schema(); metadata.extend(plan_schema.metadata().clone()); @@ -989,15 +997,15 @@ pub fn coerce_union_schema(inputs: &[Arc]) -> Result { } } let union_qualified_fields = izip!( - base_schema.iter(), + base_schema.fields(), union_datatypes.into_iter(), union_nullabilities, union_field_meta.into_iter() ) - .map(|((qualifier, field), datatype, nullable, metadata)| { + .map(|(field, datatype, nullable, metadata)| { let mut field = Field::new(field.name().clone(), datatype, nullable); field.set_metadata(metadata); - (qualifier.cloned(), field.into()) + (None, field.into()) }) .collect::>(); @@ -1041,11 +1049,12 @@ mod test { use std::sync::Arc; use arrow::datatypes::DataType::Utf8; - use arrow::datatypes::{DataType, Field, TimeUnit}; + use arrow::datatypes::{DataType, Field, Schema, TimeUnit}; use crate::analyzer::type_coercion::{ coerce_case_expression, TypeCoercion, TypeCoercionRewriter, }; + use crate::analyzer::Analyzer; use crate::test::{assert_analyzed_plan_eq, assert_analyzed_plan_with_config_eq}; use datafusion_common::config::ConfigOptions; use datafusion_common::tree_node::{TransformedResult, TreeNode}; @@ -1057,9 +1066,10 @@ mod test { cast, col, create_udaf, is_true, lit, AccumulatorFactoryFunction, AggregateUDF, BinaryExpr, Case, ColumnarValue, Expr, ExprSchemable, Filter, LogicalPlan, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, - SimpleAggregateUDF, Subquery, Volatility, + SimpleAggregateUDF, Subquery, Union, Volatility, }; use datafusion_functions_aggregate::average::AvgAccumulator; + use datafusion_sql::TableReference; fn empty() -> Arc { Arc::new(LogicalPlan::EmptyRelation(EmptyRelation { @@ -1090,6 +1100,42 @@ mod test { assert_analyzed_plan_eq(Arc::new(TypeCoercion::new()), plan, expected) } + #[test] + fn test_coerce_union() -> Result<()> { + let left_plan = Arc::new(LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new( + DFSchema::try_from_qualified_schema( + TableReference::full("datafusion", "test", "foo"), + &Schema::new(vec![Field::new("a", DataType::Int32, false)]), + ) + .unwrap(), + ), + })); + let right_plan = Arc::new(LogicalPlan::EmptyRelation(EmptyRelation { + produce_one_row: false, + schema: Arc::new( + DFSchema::try_from_qualified_schema( + TableReference::full("datafusion", "test", "foo"), + &Schema::new(vec![Field::new("a", DataType::Int64, false)]), + ) + .unwrap(), + ), + })); + let union = LogicalPlan::Union(Union::try_new_with_loose_types(vec![ + left_plan, right_plan, + ])?); + let analyzed_union = Analyzer::with_rules(vec![Arc::new(TypeCoercion::new())]) + .execute_and_check(union, &ConfigOptions::default(), |_, _| {})?; + let top_level_plan = LogicalPlan::Projection(Projection::try_new( + vec![col("a")], + Arc::new(analyzed_union), + )?); + + let expected = "Projection: a\n Union\n Projection: CAST(datafusion.test.foo.a AS Int64) AS a\n EmptyRelation\n EmptyRelation"; + assert_analyzed_plan_eq(Arc::new(TypeCoercion::new()), top_level_plan, expected) + } + fn coerce_on_output_if_viewtype(plan: LogicalPlan, expected: &str) -> Result<()> { let mut options = ConfigOptions::default(); options.optimizer.expand_views_at_output = true; diff --git a/datafusion/optimizer/src/propagate_empty_relation.rs b/datafusion/optimizer/src/propagate_empty_relation.rs index d26df073dc6fd..344707ae8dbe3 100644 --- a/datafusion/optimizer/src/propagate_empty_relation.rs +++ b/datafusion/optimizer/src/propagate_empty_relation.rs @@ -316,7 +316,7 @@ mod tests { let plan = LogicalPlanBuilder::from(left).union(right)?.build()?; - let expected = "TableScan: test"; + let expected = "Projection: a, b, c\n TableScan: test"; assert_together_optimized_plan(plan, expected, true) } @@ -406,7 +406,7 @@ mod tests { let plan = LogicalPlanBuilder::from(left).union(right)?.build()?; - let expected = "TableScan: test"; + let expected = "Projection: a, b, c\n TableScan: test"; assert_together_optimized_plan(plan, expected, true) } diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 067b23ac2fb01..93ffa313b8f70 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -654,7 +654,7 @@ explain select * FROM ( ---- logical_plan 01)Limit: skip=4, fetch=10 -02)--Sort: ordered_table.c DESC NULLS FIRST, fetch=14 +02)--Sort: c DESC NULLS FIRST, fetch=14 03)----Union 04)------Projection: CAST(ordered_table.c AS Int64) AS c 05)--------TableScan: ordered_table projection=[c] diff --git a/datafusion/sqllogictest/test_files/order.slt b/datafusion/sqllogictest/test_files/order.slt index d7da21c58ec60..80f6d1b384880 100644 --- a/datafusion/sqllogictest/test_files/order.slt +++ b/datafusion/sqllogictest/test_files/order.slt @@ -774,7 +774,7 @@ SELECT * FROM v ORDER BY 1, 2; ---- logical_plan -01)Sort: u.m ASC NULLS LAST, u.t ASC NULLS LAST +01)Sort: m ASC NULLS LAST, t ASC NULLS LAST 02)--Union 03)----SubqueryAlias: u 04)------Projection: Int64(0) AS m, m0.t @@ -925,10 +925,10 @@ ORDER BY SUM(column1) # ORDER BY with a GROUP BY clause query I -SELECT SUM(column1) - FROM foo -GROUP BY column2 -ORDER BY SUM(column1) +SELECT SUM(column1) + FROM foo +GROUP BY column2 +ORDER BY SUM(column1) ---- 0 2 @@ -940,12 +940,12 @@ ORDER BY SUM(column1) # ORDER BY with a GROUP BY clause and a HAVING clause query I -SELECT - SUM(column1) -FROM foo -GROUP BY column2 -HAVING SUM(column1) < 3 -ORDER BY SUM(column1) +SELECT + SUM(column1) +FROM foo +GROUP BY column2 +HAVING SUM(column1) < 3 +ORDER BY SUM(column1) ---- 0 2 @@ -1071,7 +1071,7 @@ physical_plan 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true statement ok -drop table ordered_table; +drop table ordered_table; # ABS(x) breaks the ordering if x's range contains both negative and positive values. @@ -1107,7 +1107,7 @@ physical_plan 05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/core/tests/data/window_2.csv]]}, projection=[c], output_ordering=[c@0 ASC NULLS LAST], file_type=csv, has_header=true statement ok -drop table ordered_table; +drop table ordered_table; # ABS(x) preserves the ordering if x's range falls into positive values. # Since x is defined as INT UNSIGNED, its range is assumed to be from 0 to INF. @@ -1239,15 +1239,14 @@ order by d, c, a, a0, b limit 2; ---- logical_plan -01)Projection: t1.b, t1.c, t1.a, t1.a0 -02)--Sort: t1.d ASC NULLS LAST, t1.c ASC NULLS LAST, t1.a ASC NULLS LAST, t1.a0 ASC NULLS LAST, t1.b ASC NULLS LAST, fetch=2 -03)----Union -04)------SubqueryAlias: t1 -05)--------Projection: ordered_table.b, ordered_table.c, ordered_table.a, Int32(NULL) AS a0, ordered_table.d -06)----------TableScan: ordered_table projection=[a, b, c, d] -07)------SubqueryAlias: t2 -08)--------Projection: ordered_table.b, ordered_table.c, Int32(NULL) AS a, ordered_table.a0, ordered_table.d -09)----------TableScan: ordered_table projection=[a0, b, c, d] +01)Sort: d ASC NULLS LAST, c ASC NULLS LAST, a ASC NULLS LAST, a0 ASC NULLS LAST, b ASC NULLS LAST, fetch=2 +02)--Union +03)----SubqueryAlias: t1 +04)------Projection: ordered_table.b, ordered_table.c, ordered_table.a, Int32(NULL) AS a0, ordered_table.d +05)--------TableScan: ordered_table projection=[a, b, c, d] +06)----SubqueryAlias: t2 +07)------Projection: ordered_table.b, ordered_table.c, Int32(NULL) AS a, ordered_table.a0, ordered_table.d +08)--------TableScan: ordered_table projection=[a0, b, c, d] physical_plan 01)ProjectionExec: expr=[b@0 as b, c@1 as c, a@2 as a, a0@3 as a0] 02)--SortPreservingMergeExec: [d@4 ASC NULLS LAST, c@1 ASC NULLS LAST, a@2 ASC NULLS LAST, a0@3 ASC NULLS LAST, b@0 ASC NULLS LAST], fetch=2 diff --git a/datafusion/sqllogictest/test_files/type_coercion.slt b/datafusion/sqllogictest/test_files/type_coercion.slt index 0900c88c15c01..2c6079bc7039d 100644 --- a/datafusion/sqllogictest/test_files/type_coercion.slt +++ b/datafusion/sqllogictest/test_files/type_coercion.slt @@ -187,7 +187,7 @@ EXPLAIN SELECT a FROM (select 1 a) x GROUP BY 1 (SELECT a FROM (select 1.1 a) x GROUP BY 1) ORDER BY 1 ---- logical_plan -01)Sort: x.a ASC NULLS LAST +01)Sort: a ASC NULLS LAST 02)--Union 03)----Projection: CAST(x.a AS Float64) AS a 04)------Aggregate: groupBy=[[x.a]], aggr=[[]] diff --git a/datafusion/sqllogictest/test_files/union.slt b/datafusion/sqllogictest/test_files/union.slt index 918c6e2811737..654bccfab5a6f 100644 --- a/datafusion/sqllogictest/test_files/union.slt +++ b/datafusion/sqllogictest/test_files/union.slt @@ -226,7 +226,7 @@ query TT EXPLAIN SELECT name FROM t1 UNION (SELECT name from t2 UNION SELECT name || '_new' from t2) ---- logical_plan -01)Aggregate: groupBy=[[t1.name]], aggr=[[]] +01)Aggregate: groupBy=[[name]], aggr=[[]] 02)--Union 03)----TableScan: t1 projection=[name] 04)----TableScan: t2 projection=[name] @@ -411,7 +411,7 @@ query TT explain SELECT c1, c9 FROM aggregate_test_100 UNION ALL SELECT c1, c3 FROM aggregate_test_100 ORDER BY c9 DESC LIMIT 5 ---- logical_plan -01)Sort: aggregate_test_100.c9 DESC NULLS FIRST, fetch=5 +01)Sort: c9 DESC NULLS FIRST, fetch=5 02)--Union 03)----Projection: aggregate_test_100.c1, CAST(aggregate_test_100.c9 AS Decimal128(20, 0)) AS c9 04)------TableScan: aggregate_test_100 projection=[c1, c9] @@ -449,7 +449,7 @@ SELECT count(*) FROM ( ---- logical_plan 01)Projection: count(Int64(1)) AS count(*) -02)--Aggregate: groupBy=[[t1.name]], aggr=[[count(Int64(1))]] +02)--Aggregate: groupBy=[[name]], aggr=[[count(Int64(1))]] 03)----Union 04)------Aggregate: groupBy=[[t1.name]], aggr=[[]] 05)--------TableScan: t1 projection=[name] @@ -601,7 +601,7 @@ UNION ALL ORDER BY c1 ---- logical_plan -01)Sort: t1.c1 ASC NULLS LAST +01)Sort: c1 ASC NULLS LAST 02)--Union 03)----TableScan: t1 projection=[c1] 04)----Projection: t2.c1a AS c1 @@ -709,6 +709,25 @@ SELECT t1.v2, t1.v0 FROM t2 NATURAL JOIN t1 SELECT t1.v2, t1.v0 FROM t2 NATURAL JOIN t1 WHERE (t1.v2 IS NULL); ---- +query IR +SELECT t1.v0, t2.v0 FROM t1,t2 + UNION ALL +SELECT t1.v0, t2.v0 FROM t1,t2 +ORDER BY v0; +---- +-1493773377 0.280145772929 +-1493773377 0.280145772929 +-1229445667 0.280145772929 +-1229445667 0.280145772929 +1541512604 0.280145772929 +1541512604 0.280145772929 +NULL 0.280145772929 +NULL 0.280145772929 +NULL 0.280145772929 +NULL 0.280145772929 +NULL 0.280145772929 +NULL 0.280145772929 + statement ok CREATE TABLE t3 ( id INT @@ -814,7 +833,7 @@ UNION ALL ORDER BY c1 ---- logical_plan -01)Sort: aggregate_test_100.c1 ASC NULLS LAST +01)Sort: c1 ASC NULLS LAST 02)--Union 03)----Filter: aggregate_test_100.c1 = Utf8("a") 04)------TableScan: aggregate_test_100 projection=[c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13], partial_filters=[aggregate_test_100.c1 = Utf8("a")] @@ -860,3 +879,39 @@ FROM ( GROUP BY combined ---- AB + + +# Test union in view +statement ok +CREATE TABLE u1 (x INT, y INT); + +statement ok +INSERT INTO u1 VALUES (3, 3), (3, 3), (1, 1); + +statement ok +CREATE TABLE u2 (y BIGINT, z BIGINT); + +statement ok +INSERT INTO u2 VALUES (20, 20), (40, 40); + +statement ok +CREATE VIEW v1 AS +SELECT y FROM u1 UNION ALL SELECT y FROM u2 ORDER BY y; + +query I +SELECT * FROM (SELECT y FROM u1 UNION ALL SELECT y FROM u2) ORDER BY y; +---- +1 +3 +3 +20 +40 + +query I +SELECT * FROM v1; +---- +1 +3 +3 +20 +40 diff --git a/datafusion/sqllogictest/test_files/union_by_name.slt b/datafusion/sqllogictest/test_files/union_by_name.slt index 63a43a36ff16d..3844dba68079f 100644 --- a/datafusion/sqllogictest/test_files/union_by_name.slt +++ b/datafusion/sqllogictest/test_files/union_by_name.slt @@ -54,13 +54,13 @@ INSERT INTO t2 VALUES (2, 2), (4, 4); # Test binding query I -SELECT t1.x FROM t1 UNION BY NAME SELECT x FROM t1 ORDER BY t1.x; +SELECT t1.x FROM t1 UNION BY NAME SELECT x FROM t1 ORDER BY x; ---- 1 3 query I -SELECT t1.x FROM t1 UNION ALL BY NAME SELECT x FROM t1 ORDER BY t1.x; +SELECT t1.x FROM t1 UNION ALL BY NAME SELECT x FROM t1 ORDER BY x; ---- 1 1 @@ -70,13 +70,13 @@ SELECT t1.x FROM t1 UNION ALL BY NAME SELECT x FROM t1 ORDER BY t1.x; 3 query I -SELECT x FROM t1 UNION BY NAME SELECT x FROM t1 ORDER BY t1.x; +SELECT x FROM t1 UNION BY NAME SELECT x FROM t1 ORDER BY x; ---- 1 3 query I -SELECT x FROM t1 UNION ALL BY NAME SELECT x FROM t1 ORDER BY t1.x; +SELECT x FROM t1 UNION ALL BY NAME SELECT x FROM t1 ORDER BY x; ---- 1 1 @@ -124,8 +124,8 @@ NULL 3 # Ambiguous name -statement error DataFusion error: Schema error: No field named t1.x. Valid fields are a, b. -SELECT x AS a FROM t1 UNION BY NAME SELECT x AS b FROM t1 ORDER BY t1.x; +statement error DataFusion error: Schema error: No field named x. Valid fields are a, b. +SELECT x AS a FROM t1 UNION BY NAME SELECT x AS b FROM t1 ORDER BY x; query II (SELECT y FROM t1 UNION ALL SELECT x FROM t1) UNION BY NAME (SELECT z FROM t2 UNION ALL SELECT y FROM t2) ORDER BY y, z; From a28f2cd6e9c9f75783b46e72d5ae3c64e61326d3 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 14 Apr 2025 16:40:37 +0800 Subject: [PATCH 057/177] Fix enforce_distribution and enforce_sorting missing fetch --- .../src/enforce_distribution.rs | 17 +++++++++++++---- .../src/enforce_sorting/mod.rs | 6 +++++- .../replace_with_order_preserving_variants.rs | 5 +++++ 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 8060b9639892a..957118d03682a 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -1036,12 +1036,14 @@ fn remove_dist_changing_operators( /// ``` fn replace_order_preserving_variants( mut context: DistributionContext, + ordering_satisfied: bool, ) -> Result<(DistributionContext, Option)> { let mut children = vec![]; let mut fetch = None; for child in context.children.into_iter() { if child.data { - let (child, inner_fetch) = replace_order_preserving_variants(child)?; + let (child, inner_fetch) = + replace_order_preserving_variants(child, ordering_satisfied)?; children.push(child); fetch = inner_fetch; } else { @@ -1053,6 +1055,13 @@ fn replace_order_preserving_variants( // Keep the fetch value of the SortPreservingMerge operator, maybe it will be used later. let fetch = context.plan.fetch(); let child_plan = Arc::clone(&context.children[0].plan); + if !ordering_satisfied { + // It's safe to unwrap because `CoalescePartitionsExec` supports `fetch`. + context.plan = CoalescePartitionsExec::new(child_plan) + .with_fetch(fetch) + .unwrap(); + return Ok((context, None)); + } context.plan = Arc::new(CoalescePartitionsExec::new(child_plan)); return Ok((context, fetch)); } else if let Some(repartition) = @@ -1312,7 +1321,7 @@ pub fn ensure_distribution( && child.data { let (replaced_child, fetch) = - replace_order_preserving_variants(child)?; + replace_order_preserving_variants(child, ordering_satisfied)?; child = replaced_child; // If ordering requirements were satisfied before repartitioning, // make sure ordering requirements are still satisfied after. @@ -1333,12 +1342,12 @@ pub fn ensure_distribution( // Operator requires specific distribution. Distribution::SinglePartition | Distribution::HashPartitioned(_) => { // Since there is no ordering requirement, preserving ordering is pointless - child = replace_order_preserving_variants(child)?.0; + child = replace_order_preserving_variants(child, false)?.0; } Distribution::UnspecifiedDistribution => { // Since ordering is lost, trying to preserve ordering is pointless if !maintains || plan.as_any().is::() { - child = replace_order_preserving_variants(child)?.0; + child = replace_order_preserving_variants(child, false)?.0; } } } diff --git a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs index 9f75fe097a951..85c7a497ed7d7 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/mod.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/mod.rs @@ -400,6 +400,7 @@ pub fn parallelize_sorts( ), )) } else if is_coalesce_partitions(&requirements.plan) { + let fetch = requirements.plan.fetch(); // There is an unnecessary `CoalescePartitionsExec` in the plan. // This will handle the recursive `CoalescePartitionsExec` plans. requirements = remove_bottleneck_in_subplan(requirements)?; @@ -408,7 +409,10 @@ pub fn parallelize_sorts( Ok(Transformed::yes( PlanWithCorrespondingCoalescePartitions::new( - Arc::new(CoalescePartitionsExec::new(Arc::clone(&requirements.plan))), + // Safe to unwrap, because `CoalescePartitionsExec` has a fetch + CoalescePartitionsExec::new(Arc::clone(&requirements.plan)) + .with_fetch(fetch) + .unwrap(), false, vec![requirements], ), diff --git a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs index 2c5c0d4d510ec..b464ec387cdfc 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs @@ -137,6 +137,11 @@ fn plan_with_order_preserving_variants( return Ok(sort_input); } else if is_coalesce_partitions(&sort_input.plan) && is_spm_better { let child = &sort_input.children[0].plan; + let mut fetch = fetch; + if let Some(coalesce_fetch) = sort_input.plan.fetch() { + // Get the min fetch between the `fetch` and the coalesce's fetch: + fetch = Some(coalesce_fetch.min(fetch.unwrap_or(usize::MAX))) + }; if let Some(ordering) = child.output_ordering() { // When the input of a `CoalescePartitionsExec` has an ordering, // replace it with a `SortPreservingMergeExec` if appropriate: From e4433049b04ca2c1e2031eb05d1a0990210f11d6 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 16 Apr 2025 15:35:01 -0400 Subject: [PATCH 058/177] Final release note touchups (#15740) --- dev/changelog/47.0.0.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/dev/changelog/47.0.0.md b/dev/changelog/47.0.0.md index 68dd5978e41c2..64ca2e157a9e3 100644 --- a/dev/changelog/47.0.0.md +++ b/dev/changelog/47.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 47.0.0 Changelog -This release consists of 362 commits from 95 contributors. See credits at the end of this changelog for more information. +This release consists of 364 commits from 94 contributors. See credits at the end of this changelog for more information. **Breaking changes:** @@ -94,6 +94,7 @@ This release consists of 362 commits from 95 contributors. See credits at the en - fix: Rewrite `date_trunc` and `from_unixtime` for the SQLite unparser [#15630](https://github.com/apache/datafusion/pull/15630) (peasee) - fix(substrait): fix regressed edge case in renaming inner struct fields [#15634](https://github.com/apache/datafusion/pull/15634) (Blizzara) - fix: normalize window ident [#15639](https://github.com/apache/datafusion/pull/15639) (chenkovsky) +- fix: unparse join without projection [#15693](https://github.com/apache/datafusion/pull/15693) (chenkovsky) **Documentation updates:** @@ -134,6 +135,7 @@ This release consists of 362 commits from 95 contributors. See credits at the en - Add coerce int96 option for Parquet to support different TimeUnits, test int96_from_spark.parquet from parquet-testing [#15537](https://github.com/apache/datafusion/pull/15537) (mbutrovich) - STRING_AGG missing functionality [#14412](https://github.com/apache/datafusion/pull/14412) (gabotechs) - doc : update RepartitionExec display tree [#15710](https://github.com/apache/datafusion/pull/15710) (getChan) +- Update version to 47.0.0, add CHANGELOG [#15731](https://github.com/apache/datafusion/pull/15731) (xudong963) **Other:** @@ -398,6 +400,7 @@ This release consists of 362 commits from 95 contributors. See credits at the en - Fix internal error in sort when hitting memory limit [#15692](https://github.com/apache/datafusion/pull/15692) (DerGut) - Update checked in Cargo.lock file to get clean CI [#15725](https://github.com/apache/datafusion/pull/15725) (alamb) - chore(deps): bump indexmap from 2.8.0 to 2.9.0 [#15732](https://github.com/apache/datafusion/pull/15732) (dependabot[bot]) +- Minor: include output partition count of `RepartitionExec` to tree explain [#15717](https://github.com/apache/datafusion/pull/15717) (2010YOUY01) ## Credits @@ -406,12 +409,12 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co ``` 48 dependabot[bot] 34 Andrew Lamb + 16 xudong.w 15 Jay Zhan 15 Qi Zhu 15 irenjj - 15 xudong.w - 12 Chen Chongchen - 12 Yongting You + 13 Chen Chongchen + 13 Yongting You 10 Tommy shu 7 Shruti Sharma 6 Alan Tang From fe4a4ca417f646be8d8929c5ab94863af278dcbf Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 21 Apr 2025 18:06:56 +0800 Subject: [PATCH 059/177] Upgrade DF47 --- Cargo.lock | 852 ++++-------------- .../src/analyzer/expand_wildcard_rule.rs | 0 datafusion/physical-plan/src/sorts/sort.rs | 1 + parquet-testing | 2 +- 4 files changed, 187 insertions(+), 668 deletions(-) delete mode 100644 datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs diff --git a/Cargo.lock b/Cargo.lock index 779d688e908bd..b6652cfbff274 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -199,9 +199,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.97" +version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" +checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" [[package]] name = "apache-avro" @@ -265,7 +265,7 @@ dependencies = [ "arrow-string", "half", "pyo3", - "rand 0.9.0", + "rand 0.9.1", ] [[package]] @@ -413,11 +413,7 @@ dependencies = [ "arrow-schema", "chrono", "half", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "lexical-core", "memchr", "num", @@ -458,11 +454,7 @@ version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7450c76ab7c5a6805be3440dc2e2096010da58f7cab301fdc996a4ee3ee74e49" dependencies = [ -<<<<<<< HEAD "bitflags 2.9.0", -======= - "bitflags 2.8.0", ->>>>>>> upstream/branch-47 "serde", ] @@ -511,9 +503,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.0.16" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1835b7f27878de8525dc71410b5a31cdcc5f230aed5ba5df968e09c201b23d" +checksum = "2bd389a4b2970a01282ee455294913c0a43724daedcd1a24c3eb0ec1c1320b66" dependencies = [ "anstyle", "bstr", @@ -618,15 +610,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-config" -<<<<<<< HEAD -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a84fe2c5e9965fba0fbc2001db252f1d57527d82a905cca85127df227bca748" -======= version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c39646d1a6b51240a1a23bb57ea4eebede7e16fbc237fdc876980233dcecb4f" ->>>>>>> upstream/branch-47 dependencies = [ "aws-credential-types", "aws-runtime", @@ -643,11 +629,7 @@ dependencies = [ "bytes", "fastrand", "hex", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "ring", "time", "tokio", @@ -670,9 +652,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.6" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +checksum = "19b756939cb2f8dc900aa6dcd505e6e2428e9cae7ff7b028c49e3946efa70878" dependencies = [ "aws-lc-sys", "zeroize", @@ -680,15 +662,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -<<<<<<< HEAD -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77926887776171ced7d662120a75998e444d3750c951abfe07f90da130514b1f" -======= -version = "0.27.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbe221bbf523b625a4dd8585c7f38166e31167ec2ca98051dbcb4c3b6e825d2" ->>>>>>> upstream/branch-47 +checksum = "0ddeb19ee86cb16ecfc871e5b0660aff6285760957aaedda6284cf0e790d3769" dependencies = [ "bindgen", "cc", @@ -724,15 +700,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -<<<<<<< HEAD -version = "1.62.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5330ad4e8a1ff49e9f26b738611caa72b105c41d41733801d1a36e8f9de936" -======= -version = "1.63.0" +version = "1.64.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cb45b83b53b5cd55ee33fd9fd8a70750255a3f286e4dca20e882052f2b256f" ->>>>>>> upstream/branch-47 +checksum = "02d4bdb0e5f80f0689e61c77ab678b2b9304af329616af38aef5b6b967b8e736" dependencies = [ "aws-credential-types", "aws-runtime", @@ -753,15 +723,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -<<<<<<< HEAD -version = "1.63.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7956b1a85d49082347a7d17daa2e32df191f3e23c03d47294b99f95413026a78" -======= -version = "1.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4d9bc075ea6238778ed3951b65d3cde8c3864282d64fdcd19f2a90c0609f1" ->>>>>>> upstream/branch-47 +checksum = "acbbb3ce8da257aedbccdcb1aadafbbb6a5fe9adf445db0e1ea897bdc7e22d08" dependencies = [ "aws-credential-types", "aws-runtime", @@ -782,15 +746,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -<<<<<<< HEAD -version = "1.63.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065c533fbe6f84962af33fcf02b0350b7c1f79285baab5924615d2be3b232855" -======= -version = "1.64.0" +version = "1.65.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819ccba087f403890fee4825eeab460e64c59345667d2b83a12cf544b581e3a7" ->>>>>>> upstream/branch-47 +checksum = "96a78a8f50a1630db757b60f679c8226a8a70ee2ab5f5e6e51dc67f6c61c7cfd" dependencies = [ "aws-credential-types", "aws-runtime", @@ -856,11 +814,7 @@ dependencies = [ "bytes-utils", "futures-core", "http 0.2.12", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "http-body 0.4.6", "once_cell", "percent-encoding", @@ -871,19 +825,15 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0497ef5d53065b7cd6a35e9c1654bd1fefeae5c52900d91d1b188b0af0f29324" +checksum = "8aff1159006441d02e57204bf57a1b890ba68bedb6904ffd2873c1c4c11c546b" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", "h2", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "hyper", "hyper-rustls", "hyper-util", @@ -927,33 +877,20 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -<<<<<<< HEAD -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6328865e36c6fd970094ead6b05efd047d3a80ec5fc3be5e743910da9f2ebf8" -======= version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0152749e17ce4d1b47c7747bdfec09dac1ccafdcbc741ebf9daa2a373356730f" ->>>>>>> upstream/branch-47 dependencies = [ "aws-smithy-async", "aws-smithy-http", "aws-smithy-http-client", -<<<<<<< HEAD -======= "aws-smithy-observability", ->>>>>>> upstream/branch-47 "aws-smithy-runtime-api", "aws-smithy-types", "bytes", "fastrand", "http 0.2.12", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "http-body 0.4.6", "http-body 1.0.1", "once_cell", @@ -1130,17 +1067,10 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ -<<<<<<< HEAD "bitflags 2.9.0", "cexpr", "clang-sys", "itertools 0.12.1", -======= - "bitflags 2.8.0", - "cexpr", - "clang-sys", - "itertools 0.10.5", ->>>>>>> upstream/branch-47 "lazy_static", "lazycell", "log", @@ -1189,15 +1119,9 @@ dependencies = [ [[package]] name = "blake3" -<<<<<<< HEAD -version = "1.7.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17679a8d69b6d7fd9cd9801a536cec9fa5e5970b69f9d4747f70b39b031f5e7" -======= -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389a099b34312839e16420d499a9cad9650541715937ffbdd40d36f49e77eeb3" ->>>>>>> upstream/branch-47 +checksum = "3888aaa89e4b2a40fca9848e400f6a658a5a3978de7be858e209cafa8be9a4a0" dependencies = [ "arrayref", "arrayvec", @@ -1267,9 +1191,9 @@ dependencies = [ [[package]] name = "borsh" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b74d67a0fc0af8e9823b79fd1c43a0900e5a8f0e0f4cc9210796bf3a820126" +checksum = "ad8646f98db542e39fc66e68a20b2144f6a732636df7c2354e74645faaa433ce" dependencies = [ "borsh-derive", "cfg_aliases", @@ -1277,9 +1201,9 @@ dependencies = [ [[package]] name = "borsh-derive" -version = "1.5.6" +version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d37ed1b2c9b78421218a0b4f6d8349132d6ec2cfeba1cfb0118b0a8e268df9e" +checksum = "fdd1d3c0c2f5833f22386f252fe8ed005c7f59fdcddeef025c01b4c3b9fd9ac3" dependencies = [ "once_cell", "proc-macro-crate", @@ -1301,9 +1225,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "4.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "a334ef7c9e23abf0ce748e8cd309037da93e606ad52eb372e4ce327a0dcfbdfd" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1311,9 +1235,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.11.3" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" dependencies = [ "memchr", "regex-automata", @@ -1407,9 +1331,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.16" +version = "1.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" +checksum = "8e3a13707ac958681c13b39b458c073d0d9bc8a22cb1b2f4c8e55eb72c13f362" dependencies = [ "jobserver", "libc", @@ -1465,9 +1389,9 @@ dependencies = [ [[package]] name = "chrono-tz-build" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94fea34d77a245229e7746bd2beb786cd2a896f306ff491fb8cecb3074b10a7" +checksum = "8f10f8c9340e31fc120ff885fcdb54a0b48e474bbd77cab557f0c30a3e569402" dependencies = [ "parse-zoneinfo", "phf_codegen", @@ -1524,15 +1448,9 @@ dependencies = [ [[package]] name = "clap" -<<<<<<< HEAD -version = "4.5.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6088f3ae8c3608d19260cd7445411865a485688711b78b5be70d78cd96136f83" -======= -version = "4.5.35" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" ->>>>>>> upstream/branch-47 +checksum = "eccb054f56cbd38340b380d4a8e69ef1f02f1af43db2f0cc817a4774d80ae071" dependencies = [ "clap_builder", "clap_derive", @@ -1540,15 +1458,9 @@ dependencies = [ [[package]] name = "clap_builder" -<<<<<<< HEAD -version = "4.5.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a7ef7f676155edfb82daa97f99441f3ebf4a58d5e32f295a56259f1b6facc8" -======= -version = "4.5.35" +version = "4.5.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" ->>>>>>> upstream/branch-47 +checksum = "efd9466fac8543255d3b1fcad4762c5e116ffe808c8a3043d4263cd4fd4862a2" dependencies = [ "anstream", "anstyle", @@ -1730,11 +1642,7 @@ dependencies = [ "anes", "cast", "ciborium", -<<<<<<< HEAD - "clap 4.5.32", -======= - "clap 4.5.35", ->>>>>>> upstream/branch-47 + "clap 4.5.37", "criterion-plot", "futures", "is-terminal", @@ -1846,9 +1754,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ "darling_core", "darling_macro", @@ -1856,9 +1764,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" dependencies = [ "fnv", "ident_case", @@ -1870,9 +1778,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.10" +version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", @@ -1901,11 +1809,7 @@ dependencies = [ [[package]] name = "datafusion" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "arrow-ipc", @@ -1975,11 +1879,7 @@ dependencies = [ [[package]] name = "datafusion-benchmarks" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "datafusion", @@ -2003,11 +1903,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "async-trait", @@ -2031,11 +1927,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "async-trait", @@ -2057,22 +1949,14 @@ dependencies = [ [[package]] name = "datafusion-cli" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "assert_cmd", "async-trait", "aws-config", "aws-credential-types", -<<<<<<< HEAD - "clap 4.5.32", -======= - "clap 4.5.35", ->>>>>>> upstream/branch-47 + "clap 4.5.37", "ctor", "datafusion", "dirs", @@ -2094,11 +1978,7 @@ dependencies = [ [[package]] name = "datafusion-common" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "ahash 0.8.11", "apache-avro", @@ -2108,12 +1988,8 @@ dependencies = [ "chrono", "half", "hashbrown 0.14.5", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", "insta", ->>>>>>> upstream/branch-47 "libc", "log", "object_store", @@ -2129,11 +2005,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "futures", "log", @@ -2142,11 +2014,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "async-compression", @@ -2282,19 +2150,11 @@ dependencies = [ [[package]] name = "datafusion-doc" -<<<<<<< HEAD -version = "46.0.1" - -[[package]] -name = "datafusion-examples" -version = "46.0.1" -======= version = "47.0.0" [[package]] name = "datafusion-examples" version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "arrow-flight", @@ -2324,11 +2184,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "chrono", @@ -2346,11 +2202,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "chrono", @@ -2362,11 +2214,7 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr-common", "env_logger", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "paste", "recursive", "serde_json", @@ -2375,30 +2223,18 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -<<<<<<< HEAD -version = "46.0.1" -dependencies = [ - "arrow", - "datafusion-common", - "indexmap 2.8.0", -======= version = "47.0.0" dependencies = [ "arrow", "datafusion-common", "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "itertools 0.14.0", "paste", ] [[package]] name = "datafusion-ffi" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "abi_stable", "arrow", @@ -2417,11 +2253,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "arrow-buffer", @@ -2450,11 +2282,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "ahash 0.8.11", "arrow", @@ -2475,11 +2303,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "ahash 0.8.11", "arrow", @@ -2492,11 +2316,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "arrow-ord", @@ -2517,11 +2337,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "async-trait", @@ -2535,11 +2351,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "datafusion-common", @@ -2555,11 +2367,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2567,11 +2375,7 @@ dependencies = [ [[package]] name = "datafusion-macros" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "datafusion-expr", "quote", @@ -2580,11 +2384,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "async-trait", @@ -2598,12 +2398,8 @@ dependencies = [ "datafusion-physical-expr", "datafusion-sql", "env_logger", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", "insta", ->>>>>>> upstream/branch-47 "itertools 0.14.0", "log", "recursive", @@ -2613,11 +2409,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "ahash 0.8.11", "arrow", @@ -2630,12 +2422,8 @@ dependencies = [ "datafusion-physical-expr-common", "half", "hashbrown 0.14.5", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", "insta", ->>>>>>> upstream/branch-47 "itertools 0.14.0", "log", "paste", @@ -2646,11 +2434,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "ahash 0.8.11", "arrow", @@ -2662,11 +2446,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "datafusion-common", @@ -2685,11 +2465,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "ahash 0.8.11", "arrow", @@ -2710,12 +2486,8 @@ dependencies = [ "futures", "half", "hashbrown 0.14.5", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", "insta", ->>>>>>> upstream/branch-47 "itertools 0.14.0", "log", "parking_lot", @@ -2729,11 +2501,7 @@ dependencies = [ [[package]] name = "datafusion-proto" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "chrono", @@ -2756,11 +2524,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "datafusion-common", @@ -2795,11 +2559,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "bigdecimal", @@ -2811,12 +2571,8 @@ dependencies = [ "datafusion-functions-nested", "datafusion-functions-window", "env_logger", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", "insta", ->>>>>>> upstream/branch-47 "log", "paste", "recursive", @@ -2827,22 +2583,14 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "arrow", "async-trait", "bigdecimal", "bytes", "chrono", -<<<<<<< HEAD - "clap 4.5.32", -======= - "clap 4.5.35", ->>>>>>> upstream/branch-47 + "clap 4.5.37", "datafusion", "env_logger", "futures", @@ -2866,11 +2614,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "async-recursion", "async-trait", @@ -2890,11 +2634,7 @@ dependencies = [ [[package]] name = "datafusion-wasmtest" -<<<<<<< HEAD -version = "46.0.1" -======= version = "47.0.0" ->>>>>>> upstream/branch-47 dependencies = [ "chrono", "console_error_panic_hook", @@ -2993,15 +2733,6 @@ dependencies = [ [[package]] name = "dunce" version = "1.0.5" -<<<<<<< HEAD -======= -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "dyn-clone" -version = "1.0.18" ->>>>>>> upstream/branch-47 source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" @@ -3073,9 +2804,9 @@ dependencies = [ [[package]] name = "env_logger" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3716d7a920fb4fac5d84e9d4bce8ceb321e9414b4409da61b07b75c1e3d0697" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" dependencies = [ "anstream", "anstyle", @@ -3092,9 +2823,9 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" dependencies = [ "libc", "windows-sys 0.59.0", @@ -3142,13 +2873,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", -<<<<<<< HEAD - "rustix 1.0.3", + "rustix 1.0.5", "windows-sys 0.59.0", -======= - "rustix 0.38.44", - "windows-sys 0.52.0", ->>>>>>> upstream/branch-47 ] [[package]] @@ -3205,7 +2931,7 @@ version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", "rustc_version", ] @@ -3441,8 +3167,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] -<<<<<<< HEAD -======= name = "globset" version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3456,24 +3180,18 @@ dependencies = [ ] [[package]] ->>>>>>> upstream/branch-47 name = "h2" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "75249d144030531f8dee69fe9cea04d3edf809a017ae445e2abdff6629e86633" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", -<<<<<<< HEAD "http 1.3.1", - "indexmap 2.8.0", -======= - "http 1.2.0", "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "slab", "tokio", "tokio-util", @@ -3482,9 +3200,9 @@ dependencies = [ [[package]] name = "half" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7db2ff139bba50379da6aa0766b52fdcb62cb5b263009b09ed58ba604e14bbd1" +checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" dependencies = [ "cfg-if", "crunchy", @@ -3638,11 +3356,7 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" name = "humantime" version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -<<<<<<< HEAD checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" -======= -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" ->>>>>>> upstream/branch-47 [[package]] name = "hyper" @@ -3654,11 +3368,7 @@ dependencies = [ "futures-channel", "futures-util", "h2", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "http-body 1.0.1", "httparse", "httpdate", @@ -3691,11 +3401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "hyper", "hyper-util", "rustls", @@ -3721,9 +3427,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "497bbc33a26fdd4af9ed9c70d63f61cf56a938375fbb32df34db9b1cd6d643f2" dependencies = [ "bytes", "futures-channel", @@ -3731,6 +3437,7 @@ dependencies = [ "http 1.3.1", "http-body 1.0.1", "hyper", + "libc", "pin-project-lite", "socket2", "tokio", @@ -3755,16 +3462,17 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.61" +version = "0.1.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" dependencies = [ "android_system_properties", "core-foundation-sys", "iana-time-zone-haiku", "js-sys", + "log", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core 0.61.0", ] [[package]] @@ -3817,9 +3525,9 @@ dependencies = [ [[package]] name = "icu_locid_transform_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" [[package]] name = "icu_normalizer" @@ -3841,9 +3549,9 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" [[package]] name = "icu_properties" @@ -3862,9 +3570,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" [[package]] name = "icu_provider" @@ -3934,15 +3642,9 @@ dependencies = [ [[package]] name = "indexmap" -<<<<<<< HEAD -version = "2.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3954d50fe15b02142bf25d3b8bdadb634ec3948f103d04ffe3031bc8fe9d7058" -======= version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" ->>>>>>> upstream/branch-47 dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -4069,33 +3771,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde", -] - -[[package]] -name = "jiff-static" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -name = "jiff" -version = "0.2.4" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" +checksum = "59ec30f7142be6fe14e1b021f50b85db8df2d4324ea6e91ec3e5dcde092021d0" dependencies = [ "jiff-static", "log", @@ -4106,9 +3784,9 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.4" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" +checksum = "526b834d727fd59d37b076b0c3236d9adde1b1729a4361e20b2026f738cc1dbe" dependencies = [ "proc-macro2", "quote", @@ -4117,10 +3795,11 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.2", "libc", ] @@ -4212,9 +3891,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.171" +version = "0.2.172" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6" +checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" [[package]] name = "libflate" @@ -4268,15 +3947,9 @@ checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" [[package]] name = "libmimalloc-sys" -<<<<<<< HEAD -version = "0.1.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07d0e07885d6a754b9c7993f2625187ad694ee985d60f23355ff0e7077261502" -======= version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec9d6fac27761dabcd4ee73571cdb06b7022dc99089acbe5435691edffaac0f4" ->>>>>>> upstream/branch-47 dependencies = [ "cc", "libc", @@ -4290,7 +3963,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ "bitflags 2.9.0", "libc", - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", ] [[package]] @@ -4301,11 +3974,7 @@ checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" dependencies = [ "anstream", "anstyle", -<<<<<<< HEAD - "clap 4.5.32", -======= - "clap 4.5.35", ->>>>>>> upstream/branch-47 + "clap 4.5.37", "escape8259", ] @@ -4332,19 +4001,9 @@ checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "linux-raw-sys" -<<<<<<< HEAD -version = "0.9.3" -======= -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9c683daf087dc577b7506e9695b3d556a9f3849903fa28186283afd6809e9" - -[[package]] -name = "litemap" -version = "0.7.4" ->>>>>>> upstream/branch-47 -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe7db12097d22ec582439daf8618b8fdd1a7bef6270e9af3b1ebcd30893cf413" +checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" @@ -4421,15 +4080,9 @@ dependencies = [ [[package]] name = "mimalloc" -<<<<<<< HEAD -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99585191385958383e13f6b822e6b6d8d9cf928e7d286ceb092da92b43c87bc1" -======= version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "995942f432bbb4822a7e9c3faa87a695185b0d09273ba85f097b54f4e458f2af" ->>>>>>> upstream/branch-47 dependencies = [ "libmimalloc-sys", ] @@ -4458,15 +4111,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -<<<<<<< HEAD -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e3e04debbb59698c15bacbb6d93584a8c0ca9cc3213cb423d31f760d8843ce5" -======= version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" ->>>>>>> upstream/branch-47 dependencies = [ "adler2", ] @@ -4633,11 +4280,11 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "objc2-core-foundation" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daeaf60f25471d26948a1c2f840e3f7d86f4109e3af4e8e4b5cd70c39690d925" +checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.0", ] [[package]] @@ -4661,15 +4308,11 @@ dependencies = [ "chrono", "form_urlencoded", "futures", - "http 1.2.0", + "http 1.3.1", "http-body-util", "humantime", "hyper", -<<<<<<< HEAD - "itertools 0.13.0", -======= "itertools 0.14.0", ->>>>>>> upstream/branch-47 "md-5", "parking_lot", "percent-encoding", @@ -4690,9 +4333,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.1" +version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d75b0bedcc4fe52caa0e03d9f1151a323e4aa5e2d78ba3580400cd3c9e2bc4bc" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "oorandom" @@ -4757,7 +4400,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "smallvec", "windows-targets 0.52.6", ] @@ -4888,11 +4531,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", ->>>>>>> upstream/branch-47 ] [[package]] @@ -5014,15 +4653,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "portable-atomic-util" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" -dependencies = [ - "portable-atomic", -] - [[package]] name = "postgres-derive" version = "0.4.6" @@ -5048,7 +4678,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.9.0", + "rand 0.9.1", "sha2", "stringprep", ] @@ -5078,7 +4708,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.8.23", + "zerocopy 0.8.24", ] [[package]] @@ -5113,9 +4743,9 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.31" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5316f57387668042f561aae71480de936257848f9c43ce528e311d89a07cadeb" +checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" dependencies = [ "proc-macro2", "syn 2.0.100", @@ -5156,9 +4786,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31971752e70b8b2686d7e46ec17fb38dad4051d94024c88df49b667caea9c84" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" dependencies = [ "unicode-ident", ] @@ -5255,15 +4885,9 @@ dependencies = [ [[package]] name = "pyo3" -<<<<<<< HEAD -version = "0.23.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7778bffd85cf38175ac1f545509665d0b9b92a198ca7941f131f85f7a4f9a872" -======= version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17da310086b068fbdcefbba30aeb3721d5bb9af8db4987d6735b2183ca567229" ->>>>>>> upstream/branch-47 dependencies = [ "cfg-if", "indoc", @@ -5279,15 +4903,9 @@ dependencies = [ [[package]] name = "pyo3-build-config" -<<<<<<< HEAD -version = "0.23.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94f6cbe86ef3bf18998d9df6e0f3fc1050a8c5efa409bf712e661a4366e010fb" -======= version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e27165889bd793000a098bb966adc4300c312497ea25cf7a690a9f0ac5aa5fc1" ->>>>>>> upstream/branch-47 dependencies = [ "once_cell", "target-lexicon", @@ -5295,15 +4913,9 @@ dependencies = [ [[package]] name = "pyo3-ffi" -<<<<<<< HEAD -version = "0.23.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f1b4c431c0bb1c8fb0a338709859eed0d030ff6daa34368d3b152a63dfdd8d" -======= version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05280526e1dbf6b420062f3ef228b78c0c54ba94e157f5cb724a609d0f2faabc" ->>>>>>> upstream/branch-47 dependencies = [ "libc", "pyo3-build-config", @@ -5311,15 +4923,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -<<<<<<< HEAD -version = "0.23.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc2201328f63c4710f68abdf653c89d8dbc2858b88c5d88b0ff38a75288a9da" -======= version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3ce5686aa4d3f63359a5100c62a127c9f15e8398e5fdeb5deef1fed5cd5f44" ->>>>>>> upstream/branch-47 dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -5329,15 +4935,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -<<<<<<< HEAD -version = "0.23.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca6726ad0f3da9c9de093d6f116a93c1a38e417ed73bf138472cf4064f72028" -======= version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4cf6faa0cbfb0ed08e89beb8103ae9724eb4750e3a78084ba4017cbe94f3855" ->>>>>>> upstream/branch-47 dependencies = [ "heck 0.5.0", "proc-macro2", @@ -5354,9 +4954,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.37.2" +version = "0.37.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +checksum = "a4ce8c88de324ff838700f36fb6ab86c96df0e3c4ab6ef3a9b2044465cce1369" dependencies = [ "memchr", "serde", @@ -5390,7 +4990,7 @@ checksum = "b820744eb4dc9b57a3398183639c511b5a26d2ed702cedd3febaa1393caa22cc" dependencies = [ "bytes", "getrandom 0.3.2", - "rand 0.9.0", + "rand 0.9.1", "ring", "rustc-hash 2.1.1", "rustls", @@ -5404,9 +5004,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" +checksum = "541d0f57c6ec747a90738a52741d3221f7960e8ac2f0ff4b1a63680e033b4ab5" dependencies = [ "cfg_aliases", "libc", @@ -5460,13 +5060,12 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", - "zerocopy 0.8.23", ] [[package]] @@ -5568,9 +5167,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b8c0c260b63a8219631167be35e6a988e9554dbd323f8bd08439c8ed1302bd1" +checksum = "d2f103c6d277498fbceb16e84d317e2a400f160f46904d5f5410848c829511a3" dependencies = [ "bitflags 2.9.0", ] @@ -5666,11 +5265,7 @@ dependencies = [ "futures-core", "futures-util", "h2", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "http-body 1.0.1", "http-body-util", "hyper", @@ -5707,15 +5302,9 @@ dependencies = [ [[package]] name = "ring" -<<<<<<< HEAD version = "0.17.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -======= -version = "0.17.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee" ->>>>>>> upstream/branch-47 dependencies = [ "cc", "cfg-if", @@ -5803,15 +5392,9 @@ dependencies = [ [[package]] name = "rust_decimal" -<<<<<<< HEAD -version = "1.37.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c24af6e7ac43c88a8a458d1139d0246fdce2f6cd2f1ac6cb51eb88b29c978af" -======= version = "1.37.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faa7de2ba56ac291bd90c6b9bece784a52ae1411f9506544b3eae36dd2356d50" ->>>>>>> upstream/branch-47 dependencies = [ "arrayvec", "borsh", @@ -5861,50 +5444,27 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.4.15", -<<<<<<< HEAD "windows-sys 0.59.0", ] [[package]] name = "rustix" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e56a18552996ac8d29ecc3b190b4fdbb2d91ca4ec396de7bbffaf43f3d637e96" +checksum = "d97817398dd4bb2e6da002002db259209759911da105da92bec29ccb12cf58bf" dependencies = [ "bitflags 2.9.0", "errno", "libc", - "linux-raw-sys 0.9.3", -======= ->>>>>>> upstream/branch-47 - "windows-sys 0.59.0", -] - -[[package]] -<<<<<<< HEAD -name = "rustls" -version = "0.23.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ee9188ac4ec04a2f0531e55d035fb2de73f18b41a63c70c2712503b6fb13c" -======= -name = "rustix" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" -dependencies = [ - "bitflags 2.8.0", - "errno", - "libc", - "linux-raw-sys 0.9.2", + "linux-raw-sys 0.9.4", "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" ->>>>>>> upstream/branch-47 +checksum = "df51b5869f3a441595eac5e8ff14d486ff285f7b8c0df8770e49c3b56351f0f0" dependencies = [ "aws-lc-rs", "once_cell", @@ -5947,15 +5507,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -<<<<<<< HEAD -version = "0.103.0" +version = "0.103.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0aa4eeac2588ffff23e9d7a7e9b3f971c5fb5b7ebc9452745e0c232c64f83b2f" -======= -version = "0.102.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" ->>>>>>> upstream/branch-47 +checksum = "fef8b8769aaccf73098557a87cd1816b4f9c7c16811c9c77142aa695c16f2c03" dependencies = [ "aws-lc-rs", "ring", @@ -6057,11 +5611,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ -<<<<<<< HEAD "bitflags 2.9.0", -======= - "bitflags 2.8.0", ->>>>>>> upstream/branch-47 "core-foundation", "core-foundation-sys", "libc", @@ -6190,11 +5740,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "serde", "serde_derive", "serde_json", @@ -6220,11 +5766,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "itoa", "ryu", "serde", @@ -6259,9 +5801,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.2" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" dependencies = [ "libc", ] @@ -6295,35 +5837,11 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcf8323ef1faaee30a44a340193b1ac6814fd9b7b4e88e9d4519a3e4abe1cfd" - -[[package]] -<<<<<<< HEAD -name = "snafu" -version = "0.8.5" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "223891c85e2a29c3fe8fb900c1fae5e69c2e42415e3177752e8718475efa5019" -dependencies = [ - "snafu-derive", -] +checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" [[package]] -name = "snafu-derive" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 2.0.100", -] - -[[package]] -======= ->>>>>>> upstream/branch-47 name = "snap" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6349,9 +5867,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8" +checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" dependencies = [ "libc", "windows-sys 0.52.0", @@ -6359,9 +5877,9 @@ dependencies = [ [[package]] name = "sqllogictest" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b2f0b80fc250ed3fdd82fc88c0ada5ad62ee1ed5314ac5474acfa52082f518" +checksum = "ee6199c1e008acc669b1e5873c138bf3ad4f8709ccd5c5d88913e664ae4f75de" dependencies = [ "async-trait", "educe", @@ -6546,9 +6064,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.55.0" +version = "0.55.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a359aeb711c1e1944c0c4178bbb2d679d39237ac5bfe28f7e0506e522e5ce6" +checksum = "048fe52a3664881ccdfdc9bdb0f4e8805f3444ee64abf299d365c54f6a2ffabb" dependencies = [ "heck 0.5.0", "pbjson", @@ -6652,11 +6170,7 @@ dependencies = [ "fastrand", "getrandom 0.3.2", "once_cell", -<<<<<<< HEAD - "rustix 1.0.3", -======= - "rustix 1.0.2", ->>>>>>> upstream/branch-47 + "rustix 1.0.5", "windows-sys 0.59.0", ] @@ -6762,8 +6276,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.100", -<<<<<<< HEAD -======= ] [[package]] @@ -6774,7 +6286,6 @@ checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", ->>>>>>> upstream/branch-47 ] [[package]] @@ -6790,9 +6301,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.40" +version = "0.3.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d9c75b47bdff86fa3334a3db91356b8d7d86a9b839dab7d0bdc5c3d3a077618" +checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" dependencies = [ "deranged", "itoa", @@ -6811,9 +6322,9 @@ checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" [[package]] name = "time-macros" -version = "0.2.21" +version = "0.2.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29aa485584182073ed57fd5004aa09c371f021325014694e432313345865fd04" +checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" dependencies = [ "num-conv", "time-core", @@ -6865,9 +6376,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "e6b88822cbe49de4185e3a4cbf8321dd487cf5fe0c5c65695fef6346371e9c48" dependencies = [ "backtrace", "bytes", @@ -6911,7 +6422,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.0", + "rand 0.9.1", "socket2", "tokio", "tokio-util", @@ -6920,15 +6431,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -<<<<<<< HEAD version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" -======= -version = "0.26.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" ->>>>>>> upstream/branch-47 dependencies = [ "rustls", "tokio", @@ -6985,11 +6490,7 @@ version = "0.22.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b4795ff5edd201c7cd6dca065ae59972ce77d1b80fa0a84d94950ece7d1474" dependencies = [ -<<<<<<< HEAD - "indexmap 2.8.0", -======= "indexmap 2.9.0", ->>>>>>> upstream/branch-47 "toml_datetime", "winnow", ] @@ -7006,11 +6507,7 @@ dependencies = [ "base64 0.22.1", "bytes", "h2", -<<<<<<< HEAD "http 1.3.1", -======= - "http 1.2.0", ->>>>>>> upstream/branch-47 "http-body 1.0.1", "http-body-util", "hyper", @@ -7568,11 +7065,11 @@ dependencies = [ [[package]] name = "whoami" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372d5b87f58ec45c384ba03563b03544dc5fadc3983e434b286913f5b4a9bb6d" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ - "redox_syscall 0.5.10", + "redox_syscall 0.5.11", "wasite", "web-sys", ] @@ -7620,23 +7117,27 @@ dependencies = [ [[package]] name = "windows-core" -version = "0.52.0" +version = "0.57.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" dependencies = [ + "windows-implement 0.57.0", + "windows-interface 0.57.0", + "windows-result 0.1.2", "windows-targets 0.52.6", ] [[package]] name = "windows-core" -version = "0.57.0" +version = "0.61.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" dependencies = [ - "windows-implement", - "windows-interface", - "windows-result 0.1.2", - "windows-targets 0.52.6", + "windows-implement 0.60.0", + "windows-interface 0.59.1", + "windows-link", + "windows-result 0.3.2", + "windows-strings 0.4.0", ] [[package]] @@ -7650,6 +7151,17 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "windows-implement" +version = "0.60.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] + [[package]] name = "windows-interface" version = "0.57.0" @@ -7662,17 +7174,19 @@ dependencies = [ ] [[package]] -name = "windows-link" -version = "0.1.1" -<<<<<<< HEAD -======= +name = "windows-interface" +version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.100", +] [[package]] -name = "windows-registry" -version = "0.2.0" ->>>>>>> upstream/branch-47 +name = "windows-link" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" @@ -7683,7 +7197,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" dependencies = [ "windows-result 0.3.2", - "windows-strings", + "windows-strings 0.3.1", "windows-targets 0.53.0", ] @@ -7714,6 +7228,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-strings" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -7928,9 +7451,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97b544156e9bebe1a0ffbc03484fc1ffe3100cbce3ffb17eac35f7cdd7ab36" +checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10" dependencies = [ "memchr", ] @@ -7972,12 +7495,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", -<<<<<<< HEAD - "rustix 1.0.3", -======= - "linux-raw-sys 0.4.15", - "rustix 0.38.44", ->>>>>>> upstream/branch-47 + "rustix 1.0.5", ] [[package]] @@ -8030,11 +7548,11 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd97444d05a4328b90e75e503a34bad781f14e28a823ad3557f0750df1ebcbc6" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" dependencies = [ - "zerocopy-derive 0.8.23", + "zerocopy-derive 0.8.24", ] [[package]] @@ -8050,9 +7568,9 @@ dependencies = [ [[package]] name = "zerocopy-derive" -version = "0.8.23" +version = "0.8.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6352c01d0edd5db859a63e2605f4ea3183ddbd15e2c4a9e7d32184df75e4f154" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" dependencies = [ "proc-macro2", "quote", @@ -8125,18 +7643,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.1" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.13+zstd.1.5.6" +version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ "cc", "pkg-config", diff --git a/datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs b/datafusion/optimizer/src/analyzer/expand_wildcard_rule.rs deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 6cfa7324ba90b..1625139362241 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1325,6 +1325,7 @@ impl ExecutionPlan for SortExec { metrics_set: self.metrics_set.clone(), preserve_partitioning: self.preserve_partitioning, cache: self.cache.clone().with_node_id(_node_id), + common_sort_prefix: self.common_sort_prefix.clone(), }; Ok(Some(Arc::new(new_plan))) } diff --git a/parquet-testing b/parquet-testing index e45cd23f784aa..6e851ddd768d6 160000 --- a/parquet-testing +++ b/parquet-testing @@ -1 +1 @@ -Subproject commit e45cd23f784aab3d6bf0701f8f4e621469ed3be7 +Subproject commit 6e851ddd768d6af741c7b15dc594874399fc3cff From dfb339d7a1914816503b7a6140b13a7aa57f45e4 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Wed, 23 Apr 2025 16:32:40 +0800 Subject: [PATCH 060/177] Fix: fetch is missing in plan_with_order_breaking_variants method --- .../replace_with_order_preserving_variants.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs index b464ec387cdfc..4af4692664877 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs @@ -34,7 +34,7 @@ use datafusion_physical_plan::execution_plan::EmissionType; use datafusion_physical_plan::repartition::RepartitionExec; use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion_physical_plan::tree_node::PlanContext; -use datafusion_physical_plan::ExecutionPlanProperties; +use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties}; use itertools::izip; @@ -194,10 +194,11 @@ fn plan_with_order_breaking_variants( let partitioning = plan.output_partitioning().clone(); sort_input.plan = Arc::new(RepartitionExec::try_new(child, partitioning)?) as _; } else if is_sort_preserving_merge(plan) { - // Replace `SortPreservingMergeExec` with a `CoalescePartitionsExec`: + // Replace `SortPreservingMergeExec` with a `CoalescePartitionsExec` + // SPM may have `fetch`, so pass it to the `CoalescePartitionsExec` let child = Arc::clone(&sort_input.children[0].plan); - let coalesce = CoalescePartitionsExec::new(child); - sort_input.plan = Arc::new(coalesce) as _; + let coalesce = CoalescePartitionsExec::new(child).with_fetch(plan.fetch()).unwrap(); + sort_input.plan = coalesce; } else { return sort_input.update_plan_from_children(); } From 656092e9f95bf7361bc272aaf1782fce7a063e7d Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Fri, 18 Apr 2025 17:56:19 +0800 Subject: [PATCH 061/177] Add fast path for optimize_projection (#15746) --- datafusion/optimizer/src/optimize_projections/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index b3a09e2dcbcc7..069677bbe328b 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -455,6 +455,17 @@ fn merge_consecutive_projections(proj: Projection) -> Result::new(); expr.iter() From d2b8c15799deec4dfcba8096e7ed2b1c8502b29e Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Sat, 19 Apr 2025 19:35:18 +0800 Subject: [PATCH 062/177] Improve `simplify_expressions` rule (#15735) * Improve simplify_expressions rule * address comments * address comments --- .../core/tests/expr_api/simplification.rs | 4 +- .../simplify_expressions/expr_simplifier.rs | 48 ++++++++++++++++--- .../simplify_expressions/simplify_exprs.rs | 9 ++-- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/datafusion/core/tests/expr_api/simplification.rs b/datafusion/core/tests/expr_api/simplification.rs index 7bb21725ef401..34e0487f312fb 100644 --- a/datafusion/core/tests/expr_api/simplification.rs +++ b/datafusion/core/tests/expr_api/simplification.rs @@ -547,9 +547,9 @@ fn test_simplify_with_cycle_count( }; let simplifier = ExprSimplifier::new(info); let (simplified_expr, count) = simplifier - .simplify_with_cycle_count(input_expr.clone()) + .simplify_with_cycle_count_transformed(input_expr.clone()) .expect("successfully evaluated"); - + let simplified_expr = simplified_expr.data; assert_eq!( simplified_expr, expected_expr, "Mismatch evaluating {input_expr}\n Expected:{expected_expr}\n Got:{simplified_expr}" diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 8e25bb7534365..b92d73175dbd5 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -188,7 +188,7 @@ impl ExprSimplifier { /// assert_eq!(expr, b_lt_2); /// ``` pub fn simplify(&self, expr: Expr) -> Result { - Ok(self.simplify_with_cycle_count(expr)?.0) + Ok(self.simplify_with_cycle_count_transformed(expr)?.0.data) } /// Like [Self::simplify], simplifies this [`Expr`] as much as possible, evaluating @@ -198,7 +198,34 @@ impl ExprSimplifier { /// /// See [Self::simplify] for details and usage examples. /// + #[deprecated( + since = "48.0.0", + note = "Use `simplify_with_cycle_count_transformed` instead" + )] + #[allow(unused_mut)] pub fn simplify_with_cycle_count(&self, mut expr: Expr) -> Result<(Expr, u32)> { + let (transformed, cycle_count) = + self.simplify_with_cycle_count_transformed(expr)?; + Ok((transformed.data, cycle_count)) + } + + /// Like [Self::simplify], simplifies this [`Expr`] as much as possible, evaluating + /// constants and applying algebraic simplifications. Additionally returns a `u32` + /// representing the number of simplification cycles performed, which can be useful for testing + /// optimizations. + /// + /// # Returns + /// + /// A tuple containing: + /// - The simplified expression wrapped in a `Transformed` indicating if changes were made + /// - The number of simplification cycles that were performed + /// + /// See [Self::simplify] for details and usage examples. + /// + pub fn simplify_with_cycle_count_transformed( + &self, + mut expr: Expr, + ) -> Result<(Transformed, u32)> { let mut simplifier = Simplifier::new(&self.info); let mut const_evaluator = ConstEvaluator::try_new(self.info.execution_props())?; let mut shorten_in_list_simplifier = ShortenInListSimplifier::new(); @@ -212,6 +239,7 @@ impl ExprSimplifier { // simplifications can enable new constant evaluation // see `Self::with_max_cycles` let mut num_cycles = 0; + let mut has_transformed = false; loop { let Transformed { data, transformed, .. @@ -221,13 +249,18 @@ impl ExprSimplifier { .transform_data(|expr| expr.rewrite(&mut guarantee_rewriter))?; expr = data; num_cycles += 1; + // Track if any transformation occurred + has_transformed = has_transformed || transformed; if !transformed || num_cycles >= self.max_simplifier_cycles { break; } } // shorten inlist should be started after other inlist rules are applied expr = expr.rewrite(&mut shorten_in_list_simplifier).data()?; - Ok((expr, num_cycles)) + Ok(( + Transformed::new_transformed(expr, has_transformed), + num_cycles, + )) } /// Apply type coercion to an [`Expr`] so that it can be @@ -392,15 +425,15 @@ impl ExprSimplifier { /// let expr = col("a").is_not_null(); /// /// // When using default maximum cycles, 2 cycles will be performed. - /// let (simplified_expr, count) = simplifier.simplify_with_cycle_count(expr.clone()).unwrap(); - /// assert_eq!(simplified_expr, lit(true)); + /// let (simplified_expr, count) = simplifier.simplify_with_cycle_count_transformed(expr.clone()).unwrap(); + /// assert_eq!(simplified_expr.data, lit(true)); /// // 2 cycles were executed, but only 1 was needed /// assert_eq!(count, 2); /// /// // Only 1 simplification pass is necessary here, so we can set the maximum cycles to 1. - /// let (simplified_expr, count) = simplifier.with_max_cycles(1).simplify_with_cycle_count(expr.clone()).unwrap(); + /// let (simplified_expr, count) = simplifier.with_max_cycles(1).simplify_with_cycle_count_transformed(expr.clone()).unwrap(); /// // Expression has been rewritten to: (c = a AND b = 1) - /// assert_eq!(simplified_expr, lit(true)); + /// assert_eq!(simplified_expr.data, lit(true)); /// // Only 1 cycle was executed /// assert_eq!(count, 1); /// @@ -3320,7 +3353,8 @@ mod tests { let simplifier = ExprSimplifier::new( SimplifyContext::new(&execution_props).with_schema(schema), ); - simplifier.simplify_with_cycle_count(expr) + let (expr, count) = simplifier.simplify_with_cycle_count_transformed(expr)?; + Ok((expr.data, count)) } fn simplify_with_cycle_count(expr: Expr) -> (Expr, u32) { diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs index e33869ca2b636..6314209dc7670 100644 --- a/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs +++ b/datafusion/optimizer/src/simplify_expressions/simplify_exprs.rs @@ -123,10 +123,11 @@ impl SimplifyExpressions { let name_preserver = NamePreserver::new(&plan); let mut rewrite_expr = |expr: Expr| { let name = name_preserver.save(&expr); - let expr = simplifier.simplify(expr)?; - // TODO it would be nice to have a way to know if the expression was simplified - // or not. For now conservatively return Transformed::yes - Ok(Transformed::yes(name.restore(expr))) + let expr = simplifier.simplify_with_cycle_count_transformed(expr)?.0; + Ok(Transformed::new_transformed( + name.restore(expr.data), + expr.transformed, + )) }; plan.map_expressions(|expr| { From 2d1062f03fba42f0c93d0a964e44e60bcdfb2605 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Wed, 23 Apr 2025 10:30:53 +0800 Subject: [PATCH 063/177] Speed up `optimize_projection` (#15787) * save * fmt --- Cargo.lock | 1 + datafusion/optimizer/Cargo.toml | 5 ++ .../benches/projection_unnecessary.rs | 79 +++++++++++++++++++ .../optimizer/src/optimize_projections/mod.rs | 24 ++++-- 4 files changed, 104 insertions(+), 5 deletions(-) create mode 100644 datafusion/optimizer/benches/projection_unnecessary.rs diff --git a/Cargo.lock b/Cargo.lock index b6652cfbff274..15ce232d54278 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2389,6 +2389,7 @@ dependencies = [ "arrow", "async-trait", "chrono", + "criterion", "ctor", "datafusion-common", "datafusion-expr", diff --git a/datafusion/optimizer/Cargo.toml b/datafusion/optimizer/Cargo.toml index 61d101aab3f8e..60358d20e2a1a 100644 --- a/datafusion/optimizer/Cargo.toml +++ b/datafusion/optimizer/Cargo.toml @@ -55,6 +55,7 @@ regex-syntax = "0.8.0" [dev-dependencies] async-trait = { workspace = true } +criterion = { workspace = true } ctor = { workspace = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-window = { workspace = true } @@ -62,3 +63,7 @@ datafusion-functions-window-common = { workspace = true } datafusion-sql = { workspace = true } env_logger = { workspace = true } insta = { workspace = true } + +[[bench]] +name = "projection_unnecessary" +harness = false diff --git a/datafusion/optimizer/benches/projection_unnecessary.rs b/datafusion/optimizer/benches/projection_unnecessary.rs new file mode 100644 index 0000000000000..100ee97542ebb --- /dev/null +++ b/datafusion/optimizer/benches/projection_unnecessary.rs @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::datatypes::{DataType, Field, Schema}; +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use datafusion_common::ToDFSchema; +use datafusion_common::{Column, TableReference}; +use datafusion_expr::{logical_plan::LogicalPlan, projection_schema, Expr}; +use datafusion_optimizer::optimize_projections::is_projection_unnecessary; +use std::sync::Arc; + +fn is_projection_unnecessary_old( + input: &LogicalPlan, + proj_exprs: &[Expr], +) -> datafusion_common::Result { + // First check if all expressions are trivial (cheaper operation than `projection_schema`) + if !proj_exprs + .iter() + .all(|expr| matches!(expr, Expr::Column(_) | Expr::Literal(_))) + { + return Ok(false); + } + let proj_schema = projection_schema(input, proj_exprs)?; + Ok(&proj_schema == input.schema()) +} + +fn create_plan_with_many_exprs(num_exprs: usize) -> (LogicalPlan, Vec) { + // Create schema with many fields + let fields = (0..num_exprs) + .map(|i| Field::new(format!("col{}", i), DataType::Int32, false)) + .collect::>(); + let schema = Schema::new(fields); + + // Create table scan + let table_scan = LogicalPlan::EmptyRelation(datafusion_expr::EmptyRelation { + produce_one_row: true, + schema: Arc::new(schema.clone().to_dfschema().unwrap()), + }); + + // Create projection expressions (just column references) + let exprs = (0..num_exprs) + .map(|i| Expr::Column(Column::new(None::, format!("col{}", i)))) + .collect(); + + (table_scan, exprs) +} + +fn benchmark_is_projection_unnecessary(c: &mut Criterion) { + let (plan, exprs) = create_plan_with_many_exprs(1000); + + let mut group = c.benchmark_group("projection_unnecessary_comparison"); + + group.bench_function("is_projection_unnecessary_new", |b| { + b.iter(|| black_box(is_projection_unnecessary(&plan, &exprs).unwrap())) + }); + + group.bench_function("is_projection_unnecessary_old", |b| { + b.iter(|| black_box(is_projection_unnecessary_old(&plan, &exprs).unwrap())) + }); + + group.finish(); +} + +criterion_group!(benches, benchmark_is_projection_unnecessary); +criterion_main!(benches); diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index 069677bbe328b..4452b2d4ce034 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -31,8 +31,7 @@ use datafusion_common::{ use datafusion_expr::expr::Alias; use datafusion_expr::Unnest; use datafusion_expr::{ - logical_plan::LogicalPlan, projection_schema, Aggregate, Distinct, Expr, Projection, - TableScan, Window, + logical_plan::LogicalPlan, Aggregate, Distinct, Expr, Projection, TableScan, Window, }; use crate::optimize_projections::required_indices::RequiredIndices; @@ -785,9 +784,24 @@ fn rewrite_projection_given_requirements( /// Projection is unnecessary, when /// - input schema of the projection, output schema of the projection are same, and /// - all projection expressions are either Column or Literal -fn is_projection_unnecessary(input: &LogicalPlan, proj_exprs: &[Expr]) -> Result { - let proj_schema = projection_schema(input, proj_exprs)?; - Ok(&proj_schema == input.schema() && proj_exprs.iter().all(is_expr_trivial)) +pub fn is_projection_unnecessary( + input: &LogicalPlan, + proj_exprs: &[Expr], +) -> Result { + // First check if the number of expressions is equal to the number of fields in the input schema. + if proj_exprs.len() != input.schema().fields().len() { + return Ok(false); + } + Ok(input.schema().iter().zip(proj_exprs.iter()).all( + |((field_relation, field_name), expr)| { + // Check if the expression is a column and if it matches the field name + if let Expr::Column(col) = expr { + col.relation.as_ref() == field_relation && col.name.eq(field_name.name()) + } else { + false + } + }, + )) } #[cfg(test)] From 738816dd9e5001a8088f5d251f7d853ffb06c560 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 24 Apr 2025 18:02:00 +0800 Subject: [PATCH 064/177] Support inferring new predicates to push down --- datafusion/expr/src/expr_rewriter/mod.rs | 19 +++++ datafusion/optimizer/src/push_down_filter.rs | 73 ++++++++++++++++++- .../replace_with_order_preserving_variants.rs | 4 +- 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rs index 90dcbce46b017..ce567ff66f7b4 100644 --- a/datafusion/expr/src/expr_rewriter/mod.rs +++ b/datafusion/expr/src/expr_rewriter/mod.rs @@ -148,6 +148,25 @@ pub fn replace_col(expr: Expr, replace_map: &HashMap<&Column, &Column>) -> Resul .data() } +pub fn replace_col_with_expr( + expr: Expr, + replace_map: &HashMap, +) -> Result { + expr.transform(|expr| { + Ok({ + if let Expr::Column(c) = &expr { + match replace_map.get(c) { + Some(new_expr) => Transformed::yes((**new_expr).to_owned()), + None => Transformed::no(expr), + } + } else { + Transformed::no(expr) + } + }) + }) + .data() +} + /// Recursively 'unnormalize' (remove all qualifiers) from an /// expression tree. /// diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index c9617514e4539..13cc74ce5752b 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -30,7 +30,7 @@ use datafusion_common::{ internal_err, plan_err, qualified_name, Column, DFSchema, Result, }; use datafusion_expr::expr::WindowFunction; -use datafusion_expr::expr_rewriter::replace_col; +use datafusion_expr::expr_rewriter::{replace_col, replace_col_with_expr}; use datafusion_expr::logical_plan::{Join, JoinType, LogicalPlan, TableScan, Union}; use datafusion_expr::utils::{ conjunction, expr_to_columns, split_conjunction, split_conjunction_owned, @@ -784,7 +784,7 @@ impl OptimizerRule for PushDownFilter { // remove duplicated filters let child_predicates = split_conjunction_owned(child_filter.predicate); - let new_predicates = parents_predicates + let mut new_predicates = parents_predicates .into_iter() .chain(child_predicates) // use IndexSet to remove dupes while preserving predicate order @@ -792,6 +792,8 @@ impl OptimizerRule for PushDownFilter { .into_iter() .collect::>(); + new_predicates = infer_predicates_from_equalities(new_predicates)?; + let Some(new_predicate) = conjunction(new_predicates) else { return plan_err!("at least one expression exists"); }; @@ -1382,6 +1384,73 @@ fn contain(e: &Expr, check_map: &HashMap) -> bool { is_contain } +/// Infers new predicates by substituting equalities. +/// For example, with predicates `t2.b = 3` and `t1.b > t2.b`, +/// we can infer `t1.b > 3`. +fn infer_predicates_from_equalities(predicates: Vec) -> Result> { + // Map from column names to their literal values (from equality predicates) + let mut equality_map: HashMap = + HashMap::with_capacity(predicates.len()); + let mut final_predicates = Vec::with_capacity(predicates.len()); + // First pass: collect column=literal equalities + for predicate in predicates.iter() { + if let Expr::BinaryExpr(BinaryExpr { + left, + op: Operator::Eq, + right, + }) = predicate + { + if let Expr::Column(col) = left.as_ref() { + // Only add to map if right side is a literal + if matches!(right.as_ref(), Expr::Literal(_)) { + equality_map.insert(col.clone(), *right.clone()); + final_predicates.push(predicate.clone()); + } + } else if let Expr::Column(col) = right.as_ref() { + // Only add to map if left side is a literal + if matches!(left.as_ref(), Expr::Literal(_)) { + equality_map.insert(col.clone(), *right.clone()); + final_predicates.push(predicate.clone()); + } + } + } + } + + // If no equality mappings found, nothing to infer + if equality_map.is_empty() { + return Ok(predicates); + } + + // Second pass: apply substitutions to create new predicates + for predicate in predicates { + // Skip equality predicates we already used for mapping + if final_predicates.contains(&predicate) { + continue; + } + + // Try to replace columns with their literal values + let mut columns_in_expr = HashSet::new(); + expr_to_columns(&predicate, &mut columns_in_expr)?; + + // Create a combined replacement map for all columns in this predicate + let replace_map: HashMap<_, _> = columns_in_expr + .into_iter() + .filter_map(|col| equality_map.get(&col).map(|lit| (col, lit))) + .collect(); + + if replace_map.is_empty() { + final_predicates.push(predicate); + continue; + } + // Apply all substitutions at once to get the fully substituted predicate + let new_pred = replace_col_with_expr(predicate, &replace_map)?; + + final_predicates.push(new_pred); + } + + Ok(final_predicates) +} + #[cfg(test)] mod tests { use std::any::Any; diff --git a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs index 4af4692664877..eef2504338bf1 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs @@ -197,7 +197,9 @@ fn plan_with_order_breaking_variants( // Replace `SortPreservingMergeExec` with a `CoalescePartitionsExec` // SPM may have `fetch`, so pass it to the `CoalescePartitionsExec` let child = Arc::clone(&sort_input.children[0].plan); - let coalesce = CoalescePartitionsExec::new(child).with_fetch(plan.fetch()).unwrap(); + let coalesce = CoalescePartitionsExec::new(child) + .with_fetch(plan.fetch()) + .unwrap(); sort_input.plan = coalesce; } else { return sort_input.update_plan_from_children(); From d029200bffd2c54e45251004503b0a7089ed2f97 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Mon, 12 May 2025 17:55:05 +0800 Subject: [PATCH 065/177] Fix: `build_predicate_expression` method doesn't process `false` expr correctly (#15995) * Fix: build_predicate_expression method doesn't process false correctly * fix test --- datafusion/physical-optimizer/src/pruning.rs | 64 ++++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-optimizer/src/pruning.rs b/datafusion/physical-optimizer/src/pruning.rs index 1dd168f181676..d077e5c886a06 100644 --- a/datafusion/physical-optimizer/src/pruning.rs +++ b/datafusion/physical-optimizer/src/pruning.rs @@ -751,6 +751,13 @@ fn is_always_true(expr: &Arc) -> bool { .unwrap_or_default() } +fn is_always_false(expr: &Arc) -> bool { + expr.as_any() + .downcast_ref::() + .map(|l| matches!(l.value(), ScalarValue::Boolean(Some(false)))) + .unwrap_or_default() +} + /// Describes which columns statistics are necessary to evaluate a /// [`PruningPredicate`]. /// @@ -1427,6 +1434,11 @@ fn build_predicate_expression( required_columns: &mut RequiredColumns, unhandled_hook: &Arc, ) -> Arc { + if is_always_false(expr) { + // Shouldn't return `unhandled_hook.handle(expr)` + // Because it will transfer false to true. + return Arc::clone(expr); + } // predicate expression can only be a binary expression let expr_any = expr.as_any(); if let Some(is_null) = expr_any.downcast_ref::() { @@ -1526,6 +1538,11 @@ fn build_predicate_expression( build_predicate_expression(&right, schema, required_columns, unhandled_hook); // simplify boolean expression if applicable let expr = match (&left_expr, op, &right_expr) { + (left, Operator::And, right) + if is_always_false(left) || is_always_false(right) => + { + Arc::new(phys_expr::Literal::new(ScalarValue::Boolean(Some(false)))) + } (left, Operator::And, _) if is_always_true(left) => right_expr, (_, Operator::And, right) if is_always_true(right) => left_expr, (left, Operator::Or, right) @@ -1533,6 +1550,9 @@ fn build_predicate_expression( { Arc::new(phys_expr::Literal::new(ScalarValue::Boolean(Some(true)))) } + (left, Operator::Or, _) if is_always_false(left) => right_expr, + (_, Operator::Or, right) if is_always_false(right) => left_expr, + _ => Arc::new(phys_expr::BinaryExpr::new(left_expr, op, right_expr)), }; return expr; @@ -1889,7 +1909,7 @@ mod tests { use super::*; use datafusion_common::test_util::batches_to_string; - use datafusion_expr::{col, lit}; + use datafusion_expr::{and, col, lit, or}; use insta::assert_snapshot; use arrow::array::Decimal128Array; @@ -3285,12 +3305,10 @@ mod tests { prune_with_expr( // false - // constant literals that do NOT refer to any columns are currently not evaluated at all, hence the result is - // "all true" lit(false), &schema, &statistics, - &[true, true, true, true, true], + &[false, false, false, false, false], ); } @@ -4871,4 +4889,42 @@ mod tests { let unhandled_hook = Arc::new(ConstantUnhandledPredicateHook::default()) as _; build_predicate_expression(&expr, schema, required_columns, &unhandled_hook) } + + #[test] + fn test_build_predicate_expression_with_false() { + let expr = lit(ScalarValue::Boolean(Some(false))); + let schema = Schema::empty(); + let res = + test_build_predicate_expression(&expr, &schema, &mut RequiredColumns::new()); + let expected = logical2physical(&expr, &schema); + assert_eq!(&res, &expected); + } + + #[test] + fn test_build_predicate_expression_with_and_false() { + let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]); + let expr = and( + col("c1").eq(lit("a")), + lit(ScalarValue::Boolean(Some(false))), + ); + let res = + test_build_predicate_expression(&expr, &schema, &mut RequiredColumns::new()); + let expected = logical2physical(&lit(ScalarValue::Boolean(Some(false))), &schema); + assert_eq!(&res, &expected); + } + + #[test] + fn test_build_predicate_expression_with_or_false() { + let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]); + let left_expr = col("c1").eq(lit("a")); + let right_expr = lit(ScalarValue::Boolean(Some(false))); + let res = test_build_predicate_expression( + &or(left_expr.clone(), right_expr.clone()), + &schema, + &mut RequiredColumns::new(), + ); + let expected = + "c1_null_count@2 != row_count@3 AND c1_min@0 <= a AND a <= c1_max@1"; + assert_eq!(res.to_string(), expected); + } } From 378ce3baf550f781adb178caa4cfe79e18daab8a Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Wed, 21 May 2025 05:52:29 -0700 Subject: [PATCH 066/177] Revert use file schema in parquet pruning (#16086) * wip * comment * Update datafusion/core/src/datasource/physical_plan/parquet.rs * remove prints * better test * fmt --- .../src/datasource/physical_plan/parquet.rs | 103 +++++++++++++++++- datafusion/datasource-parquet/src/opener.rs | 28 +++-- .../datasource-parquet/src/row_filter.rs | 4 +- datafusion/datasource-parquet/src/source.rs | 2 +- 4 files changed, 120 insertions(+), 17 deletions(-) diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rs index e9bb8b0db3682..a44060b169992 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet.rs @@ -39,7 +39,7 @@ mod tests { use crate::test::object_store::local_unpartitioned_file; use arrow::array::{ ArrayRef, AsArray, Date64Array, Int32Array, Int64Array, Int8Array, StringArray, - StructArray, + StringViewArray, StructArray, }; use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaBuilder}; use arrow::record_batch::RecordBatch; @@ -99,6 +99,7 @@ mod tests { predicate: Option, pushdown_predicate: bool, page_index_predicate: bool, + bloom_filters: bool, } impl RoundTrip { @@ -131,6 +132,11 @@ mod tests { self } + fn with_bloom_filters(mut self) -> Self { + self.bloom_filters = true; + self + } + /// run the test, returning only the resulting RecordBatches async fn round_trip_to_batches( self, @@ -155,10 +161,20 @@ mod tests { source = source .with_pushdown_filters(true) .with_reorder_filters(true); + } else { + source = source.with_pushdown_filters(false); } if self.page_index_predicate { source = source.with_enable_page_index(true); + } else { + source = source.with_enable_page_index(false); + } + + if self.bloom_filters { + source = source.with_bloom_filter_on_read(true); + } else { + source = source.with_bloom_filter_on_read(false); } Arc::new(source) @@ -816,7 +832,7 @@ mod tests { } #[tokio::test] - async fn evolved_schema_filter() { + async fn evolved_schema_column_order_filter() { let c1: ArrayRef = Arc::new(StringArray::from(vec![Some("Foo"), None, Some("bar")])); @@ -847,6 +863,88 @@ mod tests { assert_eq!(read.len(), 0); } + #[tokio::test] + async fn evolved_schema_column_type_filter_strings() { + // The table and filter have a common data type, but the file schema differs + let c1: ArrayRef = + Arc::new(StringViewArray::from(vec![Some("foo"), Some("bar")])); + let batch = create_batch(vec![("c1", c1.clone())]); + + let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Utf8, false)])); + + // Predicate should prune all row groups + let filter = col("c1").eq(lit(ScalarValue::Utf8(Some("aaa".to_string())))); + let rt = RoundTrip::new() + .with_predicate(filter) + .with_schema(schema.clone()) + .round_trip(vec![batch.clone()]) + .await; + // There should be no predicate evaluation errors + let metrics = rt.parquet_exec.metrics().unwrap(); + assert_eq!(get_value(&metrics, "predicate_evaluation_errors"), 0); + assert_eq!(get_value(&metrics, "pushdown_rows_matched"), 0); + assert_eq!(rt.batches.unwrap().len(), 0); + + // Predicate should prune no row groups + let filter = col("c1").eq(lit(ScalarValue::Utf8(Some("foo".to_string())))); + let rt = RoundTrip::new() + .with_predicate(filter) + .with_schema(schema) + .round_trip(vec![batch]) + .await; + // There should be no predicate evaluation errors + let metrics = rt.parquet_exec.metrics().unwrap(); + assert_eq!(get_value(&metrics, "predicate_evaluation_errors"), 0); + assert_eq!(get_value(&metrics, "pushdown_rows_matched"), 0); + let read = rt + .batches + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum::(); + assert_eq!(read, 2, "Expected 2 rows to match the predicate"); + } + + #[tokio::test] + async fn evolved_schema_column_type_filter_ints() { + // The table and filter have a common data type, but the file schema differs + let c1: ArrayRef = Arc::new(Int8Array::from(vec![Some(1), Some(2)])); + let batch = create_batch(vec![("c1", c1.clone())]); + + let schema = + Arc::new(Schema::new(vec![Field::new("c1", DataType::UInt64, false)])); + + // Predicate should prune all row groups + let filter = col("c1").eq(lit(ScalarValue::UInt64(Some(5)))); + let rt = RoundTrip::new() + .with_predicate(filter) + .with_schema(schema.clone()) + .round_trip(vec![batch.clone()]) + .await; + // There should be no predicate evaluation errors + let metrics = rt.parquet_exec.metrics().unwrap(); + assert_eq!(get_value(&metrics, "predicate_evaluation_errors"), 0); + assert_eq!(rt.batches.unwrap().len(), 0); + + // Predicate should prune no row groups + let filter = col("c1").eq(lit(ScalarValue::UInt64(Some(1)))); + let rt = RoundTrip::new() + .with_predicate(filter) + .with_schema(schema) + .round_trip(vec![batch]) + .await; + // There should be no predicate evaluation errors + let metrics = rt.parquet_exec.metrics().unwrap(); + assert_eq!(get_value(&metrics, "predicate_evaluation_errors"), 0); + let read = rt + .batches + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum::(); + assert_eq!(read, 2, "Expected 2 rows to match the predicate"); + } + #[tokio::test] async fn evolved_schema_disjoint_schema_filter() { let c1: ArrayRef = @@ -1629,6 +1727,7 @@ mod tests { let rt = RoundTrip::new() .with_predicate(filter.clone()) .with_pushdown_predicate() + .with_bloom_filters() .round_trip(vec![batch1]) .await; diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs index cfe8213f86e4b..4517ed885a202 100644 --- a/datafusion/datasource-parquet/src/opener.rs +++ b/datafusion/datasource-parquet/src/opener.rs @@ -56,8 +56,9 @@ pub(super) struct ParquetOpener { pub limit: Option, /// Optional predicate to apply during the scan pub predicate: Option>, - /// Schema of the output table - pub table_schema: SchemaRef, + /// Schema of the output table without partition columns. + /// This is the schema we coerce the physical file schema into. + pub logical_file_schema: SchemaRef, /// Optional hint for how large the initial request to read parquet metadata /// should be pub metadata_size_hint: Option, @@ -105,13 +106,13 @@ impl FileOpener for ParquetOpener { let batch_size = self.batch_size; let projected_schema = - SchemaRef::from(self.table_schema.project(&self.projection)?); + SchemaRef::from(self.logical_file_schema.project(&self.projection)?); let schema_adapter_factory = Arc::clone(&self.schema_adapter_factory); let schema_adapter = self .schema_adapter_factory - .create(projected_schema, Arc::clone(&self.table_schema)); + .create(projected_schema, Arc::clone(&self.logical_file_schema)); let predicate = self.predicate.clone(); - let table_schema = Arc::clone(&self.table_schema); + let logical_file_schema = Arc::clone(&self.logical_file_schema); let reorder_predicates = self.reorder_filters; let pushdown_filters = self.pushdown_filters; let coerce_int96 = self.coerce_int96; @@ -142,17 +143,20 @@ impl FileOpener for ParquetOpener { .await?; // Note about schemas: we are actually dealing with **3 different schemas** here: - // - The table schema as defined by the TableProvider. This is what the user sees, what they get when they `SELECT * FROM table`, etc. - // - The "virtual" file schema: this is the table schema minus any hive partition columns and projections. This is what the file schema is coerced to. + // - The table schema as defined by the TableProvider. + // This is what the user sees, what they get when they `SELECT * FROM table`, etc. + // - The logical file schema: this is the table schema minus any hive partition columns and projections. + // This is what the physicalfile schema is coerced to. // - The physical file schema: this is the schema as defined by the parquet file. This is what the parquet file actually contains. let mut physical_file_schema = Arc::clone(reader_metadata.schema()); // The schema loaded from the file may not be the same as the // desired schema (for example if we want to instruct the parquet // reader to read strings using Utf8View instead). Update if necessary - if let Some(merged) = - apply_file_schema_type_coercions(&table_schema, &physical_file_schema) - { + if let Some(merged) = apply_file_schema_type_coercions( + &logical_file_schema, + &physical_file_schema, + ) { physical_file_schema = Arc::new(merged); options = options.with_schema(Arc::clone(&physical_file_schema)); reader_metadata = ArrowReaderMetadata::try_new( @@ -179,7 +183,7 @@ impl FileOpener for ParquetOpener { // Build predicates for this specific file let (pruning_predicate, page_pruning_predicate) = build_pruning_predicates( &predicate, - &physical_file_schema, + &logical_file_schema, &predicate_creation_errors, ); @@ -216,7 +220,7 @@ impl FileOpener for ParquetOpener { let row_filter = row_filter::build_row_filter( &predicate, &physical_file_schema, - &table_schema, + &logical_file_schema, builder.metadata(), reorder_predicates, &file_metrics, diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs index 2d2993c29a6f2..366ad058ecc63 100644 --- a/datafusion/datasource-parquet/src/row_filter.rs +++ b/datafusion/datasource-parquet/src/row_filter.rs @@ -450,7 +450,7 @@ fn columns_sorted(_columns: &[usize], _metadata: &ParquetMetaData) -> Result, physical_file_schema: &SchemaRef, - table_schema: &SchemaRef, + logical_file_schema: &SchemaRef, metadata: &ParquetMetaData, reorder_predicates: bool, file_metrics: &ParquetFileMetrics, @@ -471,7 +471,7 @@ pub fn build_row_filter( FilterCandidateBuilder::new( Arc::clone(expr), Arc::clone(physical_file_schema), - Arc::clone(table_schema), + Arc::clone(logical_file_schema), Arc::clone(schema_adapter_factory), ) .build(metadata) diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index 6236525fcb9f4..9034afa89d8cf 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -489,7 +489,7 @@ impl FileSource for ParquetSource { .expect("Batch size must set before creating ParquetOpener"), limit: base_config.limit, predicate: self.predicate.clone(), - table_schema: Arc::clone(&base_config.file_schema), + logical_file_schema: Arc::clone(&base_config.file_schema), metadata_size_hint: self.metadata_size_hint, metrics: self.metrics().clone(), parquet_file_reader_factory, From c76c1f076cca6f1922de8ba86b98c05b6a27e7ac Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 6 Jun 2025 13:19:46 -0600 Subject: [PATCH 067/177] fix: [branch-48] Revert "Improve performance of constant aggregate window expression" (#16307) * Revert "Improve performance of constant aggregate window expression (#16234)" This reverts commit 0c3037404929fc3a3c4fbf6b9b7325d422ce10bd. * update changelog * update changelog --- .../physical-expr/src/window/aggregate.rs | 34 +------------------ .../src/window/sliding_aggregate.rs | 4 --- .../physical-expr/src/window/window_expr.rs | 11 +----- dev/changelog/48.0.0.md | 4 +-- 4 files changed, 3 insertions(+), 50 deletions(-) diff --git a/datafusion/physical-expr/src/window/aggregate.rs b/datafusion/physical-expr/src/window/aggregate.rs index dae0667afb252..9b959796136a9 100644 --- a/datafusion/physical-expr/src/window/aggregate.rs +++ b/datafusion/physical-expr/src/window/aggregate.rs @@ -34,7 +34,7 @@ use arrow::array::ArrayRef; use arrow::datatypes::FieldRef; use arrow::record_batch::RecordBatch; use datafusion_common::{DataFusionError, Result, ScalarValue}; -use datafusion_expr::{Accumulator, WindowFrame, WindowFrameBound, WindowFrameUnits}; +use datafusion_expr::{Accumulator, WindowFrame}; use datafusion_physical_expr_common::sort_expr::LexOrdering; /// A window expr that takes the form of an aggregate function. @@ -46,7 +46,6 @@ pub struct PlainAggregateWindowExpr { partition_by: Vec>, order_by: LexOrdering, window_frame: Arc, - is_constant_in_partition: bool, } impl PlainAggregateWindowExpr { @@ -57,14 +56,11 @@ impl PlainAggregateWindowExpr { order_by: &LexOrdering, window_frame: Arc, ) -> Self { - let is_constant_in_partition = - Self::is_window_constant_in_partition(order_by, &window_frame); Self { aggregate, partition_by: partition_by.to_vec(), order_by: order_by.clone(), window_frame, - is_constant_in_partition, } } @@ -89,30 +85,6 @@ impl PlainAggregateWindowExpr { ); } } - - // Returns true if every row in the partition has the same window frame. This allows - // for preventing bound + function calculation for every row due to the values being the - // same. - // - // This occurs when both bounds fall under either condition below: - // 1. Bound is unbounded (`Preceding` or `Following`) - // 2. Bound is `CurrentRow` while using `Range` units with no order by clause - // This results in an invalid range specification. Following PostgreSQL’s convention, - // we interpret this as the entire partition being used for the current window frame. - fn is_window_constant_in_partition( - order_by: &LexOrdering, - window_frame: &WindowFrame, - ) -> bool { - let is_constant_bound = |bound: &WindowFrameBound| match bound { - WindowFrameBound::CurrentRow => { - window_frame.units == WindowFrameUnits::Range && order_by.is_empty() - } - _ => bound.is_unbounded(), - }; - - is_constant_bound(&window_frame.start_bound) - && is_constant_bound(&window_frame.end_bound) - } } /// peer based evaluation based on the fact that batch is pre-sorted given the sort columns @@ -241,8 +213,4 @@ impl AggregateWindowExpr for PlainAggregateWindowExpr { accumulator.evaluate() } } - - fn is_constant_in_partition(&self) -> bool { - self.is_constant_in_partition - } } diff --git a/datafusion/physical-expr/src/window/sliding_aggregate.rs b/datafusion/physical-expr/src/window/sliding_aggregate.rs index 09d6af748755f..2b22299f9386b 100644 --- a/datafusion/physical-expr/src/window/sliding_aggregate.rs +++ b/datafusion/physical-expr/src/window/sliding_aggregate.rs @@ -210,8 +210,4 @@ impl AggregateWindowExpr for SlidingAggregateWindowExpr { accumulator.evaluate() } } - - fn is_constant_in_partition(&self) -> bool { - false - } } diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index 70a73c44ae9de..8d72604a6af50 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -186,10 +186,6 @@ pub trait AggregateWindowExpr: WindowExpr { accumulator: &mut Box, ) -> Result; - /// Indicates whether this window function always produces the same result - /// for all rows in the partition. - fn is_constant_in_partition(&self) -> bool; - /// Evaluates the window function against the batch. fn aggregate_evaluate(&self, batch: &RecordBatch) -> Result { let mut accumulator = self.get_accumulator()?; @@ -276,13 +272,8 @@ pub trait AggregateWindowExpr: WindowExpr { not_end: bool, ) -> Result { let values = self.evaluate_args(record_batch)?; - - if self.is_constant_in_partition() { - accumulator.update_batch(&values)?; - let value = accumulator.evaluate()?; - return value.to_array_of_size(record_batch.num_rows()); - } let order_bys = get_orderby_values(self.order_by_columns(record_batch)?); + let most_recent_row_order_bys = most_recent_row .map(|batch| self.order_by_columns(batch)) .transpose()? diff --git a/dev/changelog/48.0.0.md b/dev/changelog/48.0.0.md index 9cf6c03b7acf0..95f955718119e 100644 --- a/dev/changelog/48.0.0.md +++ b/dev/changelog/48.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 48.0.0 Changelog -This release consists of 267 commits from 89 contributors. See credits at the end of this changelog for more information. +This release consists of 266 commits from 88 contributors. See credits at the end of this changelog for more information. **Breaking changes:** @@ -297,7 +297,6 @@ This release consists of 267 commits from 89 contributors. See credits at the en - Simplify FileSource / SchemaAdapterFactory API [#16214](https://github.com/apache/datafusion/pull/16214) (alamb) - Add dicts to aggregation fuzz testing [#16232](https://github.com/apache/datafusion/pull/16232) (blaginin) - chore(deps): bump sysinfo from 0.35.1 to 0.35.2 [#16247](https://github.com/apache/datafusion/pull/16247) (dependabot[bot]) -- Improve performance of constant aggregate window expression [#16234](https://github.com/apache/datafusion/pull/16234) (suibianwanwank) - Support compound identifier when parsing tuples [#16225](https://github.com/apache/datafusion/pull/16225) (hozan23) - Schema adapter helper [#16108](https://github.com/apache/datafusion/pull/16108) (kosiew) - Update tpch, clickbench, sort_tpch to mark failed queries [#16182](https://github.com/apache/datafusion/pull/16182) (ding-young) @@ -397,7 +396,6 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co 1 irenjj 1 jsai28 1 m09526 - 1 suibianwanwan 1 the0ninjas 1 wiedld ``` From b5dfdbeeb31b62890ceff522d318f86a8bcdaee0 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Fri, 6 Jun 2025 18:00:18 -0600 Subject: [PATCH 068/177] feat: add metadata to literal expressions (#16170) (#16315) --- datafusion-cli/src/functions.rs | 2 +- datafusion-examples/examples/expr_api.rs | 2 +- .../examples/optimizer_rule.rs | 2 +- datafusion-examples/examples/simple_udtf.rs | 5 +- datafusion/catalog-listing/src/helpers.rs | 15 +- datafusion/core/benches/map_query_sql.rs | 7 +- datafusion/core/src/dataframe/mod.rs | 5 +- .../core/src/datasource/listing/table.rs | 2 +- datafusion/core/src/execution/context/mod.rs | 2 +- datafusion/core/src/physical_planner.rs | 9 +- .../provider_filter_pushdown.rs | 10 +- datafusion/core/tests/dataframe/mod.rs | 7 +- .../core/tests/execution/logical_plan.rs | 6 +- .../core/tests/expr_api/simplification.rs | 11 +- .../core/tests/user_defined/expr_planner.rs | 2 +- .../tests/user_defined/user_defined_plan.rs | 9 +- .../user_defined_scalar_functions.rs | 67 ++++++- .../user_defined_table_functions.rs | 2 +- .../datasource-parquet/src/row_filter.rs | 4 +- .../src/row_group_filter.rs | 37 ++-- .../expr/src/conditional_expressions.rs | 2 +- datafusion/expr/src/expr.rs | 58 +++--- datafusion/expr/src/expr_fn.rs | 6 +- datafusion/expr/src/expr_rewriter/mod.rs | 9 +- datafusion/expr/src/expr_schema.rs | 23 ++- datafusion/expr/src/lib.rs | 4 +- datafusion/expr/src/literal.rs | 47 +++-- datafusion/expr/src/logical_plan/builder.rs | 7 +- datafusion/expr/src/logical_plan/plan.rs | 22 ++- datafusion/expr/src/tree_node.rs | 4 +- datafusion/expr/src/utils.rs | 4 +- datafusion/ffi/src/udtf.rs | 2 +- datafusion/functions-aggregate/src/count.rs | 4 +- datafusion/functions-aggregate/src/planner.rs | 2 +- datafusion/functions-nested/benches/map.rs | 7 +- datafusion/functions-nested/src/array_has.rs | 4 +- .../functions-table/src/generate_series.rs | 4 +- datafusion/functions-window/src/planner.rs | 2 +- datafusion/functions/src/core/arrow_cast.rs | 2 +- datafusion/functions/src/core/getfield.rs | 4 +- .../functions/src/datetime/current_date.rs | 1 + .../functions/src/datetime/current_time.rs | 1 + datafusion/functions/src/datetime/now.rs | 1 + datafusion/functions/src/math/log.rs | 4 +- datafusion/functions/src/math/power.rs | 7 +- datafusion/functions/src/string/concat.rs | 16 +- datafusion/functions/src/string/concat_ws.rs | 12 +- datafusion/functions/src/string/contains.rs | 7 +- .../functions/src/string/starts_with.rs | 4 +- .../benches/projection_unnecessary.rs | 2 +- .../src/analyzer/resolve_grouping_function.rs | 10 +- .../optimizer/src/analyzer/type_coercion.rs | 2 +- datafusion/optimizer/src/decorrelate.rs | 15 +- datafusion/optimizer/src/eliminate_filter.rs | 4 +- .../src/eliminate_group_by_constant.rs | 2 +- datafusion/optimizer/src/eliminate_join.rs | 2 +- .../optimizer/src/optimize_projections/mod.rs | 16 +- datafusion/optimizer/src/push_down_filter.rs | 2 +- .../optimizer/src/scalar_subquery_to_join.rs | 4 +- .../simplify_expressions/expr_simplifier.rs | 145 +++++++++------ .../src/simplify_expressions/guarantees.rs | 10 +- .../src/simplify_expressions/regex.rs | 4 +- .../src/simplify_expressions/unwrap_cast.rs | 6 +- .../src/simplify_expressions/utils.rs | 56 +++--- datafusion/optimizer/src/utils.rs | 14 +- .../src/expressions/dynamic_filters.rs | 4 +- .../physical-expr/src/expressions/in_list.rs | 8 +- .../physical-expr/src/expressions/literal.rs | 45 ++++- datafusion/physical-expr/src/planner.rs | 41 ++++- .../proto/src/logical_plan/from_proto.rs | 2 +- datafusion/proto/src/logical_plan/to_proto.rs | 2 +- .../tests/cases/roundtrip_logical_plan.rs | 4 +- datafusion/proto/tests/cases/serialize.rs | 2 +- datafusion/sql/src/expr/mod.rs | 2 +- datafusion/sql/src/expr/substring.rs | 2 +- datafusion/sql/src/expr/value.rs | 27 +-- datafusion/sql/src/statement.rs | 2 +- datafusion/sql/src/unparser/expr.rs | 171 ++++++++++-------- datafusion/sql/src/unparser/plan.rs | 1 + datafusion/sql/src/unparser/utils.rs | 10 +- datafusion/sql/src/utils.rs | 4 +- datafusion/sqllogictest/test_files/array.slt | 8 +- .../test_files/tpch/plans/q16.slt.part | 2 +- .../test_files/tpch/plans/q19.slt.part | 4 +- .../test_files/tpch/plans/q22.slt.part | 4 +- .../consumer/expr/aggregate_function.rs | 2 +- .../src/logical_plan/consumer/expr/literal.rs | 2 +- .../consumer/expr/scalar_function.rs | 4 +- .../consumer/expr/window_function.rs | 2 +- .../src/logical_plan/consumer/rel/read_rel.rs | 2 +- .../src/logical_plan/producer/expr/cast.rs | 6 +- .../src/logical_plan/producer/expr/mod.rs | 4 +- .../src/logical_plan/producer/rel/read_rel.rs | 4 +- docs/source/library-user-guide/adding-udfs.md | 4 +- 94 files changed, 724 insertions(+), 421 deletions(-) diff --git a/datafusion-cli/src/functions.rs b/datafusion-cli/src/functions.rs index 911bbf34b06f4..f07dac649df98 100644 --- a/datafusion-cli/src/functions.rs +++ b/datafusion-cli/src/functions.rs @@ -322,7 +322,7 @@ pub struct ParquetMetadataFunc {} impl TableFunctionImpl for ParquetMetadataFunc { fn call(&self, exprs: &[Expr]) -> Result> { let filename = match exprs.first() { - Some(Expr::Literal(ScalarValue::Utf8(Some(s)))) => s, // single quote: parquet_metadata('x.parquet') + Some(Expr::Literal(ScalarValue::Utf8(Some(s)), _)) => s, // single quote: parquet_metadata('x.parquet') Some(Expr::Column(Column { name, .. })) => name, // double quote: parquet_metadata("x.parquet") _ => { return plan_err!( diff --git a/datafusion-examples/examples/expr_api.rs b/datafusion-examples/examples/expr_api.rs index 089b8db6a5a06..92cf33f4fdf6d 100644 --- a/datafusion-examples/examples/expr_api.rs +++ b/datafusion-examples/examples/expr_api.rs @@ -65,7 +65,7 @@ async fn main() -> Result<()> { let expr2 = Expr::BinaryExpr(BinaryExpr::new( Box::new(col("a")), Operator::Plus, - Box::new(Expr::Literal(ScalarValue::Int32(Some(5)))), + Box::new(Expr::Literal(ScalarValue::Int32(Some(5)), None)), )); assert_eq!(expr, expr2); diff --git a/datafusion-examples/examples/optimizer_rule.rs b/datafusion-examples/examples/optimizer_rule.rs index 63f17484809e2..176b1a69808c1 100644 --- a/datafusion-examples/examples/optimizer_rule.rs +++ b/datafusion-examples/examples/optimizer_rule.rs @@ -171,7 +171,7 @@ fn is_binary_eq(binary_expr: &BinaryExpr) -> bool { /// Return true if the expression is a literal or column reference fn is_lit_or_col(expr: &Expr) -> bool { - matches!(expr, Expr::Column(_) | Expr::Literal(_)) + matches!(expr, Expr::Column(_) | Expr::Literal(_, _)) } /// A simple user defined filter function diff --git a/datafusion-examples/examples/simple_udtf.rs b/datafusion-examples/examples/simple_udtf.rs index d2b2d1bf96551..b65ffb8d71748 100644 --- a/datafusion-examples/examples/simple_udtf.rs +++ b/datafusion-examples/examples/simple_udtf.rs @@ -133,7 +133,8 @@ struct LocalCsvTableFunc {} impl TableFunctionImpl for LocalCsvTableFunc { fn call(&self, exprs: &[Expr]) -> Result> { - let Some(Expr::Literal(ScalarValue::Utf8(Some(ref path)))) = exprs.first() else { + let Some(Expr::Literal(ScalarValue::Utf8(Some(ref path)), _)) = exprs.first() + else { return plan_err!("read_csv requires at least one string argument"); }; @@ -145,7 +146,7 @@ impl TableFunctionImpl for LocalCsvTableFunc { let info = SimplifyContext::new(&execution_props); let expr = ExprSimplifier::new(info).simplify(expr.clone())?; - if let Expr::Literal(ScalarValue::Int64(Some(limit))) = expr { + if let Expr::Literal(ScalarValue::Int64(Some(limit)), _) = expr { Ok(limit as usize) } else { plan_err!("Limit must be an integer") diff --git a/datafusion/catalog-listing/src/helpers.rs b/datafusion/catalog-listing/src/helpers.rs index 037c69cebd572..00e9c71df3489 100644 --- a/datafusion/catalog-listing/src/helpers.rs +++ b/datafusion/catalog-listing/src/helpers.rs @@ -61,7 +61,7 @@ pub fn expr_applicable_for_cols(col_names: &[&str], expr: &Expr) -> bool { Ok(TreeNodeRecursion::Stop) } } - Expr::Literal(_) + Expr::Literal(_, _) | Expr::Alias(_) | Expr::OuterReferenceColumn(_, _) | Expr::ScalarVariable(_, _) @@ -346,8 +346,8 @@ fn populate_partition_values<'a>( { match op { Operator::Eq => match (left.as_ref(), right.as_ref()) { - (Expr::Column(Column { ref name, .. }), Expr::Literal(val)) - | (Expr::Literal(val), Expr::Column(Column { ref name, .. })) => { + (Expr::Column(Column { ref name, .. }), Expr::Literal(val, _)) + | (Expr::Literal(val, _), Expr::Column(Column { ref name, .. })) => { if partition_values .insert(name, PartitionValue::Single(val.to_string())) .is_some() @@ -984,7 +984,7 @@ mod tests { assert_eq!( evaluate_partition_prefix( partitions, - &[col("a").eq(Expr::Literal(ScalarValue::Date32(Some(3))))], + &[col("a").eq(Expr::Literal(ScalarValue::Date32(Some(3)), None))], ), Some(Path::from("a=1970-01-04")), ); @@ -993,9 +993,10 @@ mod tests { assert_eq!( evaluate_partition_prefix( partitions, - &[col("a").eq(Expr::Literal(ScalarValue::Date64(Some( - 4 * 24 * 60 * 60 * 1000 - )))),], + &[col("a").eq(Expr::Literal( + ScalarValue::Date64(Some(4 * 24 * 60 * 60 * 1000)), + None + )),], ), Some(Path::from("a=1970-01-05")), ); diff --git a/datafusion/core/benches/map_query_sql.rs b/datafusion/core/benches/map_query_sql.rs index 97d47fc3b9079..063b8e6c86bbf 100644 --- a/datafusion/core/benches/map_query_sql.rs +++ b/datafusion/core/benches/map_query_sql.rs @@ -71,8 +71,11 @@ fn criterion_benchmark(c: &mut Criterion) { let mut value_buffer = Vec::new(); for i in 0..1000 { - key_buffer.push(Expr::Literal(ScalarValue::Utf8(Some(keys[i].clone())))); - value_buffer.push(Expr::Literal(ScalarValue::Int32(Some(values[i])))); + key_buffer.push(Expr::Literal( + ScalarValue::Utf8(Some(keys[i].clone())), + None, + )); + value_buffer.push(Expr::Literal(ScalarValue::Int32(Some(values[i])), None)); } c.bench_function("map_1000_1", |b| { b.iter(|| { diff --git a/datafusion/core/src/dataframe/mod.rs b/datafusion/core/src/dataframe/mod.rs index 69992e57ca7d0..02a18f22c9168 100644 --- a/datafusion/core/src/dataframe/mod.rs +++ b/datafusion/core/src/dataframe/mod.rs @@ -1337,7 +1337,10 @@ impl DataFrame { /// ``` pub async fn count(self) -> Result { let rows = self - .aggregate(vec![], vec![count(Expr::Literal(COUNT_STAR_EXPANSION))])? + .aggregate( + vec![], + vec![count(Expr::Literal(COUNT_STAR_EXPANSION, None))], + )? .collect() .await?; let len = *rows diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 3c87d3ee2329c..0dd6cd38ba537 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -2230,7 +2230,7 @@ mod tests { let filter_predicate = Expr::BinaryExpr(BinaryExpr::new( Box::new(Expr::Column("column1".into())), Operator::GtEq, - Box::new(Expr::Literal(ScalarValue::Int32(Some(0)))), + Box::new(Expr::Literal(ScalarValue::Int32(Some(0)), None)), )); // Create a new batch of data to insert into the table diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index 5ef666b61e547..dbe5c2c00f17e 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -1214,7 +1214,7 @@ impl SessionContext { let mut params: Vec = parameters .into_iter() .map(|e| match e { - Expr::Literal(scalar) => Ok(scalar), + Expr::Literal(scalar, _) => Ok(scalar), _ => not_impl_err!("Unsupported parameter type: {}", e), }) .collect::>()?; diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 61d1fee794723..c65fcb4c4c931 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2257,7 +2257,8 @@ mod tests { // verify that the plan correctly casts u8 to i64 // the cast from u8 to i64 for literal will be simplified, and get lit(int64(5)) // the cast here is implicit so has CastOptions with safe=true - let expected = "BinaryExpr { left: Column { name: \"c7\", index: 2 }, op: Lt, right: Literal { value: Int64(5) }, fail_on_overflow: false }"; + let expected = r#"BinaryExpr { left: Column { name: "c7", index: 2 }, op: Lt, right: Literal { value: Int64(5), field: Field { name: "5", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }"#; + assert!(format!("{exec_plan:?}").contains(expected)); Ok(()) } @@ -2282,7 +2283,7 @@ mod tests { &session_state, ); - let expected = r#"Ok(PhysicalGroupBy { expr: [(Column { name: "c1", index: 0 }, "c1"), (Column { name: "c2", index: 1 }, "c2"), (Column { name: "c3", index: 2 }, "c3")], null_expr: [(Literal { value: Utf8(NULL) }, "c1"), (Literal { value: Int64(NULL) }, "c2"), (Literal { value: Int64(NULL) }, "c3")], groups: [[false, false, false], [true, false, false], [false, true, false], [false, false, true], [true, true, false], [true, false, true], [false, true, true], [true, true, true]] })"#; + let expected = r#"Ok(PhysicalGroupBy { expr: [(Column { name: "c1", index: 0 }, "c1"), (Column { name: "c2", index: 1 }, "c2"), (Column { name: "c3", index: 2 }, "c3")], null_expr: [(Literal { value: Utf8(NULL), field: Field { name: "NULL", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }, "c1"), (Literal { value: Int64(NULL), field: Field { name: "NULL", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }, "c2"), (Literal { value: Int64(NULL), field: Field { name: "NULL", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }, "c3")], groups: [[false, false, false], [true, false, false], [false, true, false], [false, false, true], [true, true, false], [true, false, true], [false, true, true], [true, true, true]] })"#; assert_eq!(format!("{cube:?}"), expected); @@ -2309,7 +2310,7 @@ mod tests { &session_state, ); - let expected = r#"Ok(PhysicalGroupBy { expr: [(Column { name: "c1", index: 0 }, "c1"), (Column { name: "c2", index: 1 }, "c2"), (Column { name: "c3", index: 2 }, "c3")], null_expr: [(Literal { value: Utf8(NULL) }, "c1"), (Literal { value: Int64(NULL) }, "c2"), (Literal { value: Int64(NULL) }, "c3")], groups: [[true, true, true], [false, true, true], [false, false, true], [false, false, false]] })"#; + let expected = r#"Ok(PhysicalGroupBy { expr: [(Column { name: "c1", index: 0 }, "c1"), (Column { name: "c2", index: 1 }, "c2"), (Column { name: "c3", index: 2 }, "c3")], null_expr: [(Literal { value: Utf8(NULL), field: Field { name: "NULL", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }, "c1"), (Literal { value: Int64(NULL), field: Field { name: "NULL", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }, "c2"), (Literal { value: Int64(NULL), field: Field { name: "NULL", data_type: Int64, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }, "c3")], groups: [[true, true, true], [false, true, true], [false, false, true], [false, false, false]] })"#; assert_eq!(format!("{rollup:?}"), expected); @@ -2493,7 +2494,7 @@ mod tests { let execution_plan = plan(&logical_plan).await?; // verify that the plan correctly adds cast from Int64(1) to Utf8, and the const will be evaluated. - let expected = "expr: [(BinaryExpr { left: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"a\") }, fail_on_overflow: false }, op: Or, right: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"1\") }, fail_on_overflow: false }, fail_on_overflow: false }"; + let expected = "expr: [(BinaryExpr { left: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"a\"), field: Field { name: \"a\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }, op: Or, right: BinaryExpr { left: Column { name: \"c1\", index: 0 }, op: Eq, right: Literal { value: Utf8(\"1\"), field: Field { name: \"1\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }, fail_on_overflow: false }"; let actual = format!("{execution_plan:?}"); assert!(actual.contains(expected), "{}", actual); diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs index f68bcfaf15507..c80c0b4bf54ba 100644 --- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs +++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs @@ -179,12 +179,12 @@ impl TableProvider for CustomProvider { match &filters[0] { Expr::BinaryExpr(BinaryExpr { right, .. }) => { let int_value = match &**right { - Expr::Literal(ScalarValue::Int8(Some(i))) => *i as i64, - Expr::Literal(ScalarValue::Int16(Some(i))) => *i as i64, - Expr::Literal(ScalarValue::Int32(Some(i))) => *i as i64, - Expr::Literal(ScalarValue::Int64(Some(i))) => *i, + Expr::Literal(ScalarValue::Int8(Some(i)), _) => *i as i64, + Expr::Literal(ScalarValue::Int16(Some(i)), _) => *i as i64, + Expr::Literal(ScalarValue::Int32(Some(i)), _) => *i as i64, + Expr::Literal(ScalarValue::Int64(Some(i)), _) => *i, Expr::Cast(Cast { expr, data_type: _ }) => match expr.deref() { - Expr::Literal(lit_value) => match lit_value { + Expr::Literal(lit_value, _) => match lit_value { ScalarValue::Int8(Some(v)) => *v as i64, ScalarValue::Int16(Some(v)) => *v as i64, ScalarValue::Int32(Some(v)) => *v as i64, diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index aa36de1e555f9..c737d0f9c3b0f 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -1210,7 +1210,7 @@ async fn join_on_filter_datatype() -> Result<()> { let join = left.clone().join_on( right.clone(), JoinType::Inner, - Some(Expr::Literal(ScalarValue::Null)), + Some(Expr::Literal(ScalarValue::Null, None)), )?; assert_snapshot!(join.into_optimized_plan().unwrap(), @"EmptyRelation"); @@ -4527,7 +4527,10 @@ async fn consecutive_projection_same_schema() -> Result<()> { // Add `t` column full of nulls let df = df - .with_column("t", cast(Expr::Literal(ScalarValue::Null), DataType::Int32)) + .with_column( + "t", + cast(Expr::Literal(ScalarValue::Null, None), DataType::Int32), + ) .unwrap(); df.clone().show().await.unwrap(); diff --git a/datafusion/core/tests/execution/logical_plan.rs b/datafusion/core/tests/execution/logical_plan.rs index 97bb2a727bbfe..f5a8a30e01307 100644 --- a/datafusion/core/tests/execution/logical_plan.rs +++ b/datafusion/core/tests/execution/logical_plan.rs @@ -47,9 +47,9 @@ async fn count_only_nulls() -> Result<()> { let input = Arc::new(LogicalPlan::Values(Values { schema: input_schema, values: vec![ - vec![Expr::Literal(ScalarValue::Null)], - vec![Expr::Literal(ScalarValue::Null)], - vec![Expr::Literal(ScalarValue::Null)], + vec![Expr::Literal(ScalarValue::Null, None)], + vec![Expr::Literal(ScalarValue::Null, None)], + vec![Expr::Literal(ScalarValue::Null, None)], ], })); let input_col_ref = Expr::Column(Column { diff --git a/datafusion/core/tests/expr_api/simplification.rs b/datafusion/core/tests/expr_api/simplification.rs index 34e0487f312fb..91a507bdf7f09 100644 --- a/datafusion/core/tests/expr_api/simplification.rs +++ b/datafusion/core/tests/expr_api/simplification.rs @@ -282,10 +282,13 @@ fn select_date_plus_interval() -> Result<()> { let date_plus_interval_expr = to_timestamp_expr(ts_string) .cast_to(&DataType::Date32, schema)? - + Expr::Literal(ScalarValue::IntervalDayTime(Some(IntervalDayTime { - days: 123, - milliseconds: 0, - }))); + + Expr::Literal( + ScalarValue::IntervalDayTime(Some(IntervalDayTime { + days: 123, + milliseconds: 0, + })), + None, + ); let plan = LogicalPlanBuilder::from(table_scan.clone()) .project(vec![date_plus_interval_expr])? diff --git a/datafusion/core/tests/user_defined/expr_planner.rs b/datafusion/core/tests/user_defined/expr_planner.rs index 1fc6d14c5b229..07d289cab06c2 100644 --- a/datafusion/core/tests/user_defined/expr_planner.rs +++ b/datafusion/core/tests/user_defined/expr_planner.rs @@ -56,7 +56,7 @@ impl ExprPlanner for MyCustomPlanner { } BinaryOperator::Question => { Ok(PlannerResult::Planned(Expr::Alias(Alias::new( - Expr::Literal(ScalarValue::Boolean(Some(true))), + Expr::Literal(ScalarValue::Boolean(Some(true)), None), None::<&str>, format!("{} ? {}", expr.left, expr.right), )))) diff --git a/datafusion/core/tests/user_defined/user_defined_plan.rs b/datafusion/core/tests/user_defined/user_defined_plan.rs index b68ef6aca0931..4d3916c1760ea 100644 --- a/datafusion/core/tests/user_defined/user_defined_plan.rs +++ b/datafusion/core/tests/user_defined/user_defined_plan.rs @@ -912,11 +912,12 @@ impl MyAnalyzerRule { .map(|e| { e.transform(|e| { Ok(match e { - Expr::Literal(ScalarValue::Int64(i)) => { + Expr::Literal(ScalarValue::Int64(i), _) => { // transform to UInt64 - Transformed::yes(Expr::Literal(ScalarValue::UInt64( - i.map(|i| i as u64), - ))) + Transformed::yes(Expr::Literal( + ScalarValue::UInt64(i.map(|i| i as u64)), + None, + )) } _ => Transformed::no(e), }) diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index 25458efa4fa55..3e8fafc7a6361 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::hash::{DefaultHasher, Hash, Hasher}; use std::sync::Arc; -use arrow::array::{as_string_array, record_batch, Int8Array, UInt64Array}; +use arrow::array::{as_string_array, create_array, record_batch, Int8Array, UInt64Array}; use arrow::array::{ builder::BooleanBuilder, cast::AsArray, Array, ArrayRef, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray, @@ -42,9 +42,9 @@ use datafusion_common::{ }; use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyInfo}; use datafusion_expr::{ - Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody, LogicalPlanBuilder, - OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, - Signature, Volatility, + lit_with_metadata, Accumulator, ColumnarValue, CreateFunction, CreateFunctionBody, + LogicalPlanBuilder, OperateFunctionArg, ReturnFieldArgs, ScalarFunctionArgs, + ScalarUDF, ScalarUDFImpl, Signature, Volatility, }; use datafusion_functions_nested::range::range_udf; use parking_lot::Mutex; @@ -1529,6 +1529,65 @@ async fn test_metadata_based_udf() -> Result<()> { Ok(()) } +#[tokio::test] +async fn test_metadata_based_udf_with_literal() -> Result<()> { + let ctx = SessionContext::new(); + let input_metadata: HashMap = + [("modify_values".to_string(), "double_output".to_string())] + .into_iter() + .collect(); + let df = ctx.sql("select 0;").await?.select(vec![ + lit(5u64).alias_with_metadata("lit_with_doubling", Some(input_metadata.clone())), + lit(5u64).alias("lit_no_doubling"), + lit_with_metadata(5u64, Some(input_metadata)) + .alias("lit_with_double_no_alias_metadata"), + ])?; + + let output_metadata: HashMap = + [("output_metatype".to_string(), "custom_value".to_string())] + .into_iter() + .collect(); + let custom_udf = ScalarUDF::from(MetadataBasedUdf::new(output_metadata.clone())); + + let plan = LogicalPlanBuilder::from(df.into_optimized_plan()?) + .project(vec![ + custom_udf + .call(vec![col("lit_with_doubling")]) + .alias("doubled_output"), + custom_udf + .call(vec![col("lit_no_doubling")]) + .alias("not_doubled_output"), + custom_udf + .call(vec![col("lit_with_double_no_alias_metadata")]) + .alias("double_without_alias_metadata"), + ])? + .build()?; + + let actual = DataFrame::new(ctx.state(), plan).collect().await?; + + let schema = Arc::new(Schema::new(vec![ + Field::new("doubled_output", DataType::UInt64, false) + .with_metadata(output_metadata.clone()), + Field::new("not_doubled_output", DataType::UInt64, false) + .with_metadata(output_metadata.clone()), + Field::new("double_without_alias_metadata", DataType::UInt64, false) + .with_metadata(output_metadata.clone()), + ])); + + let expected = RecordBatch::try_new( + schema, + vec![ + create_array!(UInt64, [10]), + create_array!(UInt64, [5]), + create_array!(UInt64, [10]), + ], + )?; + + assert_eq!(expected, actual[0]); + + Ok(()) +} + /// This UDF is to test extension handling, both on the input and output /// sides. For the input, we will handle the data differently if there is /// the canonical extension type Bool8. For the output we will add a diff --git a/datafusion/core/tests/user_defined/user_defined_table_functions.rs b/datafusion/core/tests/user_defined/user_defined_table_functions.rs index e4aff0b00705d..2c6611f382cea 100644 --- a/datafusion/core/tests/user_defined/user_defined_table_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_table_functions.rs @@ -205,7 +205,7 @@ impl TableFunctionImpl for SimpleCsvTableFunc { let mut filepath = String::new(); for expr in exprs { match expr { - Expr::Literal(ScalarValue::Utf8(Some(ref path))) => { + Expr::Literal(ScalarValue::Utf8(Some(ref path)), _) => { filepath.clone_from(path); } expr => new_exprs.push(expr.clone()), diff --git a/datafusion/datasource-parquet/src/row_filter.rs b/datafusion/datasource-parquet/src/row_filter.rs index cde9e56c92800..db455fed61606 100644 --- a/datafusion/datasource-parquet/src/row_filter.rs +++ b/datafusion/datasource-parquet/src/row_filter.rs @@ -557,6 +557,7 @@ mod test { // Test all should fail let expr = col("timestamp_col").lt(Expr::Literal( ScalarValue::TimestampNanosecond(Some(1), Some(Arc::from("UTC"))), + None, )); let expr = logical2physical(&expr, &table_schema); let schema_adapter_factory = Arc::new(DefaultSchemaAdapterFactory); @@ -597,6 +598,7 @@ mod test { // Test all should pass let expr = col("timestamp_col").gt(Expr::Literal( ScalarValue::TimestampNanosecond(Some(0), Some(Arc::from("UTC"))), + None, )); let expr = logical2physical(&expr, &table_schema); let schema_adapter_factory = Arc::new(DefaultSchemaAdapterFactory); @@ -660,7 +662,7 @@ mod test { let expr = col("string_col") .is_not_null() - .or(col("bigint_col").gt(Expr::Literal(ScalarValue::Int64(Some(5))))); + .or(col("bigint_col").gt(Expr::Literal(ScalarValue::Int64(Some(5)), None))); let expr = logical2physical(&expr, &table_schema); assert!(can_expr_be_pushed_down_with_schemas(&expr, &table_schema)); diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index d44fa16843201..f9fb9214429de 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -1242,12 +1242,16 @@ mod tests { .run( lit("1").eq(lit("1")).and( col(r#""String""#) - .eq(Expr::Literal(ScalarValue::Utf8View(Some(String::from( - "Hello_Not_Exists", - ))))) - .or(col(r#""String""#).eq(Expr::Literal(ScalarValue::Utf8View( - Some(String::from("Hello_Not_Exists2")), - )))), + .eq(Expr::Literal( + ScalarValue::Utf8View(Some(String::from("Hello_Not_Exists"))), + None, + )) + .or(col(r#""String""#).eq(Expr::Literal( + ScalarValue::Utf8View(Some(String::from( + "Hello_Not_Exists2", + ))), + None, + ))), ), ) .await @@ -1327,15 +1331,18 @@ mod tests { // generate pruning predicate `(String = "Hello") OR (String = "the quick") OR (String = "are you")` .run( col(r#""String""#) - .eq(Expr::Literal(ScalarValue::Utf8View(Some(String::from( - "Hello", - ))))) - .or(col(r#""String""#).eq(Expr::Literal(ScalarValue::Utf8View( - Some(String::from("the quick")), - )))) - .or(col(r#""String""#).eq(Expr::Literal(ScalarValue::Utf8View( - Some(String::from("are you")), - )))), + .eq(Expr::Literal( + ScalarValue::Utf8View(Some(String::from("Hello"))), + None, + )) + .or(col(r#""String""#).eq(Expr::Literal( + ScalarValue::Utf8View(Some(String::from("the quick"))), + None, + ))) + .or(col(r#""String""#).eq(Expr::Literal( + ScalarValue::Utf8View(Some(String::from("are you"))), + None, + ))), ) .await } diff --git a/datafusion/expr/src/conditional_expressions.rs b/datafusion/expr/src/conditional_expressions.rs index 9cb51612d0cab..69525ea52137c 100644 --- a/datafusion/expr/src/conditional_expressions.rs +++ b/datafusion/expr/src/conditional_expressions.rs @@ -72,7 +72,7 @@ impl CaseBuilder { let then_types: Vec = then_expr .iter() .map(|e| match e { - Expr::Literal(_) => e.get_type(&DFSchema::empty()), + Expr::Literal(_, _) => e.get_type(&DFSchema::empty()), _ => Ok(DataType::Null), }) .collect::>>()?; diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index dcd5380b4859d..f379edf105848 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -17,7 +17,8 @@ //! Logical Expressions: [`Expr`] -use std::collections::HashSet; +use std::cmp::Ordering; +use std::collections::{BTreeMap, HashSet}; use std::fmt::{self, Display, Formatter, Write}; use std::hash::{Hash, Hasher}; use std::mem; @@ -51,7 +52,7 @@ use sqlparser::ast::{ /// BinaryExpr { /// left: Expr::Column("A"), /// op: Operator::Plus, -/// right: Expr::Literal(ScalarValue::Int32(Some(1))) +/// right: Expr::Literal(ScalarValue::Int32(Some(1)), None) /// } /// ``` /// @@ -113,10 +114,10 @@ use sqlparser::ast::{ /// # use datafusion_expr::{lit, col, Expr}; /// // All literals are strongly typed in DataFusion. To make an `i64` 42: /// let expr = lit(42i64); -/// assert_eq!(expr, Expr::Literal(ScalarValue::Int64(Some(42)))); -/// assert_eq!(expr, Expr::Literal(ScalarValue::Int64(Some(42)))); +/// assert_eq!(expr, Expr::Literal(ScalarValue::Int64(Some(42)), None)); +/// assert_eq!(expr, Expr::Literal(ScalarValue::Int64(Some(42)), None)); /// // To make a (typed) NULL: -/// let expr = Expr::Literal(ScalarValue::Int64(None)); +/// let expr = Expr::Literal(ScalarValue::Int64(None), None); /// // to make an (untyped) NULL (the optimizer will coerce this to the correct type): /// let expr = lit(ScalarValue::Null); /// ``` @@ -150,7 +151,7 @@ use sqlparser::ast::{ /// if let Expr::BinaryExpr(binary_expr) = expr { /// assert_eq!(*binary_expr.left, col("c1")); /// let scalar = ScalarValue::Int32(Some(42)); -/// assert_eq!(*binary_expr.right, Expr::Literal(scalar)); +/// assert_eq!(*binary_expr.right, Expr::Literal(scalar, None)); /// assert_eq!(binary_expr.op, Operator::Eq); /// } /// ``` @@ -194,7 +195,7 @@ use sqlparser::ast::{ /// ``` /// # use datafusion_expr::{lit, col}; /// let expr = col("c1") + lit(42); -/// assert_eq!(format!("{expr:?}"), "BinaryExpr(BinaryExpr { left: Column(Column { relation: None, name: \"c1\" }), op: Plus, right: Literal(Int32(42)) })"); +/// assert_eq!(format!("{expr:?}"), "BinaryExpr(BinaryExpr { left: Column(Column { relation: None, name: \"c1\" }), op: Plus, right: Literal(Int32(42), None) })"); /// ``` /// /// ## Use the `Display` trait (detailed expression) @@ -240,7 +241,7 @@ use sqlparser::ast::{ /// let mut scalars = HashSet::new(); /// // apply recursively visits all nodes in the expression tree /// expr.apply(|e| { -/// if let Expr::Literal(scalar) = e { +/// if let Expr::Literal(scalar, _) = e { /// scalars.insert(scalar); /// } /// // The return value controls whether to continue visiting the tree @@ -275,7 +276,7 @@ use sqlparser::ast::{ /// assert!(rewritten.transformed); /// // to 42 = 5 AND b = 6 /// assert_eq!(rewritten.data, lit(42).eq(lit(5)).and(col("b").eq(lit(6)))); -#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)] +#[derive(Clone, PartialEq, PartialOrd, Eq, Debug, Hash)] pub enum Expr { /// An expression with a specific name. Alias(Alias), @@ -283,8 +284,8 @@ pub enum Expr { Column(Column), /// A named reference to a variable in a registry. ScalarVariable(DataType, Vec), - /// A constant value. - Literal(ScalarValue), + /// A constant value along with associated metadata + Literal(ScalarValue, Option>), /// A binary expression such as "age > 21" BinaryExpr(BinaryExpr), /// LIKE expression @@ -368,7 +369,7 @@ pub enum Expr { impl Default for Expr { fn default() -> Self { - Expr::Literal(ScalarValue::Null) + Expr::Literal(ScalarValue::Null, None) } } @@ -450,13 +451,13 @@ impl Hash for Alias { } impl PartialOrd for Alias { - fn partial_cmp(&self, other: &Self) -> Option { + fn partial_cmp(&self, other: &Self) -> Option { let cmp = self.expr.partial_cmp(&other.expr); - let Some(std::cmp::Ordering::Equal) = cmp else { + let Some(Ordering::Equal) = cmp else { return cmp; }; let cmp = self.relation.partial_cmp(&other.relation); - let Some(std::cmp::Ordering::Equal) = cmp else { + let Some(Ordering::Equal) = cmp else { return cmp; }; self.name.partial_cmp(&other.name) @@ -1537,8 +1538,16 @@ impl Expr { |expr| { // f_up: unalias on up so we can remove nested aliases like // `(x as foo) as bar` - if let Expr::Alias(Alias { expr, .. }) = expr { - Ok(Transformed::yes(*expr)) + if let Expr::Alias(alias) = expr { + match alias + .metadata + .as_ref() + .map(|h| h.is_empty()) + .unwrap_or(true) + { + true => Ok(Transformed::yes(*alias.expr)), + false => Ok(Transformed::no(Expr::Alias(alias))), + } } else { Ok(Transformed::no(expr)) } @@ -2299,7 +2308,7 @@ impl HashNode for Expr { data_type.hash(state); name.hash(state); } - Expr::Literal(scalar_value) => { + Expr::Literal(scalar_value, _) => { scalar_value.hash(state); } Expr::BinaryExpr(BinaryExpr { @@ -2479,7 +2488,7 @@ impl Display for SchemaDisplay<'_> { // TODO: remove the next line after `Expr::Wildcard` is removed #[expect(deprecated)] Expr::Column(_) - | Expr::Literal(_) + | Expr::Literal(_, _) | Expr::ScalarVariable(..) | Expr::OuterReferenceColumn(..) | Expr::Placeholder(_) @@ -2738,7 +2747,7 @@ struct SqlDisplay<'a>(&'a Expr); impl Display for SqlDisplay<'_> { fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { match self.0 { - Expr::Literal(scalar) => scalar.fmt(f), + Expr::Literal(scalar, _) => scalar.fmt(f), Expr::Alias(Alias { name, .. }) => write!(f, "{name}"), Expr::Between(Between { expr, @@ -3005,7 +3014,12 @@ impl Display for Expr { write!(f, "{OUTER_REFERENCE_COLUMN_PREFIX}({c})") } Expr::ScalarVariable(_, var_names) => write!(f, "{}", var_names.join(".")), - Expr::Literal(v) => write!(f, "{v:?}"), + Expr::Literal(v, metadata) => { + match metadata.as_ref().map(|m| m.is_empty()).unwrap_or(true) { + false => write!(f, "{v:?} {:?}", metadata.as_ref().unwrap()), + true => write!(f, "{v:?}"), + } + } Expr::Case(case) => { write!(f, "CASE ")?; if let Some(e) = &case.expr { @@ -3376,7 +3390,7 @@ mod test { #[allow(deprecated)] fn format_cast() -> Result<()> { let expr = Expr::Cast(Cast { - expr: Box::new(Expr::Literal(ScalarValue::Float32(Some(1.23)))), + expr: Box::new(Expr::Literal(ScalarValue::Float32(Some(1.23)), None)), data_type: DataType::Utf8, }); let expected_canonical = "CAST(Float32(1.23) AS Utf8)"; diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index 5182ccb15c0a9..e8885ed6b7240 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -690,17 +690,17 @@ impl WindowUDFImpl for SimpleWindowUDF { pub fn interval_year_month_lit(value: &str) -> Expr { let interval = parse_interval_year_month(value).ok(); - Expr::Literal(ScalarValue::IntervalYearMonth(interval)) + Expr::Literal(ScalarValue::IntervalYearMonth(interval), None) } pub fn interval_datetime_lit(value: &str) -> Expr { let interval = parse_interval_day_time(value).ok(); - Expr::Literal(ScalarValue::IntervalDayTime(interval)) + Expr::Literal(ScalarValue::IntervalDayTime(interval), None) } pub fn interval_month_day_nano_lit(value: &str) -> Expr { let interval = parse_interval_month_day_nano(value).ok(); - Expr::Literal(ScalarValue::IntervalMonthDayNano(interval)) + Expr::Literal(ScalarValue::IntervalMonthDayNano(interval), None) } /// Extensions for configuring [`Expr::AggregateFunction`] or [`Expr::WindowFunction`] diff --git a/datafusion/expr/src/expr_rewriter/mod.rs b/datafusion/expr/src/expr_rewriter/mod.rs index 90dcbce46b017..f80b8e5a7759c 100644 --- a/datafusion/expr/src/expr_rewriter/mod.rs +++ b/datafusion/expr/src/expr_rewriter/mod.rs @@ -354,6 +354,7 @@ mod test { use std::ops::Add; use super::*; + use crate::literal::lit_with_metadata; use crate::{col, lit, Cast}; use arrow::datatypes::{DataType, Field, Schema}; use datafusion_common::tree_node::TreeNodeRewriter; @@ -383,13 +384,17 @@ mod test { // rewrites all "foo" string literals to "bar" let transformer = |expr: Expr| -> Result> { match expr { - Expr::Literal(ScalarValue::Utf8(Some(utf8_val))) => { + Expr::Literal(ScalarValue::Utf8(Some(utf8_val)), metadata) => { let utf8_val = if utf8_val == "foo" { "bar".to_string() } else { utf8_val }; - Ok(Transformed::yes(lit(utf8_val))) + Ok(Transformed::yes(lit_with_metadata( + utf8_val, + metadata + .map(|m| m.into_iter().collect::>()), + ))) } // otherwise, return None _ => Ok(Transformed::no(expr)), diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index bdf9911b006c7..1973a00a67df5 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -115,7 +115,7 @@ impl ExprSchemable for Expr { Expr::Column(c) => Ok(schema.data_type(c)?.clone()), Expr::OuterReferenceColumn(ty, _) => Ok(ty.clone()), Expr::ScalarVariable(ty, _) => Ok(ty.clone()), - Expr::Literal(l) => Ok(l.data_type()), + Expr::Literal(l, _) => Ok(l.data_type()), Expr::Case(case) => { for (_, then_expr) in &case.when_then_expr { let then_type = then_expr.get_type(schema)?; @@ -278,7 +278,7 @@ impl ExprSchemable for Expr { Expr::Column(c) => input_schema.nullable(c), Expr::OuterReferenceColumn(_, _) => Ok(true), - Expr::Literal(value) => Ok(value.is_null()), + Expr::Literal(value, _) => Ok(value.is_null()), Expr::Case(case) => { // This expression is nullable if any of the input expressions are nullable let then_nullable = case @@ -420,11 +420,18 @@ impl ExprSchemable for Expr { Expr::ScalarVariable(ty, _) => { Ok(Arc::new(Field::new(&schema_name, ty.clone(), true))) } - Expr::Literal(l) => Ok(Arc::new(Field::new( - &schema_name, - l.data_type(), - l.is_null(), - ))), + Expr::Literal(l, metadata) => { + let mut field = Field::new(&schema_name, l.data_type(), l.is_null()); + if let Some(metadata) = metadata { + field = field.with_metadata( + metadata + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + ); + } + Ok(Arc::new(field)) + } Expr::IsNull(_) | Expr::IsNotNull(_) | Expr::IsTrue(_) @@ -533,7 +540,7 @@ impl ExprSchemable for Expr { let arguments = args .iter() .map(|e| match e { - Expr::Literal(sv) => Some(sv), + Expr::Literal(sv, _) => Some(sv), _ => None, }) .collect::>(); diff --git a/datafusion/expr/src/lib.rs b/datafusion/expr/src/lib.rs index 48931d6525af5..1f44f755b2148 100644 --- a/datafusion/expr/src/lib.rs +++ b/datafusion/expr/src/lib.rs @@ -94,7 +94,9 @@ pub use function::{ AccumulatorFactoryFunction, PartitionEvaluatorFactory, ReturnTypeFunction, ScalarFunctionImplementation, StateTypeFunction, }; -pub use literal::{lit, lit_timestamp_nano, Literal, TimestampLiteral}; +pub use literal::{ + lit, lit_timestamp_nano, lit_with_metadata, Literal, TimestampLiteral, +}; pub use logical_plan::*; pub use partition_evaluator::PartitionEvaluator; pub use sqlparser; diff --git a/datafusion/expr/src/literal.rs b/datafusion/expr/src/literal.rs index 90ba5a9a693c7..48e058b8b7b1c 100644 --- a/datafusion/expr/src/literal.rs +++ b/datafusion/expr/src/literal.rs @@ -19,12 +19,37 @@ use crate::Expr; use datafusion_common::ScalarValue; +use std::collections::HashMap; /// Create a literal expression pub fn lit(n: T) -> Expr { n.lit() } +pub fn lit_with_metadata( + n: T, + metadata: impl Into>>, +) -> Expr { + let metadata = metadata.into(); + let Some(metadata) = metadata else { + return n.lit(); + }; + + let Expr::Literal(sv, prior_metadata) = n.lit() else { + unreachable!(); + }; + + let new_metadata = match prior_metadata { + Some(mut prior) => { + prior.extend(metadata); + prior + } + None => metadata.into_iter().collect(), + }; + + Expr::Literal(sv, Some(new_metadata)) +} + /// Create a literal timestamp expression pub fn lit_timestamp_nano(n: T) -> Expr { n.lit_timestamp_nano() @@ -43,37 +68,37 @@ pub trait TimestampLiteral { impl Literal for &str { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::from(*self)) + Expr::Literal(ScalarValue::from(*self), None) } } impl Literal for String { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::from(self.as_ref())) + Expr::Literal(ScalarValue::from(self.as_ref()), None) } } impl Literal for &String { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::from(self.as_ref())) + Expr::Literal(ScalarValue::from(self.as_ref()), None) } } impl Literal for Vec { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::Binary(Some((*self).to_owned()))) + Expr::Literal(ScalarValue::Binary(Some((*self).to_owned())), None) } } impl Literal for &[u8] { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::Binary(Some((*self).to_owned()))) + Expr::Literal(ScalarValue::Binary(Some((*self).to_owned())), None) } } impl Literal for ScalarValue { fn lit(&self) -> Expr { - Expr::Literal(self.clone()) + Expr::Literal(self.clone(), None) } } @@ -82,7 +107,7 @@ macro_rules! make_literal { #[doc = $DOC] impl Literal for $TYPE { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::$SCALAR(Some(self.clone()))) + Expr::Literal(ScalarValue::$SCALAR(Some(self.clone())), None) } } }; @@ -93,7 +118,7 @@ macro_rules! make_nonzero_literal { #[doc = $DOC] impl Literal for $TYPE { fn lit(&self) -> Expr { - Expr::Literal(ScalarValue::$SCALAR(Some(self.get()))) + Expr::Literal(ScalarValue::$SCALAR(Some(self.get())), None) } } }; @@ -104,10 +129,10 @@ macro_rules! make_timestamp_literal { #[doc = $DOC] impl TimestampLiteral for $TYPE { fn lit_timestamp_nano(&self) -> Expr { - Expr::Literal(ScalarValue::TimestampNanosecond( - Some((self.clone()).into()), + Expr::Literal( + ScalarValue::TimestampNanosecond(Some((self.clone()).into()), None), None, - )) + ) } } }; diff --git a/datafusion/expr/src/logical_plan/builder.rs b/datafusion/expr/src/logical_plan/builder.rs index f75e79cd66726..533e81e64f29c 100644 --- a/datafusion/expr/src/logical_plan/builder.rs +++ b/datafusion/expr/src/logical_plan/builder.rs @@ -341,8 +341,11 @@ impl LogicalPlanBuilder { // wrap cast if data type is not same as common type. for row in &mut values { for (j, field_type) in fields.iter().map(|f| f.data_type()).enumerate() { - if let Expr::Literal(ScalarValue::Null) = row[j] { - row[j] = Expr::Literal(ScalarValue::try_from(field_type)?); + if let Expr::Literal(ScalarValue::Null, metadata) = &row[j] { + row[j] = Expr::Literal( + ScalarValue::try_from(field_type)?, + metadata.clone(), + ); } else { row[j] = std::mem::take(&mut row[j]).cast_to(field_type, schema)?; } diff --git a/datafusion/expr/src/logical_plan/plan.rs b/datafusion/expr/src/logical_plan/plan.rs index 691f5684a11c3..5bc07cf6213e3 100644 --- a/datafusion/expr/src/logical_plan/plan.rs +++ b/datafusion/expr/src/logical_plan/plan.rs @@ -1305,7 +1305,7 @@ impl LogicalPlan { // Empty group_expr will return Some(1) if group_expr .iter() - .all(|expr| matches!(expr, Expr::Literal(_))) + .all(|expr| matches!(expr, Expr::Literal(_, _))) { Some(1) } else { @@ -1455,7 +1455,7 @@ impl LogicalPlan { let transformed_expr = e.transform_up(|e| { if let Expr::Placeholder(Placeholder { id, .. }) = e { let value = param_values.get_placeholders_with_values(&id)?; - Ok(Transformed::yes(Expr::Literal(value))) + Ok(Transformed::yes(Expr::Literal(value, None))) } else { Ok(Transformed::no(e)) } @@ -2698,7 +2698,9 @@ impl Union { { expr.push(Expr::Column(column)); } else { - expr.push(Expr::Literal(ScalarValue::Null).alias(column.name())); + expr.push( + Expr::Literal(ScalarValue::Null, None).alias(column.name()), + ); } } wrapped_inputs.push(Arc::new(LogicalPlan::Projection( @@ -3224,7 +3226,7 @@ impl Limit { pub fn get_skip_type(&self) -> Result { match self.skip.as_deref() { Some(expr) => match *expr { - Expr::Literal(ScalarValue::Int64(s)) => { + Expr::Literal(ScalarValue::Int64(s), _) => { // `skip = NULL` is equivalent to `skip = 0` let s = s.unwrap_or(0); if s >= 0 { @@ -3244,14 +3246,16 @@ impl Limit { pub fn get_fetch_type(&self) -> Result { match self.fetch.as_deref() { Some(expr) => match *expr { - Expr::Literal(ScalarValue::Int64(Some(s))) => { + Expr::Literal(ScalarValue::Int64(Some(s)), _) => { if s >= 0 { Ok(FetchType::Literal(Some(s as usize))) } else { plan_err!("LIMIT must be >= 0, '{}' was provided", s) } } - Expr::Literal(ScalarValue::Int64(None)) => Ok(FetchType::Literal(None)), + Expr::Literal(ScalarValue::Int64(None), _) => { + Ok(FetchType::Literal(None)) + } _ => Ok(FetchType::UnsupportedExpr), }, None => Ok(FetchType::Literal(None)), @@ -4539,7 +4543,7 @@ mod tests { let col = schema.field_names()[0].clone(); let filter = Filter::try_new( - Expr::Column(col.into()).eq(Expr::Literal(ScalarValue::Int32(Some(1)))), + Expr::Column(col.into()).eq(Expr::Literal(ScalarValue::Int32(Some(1)), None)), scan, ) .unwrap(); @@ -4666,12 +4670,14 @@ mod tests { skip: None, fetch: Some(Box::new(Expr::Literal( ScalarValue::new_ten(&DataType::UInt32).unwrap(), + None, ))), input: Arc::clone(&input), }), LogicalPlan::Limit(Limit { skip: Some(Box::new(Expr::Literal( ScalarValue::new_ten(&DataType::UInt32).unwrap(), + None, ))), fetch: None, input: Arc::clone(&input), @@ -4679,9 +4685,11 @@ mod tests { LogicalPlan::Limit(Limit { skip: Some(Box::new(Expr::Literal( ScalarValue::new_one(&DataType::UInt32).unwrap(), + None, ))), fetch: Some(Box::new(Expr::Literal( ScalarValue::new_ten(&DataType::UInt32).unwrap(), + None, ))), input, }), diff --git a/datafusion/expr/src/tree_node.rs b/datafusion/expr/src/tree_node.rs index bfdc193945763..f953aec5a1e39 100644 --- a/datafusion/expr/src/tree_node.rs +++ b/datafusion/expr/src/tree_node.rs @@ -73,7 +73,7 @@ impl TreeNode for Expr { // Treat OuterReferenceColumn as a leaf expression | Expr::OuterReferenceColumn(_, _) | Expr::ScalarVariable(_, _) - | Expr::Literal(_) + | Expr::Literal(_, _) | Expr::Exists { .. } | Expr::ScalarSubquery(_) | Expr::Wildcard { .. } @@ -126,7 +126,7 @@ impl TreeNode for Expr { | Expr::Exists { .. } | Expr::ScalarSubquery(_) | Expr::ScalarVariable(_, _) - | Expr::Literal(_) => Transformed::no(self), + | Expr::Literal(_, _) => Transformed::no(self), Expr::Unnest(Unnest { expr, .. }) => expr .map_elements(f)? .update_data(|expr| Expr::Unnest(Unnest { expr })), diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index 6f44e37d0523b..b7851e5300998 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -276,7 +276,7 @@ pub fn expr_to_columns(expr: &Expr, accum: &mut HashSet) -> Result<()> { Expr::Unnest(_) | Expr::ScalarVariable(_, _) | Expr::Alias(_) - | Expr::Literal(_) + | Expr::Literal(_, _) | Expr::BinaryExpr { .. } | Expr::Like { .. } | Expr::SimilarTo { .. } @@ -785,7 +785,7 @@ pub(crate) fn find_column_indexes_referenced_by_expr( indexes.push(idx); } } - Expr::Literal(_) => { + Expr::Literal(_, _) => { indexes.push(usize::MAX); } _ => {} diff --git a/datafusion/ffi/src/udtf.rs b/datafusion/ffi/src/udtf.rs index 08bc4d0cd83b4..ceedec2599a29 100644 --- a/datafusion/ffi/src/udtf.rs +++ b/datafusion/ffi/src/udtf.rs @@ -214,7 +214,7 @@ mod tests { let args = args .iter() .map(|arg| { - if let Expr::Literal(scalar) = arg { + if let Expr::Literal(scalar, _) = arg { Ok(scalar) } else { exec_err!("Expected only literal arguments to table udf") diff --git a/datafusion/functions-aggregate/src/count.rs b/datafusion/functions-aggregate/src/count.rs index f375a68d94585..6b7199c44b329 100644 --- a/datafusion/functions-aggregate/src/count.rs +++ b/datafusion/functions-aggregate/src/count.rs @@ -101,7 +101,7 @@ pub fn count_distinct(expr: Expr) -> Expr { /// let expr = col(expr.schema_name().to_string()); /// ``` pub fn count_all() -> Expr { - count(Expr::Literal(COUNT_STAR_EXPANSION)).alias("count(*)") + count(Expr::Literal(COUNT_STAR_EXPANSION, None)).alias("count(*)") } /// Creates window aggregation to count all rows. @@ -126,7 +126,7 @@ pub fn count_all() -> Expr { pub fn count_all_window() -> Expr { Expr::from(WindowFunction::new( WindowFunctionDefinition::AggregateUDF(count_udaf()), - vec![Expr::Literal(COUNT_STAR_EXPANSION)], + vec![Expr::Literal(COUNT_STAR_EXPANSION, None)], )) } diff --git a/datafusion/functions-aggregate/src/planner.rs b/datafusion/functions-aggregate/src/planner.rs index c8cb841189954..f0e37f6b1dbe4 100644 --- a/datafusion/functions-aggregate/src/planner.rs +++ b/datafusion/functions-aggregate/src/planner.rs @@ -100,7 +100,7 @@ impl ExprPlanner for AggregateFunctionPlanner { let new_expr = Expr::AggregateFunction(AggregateFunction::new_udf( func, - vec![Expr::Literal(COUNT_STAR_EXPANSION)], + vec![Expr::Literal(COUNT_STAR_EXPANSION, None)], distinct, filter, order_by, diff --git a/datafusion/functions-nested/benches/map.rs b/datafusion/functions-nested/benches/map.rs index a752a47bcbaa5..55dd7ad144605 100644 --- a/datafusion/functions-nested/benches/map.rs +++ b/datafusion/functions-nested/benches/map.rs @@ -58,8 +58,11 @@ fn criterion_benchmark(c: &mut Criterion) { let values = values(&mut rng); let mut buffer = Vec::new(); for i in 0..1000 { - buffer.push(Expr::Literal(ScalarValue::Utf8(Some(keys[i].clone())))); - buffer.push(Expr::Literal(ScalarValue::Int32(Some(values[i])))); + buffer.push(Expr::Literal( + ScalarValue::Utf8(Some(keys[i].clone())), + None, + )); + buffer.push(Expr::Literal(ScalarValue::Int32(Some(values[i])), None)); } let planner = NestedFunctionPlanner {}; diff --git a/datafusion/functions-nested/src/array_has.rs b/datafusion/functions-nested/src/array_has.rs index 5ef1491313b13..3b9b705e72c5e 100644 --- a/datafusion/functions-nested/src/array_has.rs +++ b/datafusion/functions-nested/src/array_has.rs @@ -133,7 +133,7 @@ impl ScalarUDFImpl for ArrayHas { // if the haystack is a constant list, we can use an inlist expression which is more // efficient because the haystack is not varying per-row - if let Expr::Literal(ScalarValue::List(array)) = haystack { + if let Expr::Literal(ScalarValue::List(array), _) = haystack { // TODO: support LargeList // (not supported by `convert_array_to_scalar_vec`) // (FixedSizeList not supported either, but seems to have worked fine when attempting to @@ -147,7 +147,7 @@ impl ScalarUDFImpl for ArrayHas { let list = scalar_values .into_iter() .flatten() - .map(Expr::Literal) + .map(|v| Expr::Literal(v, None)) .collect(); return Ok(ExprSimplifyResult::Simplified(Expr::InList(InList { diff --git a/datafusion/functions-table/src/generate_series.rs b/datafusion/functions-table/src/generate_series.rs index ee95567ab73dc..ffb93cf59b16e 100644 --- a/datafusion/functions-table/src/generate_series.rs +++ b/datafusion/functions-table/src/generate_series.rs @@ -199,8 +199,8 @@ impl TableFunctionImpl for GenerateSeriesFuncImpl { let mut normalize_args = Vec::new(); for expr in exprs { match expr { - Expr::Literal(ScalarValue::Null) => {} - Expr::Literal(ScalarValue::Int64(Some(n))) => normalize_args.push(*n), + Expr::Literal(ScalarValue::Null, _) => {} + Expr::Literal(ScalarValue::Int64(Some(n)), _) => normalize_args.push(*n), _ => return plan_err!("First argument must be an integer literal"), }; } diff --git a/datafusion/functions-window/src/planner.rs b/datafusion/functions-window/src/planner.rs index 8fca0114f65ec..091737bb9c156 100644 --- a/datafusion/functions-window/src/planner.rs +++ b/datafusion/functions-window/src/planner.rs @@ -97,7 +97,7 @@ impl ExprPlanner for WindowFunctionPlanner { let new_expr = Expr::from(WindowFunction::new( func_def, - vec![Expr::Literal(COUNT_STAR_EXPANSION)], + vec![Expr::Literal(COUNT_STAR_EXPANSION, None)], )) .partition_by(partition_by) .order_by(order_by) diff --git a/datafusion/functions/src/core/arrow_cast.rs b/datafusion/functions/src/core/arrow_cast.rs index 2d769dfa56579..e9dee09e74bf0 100644 --- a/datafusion/functions/src/core/arrow_cast.rs +++ b/datafusion/functions/src/core/arrow_cast.rs @@ -177,7 +177,7 @@ impl ScalarUDFImpl for ArrowCastFunc { fn data_type_from_args(args: &[Expr]) -> Result { let [_, type_arg] = take_function_args("arrow_cast", args)?; - let Expr::Literal(ScalarValue::Utf8(Some(val))) = type_arg else { + let Expr::Literal(ScalarValue::Utf8(Some(val)), _) = type_arg else { return exec_err!( "arrow_cast requires its second argument to be a constant string, got {:?}", type_arg diff --git a/datafusion/functions/src/core/getfield.rs b/datafusion/functions/src/core/getfield.rs index de87308ef3c49..2f39132871bb5 100644 --- a/datafusion/functions/src/core/getfield.rs +++ b/datafusion/functions/src/core/getfield.rs @@ -108,7 +108,7 @@ impl ScalarUDFImpl for GetFieldFunc { let [base, field_name] = take_function_args(self.name(), args)?; let name = match field_name { - Expr::Literal(name) => name, + Expr::Literal(name, _) => name, other => &ScalarValue::Utf8(Some(other.schema_name().to_string())), }; @@ -118,7 +118,7 @@ impl ScalarUDFImpl for GetFieldFunc { fn schema_name(&self, args: &[Expr]) -> Result { let [base, field_name] = take_function_args(self.name(), args)?; let name = match field_name { - Expr::Literal(name) => name, + Expr::Literal(name, _) => name, other => &ScalarValue::Utf8(Some(other.schema_name().to_string())), }; diff --git a/datafusion/functions/src/datetime/current_date.rs b/datafusion/functions/src/datetime/current_date.rs index 9998e7d3758e0..2bda1f262abe0 100644 --- a/datafusion/functions/src/datetime/current_date.rs +++ b/datafusion/functions/src/datetime/current_date.rs @@ -108,6 +108,7 @@ impl ScalarUDFImpl for CurrentDateFunc { ); Ok(ExprSimplifyResult::Simplified(Expr::Literal( ScalarValue::Date32(days), + None, ))) } diff --git a/datafusion/functions/src/datetime/current_time.rs b/datafusion/functions/src/datetime/current_time.rs index c416d0240b13c..9b9d3997e9d75 100644 --- a/datafusion/functions/src/datetime/current_time.rs +++ b/datafusion/functions/src/datetime/current_time.rs @@ -96,6 +96,7 @@ impl ScalarUDFImpl for CurrentTimeFunc { let nano = now_ts.timestamp_nanos_opt().map(|ts| ts % 86400000000000); Ok(ExprSimplifyResult::Simplified(Expr::Literal( ScalarValue::Time64Nanosecond(nano), + None, ))) } diff --git a/datafusion/functions/src/datetime/now.rs b/datafusion/functions/src/datetime/now.rs index 30b4d4ca9c76f..ffb3aed5a9606 100644 --- a/datafusion/functions/src/datetime/now.rs +++ b/datafusion/functions/src/datetime/now.rs @@ -108,6 +108,7 @@ impl ScalarUDFImpl for NowFunc { .timestamp_nanos_opt(); Ok(ExprSimplifyResult::Simplified(Expr::Literal( ScalarValue::TimestampNanosecond(now_ts, Some("+00:00".into())), + None, ))) } diff --git a/datafusion/functions/src/math/log.rs b/datafusion/functions/src/math/log.rs index ee52c035ac81d..23e267a323b91 100644 --- a/datafusion/functions/src/math/log.rs +++ b/datafusion/functions/src/math/log.rs @@ -210,7 +210,9 @@ impl ScalarUDFImpl for LogFunc { }; match number { - Expr::Literal(value) if value == ScalarValue::new_one(&number_datatype)? => { + Expr::Literal(value, _) + if value == ScalarValue::new_one(&number_datatype)? => + { Ok(ExprSimplifyResult::Simplified(lit(ScalarValue::new_zero( &info.get_data_type(&base)?, )?))) diff --git a/datafusion/functions/src/math/power.rs b/datafusion/functions/src/math/power.rs index bd1ae7c316c1a..465844704f591 100644 --- a/datafusion/functions/src/math/power.rs +++ b/datafusion/functions/src/math/power.rs @@ -156,12 +156,15 @@ impl ScalarUDFImpl for PowerFunc { let exponent_type = info.get_data_type(&exponent)?; match exponent { - Expr::Literal(value) if value == ScalarValue::new_zero(&exponent_type)? => { + Expr::Literal(value, _) + if value == ScalarValue::new_zero(&exponent_type)? => + { Ok(ExprSimplifyResult::Simplified(Expr::Literal( ScalarValue::new_one(&info.get_data_type(&base)?)?, + None, ))) } - Expr::Literal(value) if value == ScalarValue::new_one(&exponent_type)? => { + Expr::Literal(value, _) if value == ScalarValue::new_one(&exponent_type)? => { Ok(ExprSimplifyResult::Simplified(base)) } Expr::ScalarFunction(ScalarFunction { func, mut args }) diff --git a/datafusion/functions/src/string/concat.rs b/datafusion/functions/src/string/concat.rs index 773c316422b70..64a527eac1988 100644 --- a/datafusion/functions/src/string/concat.rs +++ b/datafusion/functions/src/string/concat.rs @@ -295,7 +295,7 @@ pub fn simplify_concat(args: Vec) -> Result { let data_types: Vec<_> = args .iter() .filter_map(|expr| match expr { - Expr::Literal(l) => Some(l.data_type()), + Expr::Literal(l, _) => Some(l.data_type()), _ => None, }) .collect(); @@ -304,25 +304,25 @@ pub fn simplify_concat(args: Vec) -> Result { for arg in args.clone() { match arg { - Expr::Literal(ScalarValue::Utf8(None)) => {} - Expr::Literal(ScalarValue::LargeUtf8(None)) => { + Expr::Literal(ScalarValue::Utf8(None), _) => {} + Expr::Literal(ScalarValue::LargeUtf8(None), _) => { } - Expr::Literal(ScalarValue::Utf8View(None)) => { } + Expr::Literal(ScalarValue::Utf8View(None), _) => { } // filter out `null` args // All literals have been converted to Utf8 or LargeUtf8 in type_coercion. // Concatenate it with the `contiguous_scalar`. - Expr::Literal(ScalarValue::Utf8(Some(v))) => { + Expr::Literal(ScalarValue::Utf8(Some(v)), _) => { contiguous_scalar += &v; } - Expr::Literal(ScalarValue::LargeUtf8(Some(v))) => { + Expr::Literal(ScalarValue::LargeUtf8(Some(v)), _) => { contiguous_scalar += &v; } - Expr::Literal(ScalarValue::Utf8View(Some(v))) => { + Expr::Literal(ScalarValue::Utf8View(Some(v)), _) => { contiguous_scalar += &v; } - Expr::Literal(x) => { + Expr::Literal(x, _) => { return internal_err!( "The scalar {x} should be casted to string type during the type coercion." ) diff --git a/datafusion/functions/src/string/concat_ws.rs b/datafusion/functions/src/string/concat_ws.rs index 2a2f9429f8fc3..1f45f8501e1f4 100644 --- a/datafusion/functions/src/string/concat_ws.rs +++ b/datafusion/functions/src/string/concat_ws.rs @@ -312,6 +312,7 @@ fn simplify_concat_ws(delimiter: &Expr, args: &[Expr]) -> Result { match delimiter { // when the delimiter is an empty string, @@ -336,8 +337,8 @@ fn simplify_concat_ws(delimiter: &Expr, args: &[Expr]) -> Result {} - Expr::Literal(ScalarValue::Utf8(Some(v)) | ScalarValue::LargeUtf8(Some(v)) | ScalarValue::Utf8View(Some(v))) => { + Expr::Literal(ScalarValue::Utf8(None) | ScalarValue::LargeUtf8(None) | ScalarValue::Utf8View(None), _) => {} + Expr::Literal(ScalarValue::Utf8(Some(v)) | ScalarValue::LargeUtf8(Some(v)) | ScalarValue::Utf8View(Some(v)), _) => { match contiguous_scalar { None => contiguous_scalar = Some(v.to_string()), Some(mut pre) => { @@ -347,7 +348,7 @@ fn simplify_concat_ws(delimiter: &Expr, args: &[Expr]) -> Result return internal_err!("The scalar {s} should be casted to string type during the type coercion."), + Expr::Literal(s, _) => return internal_err!("The scalar {s} should be casted to string type during the type coercion."), // If the arg is not a literal, we should first push the current `contiguous_scalar` // to the `new_args` and reset it to None. // Then pushing this arg to the `new_args`. @@ -374,10 +375,11 @@ fn simplify_concat_ws(delimiter: &Expr, args: &[Expr]) -> Result Ok(ExprSimplifyResult::Simplified(Expr::Literal( ScalarValue::Utf8(None), + None, ))), } } - Expr::Literal(d) => internal_err!( + Expr::Literal(d, _) => internal_err!( "The scalar {d} should be casted to string type during the type coercion." ), _ => { @@ -394,7 +396,7 @@ fn simplify_concat_ws(delimiter: &Expr, args: &[Expr]) -> Result bool { match expr { - Expr::Literal(v) => v.is_null(), + Expr::Literal(v, _) => v.is_null(), _ => false, } } diff --git a/datafusion/functions/src/string/contains.rs b/datafusion/functions/src/string/contains.rs index b74be15466265..215f8f7a25b91 100644 --- a/datafusion/functions/src/string/contains.rs +++ b/datafusion/functions/src/string/contains.rs @@ -191,8 +191,11 @@ mod test { #[test] fn test_contains_api() { let expr = contains( - Expr::Literal(ScalarValue::Utf8(Some("the quick brown fox".to_string()))), - Expr::Literal(ScalarValue::Utf8(Some("row".to_string()))), + Expr::Literal( + ScalarValue::Utf8(Some("the quick brown fox".to_string())), + None, + ), + Expr::Literal(ScalarValue::Utf8(Some("row".to_string())), None), ); assert_eq!( expr.to_string(), diff --git a/datafusion/functions/src/string/starts_with.rs b/datafusion/functions/src/string/starts_with.rs index a59d7080a5804..ecab1af132e03 100644 --- a/datafusion/functions/src/string/starts_with.rs +++ b/datafusion/functions/src/string/starts_with.rs @@ -130,7 +130,7 @@ impl ScalarUDFImpl for StartsWithFunc { args: Vec, info: &dyn SimplifyInfo, ) -> Result { - if let Expr::Literal(scalar_value) = &args[1] { + if let Expr::Literal(scalar_value, _) = &args[1] { // Convert starts_with(col, 'prefix') to col LIKE 'prefix%' with proper escaping // Example: starts_with(col, 'ja%') -> col LIKE 'ja\%%' // 1. 'ja%' (input pattern) @@ -142,7 +142,7 @@ impl ScalarUDFImpl for StartsWithFunc { | ScalarValue::Utf8View(Some(pattern)) => { let escaped_pattern = pattern.replace("%", "\\%"); let like_pattern = format!("{escaped_pattern}%"); - Expr::Literal(ScalarValue::Utf8(Some(like_pattern))) + Expr::Literal(ScalarValue::Utf8(Some(like_pattern)), None) } _ => return Ok(ExprSimplifyResult::Original(args)), }; diff --git a/datafusion/optimizer/benches/projection_unnecessary.rs b/datafusion/optimizer/benches/projection_unnecessary.rs index ee7889eb33213..c9f248fe49b5a 100644 --- a/datafusion/optimizer/benches/projection_unnecessary.rs +++ b/datafusion/optimizer/benches/projection_unnecessary.rs @@ -30,7 +30,7 @@ fn is_projection_unnecessary_old( // First check if all expressions are trivial (cheaper operation than `projection_schema`) if !proj_exprs .iter() - .all(|expr| matches!(expr, Expr::Column(_) | Expr::Literal(_))) + .all(|expr| matches!(expr, Expr::Column(_) | Expr::Literal(_, _))) { return Ok(false); } diff --git a/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs b/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs index f8a8185636090..fa7ff1b8b19d6 100644 --- a/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs +++ b/datafusion/optimizer/src/analyzer/resolve_grouping_function.rs @@ -189,19 +189,19 @@ fn grouping_function_on_id( // Postgres allows grouping function for group by without grouping sets, the result is then // always 0 if !is_grouping_set { - return Ok(Expr::Literal(ScalarValue::from(0i32))); + return Ok(Expr::Literal(ScalarValue::from(0i32), None)); } let group_by_expr_count = group_by_expr.len(); let literal = |value: usize| { if group_by_expr_count < 8 { - Expr::Literal(ScalarValue::from(value as u8)) + Expr::Literal(ScalarValue::from(value as u8), None) } else if group_by_expr_count < 16 { - Expr::Literal(ScalarValue::from(value as u16)) + Expr::Literal(ScalarValue::from(value as u16), None) } else if group_by_expr_count < 32 { - Expr::Literal(ScalarValue::from(value as u32)) + Expr::Literal(ScalarValue::from(value as u32), None) } else { - Expr::Literal(ScalarValue::from(value as u64)) + Expr::Literal(ScalarValue::from(value as u64), None) } }; diff --git a/datafusion/optimizer/src/analyzer/type_coercion.rs b/datafusion/optimizer/src/analyzer/type_coercion.rs index 7034982956aee..b5a3e9a2d5853 100644 --- a/datafusion/optimizer/src/analyzer/type_coercion.rs +++ b/datafusion/optimizer/src/analyzer/type_coercion.rs @@ -579,7 +579,7 @@ impl TreeNodeRewriter for TypeCoercionRewriter<'_> { Expr::Alias(_) | Expr::Column(_) | Expr::ScalarVariable(_, _) - | Expr::Literal(_) + | Expr::Literal(_, _) | Expr::SimilarTo(_) | Expr::IsNotNull(_) | Expr::IsNull(_) diff --git a/datafusion/optimizer/src/decorrelate.rs b/datafusion/optimizer/src/decorrelate.rs index 1378b53fa73f0..63236787743a4 100644 --- a/datafusion/optimizer/src/decorrelate.rs +++ b/datafusion/optimizer/src/decorrelate.rs @@ -494,9 +494,12 @@ fn agg_exprs_evaluation_result_on_empty_batch( let new_expr = match expr { Expr::AggregateFunction(expr::AggregateFunction { func, .. }) => { if func.name() == "count" { - Transformed::yes(Expr::Literal(ScalarValue::Int64(Some(0)))) + Transformed::yes(Expr::Literal( + ScalarValue::Int64(Some(0)), + None, + )) } else { - Transformed::yes(Expr::Literal(ScalarValue::Null)) + Transformed::yes(Expr::Literal(ScalarValue::Null, None)) } } _ => Transformed::no(expr), @@ -587,10 +590,10 @@ fn filter_exprs_evaluation_result_on_empty_batch( let result_expr = simplifier.simplify(result_expr)?; match &result_expr { // evaluate to false or null on empty batch, no need to pull up - Expr::Literal(ScalarValue::Null) - | Expr::Literal(ScalarValue::Boolean(Some(false))) => None, + Expr::Literal(ScalarValue::Null, _) + | Expr::Literal(ScalarValue::Boolean(Some(false)), _) => None, // evaluate to true on empty batch, need to pull up the expr - Expr::Literal(ScalarValue::Boolean(Some(true))) => { + Expr::Literal(ScalarValue::Boolean(Some(true)), _) => { for (name, exprs) in input_expr_result_map_for_count_bug { expr_result_map_for_count_bug.insert(name.clone(), exprs.clone()); } @@ -605,7 +608,7 @@ fn filter_exprs_evaluation_result_on_empty_batch( Box::new(result_expr.clone()), Box::new(input_expr.clone()), )], - else_expr: Some(Box::new(Expr::Literal(ScalarValue::Null))), + else_expr: Some(Box::new(Expr::Literal(ScalarValue::Null, None))), }); let expr_key = new_expr.schema_name().to_string(); expr_result_map_for_count_bug.insert(expr_key, new_expr); diff --git a/datafusion/optimizer/src/eliminate_filter.rs b/datafusion/optimizer/src/eliminate_filter.rs index 452df6e8331f8..e28771be548b3 100644 --- a/datafusion/optimizer/src/eliminate_filter.rs +++ b/datafusion/optimizer/src/eliminate_filter.rs @@ -60,7 +60,7 @@ impl OptimizerRule for EliminateFilter { ) -> Result> { match plan { LogicalPlan::Filter(Filter { - predicate: Expr::Literal(ScalarValue::Boolean(v)), + predicate: Expr::Literal(ScalarValue::Boolean(v), _), input, .. }) => match v { @@ -122,7 +122,7 @@ mod tests { #[test] fn filter_null() -> Result<()> { - let filter_expr = Expr::Literal(ScalarValue::Boolean(None)); + let filter_expr = Expr::Literal(ScalarValue::Boolean(None), None); let table_scan = test_table_scan().unwrap(); let plan = LogicalPlanBuilder::from(table_scan) diff --git a/datafusion/optimizer/src/eliminate_group_by_constant.rs b/datafusion/optimizer/src/eliminate_group_by_constant.rs index 604f083b37090..9c47ce024f91a 100644 --- a/datafusion/optimizer/src/eliminate_group_by_constant.rs +++ b/datafusion/optimizer/src/eliminate_group_by_constant.rs @@ -101,7 +101,7 @@ fn is_constant_expression(expr: &Expr) -> bool { Expr::BinaryExpr(e) => { is_constant_expression(&e.left) && is_constant_expression(&e.right) } - Expr::Literal(_) => true, + Expr::Literal(_, _) => true, Expr::ScalarFunction(e) => { matches!( e.func.signature().volatility, diff --git a/datafusion/optimizer/src/eliminate_join.rs b/datafusion/optimizer/src/eliminate_join.rs index 2aad889b2fcbe..dfc3a220d0f9d 100644 --- a/datafusion/optimizer/src/eliminate_join.rs +++ b/datafusion/optimizer/src/eliminate_join.rs @@ -54,7 +54,7 @@ impl OptimizerRule for EliminateJoin { match plan { LogicalPlan::Join(join) if join.join_type == Inner && join.on.is_empty() => { match join.filter { - Some(Expr::Literal(ScalarValue::Boolean(Some(false)))) => Ok( + Some(Expr::Literal(ScalarValue::Boolean(Some(false)), _)) => Ok( Transformed::yes(LogicalPlan::EmptyRelation(EmptyRelation { produce_one_row: false, schema: join.schema, diff --git a/datafusion/optimizer/src/optimize_projections/mod.rs b/datafusion/optimizer/src/optimize_projections/mod.rs index ba583a8d7123a..d0457e7090262 100644 --- a/datafusion/optimizer/src/optimize_projections/mod.rs +++ b/datafusion/optimizer/src/optimize_projections/mod.rs @@ -533,7 +533,7 @@ fn merge_consecutive_projections(proj: Projection) -> Result bool { - matches!(expr, Expr::Column(_) | Expr::Literal(_)) + matches!(expr, Expr::Column(_) | Expr::Literal(_, _)) } /// Rewrites a projection expression using the projection before it (i.e. its input) @@ -583,8 +583,18 @@ fn is_expr_trivial(expr: &Expr) -> bool { fn rewrite_expr(expr: Expr, input: &Projection) -> Result> { expr.transform_up(|expr| { match expr { - // remove any intermediate aliases - Expr::Alias(alias) => Ok(Transformed::yes(*alias.expr)), + // remove any intermediate aliases if they do not carry metadata + Expr::Alias(alias) => { + match alias + .metadata + .as_ref() + .map(|h| h.is_empty()) + .unwrap_or(true) + { + true => Ok(Transformed::yes(*alias.expr)), + false => Ok(Transformed::no(Expr::Alias(alias))), + } + } Expr::Column(col) => { // Find index of column: let idx = input.schema.index_of_column(&col)?; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 7c352031bce6a..1c1996d6a2415 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -254,7 +254,7 @@ fn can_evaluate_as_join_condition(predicate: &Expr) -> Result { let mut is_evaluate = true; predicate.apply(|expr| match expr { Expr::Column(_) - | Expr::Literal(_) + | Expr::Literal(_, _) | Expr::Placeholder(_) | Expr::ScalarVariable(_, _) => Ok(TreeNodeRecursion::Jump), Expr::Exists { .. } diff --git a/datafusion/optimizer/src/scalar_subquery_to_join.rs b/datafusion/optimizer/src/scalar_subquery_to_join.rs index 897e07cb987ed..2f9a2f6bb9ed8 100644 --- a/datafusion/optimizer/src/scalar_subquery_to_join.rs +++ b/datafusion/optimizer/src/scalar_subquery_to_join.rs @@ -335,7 +335,7 @@ fn build_join( .join_on( sub_query_alias, JoinType::Left, - vec![Expr::Literal(ScalarValue::Boolean(Some(true)))], + vec![Expr::Literal(ScalarValue::Boolean(Some(true)), None)], )? .build()? } @@ -365,7 +365,7 @@ fn build_join( ), ( Box::new(Expr::Not(Box::new(filter.clone()))), - Box::new(Expr::Literal(ScalarValue::Null)), + Box::new(Expr::Literal(ScalarValue::Null, None)), ), ], else_expr: Some(Box::new(Expr::Column(Column::new_unqualified( diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index fa565a973f6bf..e91aea3305bea 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -18,7 +18,7 @@ //! Expression simplification API use std::borrow::Cow; -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::ops::Not; use arrow::{ @@ -477,7 +477,7 @@ impl TreeNodeRewriter for Canonicalizer { }))) } // - (Expr::Literal(_a), Expr::Column(_b), Some(swapped_op)) => { + (Expr::Literal(_a, _), Expr::Column(_b), Some(swapped_op)) => { Ok(Transformed::yes(Expr::BinaryExpr(BinaryExpr { left: right, op: swapped_op, @@ -523,9 +523,9 @@ struct ConstEvaluator<'a> { #[allow(clippy::large_enum_variant)] enum ConstSimplifyResult { // Expr was simplified and contains the new expression - Simplified(ScalarValue), + Simplified(ScalarValue, Option>), // Expr was not simplified and original value is returned - NotSimplified(ScalarValue), + NotSimplified(ScalarValue, Option>), // Evaluation encountered an error, contains the original expression SimplifyRuntimeError(DataFusionError, Expr), } @@ -567,11 +567,11 @@ impl TreeNodeRewriter for ConstEvaluator<'_> { // any error is countered during simplification, return the original // so that normal evaluation can occur Some(true) => match self.evaluate_to_scalar(expr) { - ConstSimplifyResult::Simplified(s) => { - Ok(Transformed::yes(Expr::Literal(s))) + ConstSimplifyResult::Simplified(s, m) => { + Ok(Transformed::yes(Expr::Literal(s, m))) } - ConstSimplifyResult::NotSimplified(s) => { - Ok(Transformed::no(Expr::Literal(s))) + ConstSimplifyResult::NotSimplified(s, m) => { + Ok(Transformed::no(Expr::Literal(s, m))) } ConstSimplifyResult::SimplifyRuntimeError(_, expr) => { Ok(Transformed::yes(expr)) @@ -640,7 +640,7 @@ impl<'a> ConstEvaluator<'a> { Expr::ScalarFunction(ScalarFunction { func, .. }) => { Self::volatility_ok(func.signature().volatility) } - Expr::Literal(_) + Expr::Literal(_, _) | Expr::Alias(..) | Expr::Unnest(_) | Expr::BinaryExpr { .. } @@ -666,8 +666,8 @@ impl<'a> ConstEvaluator<'a> { /// Internal helper to evaluates an Expr pub(crate) fn evaluate_to_scalar(&mut self, expr: Expr) -> ConstSimplifyResult { - if let Expr::Literal(s) = expr { - return ConstSimplifyResult::NotSimplified(s); + if let Expr::Literal(s, m) = expr { + return ConstSimplifyResult::NotSimplified(s, m); } let phys_expr = @@ -675,6 +675,18 @@ impl<'a> ConstEvaluator<'a> { Ok(e) => e, Err(err) => return ConstSimplifyResult::SimplifyRuntimeError(err, expr), }; + let metadata = phys_expr + .return_field(self.input_batch.schema_ref()) + .ok() + .and_then(|f| { + let m = f.metadata(); + match m.is_empty() { + true => None, + false => { + Some(m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()) + } + } + }); let col_val = match phys_expr.evaluate(&self.input_batch) { Ok(v) => v, Err(err) => return ConstSimplifyResult::SimplifyRuntimeError(err, expr), @@ -687,13 +699,15 @@ impl<'a> ConstEvaluator<'a> { expr, ) } else if as_list_array(&a).is_ok() { - ConstSimplifyResult::Simplified(ScalarValue::List( - a.as_list::().to_owned().into(), - )) + ConstSimplifyResult::Simplified( + ScalarValue::List(a.as_list::().to_owned().into()), + metadata, + ) } else if as_large_list_array(&a).is_ok() { - ConstSimplifyResult::Simplified(ScalarValue::LargeList( - a.as_list::().to_owned().into(), - )) + ConstSimplifyResult::Simplified( + ScalarValue::LargeList(a.as_list::().to_owned().into()), + metadata, + ) } else { // Non-ListArray match ScalarValue::try_from_array(&a, 0) { @@ -705,7 +719,7 @@ impl<'a> ConstEvaluator<'a> { expr, ) } else { - ConstSimplifyResult::Simplified(s) + ConstSimplifyResult::Simplified(s, metadata) } } Err(err) => ConstSimplifyResult::SimplifyRuntimeError(err, expr), @@ -723,7 +737,7 @@ impl<'a> ConstEvaluator<'a> { expr, ) } else { - ConstSimplifyResult::Simplified(s) + ConstSimplifyResult::Simplified(s, metadata) } } } @@ -1138,9 +1152,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { && !info.get_data_type(&left)?.is_floating() && is_one(&right) => { - Transformed::yes(Expr::Literal(ScalarValue::new_zero( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_zero(&info.get_data_type(&left)?)?, + None, + )) } // @@ -1181,9 +1196,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { op: BitwiseAnd, right, }) if is_negative_of(&left, &right) && !info.nullable(&right)? => { - Transformed::yes(Expr::Literal(ScalarValue::new_zero( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_zero(&info.get_data_type(&left)?)?, + None, + )) } // A & !A -> 0 (if A not nullable) @@ -1192,9 +1208,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { op: BitwiseAnd, right, }) if is_negative_of(&right, &left) && !info.nullable(&left)? => { - Transformed::yes(Expr::Literal(ScalarValue::new_zero( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_zero(&info.get_data_type(&left)?)?, + None, + )) } // (..A..) & A --> (..A..) @@ -1267,9 +1284,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { op: BitwiseOr, right, }) if is_negative_of(&left, &right) && !info.nullable(&right)? => { - Transformed::yes(Expr::Literal(ScalarValue::new_negative_one( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_negative_one(&info.get_data_type(&left)?)?, + None, + )) } // A | !A -> -1 (if A not nullable) @@ -1278,9 +1296,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { op: BitwiseOr, right, }) if is_negative_of(&right, &left) && !info.nullable(&left)? => { - Transformed::yes(Expr::Literal(ScalarValue::new_negative_one( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_negative_one(&info.get_data_type(&left)?)?, + None, + )) } // (..A..) | A --> (..A..) @@ -1353,9 +1372,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { op: BitwiseXor, right, }) if is_negative_of(&left, &right) && !info.nullable(&right)? => { - Transformed::yes(Expr::Literal(ScalarValue::new_negative_one( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_negative_one(&info.get_data_type(&left)?)?, + None, + )) } // A ^ !A -> -1 (if A not nullable) @@ -1364,9 +1384,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { op: BitwiseXor, right, }) if is_negative_of(&right, &left) && !info.nullable(&left)? => { - Transformed::yes(Expr::Literal(ScalarValue::new_negative_one( - &info.get_data_type(&left)?, - )?)) + Transformed::yes(Expr::Literal( + ScalarValue::new_negative_one(&info.get_data_type(&left)?)?, + None, + )) } // (..A..) ^ A --> (the expression without A, if number of A is odd, otherwise one A) @@ -1377,7 +1398,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { }) if expr_contains(&left, &right, BitwiseXor) => { let expr = delete_xor_in_complex_expr(&left, &right, false); Transformed::yes(if expr == *right { - Expr::Literal(ScalarValue::new_zero(&info.get_data_type(&right)?)?) + Expr::Literal( + ScalarValue::new_zero(&info.get_data_type(&right)?)?, + None, + ) } else { expr }) @@ -1391,7 +1415,10 @@ impl TreeNodeRewriter for Simplifier<'_, S> { }) if expr_contains(&right, &left, BitwiseXor) => { let expr = delete_xor_in_complex_expr(&right, &left, true); Transformed::yes(if expr == *left { - Expr::Literal(ScalarValue::new_zero(&info.get_data_type(&left)?)?) + Expr::Literal( + ScalarValue::new_zero(&info.get_data_type(&left)?)?, + None, + ) } else { expr }) @@ -1642,7 +1669,7 @@ impl TreeNodeRewriter for Simplifier<'_, S> { expr, list, negated, - }) if list.is_empty() && *expr != Expr::Literal(ScalarValue::Null) => { + }) if list.is_empty() && *expr != Expr::Literal(ScalarValue::Null, None) => { Transformed::yes(lit(negated)) } @@ -1868,7 +1895,7 @@ impl TreeNodeRewriter for Simplifier<'_, S> { .into_iter() .map(|right| { match right { - Expr::Literal(right_lit_value) => { + Expr::Literal(right_lit_value, _) => { // if the right_lit_value can be casted to the type of internal_left_expr // we need to unwrap the cast for cast/try_cast expr, and add cast to the literal let Some(value) = try_cast_literal_to_type(&right_lit_value, &expr_type) else { @@ -1902,18 +1929,18 @@ impl TreeNodeRewriter for Simplifier<'_, S> { fn as_string_scalar(expr: &Expr) -> Option<(DataType, &Option)> { match expr { - Expr::Literal(ScalarValue::Utf8(s)) => Some((DataType::Utf8, s)), - Expr::Literal(ScalarValue::LargeUtf8(s)) => Some((DataType::LargeUtf8, s)), - Expr::Literal(ScalarValue::Utf8View(s)) => Some((DataType::Utf8View, s)), + Expr::Literal(ScalarValue::Utf8(s), _) => Some((DataType::Utf8, s)), + Expr::Literal(ScalarValue::LargeUtf8(s), _) => Some((DataType::LargeUtf8, s)), + Expr::Literal(ScalarValue::Utf8View(s), _) => Some((DataType::Utf8View, s)), _ => None, } } fn to_string_scalar(data_type: DataType, value: Option) -> Expr { match data_type { - DataType::Utf8 => Expr::Literal(ScalarValue::Utf8(value)), - DataType::LargeUtf8 => Expr::Literal(ScalarValue::LargeUtf8(value)), - DataType::Utf8View => Expr::Literal(ScalarValue::Utf8View(value)), + DataType::Utf8 => Expr::Literal(ScalarValue::Utf8(value), None), + DataType::LargeUtf8 => Expr::Literal(ScalarValue::LargeUtf8(value), None), + DataType::Utf8View => Expr::Literal(ScalarValue::Utf8View(value), None), _ => unreachable!(), } } @@ -1959,12 +1986,12 @@ fn as_inlist(expr: &Expr) -> Option> { Expr::InList(inlist) => Some(Cow::Borrowed(inlist)), Expr::BinaryExpr(BinaryExpr { left, op, right }) if *op == Operator::Eq => { match (left.as_ref(), right.as_ref()) { - (Expr::Column(_), Expr::Literal(_)) => Some(Cow::Owned(InList { + (Expr::Column(_), Expr::Literal(_, _)) => Some(Cow::Owned(InList { expr: left.clone(), list: vec![*right.clone()], negated: false, })), - (Expr::Literal(_), Expr::Column(_)) => Some(Cow::Owned(InList { + (Expr::Literal(_, _), Expr::Column(_)) => Some(Cow::Owned(InList { expr: right.clone(), list: vec![*left.clone()], negated: false, @@ -1984,12 +2011,12 @@ fn to_inlist(expr: Expr) -> Option { op: Operator::Eq, right, }) => match (left.as_ref(), right.as_ref()) { - (Expr::Column(_), Expr::Literal(_)) => Some(InList { + (Expr::Column(_), Expr::Literal(_, _)) => Some(InList { expr: left, list: vec![*right], negated: false, }), - (Expr::Literal(_), Expr::Column(_)) => Some(InList { + (Expr::Literal(_, _), Expr::Column(_)) => Some(InList { expr: right, list: vec![*left], negated: false, @@ -2408,7 +2435,7 @@ mod tests { #[test] fn test_simplify_multiply_by_null() { - let null = Expr::Literal(ScalarValue::Null); + let null = Expr::Literal(ScalarValue::Null, None); // A * null --> null { let expr = col("c2") * null.clone(); @@ -4543,10 +4570,10 @@ mod tests { // The simplifier removes the cast. assert_eq!( simplify(coerced), - col("c5").eq(Expr::Literal(ScalarValue::FixedSizeBinary( - 3, - Some(bytes.to_vec()), - ))) + col("c5").eq(Expr::Literal( + ScalarValue::FixedSizeBinary(3, Some(bytes.to_vec()),), + None + )) ); } diff --git a/datafusion/optimizer/src/simplify_expressions/guarantees.rs b/datafusion/optimizer/src/simplify_expressions/guarantees.rs index 2c11632ad6d26..bbb023cfbad9f 100644 --- a/datafusion/optimizer/src/simplify_expressions/guarantees.rs +++ b/datafusion/optimizer/src/simplify_expressions/guarantees.rs @@ -84,7 +84,7 @@ impl TreeNodeRewriter for GuaranteeRewriter<'_> { low, high, }) => { - if let (Some(interval), Expr::Literal(low), Expr::Literal(high)) = ( + if let (Some(interval), Expr::Literal(low, _), Expr::Literal(high, _)) = ( self.guarantees.get(inner.as_ref()), low.as_ref(), high.as_ref(), @@ -115,7 +115,7 @@ impl TreeNodeRewriter for GuaranteeRewriter<'_> { .get(left.as_ref()) .map(|interval| Cow::Borrowed(*interval)) .or_else(|| { - if let Expr::Literal(value) = left.as_ref() { + if let Expr::Literal(value, _) = left.as_ref() { Some(Cow::Owned(value.clone().into())) } else { None @@ -126,7 +126,7 @@ impl TreeNodeRewriter for GuaranteeRewriter<'_> { .get(right.as_ref()) .map(|interval| Cow::Borrowed(*interval)) .or_else(|| { - if let Expr::Literal(value) = right.as_ref() { + if let Expr::Literal(value, _) = right.as_ref() { Some(Cow::Owned(value.clone().into())) } else { None @@ -168,7 +168,7 @@ impl TreeNodeRewriter for GuaranteeRewriter<'_> { let new_list: Vec = list .iter() .filter_map(|expr| { - if let Expr::Literal(item) = expr { + if let Expr::Literal(item, _) = expr { match interval .contains(NullableInterval::from(item.clone())) { @@ -415,7 +415,7 @@ mod tests { let mut rewriter = GuaranteeRewriter::new(guarantees.iter()); let output = col("x").rewrite(&mut rewriter).data().unwrap(); - assert_eq!(output, Expr::Literal(scalar.clone())); + assert_eq!(output, Expr::Literal(scalar.clone(), None)); } } diff --git a/datafusion/optimizer/src/simplify_expressions/regex.rs b/datafusion/optimizer/src/simplify_expressions/regex.rs index ec6485bf4b443..82c5ea3d8d820 100644 --- a/datafusion/optimizer/src/simplify_expressions/regex.rs +++ b/datafusion/optimizer/src/simplify_expressions/regex.rs @@ -46,7 +46,7 @@ pub fn simplify_regex_expr( ) -> Result { let mode = OperatorMode::new(&op); - if let Expr::Literal(ScalarValue::Utf8(Some(pattern))) = right.as_ref() { + if let Expr::Literal(ScalarValue::Utf8(Some(pattern)), _) = right.as_ref() { // Handle the special case for ".*" pattern if pattern == ANY_CHAR_REGEX_PATTERN { let new_expr = if mode.not { @@ -121,7 +121,7 @@ impl OperatorMode { let like = Like { negated: self.not, expr, - pattern: Box::new(Expr::Literal(ScalarValue::from(pattern))), + pattern: Box::new(Expr::Literal(ScalarValue::from(pattern), None)), escape_char: None, case_insensitive: self.i, }; diff --git a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs index b70b19bae6df2..7c8ff8305e843 100644 --- a/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs +++ b/datafusion/optimizer/src/simplify_expressions/unwrap_cast.rs @@ -76,7 +76,7 @@ pub(super) fn unwrap_cast_in_comparison_for_binary( match (cast_expr, literal) { ( Expr::TryCast(TryCast { expr, .. }) | Expr::Cast(Cast { expr, .. }), - Expr::Literal(lit_value), + Expr::Literal(lit_value, _), ) => { let Ok(expr_type) = info.get_data_type(&expr) else { return internal_err!("Can't get the data type of the expr {:?}", &expr); @@ -126,7 +126,7 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_binary< | Expr::Cast(Cast { expr: left_expr, .. }), - Expr::Literal(lit_val), + Expr::Literal(lit_val, _), ) => { let Ok(expr_type) = info.get_data_type(left_expr) else { return false; @@ -183,7 +183,7 @@ pub(super) fn is_cast_expr_and_support_unwrap_cast_in_comparison_for_inlist< } match right { - Expr::Literal(lit_val) + Expr::Literal(lit_val, _) if try_cast_literal_to_type(lit_val, &expr_type).is_some() => {} _ => return false, } diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index cf182175e48ee..4df0e125eb18c 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -139,34 +139,34 @@ pub fn delete_xor_in_complex_expr(expr: &Expr, needle: &Expr, is_left: bool) -> pub fn is_zero(s: &Expr) -> bool { match s { - Expr::Literal(ScalarValue::Int8(Some(0))) - | Expr::Literal(ScalarValue::Int16(Some(0))) - | Expr::Literal(ScalarValue::Int32(Some(0))) - | Expr::Literal(ScalarValue::Int64(Some(0))) - | Expr::Literal(ScalarValue::UInt8(Some(0))) - | Expr::Literal(ScalarValue::UInt16(Some(0))) - | Expr::Literal(ScalarValue::UInt32(Some(0))) - | Expr::Literal(ScalarValue::UInt64(Some(0))) => true, - Expr::Literal(ScalarValue::Float32(Some(v))) if *v == 0. => true, - Expr::Literal(ScalarValue::Float64(Some(v))) if *v == 0. => true, - Expr::Literal(ScalarValue::Decimal128(Some(v), _p, _s)) if *v == 0 => true, + Expr::Literal(ScalarValue::Int8(Some(0)), _) + | Expr::Literal(ScalarValue::Int16(Some(0)), _) + | Expr::Literal(ScalarValue::Int32(Some(0)), _) + | Expr::Literal(ScalarValue::Int64(Some(0)), _) + | Expr::Literal(ScalarValue::UInt8(Some(0)), _) + | Expr::Literal(ScalarValue::UInt16(Some(0)), _) + | Expr::Literal(ScalarValue::UInt32(Some(0)), _) + | Expr::Literal(ScalarValue::UInt64(Some(0)), _) => true, + Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 0. => true, + Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 0. => true, + Expr::Literal(ScalarValue::Decimal128(Some(v), _p, _s), _) if *v == 0 => true, _ => false, } } pub fn is_one(s: &Expr) -> bool { match s { - Expr::Literal(ScalarValue::Int8(Some(1))) - | Expr::Literal(ScalarValue::Int16(Some(1))) - | Expr::Literal(ScalarValue::Int32(Some(1))) - | Expr::Literal(ScalarValue::Int64(Some(1))) - | Expr::Literal(ScalarValue::UInt8(Some(1))) - | Expr::Literal(ScalarValue::UInt16(Some(1))) - | Expr::Literal(ScalarValue::UInt32(Some(1))) - | Expr::Literal(ScalarValue::UInt64(Some(1))) => true, - Expr::Literal(ScalarValue::Float32(Some(v))) if *v == 1. => true, - Expr::Literal(ScalarValue::Float64(Some(v))) if *v == 1. => true, - Expr::Literal(ScalarValue::Decimal128(Some(v), _p, s)) => { + Expr::Literal(ScalarValue::Int8(Some(1)), _) + | Expr::Literal(ScalarValue::Int16(Some(1)), _) + | Expr::Literal(ScalarValue::Int32(Some(1)), _) + | Expr::Literal(ScalarValue::Int64(Some(1)), _) + | Expr::Literal(ScalarValue::UInt8(Some(1)), _) + | Expr::Literal(ScalarValue::UInt16(Some(1)), _) + | Expr::Literal(ScalarValue::UInt32(Some(1)), _) + | Expr::Literal(ScalarValue::UInt64(Some(1)), _) => true, + Expr::Literal(ScalarValue::Float32(Some(v)), _) if *v == 1. => true, + Expr::Literal(ScalarValue::Float64(Some(v)), _) if *v == 1. => true, + Expr::Literal(ScalarValue::Decimal128(Some(v), _p, s), _) => { *s >= 0 && POWS_OF_TEN .get(*s as usize) @@ -179,7 +179,7 @@ pub fn is_one(s: &Expr) -> bool { pub fn is_true(expr: &Expr) -> bool { match expr { - Expr::Literal(ScalarValue::Boolean(Some(v))) => *v, + Expr::Literal(ScalarValue::Boolean(Some(v)), _) => *v, _ => false, } } @@ -187,24 +187,24 @@ pub fn is_true(expr: &Expr) -> bool { /// returns true if expr is a /// `Expr::Literal(ScalarValue::Boolean(v))` , false otherwise pub fn is_bool_lit(expr: &Expr) -> bool { - matches!(expr, Expr::Literal(ScalarValue::Boolean(_))) + matches!(expr, Expr::Literal(ScalarValue::Boolean(_), _)) } /// Return a literal NULL value of Boolean data type pub fn lit_bool_null() -> Expr { - Expr::Literal(ScalarValue::Boolean(None)) + Expr::Literal(ScalarValue::Boolean(None), None) } pub fn is_null(expr: &Expr) -> bool { match expr { - Expr::Literal(v) => v.is_null(), + Expr::Literal(v, _) => v.is_null(), _ => false, } } pub fn is_false(expr: &Expr) -> bool { match expr { - Expr::Literal(ScalarValue::Boolean(Some(v))) => !(*v), + Expr::Literal(ScalarValue::Boolean(Some(v)), _) => !(*v), _ => false, } } @@ -247,7 +247,7 @@ pub fn is_negative_of(not_expr: &Expr, expr: &Expr) -> bool { /// `Expr::Literal(ScalarValue::Boolean(v))`. pub fn as_bool_lit(expr: &Expr) -> Result> { match expr { - Expr::Literal(ScalarValue::Boolean(v)) => Ok(*v), + Expr::Literal(ScalarValue::Boolean(v), _) => Ok(*v), _ => internal_err!("Expected boolean literal, got {expr:?}"), } } diff --git a/datafusion/optimizer/src/utils.rs b/datafusion/optimizer/src/utils.rs index 135f37dd9883b..0aa0bf3ea430d 100644 --- a/datafusion/optimizer/src/utils.rs +++ b/datafusion/optimizer/src/utils.rs @@ -163,7 +163,11 @@ mod tests { (Expr::IsNotNull(Box::new(col("a"))), true), // a = NULL ( - binary_expr(col("a"), Operator::Eq, Expr::Literal(ScalarValue::Null)), + binary_expr( + col("a"), + Operator::Eq, + Expr::Literal(ScalarValue::Null, None), + ), true, ), // a > 8 @@ -226,12 +230,16 @@ mod tests { ), // a IN (NULL) ( - in_list(col("a"), vec![Expr::Literal(ScalarValue::Null)], false), + in_list( + col("a"), + vec![Expr::Literal(ScalarValue::Null, None)], + false, + ), true, ), // a NOT IN (NULL) ( - in_list(col("a"), vec![Expr::Literal(ScalarValue::Null)], true), + in_list(col("a"), vec![Expr::Literal(ScalarValue::Null, None)], true), true, ), ]; diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rs index 9785203a70208..756fb638af2b5 100644 --- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs +++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs @@ -342,7 +342,7 @@ mod test { ) .unwrap(); let snap = dynamic_filter_1.snapshot().unwrap().unwrap(); - insta::assert_snapshot!(format!("{snap:?}"), @r#"BinaryExpr { left: Column { name: "a", index: 0 }, op: Eq, right: Literal { value: Int32(42) }, fail_on_overflow: false }"#); + insta::assert_snapshot!(format!("{snap:?}"), @r#"BinaryExpr { left: Column { name: "a", index: 0 }, op: Eq, right: Literal { value: Int32(42), field: Field { name: "42", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }"#); let dynamic_filter_2 = reassign_predicate_columns( Arc::clone(&dynamic_filter) as Arc, &filter_schema_2, @@ -350,7 +350,7 @@ mod test { ) .unwrap(); let snap = dynamic_filter_2.snapshot().unwrap().unwrap(); - insta::assert_snapshot!(format!("{snap:?}"), @r#"BinaryExpr { left: Column { name: "a", index: 1 }, op: Eq, right: Literal { value: Int32(42) }, fail_on_overflow: false }"#); + insta::assert_snapshot!(format!("{snap:?}"), @r#"BinaryExpr { left: Column { name: "a", index: 1 }, op: Eq, right: Literal { value: Int32(42), field: Field { name: "42", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, fail_on_overflow: false }"#); // Both filters allow evaluating the same expression let batch_1 = RecordBatch::try_new( Arc::clone(&filter_schema_1), diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 469f7bbee3173..a1a14b2f30ff5 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -1451,7 +1451,7 @@ mod tests { let sql_string = fmt_sql(expr.as_ref()).to_string(); let display_string = expr.to_string(); assert_eq!(sql_string, "a IN (a, b)"); - assert_eq!(display_string, "Use a@0 IN (SET) ([Literal { value: Utf8(\"a\") }, Literal { value: Utf8(\"b\") }])"); + assert_eq!(display_string, "Use a@0 IN (SET) ([Literal { value: Utf8(\"a\"), field: Field { name: \"a\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(\"b\"), field: Field { name: \"b\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }])"); // Test: a NOT IN ('a', 'b') let list = vec![lit("a"), lit("b")]; @@ -1459,7 +1459,7 @@ mod tests { let sql_string = fmt_sql(expr.as_ref()).to_string(); let display_string = expr.to_string(); assert_eq!(sql_string, "a NOT IN (a, b)"); - assert_eq!(display_string, "a@0 NOT IN (SET) ([Literal { value: Utf8(\"a\") }, Literal { value: Utf8(\"b\") }])"); + assert_eq!(display_string, "a@0 NOT IN (SET) ([Literal { value: Utf8(\"a\"), field: Field { name: \"a\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(\"b\"), field: Field { name: \"b\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }])"); // Test: a IN ('a', 'b', NULL) let list = vec![lit("a"), lit("b"), lit(ScalarValue::Utf8(None))]; @@ -1467,7 +1467,7 @@ mod tests { let sql_string = fmt_sql(expr.as_ref()).to_string(); let display_string = expr.to_string(); assert_eq!(sql_string, "a IN (a, b, NULL)"); - assert_eq!(display_string, "Use a@0 IN (SET) ([Literal { value: Utf8(\"a\") }, Literal { value: Utf8(\"b\") }, Literal { value: Utf8(NULL) }])"); + assert_eq!(display_string, "Use a@0 IN (SET) ([Literal { value: Utf8(\"a\"), field: Field { name: \"a\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(\"b\"), field: Field { name: \"b\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(NULL), field: Field { name: \"NULL\", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }])"); // Test: a NOT IN ('a', 'b', NULL) let list = vec![lit("a"), lit("b"), lit(ScalarValue::Utf8(None))]; @@ -1475,7 +1475,7 @@ mod tests { let sql_string = fmt_sql(expr.as_ref()).to_string(); let display_string = expr.to_string(); assert_eq!(sql_string, "a NOT IN (a, b, NULL)"); - assert_eq!(display_string, "a@0 NOT IN (SET) ([Literal { value: Utf8(\"a\") }, Literal { value: Utf8(\"b\") }, Literal { value: Utf8(NULL) }])"); + assert_eq!(display_string, "a@0 NOT IN (SET) ([Literal { value: Utf8(\"a\"), field: Field { name: \"a\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(\"b\"), field: Field { name: \"b\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(NULL), field: Field { name: \"NULL\", data_type: Utf8, nullable: true, dict_id: 0, dict_is_ordered: false, metadata: {} } }])"); Ok(()) } diff --git a/datafusion/physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rs index 6f7caaea8d45f..0d4d62ef47197 100644 --- a/datafusion/physical-expr/src/expressions/literal.rs +++ b/datafusion/physical-expr/src/expressions/literal.rs @@ -18,11 +18,13 @@ //! Literal expressions for physical operations use std::any::Any; +use std::collections::HashMap; use std::hash::Hash; use std::sync::Arc; use crate::physical_expr::PhysicalExpr; +use arrow::datatypes::{Field, FieldRef}; use arrow::{ datatypes::{DataType, Schema}, record_batch::RecordBatch, @@ -34,15 +36,48 @@ use datafusion_expr_common::interval_arithmetic::Interval; use datafusion_expr_common::sort_properties::{ExprProperties, SortProperties}; /// Represents a literal value -#[derive(Debug, PartialEq, Eq, Hash)] +#[derive(Debug, PartialEq, Eq)] pub struct Literal { value: ScalarValue, + field: FieldRef, +} + +impl Hash for Literal { + fn hash(&self, state: &mut H) { + self.value.hash(state); + let metadata = self.field.metadata(); + let mut keys = metadata.keys().collect::>(); + keys.sort(); + for key in keys { + key.hash(state); + metadata.get(key).unwrap().hash(state); + } + } } impl Literal { /// Create a literal value expression pub fn new(value: ScalarValue) -> Self { - Self { value } + Self::new_with_metadata(value, None) + } + + /// Create a literal value expression + pub fn new_with_metadata( + value: ScalarValue, + metadata: impl Into>>, + ) -> Self { + let metadata = metadata.into(); + let mut field = + Field::new(format!("{value}"), value.data_type(), value.is_null()); + + if let Some(metadata) = metadata { + field = field.with_metadata(metadata); + } + + Self { + value, + field: field.into(), + } } /// Get the scalar value @@ -71,6 +106,10 @@ impl PhysicalExpr for Literal { Ok(self.value.is_null()) } + fn return_field(&self, _input_schema: &Schema) -> Result { + Ok(Arc::clone(&self.field)) + } + fn evaluate(&self, _batch: &RecordBatch) -> Result { Ok(ColumnarValue::Scalar(self.value.clone())) } @@ -102,7 +141,7 @@ impl PhysicalExpr for Literal { /// Create a literal expression pub fn lit(value: T) -> Arc { match value.lit() { - Expr::Literal(v) => Arc::new(Literal::new(v)), + Expr::Literal(v, _) => Arc::new(Literal::new(v)), _ => unreachable!(), } } diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index 8660bff796d5a..6f1417ec23bf9 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::sync::Arc; use crate::ScalarFunctionExpr; @@ -111,14 +112,42 @@ pub fn create_physical_expr( let input_schema: &Schema = &input_dfschema.into(); match e { - Expr::Alias(Alias { expr, .. }) => { - Ok(create_physical_expr(expr, input_dfschema, execution_props)?) + Expr::Alias(Alias { expr, metadata, .. }) => { + if let Expr::Literal(v, prior_metadata) = expr.as_ref() { + let mut new_metadata = prior_metadata + .as_ref() + .map(|m| { + m.iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>() + }) + .unwrap_or_default(); + if let Some(metadata) = metadata { + new_metadata.extend(metadata.clone()); + } + let new_metadata = match new_metadata.is_empty() { + true => None, + false => Some(new_metadata), + }; + + Ok(Arc::new(Literal::new_with_metadata( + v.clone(), + new_metadata, + ))) + } else { + Ok(create_physical_expr(expr, input_dfschema, execution_props)?) + } } Expr::Column(c) => { let idx = input_dfschema.index_of_column(c)?; Ok(Arc::new(Column::new(&c.name, idx))) } - Expr::Literal(value) => Ok(Arc::new(Literal::new(value.clone()))), + Expr::Literal(value, metadata) => Ok(Arc::new(Literal::new_with_metadata( + value.clone(), + metadata + .as_ref() + .map(|m| m.iter().map(|(k, v)| (k.clone(), v.clone())).collect()), + ))), Expr::ScalarVariable(_, variable_names) => { if is_system_variables(variable_names) { match execution_props.get_var_provider(VarType::System) { @@ -168,7 +197,7 @@ pub fn create_physical_expr( let binary_op = binary_expr( expr.as_ref().clone(), Operator::IsNotDistinctFrom, - Expr::Literal(ScalarValue::Boolean(None)), + Expr::Literal(ScalarValue::Boolean(None), None), ); create_physical_expr(&binary_op, input_dfschema, execution_props) } @@ -176,7 +205,7 @@ pub fn create_physical_expr( let binary_op = binary_expr( expr.as_ref().clone(), Operator::IsDistinctFrom, - Expr::Literal(ScalarValue::Boolean(None)), + Expr::Literal(ScalarValue::Boolean(None), None), ); create_physical_expr(&binary_op, input_dfschema, execution_props) } @@ -347,7 +376,7 @@ pub fn create_physical_expr( list, negated, }) => match expr.as_ref() { - Expr::Literal(ScalarValue::Utf8(None)) => { + Expr::Literal(ScalarValue::Utf8(None), _) => { Ok(expressions::lit(ScalarValue::Boolean(None))) } _ => { diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index 38546fa38064d..1b5527c14a494 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -268,7 +268,7 @@ pub fn parse_expr( ExprType::Column(column) => Ok(Expr::Column(column.into())), ExprType::Literal(literal) => { let scalar_value: ScalarValue = literal.try_into()?; - Ok(Expr::Literal(scalar_value)) + Ok(Expr::Literal(scalar_value, None)) } ExprType::WindowExpr(expr) => { let window_function = expr diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 18073516610c6..7f089b1c84676 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -217,7 +217,7 @@ pub fn serialize_expr( expr_type: Some(ExprType::Alias(alias)), } } - Expr::Literal(value) => { + Expr::Literal(value, _) => { let pb_value: protobuf::ScalarValue = value.try_into()?; protobuf::LogicalExprNode { expr_type: Some(ExprType::Literal(pb_value)), diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index 3edf152f4c71c..993cc6f87ca30 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -1968,7 +1968,7 @@ fn roundtrip_case_with_null() { let test_expr = Expr::Case(Case::new( Some(Box::new(lit(1.0_f32))), vec![(Box::new(lit(2.0_f32)), Box::new(lit(3.0_f32)))], - Some(Box::new(Expr::Literal(ScalarValue::Null))), + Some(Box::new(Expr::Literal(ScalarValue::Null, None))), )); let ctx = SessionContext::new(); @@ -1977,7 +1977,7 @@ fn roundtrip_case_with_null() { #[test] fn roundtrip_null_literal() { - let test_expr = Expr::Literal(ScalarValue::Null); + let test_expr = Expr::Literal(ScalarValue::Null, None); let ctx = SessionContext::new(); roundtrip_expr_test(test_expr, ctx); diff --git a/datafusion/proto/tests/cases/serialize.rs b/datafusion/proto/tests/cases/serialize.rs index ed99150831e7a..c9ef4377d43b1 100644 --- a/datafusion/proto/tests/cases/serialize.rs +++ b/datafusion/proto/tests/cases/serialize.rs @@ -256,7 +256,7 @@ fn test_expression_serialization_roundtrip() { use datafusion_proto::logical_plan::from_proto::parse_expr; let ctx = SessionContext::new(); - let lit = Expr::Literal(ScalarValue::Utf8(None)); + let lit = Expr::Literal(ScalarValue::Utf8(None), None); for function in string::functions() { // default to 4 args (though some exprs like substr have error checking) let num_args = 4; diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index eadf66a91ef3d..e92869873731f 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -215,7 +215,7 @@ impl SqlToRel<'_, S> { } SQLExpr::Extract { field, expr, .. } => { let mut extract_args = vec![ - Expr::Literal(ScalarValue::from(format!("{field}"))), + Expr::Literal(ScalarValue::from(format!("{field}")), None), self.sql_expr_to_logical_expr(*expr, schema, planner_context)?, ]; diff --git a/datafusion/sql/src/expr/substring.rs b/datafusion/sql/src/expr/substring.rs index 59c78bc713cc4..8f6e77e035c12 100644 --- a/datafusion/sql/src/expr/substring.rs +++ b/datafusion/sql/src/expr/substring.rs @@ -51,7 +51,7 @@ impl SqlToRel<'_, S> { (None, Some(for_expr)) => { let arg = self.sql_expr_to_logical_expr(*expr, schema, planner_context)?; - let from_logic = Expr::Literal(ScalarValue::Int64(Some(1))); + let from_logic = Expr::Literal(ScalarValue::Int64(Some(1)), None); let for_logic = self.sql_expr_to_logical_expr(*for_expr, schema, planner_context)?; vec![arg, from_logic, for_logic] diff --git a/datafusion/sql/src/expr/value.rs b/datafusion/sql/src/expr/value.rs index b77f5eaf45da2..7075a1afd9dd0 100644 --- a/datafusion/sql/src/expr/value.rs +++ b/datafusion/sql/src/expr/value.rs @@ -50,7 +50,7 @@ impl SqlToRel<'_, S> { match value { Value::Number(n, _) => self.parse_sql_number(&n, false), Value::SingleQuotedString(s) | Value::DoubleQuotedString(s) => Ok(lit(s)), - Value::Null => Ok(Expr::Literal(ScalarValue::Null)), + Value::Null => Ok(Expr::Literal(ScalarValue::Null, None)), Value::Boolean(n) => Ok(lit(n)), Value::Placeholder(param) => { Self::create_placeholder_expr(param, param_data_types) @@ -380,11 +380,10 @@ fn parse_decimal(unsigned_number: &str, negative: bool) -> Result { int_val ) })?; - Ok(Expr::Literal(ScalarValue::Decimal128( - Some(val), - precision as u8, - scale as i8, - ))) + Ok(Expr::Literal( + ScalarValue::Decimal128(Some(val), precision as u8, scale as i8), + None, + )) } else if precision <= DECIMAL256_MAX_PRECISION as u64 { let val = bigint_to_i256(&int_val).ok_or_else(|| { // Failures are unexpected here as we have already checked the precision @@ -393,11 +392,10 @@ fn parse_decimal(unsigned_number: &str, negative: bool) -> Result { int_val ) })?; - Ok(Expr::Literal(ScalarValue::Decimal256( - Some(val), - precision as u8, - scale as i8, - ))) + Ok(Expr::Literal( + ScalarValue::Decimal256(Some(val), precision as u8, scale as i8), + None, + )) } else { not_impl_err!( "Decimal precision {} exceeds the maximum supported precision: {}", @@ -483,10 +481,13 @@ mod tests { ]; for (input, expect) in cases { let output = parse_decimal(input, true).unwrap(); - assert_eq!(output, Expr::Literal(expect.arithmetic_negate().unwrap())); + assert_eq!( + output, + Expr::Literal(expect.arithmetic_negate().unwrap(), None) + ); let output = parse_decimal(input, false).unwrap(); - assert_eq!(output, Expr::Literal(expect)); + assert_eq!(output, Expr::Literal(expect, None)); } // scale < i8::MIN diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 458b3ac132179..dafb0346485e2 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -2065,7 +2065,7 @@ impl SqlToRel<'_, S> { .cloned() .unwrap_or_else(|| { // If there is no default for the column, then the default is NULL - Expr::Literal(ScalarValue::Null) + Expr::Literal(ScalarValue::Null, None) }) .cast_to(target_field.data_type(), &DFSchema::empty())?, }; diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 661e8581ac06a..cce14894acaf7 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -187,7 +187,7 @@ impl Unparser<'_> { Expr::Cast(Cast { expr, data_type }) => { Ok(self.cast_to_sql(expr, data_type)?) } - Expr::Literal(value) => Ok(self.scalar_to_sql(value)?), + Expr::Literal(value, _) => Ok(self.scalar_to_sql(value)?), Expr::Alias(Alias { expr, name: _, .. }) => self.expr_to_sql_inner(expr), Expr::WindowFunction(window_fun) => { let WindowFunction { @@ -602,7 +602,7 @@ impl Unparser<'_> { .chunks_exact(2) .map(|chunk| { let key = match &chunk[0] { - Expr::Literal(ScalarValue::Utf8(Some(s))) => self.new_ident_quoted_if_needs(s.to_string()), + Expr::Literal(ScalarValue::Utf8(Some(s)), _) => self.new_ident_quoted_if_needs(s.to_string()), _ => return internal_err!("named_struct expects even arguments to be strings, but received: {:?}", &chunk[0]) }; @@ -631,7 +631,7 @@ impl Unparser<'_> { }; let field = match &args[1] { - Expr::Literal(lit) => self.new_ident_quoted_if_needs(lit.to_string()), + Expr::Literal(lit, _) => self.new_ident_quoted_if_needs(lit.to_string()), _ => { return internal_err!( "get_field expects second argument to be a string, but received: {:?}", @@ -1911,87 +1911,87 @@ mod tests { r#"a LIKE 'foo' ESCAPE 'o'"#, ), ( - Expr::Literal(ScalarValue::Date64(Some(0))), + Expr::Literal(ScalarValue::Date64(Some(0)), None), r#"CAST('1970-01-01 00:00:00' AS DATETIME)"#, ), ( - Expr::Literal(ScalarValue::Date64(Some(10000))), + Expr::Literal(ScalarValue::Date64(Some(10000)), None), r#"CAST('1970-01-01 00:00:10' AS DATETIME)"#, ), ( - Expr::Literal(ScalarValue::Date64(Some(-10000))), + Expr::Literal(ScalarValue::Date64(Some(-10000)), None), r#"CAST('1969-12-31 23:59:50' AS DATETIME)"#, ), ( - Expr::Literal(ScalarValue::Date32(Some(0))), + Expr::Literal(ScalarValue::Date32(Some(0)), None), r#"CAST('1970-01-01' AS DATE)"#, ), ( - Expr::Literal(ScalarValue::Date32(Some(10))), + Expr::Literal(ScalarValue::Date32(Some(10)), None), r#"CAST('1970-01-11' AS DATE)"#, ), ( - Expr::Literal(ScalarValue::Date32(Some(-1))), + Expr::Literal(ScalarValue::Date32(Some(-1)), None), r#"CAST('1969-12-31' AS DATE)"#, ), ( - Expr::Literal(ScalarValue::TimestampSecond(Some(10001), None)), + Expr::Literal(ScalarValue::TimestampSecond(Some(10001), None), None), r#"CAST('1970-01-01 02:46:41' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampSecond( - Some(10001), - Some("+08:00".into()), - )), + Expr::Literal( + ScalarValue::TimestampSecond(Some(10001), Some("+08:00".into())), + None, + ), r#"CAST('1970-01-01 10:46:41 +08:00' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampMillisecond(Some(10001), None)), + Expr::Literal(ScalarValue::TimestampMillisecond(Some(10001), None), None), r#"CAST('1970-01-01 00:00:10.001' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampMillisecond( - Some(10001), - Some("+08:00".into()), - )), + Expr::Literal( + ScalarValue::TimestampMillisecond(Some(10001), Some("+08:00".into())), + None, + ), r#"CAST('1970-01-01 08:00:10.001 +08:00' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampMicrosecond(Some(10001), None)), + Expr::Literal(ScalarValue::TimestampMicrosecond(Some(10001), None), None), r#"CAST('1970-01-01 00:00:00.010001' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampMicrosecond( - Some(10001), - Some("+08:00".into()), - )), + Expr::Literal( + ScalarValue::TimestampMicrosecond(Some(10001), Some("+08:00".into())), + None, + ), r#"CAST('1970-01-01 08:00:00.010001 +08:00' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampNanosecond(Some(10001), None)), + Expr::Literal(ScalarValue::TimestampNanosecond(Some(10001), None), None), r#"CAST('1970-01-01 00:00:00.000010001' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::TimestampNanosecond( - Some(10001), - Some("+08:00".into()), - )), + Expr::Literal( + ScalarValue::TimestampNanosecond(Some(10001), Some("+08:00".into())), + None, + ), r#"CAST('1970-01-01 08:00:00.000010001 +08:00' AS TIMESTAMP)"#, ), ( - Expr::Literal(ScalarValue::Time32Second(Some(10001))), + Expr::Literal(ScalarValue::Time32Second(Some(10001)), None), r#"CAST('02:46:41' AS TIME)"#, ), ( - Expr::Literal(ScalarValue::Time32Millisecond(Some(10001))), + Expr::Literal(ScalarValue::Time32Millisecond(Some(10001)), None), r#"CAST('00:00:10.001' AS TIME)"#, ), ( - Expr::Literal(ScalarValue::Time64Microsecond(Some(10001))), + Expr::Literal(ScalarValue::Time64Microsecond(Some(10001)), None), r#"CAST('00:00:00.010001' AS TIME)"#, ), ( - Expr::Literal(ScalarValue::Time64Nanosecond(Some(10001))), + Expr::Literal(ScalarValue::Time64Nanosecond(Some(10001)), None), r#"CAST('00:00:00.000010001' AS TIME)"#, ), (sum(col("a")), r#"sum(a)"#), @@ -2136,19 +2136,17 @@ mod tests { (col("need quoted").eq(lit(1)), r#"("need quoted" = 1)"#), // See test_interval_scalar_to_expr for interval literals ( - (col("a") + col("b")).gt(Expr::Literal(ScalarValue::Decimal128( - Some(100123), - 28, - 3, - ))), + (col("a") + col("b")).gt(Expr::Literal( + ScalarValue::Decimal128(Some(100123), 28, 3), + None, + )), r#"((a + b) > 100.123)"#, ), ( - (col("a") + col("b")).gt(Expr::Literal(ScalarValue::Decimal256( - Some(100123.into()), - 28, - 3, - ))), + (col("a") + col("b")).gt(Expr::Literal( + ScalarValue::Decimal256(Some(100123.into()), 28, 3), + None, + )), r#"((a + b) > 100.123)"#, ), ( @@ -2184,28 +2182,39 @@ mod tests { "MAP {'a': 1, 'b': 2}", ), ( - Expr::Literal(ScalarValue::Dictionary( - Box::new(DataType::Int32), - Box::new(ScalarValue::Utf8(Some("foo".into()))), - )), + Expr::Literal( + ScalarValue::Dictionary( + Box::new(DataType::Int32), + Box::new(ScalarValue::Utf8(Some("foo".into()))), + ), + None, + ), "'foo'", ), ( - Expr::Literal(ScalarValue::List(Arc::new( - ListArray::from_iter_primitive::(vec![Some(vec![ + Expr::Literal( + ScalarValue::List(Arc::new(ListArray::from_iter_primitive::< + Int32Type, + _, + _, + >(vec![Some(vec![ Some(1), Some(2), Some(3), - ])]), - ))), + ])]))), + None, + ), "[1, 2, 3]", ), ( - Expr::Literal(ScalarValue::LargeList(Arc::new( - LargeListArray::from_iter_primitive::(vec![Some( - vec![Some(1), Some(2), Some(3)], - )]), - ))), + Expr::Literal( + ScalarValue::LargeList(Arc::new( + LargeListArray::from_iter_primitive::(vec![ + Some(vec![Some(1), Some(2), Some(3)]), + ]), + )), + None, + ), "[1, 2, 3]", ), ( @@ -2510,11 +2519,17 @@ mod tests { #[test] fn test_float_scalar_to_expr() { let tests = [ - (Expr::Literal(ScalarValue::Float64(Some(3f64))), "3.0"), - (Expr::Literal(ScalarValue::Float64(Some(3.1f64))), "3.1"), - (Expr::Literal(ScalarValue::Float32(Some(-2f32))), "-2.0"), + (Expr::Literal(ScalarValue::Float64(Some(3f64)), None), "3.0"), ( - Expr::Literal(ScalarValue::Float32(Some(-2.989f32))), + Expr::Literal(ScalarValue::Float64(Some(3.1f64)), None), + "3.1", + ), + ( + Expr::Literal(ScalarValue::Float32(Some(-2f32)), None), + "-2.0", + ), + ( + Expr::Literal(ScalarValue::Float32(Some(-2.989f32)), None), "-2.989", ), ]; @@ -2534,18 +2549,20 @@ mod tests { let tests = [ ( Expr::Cast(Cast { - expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some( - "blah".to_string(), - )))), + expr: Box::new(Expr::Literal( + ScalarValue::Utf8(Some("blah".to_string())), + None, + )), data_type: DataType::Binary, }), "'blah'", ), ( Expr::Cast(Cast { - expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some( - "blah".to_string(), - )))), + expr: Box::new(Expr::Literal( + ScalarValue::Utf8(Some("blah".to_string())), + None, + )), data_type: DataType::BinaryView, }), "'blah'", @@ -2637,7 +2654,10 @@ mod tests { let expr = ScalarUDF::new_from_impl( datafusion_functions::datetime::date_part::DatePartFunc::new(), ) - .call(vec![Expr::Literal(ScalarValue::new_utf8(unit)), col("x")]); + .call(vec![ + Expr::Literal(ScalarValue::new_utf8(unit), None), + col("x"), + ]); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2757,10 +2777,10 @@ mod tests { (&mysql_dialect, "DATETIME"), ] { let unparser = Unparser::new(dialect); - let expr = Expr::Literal(ScalarValue::TimestampMillisecond( - Some(1738285549123), + let expr = Expr::Literal( + ScalarValue::TimestampMillisecond(Some(1738285549123), None), None, - )); + ); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2828,9 +2848,10 @@ mod tests { fn test_cast_value_to_dict_expr() { let tests = [( Expr::Cast(Cast { - expr: Box::new(Expr::Literal(ScalarValue::Utf8(Some( - "variation".to_string(), - )))), + expr: Box::new(Expr::Literal( + ScalarValue::Utf8(Some("variation".to_string())), + None, + )), data_type: DataType::Dictionary(Box::new(Int8), Box::new(DataType::Utf8)), }), "'variation'", @@ -2868,7 +2889,7 @@ mod tests { expr: Box::new(col("a")), data_type: DataType::Float64, }), - Expr::Literal(ScalarValue::Int64(Some(2))), + Expr::Literal(ScalarValue::Int64(Some(2)), None), ], }); let ast = unparser.expr_to_sql(&expr)?; @@ -3008,7 +3029,7 @@ mod tests { datafusion_functions::datetime::date_trunc::DateTruncFunc::new(), )), args: vec![ - Expr::Literal(ScalarValue::Utf8(Some(precision.to_string()))), + Expr::Literal(ScalarValue::Utf8(Some(precision.to_string())), None), col("date_col"), ], }); diff --git a/datafusion/sql/src/unparser/plan.rs b/datafusion/sql/src/unparser/plan.rs index e89e25ddb15ac..f6677617031f6 100644 --- a/datafusion/sql/src/unparser/plan.rs +++ b/datafusion/sql/src/unparser/plan.rs @@ -1078,6 +1078,7 @@ impl Unparser<'_> { if project_vec.is_empty() { builder = builder.project(vec![Expr::Literal( ScalarValue::Int64(Some(1)), + None, )])?; } else { let project_columns = project_vec diff --git a/datafusion/sql/src/unparser/utils.rs b/datafusion/sql/src/unparser/utils.rs index c36ffbfe5ecfb..89fa392c183f3 100644 --- a/datafusion/sql/src/unparser/utils.rs +++ b/datafusion/sql/src/unparser/utils.rs @@ -422,7 +422,7 @@ pub(crate) fn date_part_to_sql( match (style, date_part_args.len()) { (DateFieldExtractStyle::Extract, 2) => { let date_expr = unparser.expr_to_sql(&date_part_args[1])?; - if let Expr::Literal(ScalarValue::Utf8(Some(field))) = &date_part_args[0] { + if let Expr::Literal(ScalarValue::Utf8(Some(field)), _) = &date_part_args[0] { let field = match field.to_lowercase().as_str() { "year" => ast::DateTimeField::Year, "month" => ast::DateTimeField::Month, @@ -443,7 +443,7 @@ pub(crate) fn date_part_to_sql( (DateFieldExtractStyle::Strftime, 2) => { let column = unparser.expr_to_sql(&date_part_args[1])?; - if let Expr::Literal(ScalarValue::Utf8(Some(field))) = &date_part_args[0] { + if let Expr::Literal(ScalarValue::Utf8(Some(field)), _) = &date_part_args[0] { let field = match field.to_lowercase().as_str() { "year" => "%Y", "month" => "%m", @@ -531,7 +531,7 @@ pub(crate) fn sqlite_from_unixtime_to_sql( "datetime", &[ from_unixtime_args[0].clone(), - Expr::Literal(ScalarValue::Utf8(Some("unixepoch".to_string()))), + Expr::Literal(ScalarValue::Utf8(Some("unixepoch".to_string())), None), ], )?)) } @@ -554,7 +554,7 @@ pub(crate) fn sqlite_date_trunc_to_sql( ); } - if let Expr::Literal(ScalarValue::Utf8(Some(unit))) = &date_trunc_args[0] { + if let Expr::Literal(ScalarValue::Utf8(Some(unit)), _) = &date_trunc_args[0] { let format = match unit.to_lowercase().as_str() { "year" => "%Y", "month" => "%Y-%m", @@ -568,7 +568,7 @@ pub(crate) fn sqlite_date_trunc_to_sql( return Ok(Some(unparser.scalar_function_to_sql( "strftime", &[ - Expr::Literal(ScalarValue::Utf8(Some(format.to_string()))), + Expr::Literal(ScalarValue::Utf8(Some(format.to_string())), None), date_trunc_args[1].clone(), ], )?)); diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index 067da40cf9a8e..52832e1324bef 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -198,7 +198,7 @@ pub(crate) fn resolve_positions_to_exprs( match expr { // sql_expr_to_logical_expr maps number to i64 // https://github.com/apache/datafusion/blob/8d175c759e17190980f270b5894348dc4cff9bbf/datafusion/src/sql/planner.rs#L882-L887 - Expr::Literal(ScalarValue::Int64(Some(position))) + Expr::Literal(ScalarValue::Int64(Some(position)), _) if position > 0_i64 && position <= select_exprs.len() as i64 => { let index = (position - 1) as usize; @@ -208,7 +208,7 @@ pub(crate) fn resolve_positions_to_exprs( _ => select_expr.clone(), }) } - Expr::Literal(ScalarValue::Int64(Some(position))) => plan_err!( + Expr::Literal(ScalarValue::Int64(Some(position)), _) => plan_err!( "Cannot find column with position {} in SELECT clause. Valid columns: 1 to {}", position, select_exprs.len() ), diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index d89ba600d7a6b..ac96daed0d44a 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -6032,7 +6032,7 @@ physical_plan 04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] 05)--------ProjectionExec: expr=[] 06)----------CoalesceBatchesExec: target_batch_size=8192 -07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278") }, Literal { value: Utf8View("a") }, Literal { value: Utf8View("b") }, Literal { value: Utf8View("c") }]) +07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278"), field: Field { name: "7f4b18de3cfeb9b4ac78c381ee2ad278", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("a"), field: Field { name: "a", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("b"), field: Field { name: "b", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("c"), field: Field { name: "c", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) 08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 09)----------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192] @@ -6061,7 +6061,7 @@ physical_plan 04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] 05)--------ProjectionExec: expr=[] 06)----------CoalesceBatchesExec: target_batch_size=8192 -07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278") }, Literal { value: Utf8View("a") }, Literal { value: Utf8View("b") }, Literal { value: Utf8View("c") }]) +07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278"), field: Field { name: "7f4b18de3cfeb9b4ac78c381ee2ad278", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("a"), field: Field { name: "a", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("b"), field: Field { name: "b", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("c"), field: Field { name: "c", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) 08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 09)----------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192] @@ -6090,7 +6090,7 @@ physical_plan 04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] 05)--------ProjectionExec: expr=[] 06)----------CoalesceBatchesExec: target_batch_size=8192 -07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278") }, Literal { value: Utf8View("a") }, Literal { value: Utf8View("b") }, Literal { value: Utf8View("c") }]) +07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278"), field: Field { name: "7f4b18de3cfeb9b4ac78c381ee2ad278", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("a"), field: Field { name: "a", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("b"), field: Field { name: "b", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("c"), field: Field { name: "c", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) 08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 09)----------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192] @@ -6150,7 +6150,7 @@ physical_plan 04)------AggregateExec: mode=Partial, gby=[], aggr=[count(Int64(1))] 05)--------ProjectionExec: expr=[] 06)----------CoalesceBatchesExec: target_batch_size=8192 -07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278") }, Literal { value: Utf8View("a") }, Literal { value: Utf8View("b") }, Literal { value: Utf8View("c") }]) +07)------------FilterExec: substr(md5(CAST(value@0 AS Utf8)), 1, 32) IN ([Literal { value: Utf8View("7f4b18de3cfeb9b4ac78c381ee2ad278"), field: Field { name: "7f4b18de3cfeb9b4ac78c381ee2ad278", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("a"), field: Field { name: "a", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("b"), field: Field { name: "b", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("c"), field: Field { name: "c", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) 08)--------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 09)----------------LazyMemoryExec: partitions=1, batch_generators=[generate_series: start=1, end=100000, batch_size=8192] diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part index edc452284cf99..cd2f407387edd 100644 --- a/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part +++ b/datafusion/sqllogictest/test_files/tpch/plans/q16.slt.part @@ -88,7 +88,7 @@ physical_plan 21)----------------------------------CoalesceBatchesExec: target_batch_size=8192 22)------------------------------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4 23)--------------------------------------CoalesceBatchesExec: target_batch_size=8192 -24)----------------------------------------FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49) }, Literal { value: Int32(14) }, Literal { value: Int32(23) }, Literal { value: Int32(45) }, Literal { value: Int32(19) }, Literal { value: Int32(3) }, Literal { value: Int32(36) }, Literal { value: Int32(9) }]) +24)----------------------------------------FilterExec: p_brand@1 != Brand#45 AND p_type@2 NOT LIKE MEDIUM POLISHED% AND Use p_size@3 IN (SET) ([Literal { value: Int32(49), field: Field { name: "49", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(14), field: Field { name: "14", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(23), field: Field { name: "23", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(45), field: Field { name: "45", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(19), field: Field { name: "19", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(3), field: Field { name: "3", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(36), field: Field { name: "36", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Int32(9), field: Field { name: "9", data_type: Int32, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) 25)------------------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 26)--------------------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl]]}, projection=[p_partkey, p_brand, p_type, p_size], file_type=csv, has_header=false 27)--------------------------CoalesceBatchesExec: target_batch_size=8192 diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part index 3b15fb3d8e533..ace2081eb18fe 100644 --- a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part +++ b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part @@ -69,7 +69,7 @@ physical_plan 03)----CoalescePartitionsExec 04)------AggregateExec: mode=Partial, gby=[], aggr=[sum(lineitem.l_extendedprice * Int64(1) - lineitem.l_discount)] 05)--------CoalesceBatchesExec: target_batch_size=8192 -06)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE") }, Literal { value: Utf8View("SM BOX") }, Literal { value: Utf8View("SM PACK") }, Literal { value: Utf8View("SM PKG") }]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG") }, Literal { value: Utf8View("MED BOX") }, Literal { value: Utf8View("MED PKG") }, Literal { value: Utf8View("MED PACK") }]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE") }, Literal { value: Utf8View("LG BOX") }, Literal { value: Utf8View("LG PACK") }, Literal { value: Utf8View("LG PKG") }]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@2, l_discount@3] +06)----------HashJoinExec: mode=Partitioned, join_type=Inner, on=[(l_partkey@0, p_partkey@0)], filter=p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "SM CASE", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "SM BOX", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "SM PACK", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "SM PKG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(100),15,2 AND l_quantity@0 <= Some(1100),15,2 AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "MED BAG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "MED BOX", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "MED PKG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "MED PACK", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(1000),15,2 AND l_quantity@0 <= Some(2000),15,2 AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "LG CASE", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "LG BOX", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "LG PACK", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "LG PKG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND l_quantity@0 >= Some(2000),15,2 AND l_quantity@0 <= Some(3000),15,2 AND p_size@2 <= 15, projection=[l_extendedprice@2, l_discount@3] 07)------------CoalesceBatchesExec: target_batch_size=8192 08)--------------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4 09)----------------CoalesceBatchesExec: target_batch_size=8192 @@ -78,6 +78,6 @@ physical_plan 12)------------CoalesceBatchesExec: target_batch_size=8192 13)--------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4 14)----------------CoalesceBatchesExec: target_batch_size=8192 -15)------------------FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE") }, Literal { value: Utf8View("SM BOX") }, Literal { value: Utf8View("SM PACK") }, Literal { value: Utf8View("SM PKG") }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG") }, Literal { value: Utf8View("MED BOX") }, Literal { value: Utf8View("MED PKG") }, Literal { value: Utf8View("MED PACK") }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE") }, Literal { value: Utf8View("LG BOX") }, Literal { value: Utf8View("LG PACK") }, Literal { value: Utf8View("LG PKG") }]) AND p_size@2 <= 15) AND p_size@2 >= 1 +15)------------------FilterExec: (p_brand@1 = Brand#12 AND p_container@3 IN ([Literal { value: Utf8View("SM CASE"), field: Field { name: "SM CASE", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM BOX"), field: Field { name: "SM BOX", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PACK"), field: Field { name: "SM PACK", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("SM PKG"), field: Field { name: "SM PKG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 5 OR p_brand@1 = Brand#23 AND p_container@3 IN ([Literal { value: Utf8View("MED BAG"), field: Field { name: "MED BAG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED BOX"), field: Field { name: "MED BOX", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PKG"), field: Field { name: "MED PKG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("MED PACK"), field: Field { name: "MED PACK", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 10 OR p_brand@1 = Brand#34 AND p_container@3 IN ([Literal { value: Utf8View("LG CASE"), field: Field { name: "LG CASE", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG BOX"), field: Field { name: "LG BOX", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PACK"), field: Field { name: "LG PACK", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("LG PKG"), field: Field { name: "LG PKG", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) AND p_size@2 <= 15) AND p_size@2 >= 1 16)--------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 17)----------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/part.tbl]]}, projection=[p_partkey, p_brand, p_size, p_container], file_type=csv, has_header=false diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part index 828bf967d8f4a..6af91b4aaa427 100644 --- a/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part +++ b/datafusion/sqllogictest/test_files/tpch/plans/q22.slt.part @@ -90,7 +90,7 @@ physical_plan 14)--------------------------CoalesceBatchesExec: target_batch_size=8192 15)----------------------------RepartitionExec: partitioning=Hash([c_custkey@0], 4), input_partitions=4 16)------------------------------CoalesceBatchesExec: target_batch_size=8192 -17)--------------------------------FilterExec: substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13") }, Literal { value: Utf8View("31") }, Literal { value: Utf8View("23") }, Literal { value: Utf8View("29") }, Literal { value: Utf8View("30") }, Literal { value: Utf8View("18") }, Literal { value: Utf8View("17") }]) +17)--------------------------------FilterExec: substr(c_phone@1, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "13", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "31", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "23", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "29", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "30", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "18", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "17", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]) 18)----------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 19)------------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_custkey, c_phone, c_acctbal], file_type=csv, has_header=false 20)--------------------------CoalesceBatchesExec: target_batch_size=8192 @@ -100,6 +100,6 @@ physical_plan 24)----------------------CoalescePartitionsExec 25)------------------------AggregateExec: mode=Partial, gby=[], aggr=[avg(customer.c_acctbal)] 26)--------------------------CoalesceBatchesExec: target_batch_size=8192 -27)----------------------------FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13") }, Literal { value: Utf8View("31") }, Literal { value: Utf8View("23") }, Literal { value: Utf8View("29") }, Literal { value: Utf8View("30") }, Literal { value: Utf8View("18") }, Literal { value: Utf8View("17") }]), projection=[c_acctbal@1] +27)----------------------------FilterExec: c_acctbal@1 > Some(0),15,2 AND substr(c_phone@0, 1, 2) IN ([Literal { value: Utf8View("13"), field: Field { name: "13", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("31"), field: Field { name: "31", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("23"), field: Field { name: "23", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("29"), field: Field { name: "29", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("30"), field: Field { name: "30", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("18"), field: Field { name: "18", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8View("17"), field: Field { name: "17", data_type: Utf8View, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }]), projection=[c_acctbal@1] 28)------------------------------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 29)--------------------------------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/customer.tbl]]}, projection=[c_phone, c_acctbal], file_type=csv, has_header=false diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/aggregate_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/aggregate_function.rs index 7687d9f7642ab..114fe1e7aecd5 100644 --- a/datafusion/substrait/src/logical_plan/consumer/expr/aggregate_function.rs +++ b/datafusion/substrait/src/logical_plan/consumer/expr/aggregate_function.rs @@ -60,7 +60,7 @@ pub async fn from_substrait_agg_func( // we inject a dummy argument that does not affect the query, but allows // us to bypass this limitation. let args = if udaf.name() == "count" && args.is_empty() { - vec![Expr::Literal(ScalarValue::Int64(Some(1)))] + vec![Expr::Literal(ScalarValue::Int64(Some(1)), None)] } else { args }; diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs index 5adc137d9a43a..d054e52675545 100644 --- a/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs +++ b/datafusion/substrait/src/logical_plan/consumer/expr/literal.rs @@ -51,7 +51,7 @@ pub async fn from_literal( expr: &Literal, ) -> datafusion::common::Result { let scalar_value = from_substrait_literal_without_names(consumer, expr)?; - Ok(Expr::Literal(scalar_value)) + Ok(Expr::Literal(scalar_value, None)) } pub(crate) fn from_substrait_literal_without_names( diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs index 027b61124ead0..7797c935211fe 100644 --- a/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs +++ b/datafusion/substrait/src/logical_plan/consumer/expr/scalar_function.rs @@ -261,7 +261,7 @@ impl BuiltinExprBuilder { .await?; match escape_char_expr { - Expr::Literal(ScalarValue::Utf8(escape_char_string)) => { + Expr::Literal(ScalarValue::Utf8(escape_char_string), _) => { // Convert Option to Option escape_char_string.and_then(|s| s.chars().next()) } @@ -337,7 +337,7 @@ mod tests { fn int64_literals(integers: &[i64]) -> Vec { integers .iter() - .map(|value| Expr::Literal(ScalarValue::Int64(Some(*value)))) + .map(|value| Expr::Literal(ScalarValue::Int64(Some(*value)), None)) .collect() } diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/window_function.rs b/datafusion/substrait/src/logical_plan/consumer/expr/window_function.rs index 4a7fde256b6cf..80b643a547ee6 100644 --- a/datafusion/substrait/src/logical_plan/consumer/expr/window_function.rs +++ b/datafusion/substrait/src/logical_plan/consumer/expr/window_function.rs @@ -94,7 +94,7 @@ pub async fn from_window_function( // we inject a dummy argument that does not affect the query, but allows // us to bypass this limitation. let args = if fun.name() == "count" && window.arguments.is_empty() { - vec![Expr::Literal(ScalarValue::Int64(Some(1)))] + vec![Expr::Literal(ScalarValue::Int64(Some(1)), None)] } else { from_substrait_func_args(consumer, &window.arguments, input_schema).await? }; diff --git a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs index 47af44c692aeb..f1cbd16d2d8f2 100644 --- a/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs +++ b/datafusion/substrait/src/logical_plan/consumer/rel/read_rel.rs @@ -136,7 +136,7 @@ pub async fn from_read_rel( lit, &named_struct.names, &mut name_idx, - )?)) + )?, None)) }) .collect::>()?; if name_idx != named_struct.names.len() { diff --git a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs index b69474f09ee43..9741dcdd10951 100644 --- a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs +++ b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs @@ -31,7 +31,7 @@ pub fn from_cast( ) -> datafusion::common::Result { let Cast { expr, data_type } = cast; // since substrait Null must be typed, so if we see a cast(null, dt), we make it a typed null - if let Expr::Literal(lit) = expr.as_ref() { + if let Expr::Literal(lit, _) = expr.as_ref() { // only the untyped(a null scalar value) null literal need this special handling // since all other kind of nulls are already typed and can be handled by substrait // e.g. null:: or null:: @@ -92,7 +92,7 @@ mod tests { let empty_schema = DFSchemaRef::new(DFSchema::empty()); let field = Field::new("out", DataType::Int32, false); - let expr = Expr::Literal(ScalarValue::Null) + let expr = Expr::Literal(ScalarValue::Null, None) .cast_to(&DataType::Int32, &empty_schema) .unwrap(); @@ -119,7 +119,7 @@ mod tests { } // a typed null should not be folded - let expr = Expr::Literal(ScalarValue::Int64(None)) + let expr = Expr::Literal(ScalarValue::Int64(None), None) .cast_to(&DataType::Int32, &empty_schema) .unwrap(); diff --git a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs index fbc4d3754df0b..42e1f962f1d1f 100644 --- a/datafusion/substrait/src/logical_plan/producer/expr/mod.rs +++ b/datafusion/substrait/src/logical_plan/producer/expr/mod.rs @@ -109,7 +109,7 @@ pub fn to_substrait_rex( Expr::ScalarVariable(_, _) => { not_impl_err!("Cannot convert {expr:?} to Substrait") } - Expr::Literal(expr) => producer.handle_literal(expr), + Expr::Literal(expr, _) => producer.handle_literal(expr), Expr::BinaryExpr(expr) => producer.handle_binary_expr(expr, schema), Expr::Like(expr) => producer.handle_like(expr, schema), Expr::SimilarTo(_) => not_impl_err!("Cannot convert {expr:?} to Substrait"), @@ -172,7 +172,7 @@ mod tests { let state = SessionStateBuilder::default().build(); // One expression, empty input schema - let expr = Expr::Literal(ScalarValue::Int32(Some(42))); + let expr = Expr::Literal(ScalarValue::Int32(Some(42)), None); let field = Field::new("out", DataType::Int32, false); let empty_schema = DFSchemaRef::new(DFSchema::empty()); let substrait = diff --git a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs index e4e0ab11c65ac..212874e7913b5 100644 --- a/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs +++ b/datafusion/substrait/src/logical_plan/producer/rel/read_rel.rs @@ -115,10 +115,10 @@ pub fn from_values( let fields = row .iter() .map(|v| match v { - Expr::Literal(sv) => to_substrait_literal(producer, sv), + Expr::Literal(sv, _) => to_substrait_literal(producer, sv), Expr::Alias(alias) => match alias.expr.as_ref() { // The schema gives us the names, so we can skip aliases - Expr::Literal(sv) => to_substrait_literal(producer, sv), + Expr::Literal(sv, _) => to_substrait_literal(producer, sv), _ => Err(substrait_datafusion_err!( "Only literal types can be aliased in Virtual Tables, got: {}", alias.expr.variant_name() )), diff --git a/docs/source/library-user-guide/adding-udfs.md b/docs/source/library-user-guide/adding-udfs.md index 8fb8a59fb8609..cd40e664239a0 100644 --- a/docs/source/library-user-guide/adding-udfs.md +++ b/docs/source/library-user-guide/adding-udfs.md @@ -1076,7 +1076,7 @@ pub struct EchoFunction {} impl TableFunctionImpl for EchoFunction { fn call(&self, exprs: &[Expr]) -> Result> { - let Some(Expr::Literal(ScalarValue::Int64(Some(value)))) = exprs.get(0) else { + let Some(Expr::Literal(ScalarValue::Int64(Some(value)), _)) = exprs.get(0) else { return plan_err!("First argument must be an integer"); }; @@ -1117,7 +1117,7 @@ With the UDTF implemented, you can register it with the `SessionContext`: # # impl TableFunctionImpl for EchoFunction { # fn call(&self, exprs: &[Expr]) -> Result> { -# let Some(Expr::Literal(ScalarValue::Int64(Some(value)))) = exprs.get(0) else { +# let Some(Expr::Literal(ScalarValue::Int64(Some(value)), _)) = exprs.get(0) else { # return plan_err!("First argument must be an integer"); # }; # From 33a32d4382bee7e3c705d0f55d05c24a115a2f98 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 7 Jun 2025 01:55:50 -0400 Subject: [PATCH 069/177] [branch-48] Update CHANGELOG for latest 48.0.0 release (#16314) * [branch-48] Update CHANGELOG for latest 48.0.0 release * prettier --- dev/changelog/48.0.0.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dev/changelog/48.0.0.md b/dev/changelog/48.0.0.md index 95f955718119e..42f128bcb7b51 100644 --- a/dev/changelog/48.0.0.md +++ b/dev/changelog/48.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 48.0.0 Changelog -This release consists of 266 commits from 88 contributors. See credits at the end of this changelog for more information. +This release consists of 269 commits from 89 contributors. See credits at the end of this changelog for more information. **Breaking changes:** @@ -94,6 +94,7 @@ This release consists of 266 commits from 88 contributors. See credits at the en - fix: metadata of join schema [#16221](https://github.com/apache/datafusion/pull/16221) (chenkovsky) - fix: add missing row count limits to TPC-H queries [#16230](https://github.com/apache/datafusion/pull/16230) (0ax1) - fix: NaN semantics in GROUP BY [#16256](https://github.com/apache/datafusion/pull/16256) (chenkovsky) +- fix: [branch-48] Revert "Improve performance of constant aggregate window expression" [#16307](https://github.com/apache/datafusion/pull/16307) (andygrove) **Documentation updates:** @@ -297,6 +298,7 @@ This release consists of 266 commits from 88 contributors. See credits at the en - Simplify FileSource / SchemaAdapterFactory API [#16214](https://github.com/apache/datafusion/pull/16214) (alamb) - Add dicts to aggregation fuzz testing [#16232](https://github.com/apache/datafusion/pull/16232) (blaginin) - chore(deps): bump sysinfo from 0.35.1 to 0.35.2 [#16247](https://github.com/apache/datafusion/pull/16247) (dependabot[bot]) +- Improve performance of constant aggregate window expression [#16234](https://github.com/apache/datafusion/pull/16234) (suibianwanwank) - Support compound identifier when parsing tuples [#16225](https://github.com/apache/datafusion/pull/16225) (hozan23) - Schema adapter helper [#16108](https://github.com/apache/datafusion/pull/16108) (kosiew) - Update tpch, clickbench, sort_tpch to mark failed queries [#16182](https://github.com/apache/datafusion/pull/16182) (ding-young) @@ -304,6 +306,7 @@ This release consists of 266 commits from 88 contributors. See credits at the en - Handle dicts for distinct count [#15871](https://github.com/apache/datafusion/pull/15871) (blaginin) - Add `--substrait-round-trip` option in sqllogictests [#16183](https://github.com/apache/datafusion/pull/16183) (gabotechs) - Minor: fix upgrade papercut `pub use PruningStatistics` [#16264](https://github.com/apache/datafusion/pull/16264) (alamb) +- chore: update DF48 changelog [#16269](https://github.com/apache/datafusion/pull/16269) (xudong963) ## Credits @@ -312,7 +315,7 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co ``` 30 dependabot[bot] 29 Andrew Lamb - 16 xudong.w + 17 xudong.w 14 Adrian Garcia Badaracco 10 Chen Chongchen 8 Gabriel @@ -327,13 +330,13 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co 4 Nuno Faria 4 Yongting You 4 logan-keede + 3 Andy Grove 3 Christian 3 Daniël Heres 3 Liam Bao 3 Phillip LeBlanc 3 Piotr Findeisen 3 ding-young - 2 Andy Grove 2 Atahan Yorgancı 2 Brayan Jules 2 Georgi Krastev @@ -396,6 +399,7 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co 1 irenjj 1 jsai28 1 m09526 + 1 suibianwanwan 1 the0ninjas 1 wiedld ``` From a13a6feeca5c7264d4f48446446f6b83fe9328a7 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 10 Jun 2025 18:36:06 +0800 Subject: [PATCH 070/177] Simplify filter predicates --- datafusion/expr/src/expr.rs | 8 + datafusion/optimizer/src/lib.rs | 1 + datafusion/optimizer/src/push_down_filter.rs | 12 +- .../optimizer/src/simplify_predicates.rs | 194 ++++++++++++++++ datafusion/sqllogictest/bin/sqllogictests.rs | 2 +- .../test_files/simplify_predicates.slt | 212 ++++++++++++++++++ 6 files changed, 427 insertions(+), 2 deletions(-) create mode 100644 datafusion/optimizer/src/simplify_predicates.rs create mode 100644 datafusion/sqllogictest/test_files/simplify_predicates.slt diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index 9f6855b698243..de5fc7cdde1e4 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -1827,6 +1827,14 @@ impl Expr { _ => None, } } + + /// Check if the Expr is literal + pub fn is_literal(&self) -> bool { + match self { + Expr::Literal(_) => true, + _ => false, + } + } } impl Normalizeable for Expr { diff --git a/datafusion/optimizer/src/lib.rs b/datafusion/optimizer/src/lib.rs index 893cb249a2a86..1464a12c8a16c 100644 --- a/datafusion/optimizer/src/lib.rs +++ b/datafusion/optimizer/src/lib.rs @@ -60,6 +60,7 @@ pub mod push_down_limit; pub mod replace_distinct_aggregate; pub mod scalar_subquery_to_join; pub mod simplify_expressions; +mod simplify_predicates; pub mod single_distinct_to_groupby; pub mod utils; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 13cc74ce5752b..60f4652eeac9a 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -41,7 +41,7 @@ use datafusion_expr::{ use crate::optimizer::ApplyOrder; use crate::utils::{has_all_column_refs, is_restrict_null_predicate}; -use crate::{OptimizerConfig, OptimizerRule}; +use crate::{simplify_predicates::simplify_predicates, OptimizerConfig, OptimizerRule}; /// Optimizer rule for pushing (moving) filter expressions down in a plan so /// they are applied as early as possible. @@ -778,6 +778,16 @@ impl OptimizerRule for PushDownFilter { return Ok(Transformed::no(plan)); }; + let predicate = split_conjunction_owned(filter.predicate.clone()); + let old_predicate_len = predicate.len(); + let new_predicates = simplify_predicates(predicate)?; + if old_predicate_len != new_predicates.len() { + let Some(new_predicate) = conjunction(new_predicates) else { + return plan_err!("at least one expression exists"); + }; + filter.predicate = new_predicate; + } + match Arc::unwrap_or_clone(filter.input) { LogicalPlan::Filter(child_filter) => { let parents_predicates = split_conjunction_owned(filter.predicate); diff --git a/datafusion/optimizer/src/simplify_predicates.rs b/datafusion/optimizer/src/simplify_predicates.rs new file mode 100644 index 0000000000000..1516d3f46d91a --- /dev/null +++ b/datafusion/optimizer/src/simplify_predicates.rs @@ -0,0 +1,194 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use datafusion_common::{Column, Result, ScalarValue}; +use datafusion_expr::{BinaryExpr, Cast, Expr, Operator}; +use std::collections::BTreeMap; + +pub(crate) fn simplify_predicates(predicates: Vec) -> Result> { + // Early return for simple cases + if predicates.len() <= 1 { + return Ok(predicates); + } + + // Group predicates by their column reference + let mut column_predicates: BTreeMap> = BTreeMap::new(); + let mut other_predicates = Vec::new(); + + for pred in predicates { + match &pred { + Expr::BinaryExpr(BinaryExpr { left, op, right }) + if matches!( + op, + Operator::Gt + | Operator::GtEq + | Operator::Lt + | Operator::LtEq + | Operator::Eq + ) => + { + let left_col = extract_column_from_expr(left); + let right_col = extract_column_from_expr(right); + let left_lit = left.is_literal(); + let right_lit = right.is_literal(); + if let (Some(col), true) = (&left_col, right_lit) { + column_predicates.entry(col.clone()).or_default().push(pred); + } else if let (true, Some(col)) = (left_lit, &right_col) { + column_predicates.entry(col.clone()).or_default().push(pred); + } else { + other_predicates.push(pred); + } + } + _ => other_predicates.push(pred), + } + } + + // Process each column's predicates to remove redundancies + let mut result = other_predicates; + for (_, preds) in column_predicates { + let simplified = simplify_column_predicates(preds)?; + result.extend(simplified); + } + + Ok(result) +} + +fn simplify_column_predicates(predicates: Vec) -> Result> { + if predicates.len() <= 1 { + return Ok(predicates); + } + + // Group by operator type, but combining similar operators + let mut greater_predicates = Vec::new(); // Combines > and >= + let mut less_predicates = Vec::new(); // Combines < and <= + let mut eq_predicates = Vec::new(); + + for pred in predicates { + match &pred { + Expr::BinaryExpr(BinaryExpr { left: _, op, right }) => { + let right_is_literal = right.is_literal(); + match (op, right_is_literal) { + (Operator::Gt, true) + | (Operator::Lt, false) + | (Operator::GtEq, true) + | (Operator::LtEq, false) => greater_predicates.push(pred), + (Operator::Lt, true) + | (Operator::Gt, false) + | (Operator::LtEq, true) + | (Operator::GtEq, false) => less_predicates.push(pred), + (Operator::Eq, _) => eq_predicates.push(pred), + _ => unreachable!("Unexpected operator: {}", op), + } + } + _ => unreachable!("Unexpected predicate {}", pred.to_string()), + } + } + + let mut result = Vec::new(); + + // If we have equality predicates, they're the most restrictive + if !eq_predicates.is_empty() { + if eq_predicates.len() > 1 { + result.push(Expr::Literal(ScalarValue::Boolean(Some(false)))); + } else { + result.push(eq_predicates[0].clone()); + } + } else { + // Handle all greater-than-style predicates (keep the most restrictive - highest value) + if !greater_predicates.is_empty() { + if let Some(most_restrictive) = + find_most_restrictive_predicate(&greater_predicates, true)? + { + result.push(most_restrictive); + } else { + result.extend(greater_predicates); + } + } + + // Handle all less-than-style predicates (keep the most restrictive - lowest value) + if !less_predicates.is_empty() { + if let Some(most_restrictive) = + find_most_restrictive_predicate(&less_predicates, false)? + { + result.push(most_restrictive); + } else { + result.extend(less_predicates); + } + } + } + + Ok(result) +} + +fn find_most_restrictive_predicate( + predicates: &[Expr], + find_greater: bool, +) -> Result> { + if predicates.is_empty() { + return Ok(None); + } + + let mut most_restrictive = predicates[0].clone(); + let mut best_value: Option = None; + + for pred in predicates { + if let Expr::BinaryExpr(BinaryExpr { left, op: _, right }) = pred { + // Extract the literal value based on which side has it + let mut scalar_value = None; + if right.is_literal() { + if let Expr::Literal(scalar) = right.as_ref() { + scalar_value = Some(scalar.clone()); + } + } else if left.is_literal() { + if let Expr::Literal(scalar) = left.as_ref() { + scalar_value = Some(scalar.clone()); + } + } + + if let Some(scalar) = scalar_value { + if let Some(current_best) = &best_value { + if let Some(comparison) = scalar.partial_cmp(current_best) { + let is_better = if find_greater { + comparison == std::cmp::Ordering::Greater + } else { + comparison == std::cmp::Ordering::Less + }; + + if is_better { + best_value = Some(scalar); + most_restrictive = pred.clone(); + } + } + } else { + best_value = Some(scalar); + most_restrictive = pred.clone(); + } + } + } + } + + Ok(Some(most_restrictive)) +} + +fn extract_column_from_expr(expr: &Expr) -> Option { + match expr { + Expr::Column(col) => Some(col.clone()), + // Handle cases where the column might be wrapped in a cast or other operation + Expr::Cast(Cast { expr, .. }) => extract_column_from_expr(expr), + _ => None, + } +} diff --git a/datafusion/sqllogictest/bin/sqllogictests.rs b/datafusion/sqllogictest/bin/sqllogictests.rs index bfb119009fe3c..21dfe2ee08f4e 100644 --- a/datafusion/sqllogictest/bin/sqllogictests.rs +++ b/datafusion/sqllogictest/bin/sqllogictests.rs @@ -169,7 +169,7 @@ async fn run_tests() -> Result<()> { .join() }) // run up to num_cpus streams in parallel - .buffer_unordered(num_cpus::get()) + .buffer_unordered(get_available_parallelism()) .flat_map(|result| { // Filter out any Ok() leaving only the DataFusionErrors futures::stream::iter(match result { diff --git a/datafusion/sqllogictest/test_files/simplify_predicates.slt b/datafusion/sqllogictest/test_files/simplify_predicates.slt new file mode 100644 index 0000000000000..6d3eefdfc8213 --- /dev/null +++ b/datafusion/sqllogictest/test_files/simplify_predicates.slt @@ -0,0 +1,212 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Test cases for predicate simplification feature +# Basic redundant comparison simplification + +statement ok +set datafusion.explain.logical_plan_only=true; + +statement ok +CREATE TABLE test_data ( + int_col INT, + float_col FLOAT, + str_col VARCHAR, + date_col DATE, + bool_col BOOLEAN +); + +# x > 5 AND x > 6 should simplify to x > 6 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col > 5 AND int_col > 6; +---- +logical_plan +01)Filter: test_data.int_col > Int32(6) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x > 5 AND x >= 6 should simplify to x >= 6 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col > 5 AND int_col >= 6; +---- +logical_plan +01)Filter: test_data.int_col >= Int32(6) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x < 10 AND x <= 8 should simplify to x <= 8 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col < 10 AND int_col <= 8; +---- +logical_plan +01)Filter: test_data.int_col <= Int32(8) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x > 5 AND x > 6 AND x > 7 should simplify to x > 7 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col > 5 AND int_col > 6 AND int_col > 7; +---- +logical_plan +01)Filter: test_data.int_col > Int32(7) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x > 5 AND y < 10 AND x > 6 AND y < 8 should simplify to x > 6 AND y < 8 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col > 5 AND float_col < 10 AND int_col > 6 AND float_col < 8; +---- +logical_plan +01)Filter: test_data.float_col < Float32(8) AND test_data.int_col > Int32(6) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + + +# x = 7 AND x > 5 should simplify to x = 7 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col = 7 AND int_col > 5; +---- +logical_plan +01)Filter: test_data.int_col = Int32(7) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# str_col > 'apple' AND str_col > 'banana' should simplify to str_col > 'banana' +query TT +EXPLAIN SELECT * FROM test_data WHERE str_col > 'apple' AND str_col > 'banana'; +---- +logical_plan +01)Filter: test_data.str_col > Utf8("banana") +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# date_col > '2023-01-01' AND date_col > '2023-02-01' should simplify to date_col > '2023-02-01' +query TT +EXPLAIN SELECT * FROM test_data WHERE date_col > '2023-01-01' AND date_col > '2023-02-01'; +---- +logical_plan +01)Filter: test_data.date_col > Date32("2023-02-01") +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +query TT +EXPLAIN SELECT * FROM test_data WHERE bool_col = true AND bool_col = false; +---- +logical_plan +01)Filter: test_data.bool_col AND NOT test_data.bool_col +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + + +# This shouldn't be simplified since they're different relationships +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col > float_col AND int_col > 5; +---- +logical_plan +01)Filter: CAST(test_data.int_col AS Float32) > test_data.float_col AND test_data.int_col > Int32(5) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# Should simplify the int_col predicates but preserve the others +query TT +EXPLAIN SELECT * FROM test_data +WHERE int_col > 5 + AND int_col > 10 + AND str_col LIKE 'A%' + AND float_col BETWEEN 1 AND 100; +---- +logical_plan +01)Filter: test_data.str_col LIKE Utf8("A%") AND test_data.float_col >= Float32(1) AND test_data.float_col <= Float32(100) AND test_data.int_col > Int32(10) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +statement ok +CREATE TABLE test_data2 ( + id INT, + value INT +); + +query TT +EXPLAIN SELECT t1.int_col, t2.value +FROM test_data t1 +JOIN test_data2 t2 ON t1.int_col = t2.id +WHERE t1.int_col > 5 + AND t1.int_col > 10 + AND t2.value < 100 + AND t2.value < 50; +---- +logical_plan +01)Projection: t1.int_col, t2.value +02)--Inner Join: t1.int_col = t2.id +03)----SubqueryAlias: t1 +04)------Filter: test_data.int_col > Int32(10) +05)--------TableScan: test_data projection=[int_col] +06)----SubqueryAlias: t2 +07)------Filter: test_data2.value < Int32(50) AND test_data2.id > Int32(10) +08)--------TableScan: test_data2 projection=[id, value] + +# Case 13: Handling negated predicates +# NOT (x < 10) AND NOT (x < 5) should simplify to NOT (x < 10) +query TT +EXPLAIN SELECT * FROM test_data WHERE NOT (int_col < 10) AND NOT (int_col < 5); +---- +logical_plan +01)Filter: test_data.int_col >= Int32(10) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x > 5 AND x < 10 should be preserved (can't be simplified) +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col > 5 AND int_col < 10; +---- +logical_plan +01)Filter: test_data.int_col > Int32(5) AND test_data.int_col < Int32(10) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# 5 < x AND 3 < x should simplify to 5 < x +query TT +EXPLAIN SELECT * FROM test_data WHERE 5 < int_col AND 3 < int_col; +---- +logical_plan +01)Filter: test_data.int_col > Int32(5) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# CAST(x AS FLOAT) > 5.0 AND CAST(x AS FLOAT) > 6.0 should simplify +query TT +EXPLAIN SELECT * FROM test_data WHERE CAST(int_col AS FLOAT) > 5.0 AND CAST(int_col AS FLOAT) > 6.0; +---- +logical_plan +01)Filter: CAST(CAST(test_data.int_col AS Float32) AS Float64) > Float64(6) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x = 5 AND x = 6 (logically impossible) +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col = 5 AND int_col = 6; +---- +logical_plan EmptyRelation + +# (x > 5 OR y < 10) AND (x > 6 OR y < 8) +# This is more complex but could still benefit from some simplification +query TT +EXPLAIN SELECT * FROM test_data +WHERE (int_col > 5 OR float_col < 10) + AND (int_col > 6 OR float_col < 8); +---- +logical_plan +01)Filter: (test_data.int_col > Int32(5) OR test_data.float_col < Float32(10)) AND (test_data.int_col > Int32(6) OR test_data.float_col < Float32(8)) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# Case 20: Combination of AND and OR with simplifiable predicates +query TT +EXPLAIN SELECT * FROM test_data +WHERE (int_col > 5 AND int_col > 6) + OR (float_col < 10 AND float_col < 8); +---- +logical_plan +01)Filter: test_data.int_col > Int32(5) AND test_data.int_col > Int32(6) OR test_data.float_col < Float32(10) AND test_data.float_col < Float32(8) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +statement ok +set datafusion.explain.logical_plan_only=false; From e5e5c4833f78209d462a3740fd3b2ac2a96adefb Mon Sep 17 00:00:00 2001 From: xudong963 Date: Tue, 24 Jun 2025 10:21:19 +0800 Subject: [PATCH 071/177] Upgrade DF48 --- Cargo.lock | 903 +++++++++--------- datafusion/datasource/src/source.rs | 5 +- datafusion/expr/src/expr.rs | 2 +- datafusion/optimizer/src/push_down_filter.rs | 8 +- .../optimizer/src/simplify_predicates.rs | 6 +- .../src/enforce_distribution.rs | 8 +- .../replace_with_order_preserving_variants.rs | 2 +- datafusion/physical-plan/src/work_table.rs | 2 +- 8 files changed, 452 insertions(+), 484 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 79888856b7600..fe67ae9bea784 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -61,9 +61,9 @@ dependencies = [ [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "adler32" @@ -149,9 +149,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933" dependencies = [ "anstyle", "anstyle-parse", @@ -164,36 +164,36 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9" dependencies = [ "windows-sys 0.59.0", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882" dependencies = [ "anstyle", - "once_cell", + "once_cell_polyfill", "windows-sys 0.59.0", ] @@ -295,7 +295,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "num", ] @@ -454,7 +454,7 @@ version = "55.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73a47aa0c771b5381de2b7f16998d351a6f4eb839f1e13d48353e17e873d969b" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "serde", "serde_json", ] @@ -552,7 +552,7 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -574,7 +574,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -585,7 +585,7 @@ checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -605,15 +605,15 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-config" -version = "1.6.3" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02a18fd934af6ae7ca52410d4548b98eb895aab0f1ea417d168d85db1434a141" +checksum = "455e9fb7743c6f6267eb2830ccc08686fbb3d13c9a689369562fd4d4ef9ea462" dependencies = [ "aws-credential-types", "aws-runtime", @@ -653,9 +653,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.12.6" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dabb68eb3a7aa08b46fddfd59a3d55c978243557a90ab804769f7e20e67d2b01" +checksum = "93fcc8f365936c834db5514fc45aee5b1202d677e6b40e48468aaaa8183ca8c7" dependencies = [ "aws-lc-sys", "zeroize", @@ -663,9 +663,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbe221bbf523b625a4dd8585c7f38166e31167ec2ca98051dbcb4c3b6e825d2" +checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" dependencies = [ "bindgen", "cc", @@ -676,9 +676,9 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.5.7" +version = "1.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c4063282c69991e57faab9e5cb21ae557e59f5b0fb285c196335243df8dc25c" +checksum = "4f6c68419d8ba16d9a7463671593c54f81ba58cab466e9b759418da606dcc2e2" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -700,9 +700,9 @@ dependencies = [ [[package]] name = "aws-sdk-sso" -version = "1.63.0" +version = "1.73.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1cb45b83b53b5cd55ee33fd9fd8a70750255a3f286e4dca20e882052f2b256f" +checksum = "b2ac1674cba7872061a29baaf02209fefe499ff034dfd91bd4cc59e4d7741489" dependencies = [ "aws-credential-types", "aws-runtime", @@ -722,9 +722,9 @@ dependencies = [ [[package]] name = "aws-sdk-ssooidc" -version = "1.64.0" +version = "1.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4d9bc075ea6238778ed3951b65d3cde8c3864282d64fdcd19f2a90c0609f1" +checksum = "3a6a22f077f5fd3e3c0270d4e1a110346cddf6769e9433eb9e6daceb4ca3b149" dependencies = [ "aws-credential-types", "aws-runtime", @@ -744,9 +744,9 @@ dependencies = [ [[package]] name = "aws-sdk-sts" -version = "1.64.0" +version = "1.75.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819ccba087f403890fee4825eeab460e64c59345667d2b83a12cf544b581e3a7" +checksum = "e3258fa707f2f585ee3049d9550954b959002abd59176975150a01d5cf38ae3f" dependencies = [ "aws-credential-types", "aws-runtime", @@ -767,9 +767,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.3.2" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3734aecf9ff79aa401a6ca099d076535ab465ff76b46440cf567c8e70b65dc13" +checksum = "ddfb9021f581b71870a17eac25b52335b82211cdc092e02b6876b2bcefa61666" dependencies = [ "aws-credential-types", "aws-smithy-http", @@ -780,8 +780,7 @@ dependencies = [ "hex", "hmac", "http 0.2.12", - "http 1.2.0", - "once_cell", + "http 1.3.1", "percent-encoding", "sha2", "time", @@ -821,9 +820,9 @@ dependencies = [ [[package]] name = "aws-smithy-http-client" -version = "1.0.0" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0497ef5d53065b7cd6a35e9c1654bd1fefeae5c52900d91d1b188b0af0f29324" +checksum = "7f491388e741b7ca73b24130ff464c1478acc34d5b331b7dd0a2ee4643595a15" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -844,9 +843,9 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.61.3" +version = "0.61.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92144e45819cae7dc62af23eac5a038a58aa544432d2102609654376a900bd07" +checksum = "a16e040799d29c17412943bdbf488fd75db04112d0c0d4b9290bacf5ae0014b9" dependencies = [ "aws-smithy-types", ] @@ -896,9 +895,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e5d9e3a80a18afa109391fb5ad09c3daf887b516c6fd805a157c6ea7994a57" +checksum = "bd8531b6d8882fd8f48f82a9754e682e29dd44cff27154af51fa3eb730f59efb" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -913,9 +912,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.3.1" +version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40076bd09fadbc12d5e026ae080d0930defa606856186e31d83ccc6a255eeaf3" +checksum = "d498595448e43de7f4296b7b7a18a8a02c61ec9349128c80a368f7c3b4ab11a8" dependencies = [ "base64-simd", "bytes", @@ -936,9 +935,9 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.9" +version = "0.60.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" +checksum = "3db87b96cb1b16c024980f133968d52882ca0daaee3a086c6decc500f6c99728" dependencies = [ "xmlparser", ] @@ -1061,7 +1060,7 @@ version = "0.69.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cexpr", "clang-sys", "itertools 0.12.1", @@ -1074,7 +1073,7 @@ dependencies = [ "regex", "rustc-hash 1.1.0", "shlex", - "syn 2.0.101", + "syn 2.0.104", "which", ] @@ -1086,9 +1085,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] name = "bitvec" @@ -1203,7 +1202,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1219,9 +1218,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "4.0.2" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74fa05ad7d803d413eb8380983b092cbbaf9a85f151b871360e7b00cd7060b37" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -1240,9 +1239,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "793db76d6187cd04dff33004d8e6c9cc4e05cd330500379d2394209271b4aeee" [[package]] name = "bytecheck" @@ -1325,9 +1324,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.14" +version = "1.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c3d1b2e905a3a7b00a6141adb0e4c0bb941d11caf55349d863942a1cc44e3c9" +checksum = "d487aa071b5f64da6f19a3e848e3578944b726ee5a4854b82172f02aa876bfdc" dependencies = [ "jobserver", "libc", @@ -1345,9 +1344,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] name = "cfg_aliases" @@ -1426,7 +1425,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading 0.8.7", + "libloading 0.8.8", ] [[package]] @@ -1442,9 +1441,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.35" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8aa86934b44c19c50f87cc2790e19f54f7a67aedb64101c2e1a2e5ecfb73944" +checksum = "40b6887a1d8685cebccf115538db5c0efe625ccac9696ad45c409d96566e910f" dependencies = [ "clap_builder", "clap_derive", @@ -1452,9 +1451,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.35" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2414dbb2dd0695280da6ea9261e327479e9d37b0630f6b53ba2a11c60c679fd9" +checksum = "e0c66c08ce9f0c698cbce5c0279d0bb6ac936d8674174fe48f736533b964f59e" dependencies = [ "anstream", "anstyle", @@ -1464,21 +1463,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.32" +version = "4.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09176aae279615badda0765c0c0b3f6ed53f4709118af73cf4655d85d1530cd7" +checksum = "d2c7947ae4cc3d851207c1adb5b5e260ff0cca11446b1d6d1423788e442257ce" dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "clipboard-win" @@ -1500,9 +1499,9 @@ dependencies = [ [[package]] name = "colorchoice" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" [[package]] name = "comfy-table" @@ -1511,7 +1510,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.1", ] [[package]] @@ -1523,7 +1522,7 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.0", + "unicode-width 0.2.1", "windows-sys 0.59.0", ] @@ -1571,9 +1570,9 @@ checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" [[package]] name = "core-foundation" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -1636,7 +1635,7 @@ dependencies = [ "anes", "cast", "ciborium", - "clap 4.5.35", + "clap 4.5.40", "criterion-plot", "futures", "is-terminal", @@ -1773,7 +1772,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1784,7 +1783,7 @@ checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ "darling_core", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -1955,7 +1954,7 @@ dependencies = [ "async-trait", "aws-config", "aws-credential-types", - "clap 4.5.35", + "clap 4.5.40", "ctor", "datafusion", "dirs", @@ -2382,7 +2381,7 @@ version = "48.0.0" dependencies = [ "datafusion-expr", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2431,7 +2430,7 @@ dependencies = [ "itertools 0.14.0", "log", "paste", - "petgraph 0.8.1", + "petgraph 0.8.2", "rand 0.9.1", "rstest", ] @@ -2607,7 +2606,7 @@ dependencies = [ "bigdecimal", "bytes", "chrono", - "clap 4.5.35", + "clap 4.5.40", "datafusion", "datafusion-spark", "datafusion-substrait", @@ -2718,7 +2717,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2729,7 +2728,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2785,7 +2784,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2823,7 +2822,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -2857,12 +2856,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.10" +version = "0.3.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2907,8 +2906,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix 0.38.44", - "windows-sys 0.52.0", + "rustix 1.0.7", + "windows-sys 0.59.0", ] [[package]] @@ -2965,15 +2964,15 @@ version = "25.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "rustc_version", ] [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" dependencies = [ "crc32fast", "libz-rs-sys", @@ -3012,9 +3011,9 @@ dependencies = [ [[package]] name = "fs-err" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f89bda4c2a21204059a977ed3bfe746677dfd137b83c339e702b0ac91d482aa" +checksum = "88d7be93788013f265201256d58f04936a8079ad5dc898743aa20525f503b683" dependencies = [ "autocfg", ] @@ -3087,7 +3086,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3170,15 +3169,15 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi 0.11.1+wasi-snapshot-preview1", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", "js-sys", @@ -3215,9 +3214,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.8" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5017294ff4bb30944501348f6f8e42e6ad28f42c8bbef7a74029aff064a4e3c2" +checksum = "a9421a676d1b147b16b82c9225157dc629087ef8ec4d5e2960f9437a90dac0a5" dependencies = [ "atomic-waker", "bytes", @@ -3264,9 +3263,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.15.3" +version = "0.15.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" dependencies = [ "allocator-api2", "equivalent", @@ -3290,9 +3289,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.4.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -3430,11 +3429,10 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.5" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "futures-util", "http 1.3.1", "hyper", "hyper-util", @@ -3461,17 +3459,21 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.10" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" +checksum = "dc2fdfdbff08affe55bb779f33b053aa1fe5dd5b54c257343c17edfa55711bdb" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", + "futures-core", "futures-util", "http 1.3.1", "http-body 1.0.1", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", "tokio", @@ -3506,7 +3508,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.52.0", + "windows-core", ] [[package]] @@ -3544,26 +3546,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "icu_locid_transform" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" -dependencies = [ - "displaydoc", - "icu_locid", - "icu_locid_transform_data", - "icu_provider", - "tinystr", - "zerovec", -] - -[[package]] -name = "icu_locid_transform_data" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - [[package]] name = "icu_normalizer" version = "2.0.0" @@ -3581,9 +3563,9 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "1.5.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" [[package]] name = "icu_properties" @@ -3603,9 +3585,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "1.5.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" [[package]] name = "icu_provider" @@ -3669,7 +3651,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" dependencies = [ "equivalent", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "serde", ] @@ -3682,7 +3664,7 @@ dependencies = [ "console", "number_prefix", "portable-atomic", - "unicode-width 0.2.0", + "unicode-width 0.2.1", "web-time", ] @@ -3730,6 +3712,16 @@ version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is-terminal" version = "0.4.16" @@ -3791,9 +3783,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.4" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d699bc6dfc879fb1bf9bdff0d4c56f0884fc6f0d0eb0fba397a6d00cd9a6b85e" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" dependencies = [ "jiff-static", "log", @@ -3804,13 +3796,13 @@ dependencies = [ [[package]] name = "jiff-static" -version = "0.2.4" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d16e75759ee0aa64c57a56acbf43916987b20c77373cb7e808979e02b93c9f9" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -3819,6 +3811,7 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ + "getrandom 0.3.3", "libc", ] @@ -3910,9 +3903,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.172" +version = "0.2.174" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libflate" @@ -3950,12 +3943,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a793df0d7afeac54f95b471d3af7f0d4fb975699f972341a4b76988d49cdf0c" +checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ "cfg-if", - "windows-targets 0.53.0", + "windows-targets 0.53.2", ] [[package]] @@ -3966,9 +3959,9 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libmimalloc-sys" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec9d6fac27761dabcd4ee73571cdb06b7022dc99089acbe5435691edffaac0f4" +checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d" dependencies = [ "cc", "libc", @@ -3980,9 +3973,9 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "libc", - "redox_syscall 0.5.8", + "redox_syscall 0.5.13", ] [[package]] @@ -3993,15 +3986,15 @@ checksum = "5297962ef19edda4ce33aaa484386e0a5b3d7f2f4e037cbeee00503ef6b29d33" dependencies = [ "anstream", "anstyle", - "clap 4.5.35", + "clap 4.5.40", "escape8259", ] [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" dependencies = [ "zlib-rs", ] @@ -4020,15 +4013,15 @@ checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" [[package]] name = "litemap" -version = "0.7.4" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" dependencies = [ "autocfg", "scopeguard", @@ -4048,11 +4041,11 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.11.3" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" dependencies = [ - "twox-hash 1.6.3", + "twox-hash", ] [[package]] @@ -4084,9 +4077,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memoffset" @@ -4099,9 +4092,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.46" +version = "0.1.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "995942f432bbb4822a7e9c3faa87a695185b0d09273ba85f097b54f4e458f2af" +checksum = "b1791cbe101e95af5764f06f20f6760521f7158f69dbf9d6baf941ee1bf6bc40" dependencies = [ "libmimalloc-sys", ] @@ -4130,22 +4123,22 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", ] [[package]] name = "mio" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.52.0", + "wasi 0.11.1+wasi-snapshot-preview1", + "windows-sys 0.59.0", ] [[package]] @@ -4169,7 +4162,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cfg-if", "cfg_aliases", "libc", @@ -4303,7 +4296,17 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", +] + +[[package]] +name = "objc2-io-kit" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +dependencies = [ + "libc", + "objc2-core-foundation", ] [[package]] @@ -4317,9 +4320,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d94ac16b433c0ccf75326388c893d2835ab7457ea35ab8ba5d745c053ef5fa16" +checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4" dependencies = [ "async-trait", "base64 0.22.1", @@ -4358,6 +4361,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" + [[package]] name = "oorandom" version = "11.1.5" @@ -4399,15 +4408,15 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "owo-colors" -version = "4.1.0" +version = "4.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb37767f6569cd834a413442455e0f066d0d522de8630436e2a1761d9726ba56" +checksum = "26995317201fa17f3656c36716aed4a7c81743a9634ac4c99c0eeda495db0cec" [[package]] name = "parking_lot" -version = "0.12.3" +version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", "parking_lot_core", @@ -4415,13 +4424,13 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.10" +version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.8", + "redox_syscall 0.5.13", "smallvec", "windows-targets 0.52.6", ] @@ -4447,7 +4456,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "lz4_flex", "num", "num-bigint", @@ -4458,7 +4467,7 @@ dependencies = [ "snap", "thrift", "tokio", - "twox-hash 2.1.0", + "twox-hash", "zstd", ] @@ -4484,7 +4493,7 @@ dependencies = [ "regex", "regex-syntax", "structmeta", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4557,12 +4566,12 @@ dependencies = [ [[package]] name = "petgraph" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a98c6720655620a521dcc722d0ad66cd8afd5d86e34a89ef691c50b7b24de06" +checksum = "54acf3a685220b533e437e264e4d932cfbdc4cc7ec0cd232ed73c08d03b8a7ca" dependencies = [ "fixedbitset", - "hashbrown 0.15.3", + "hashbrown 0.15.4", "indexmap 2.9.0", "serde", ] @@ -4622,7 +4631,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4673,9 +4682,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "portable-atomic-util" @@ -4695,7 +4704,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4750,7 +4759,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy 0.7.35", + "zerocopy", ] [[package]] @@ -4785,12 +4794,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.32" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +checksum = "061c1221631e079b26479d25bbf2275bfe5917ae8419cd7e34f13bfc2aa7539a" dependencies = [ "proc-macro2", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4861,7 +4870,7 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.101", + "syn 2.0.104", "tempfile", ] @@ -4875,7 +4884,7 @@ dependencies = [ "itertools 0.14.0", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4972,7 +4981,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4985,7 +4994,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -4996,9 +5005,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.37.2" +version = "0.37.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" dependencies = [ "memchr", "serde", @@ -5006,9 +5015,9 @@ dependencies = [ [[package]] name = "quinn" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef" +checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" dependencies = [ "bytes", "cfg_aliases", @@ -5026,13 +5035,14 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.9" +version = "0.11.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d" +checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" dependencies = [ "bytes", - "getrandom 0.2.15", - "rand 0.8.5", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.1", "ring", "rustc-hash 2.1.1", "rustls", @@ -5046,9 +5056,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.10" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944" +checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" dependencies = [ "cfg_aliases", "libc", @@ -5069,9 +5079,9 @@ dependencies = [ [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radium" @@ -5145,8 +5155,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.1", - "zerocopy 0.8.18", + "getrandom 0.3.3", ] [[package]] @@ -5196,7 +5205,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5210,11 +5219,11 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.8" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] [[package]] @@ -5228,6 +5237,26 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "ref-cast" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "regex" version = "1.11.1" @@ -5269,7 +5298,7 @@ version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ef7fa9ed0256d64a688a3747d0fef7a88851c18a5e1d57f115f38ec2e09366" dependencies = [ - "hashbrown 0.15.3", + "hashbrown 0.15.4", "memchr", ] @@ -5299,9 +5328,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.15" +version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d19c46a6fdd48bc4dab94b6103fccc55d34c67cc0ad04653aad4ea2a07cd7bbb" +checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ "base64 0.22.1", "bytes", @@ -5314,17 +5343,13 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "ipnet", "js-sys", "log", - "mime", - "once_cell", "percent-encoding", "pin-project-lite", "quinn", "rustls", "rustls-native-certs", - "rustls-pemfile", "rustls-pki-types", "serde", "serde_json", @@ -5334,13 +5359,13 @@ dependencies = [ "tokio-rustls", "tokio-util", "tower 0.5.2", + "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "wasm-streams", "web-sys", - "windows-registry", ] [[package]] @@ -5418,7 +5443,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.101", + "syn 2.0.104", "unicode-ident", ] @@ -5430,14 +5455,14 @@ checksum = "b3a8fb4672e840a587a66fc577a5491375df51ddb88f2a2c2a792598c326fe14" dependencies = [ "quote", "rand 0.8.5", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "rust_decimal" -version = "1.37.1" +version = "1.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7de2ba56ac291bd90c6b9bece784a52ae1411f9506544b3eae36dd2356d50" +checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" dependencies = [ "arrayvec", "borsh", @@ -5452,9 +5477,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" [[package]] name = "rustc-hash" @@ -5483,7 +5508,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -5492,11 +5517,11 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.2" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7178faa4b75a30e269c71e61c353ce2748cf3d76f0c44c393f4e60abf49b825" +checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "errno", "libc", "linux-raw-sys 0.9.4", @@ -5505,9 +5530,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.23" +version = "0.23.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395" +checksum = "7160e3e10bf4535308537f3c4e1641468cd0e485175d6163087c0393c7d46643" dependencies = [ "aws-lc-rs", "once_cell", @@ -5551,9 +5576,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.8" +version = "0.103.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435" dependencies = [ "aws-lc-rs", "ring", @@ -5563,9 +5588,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" [[package]] name = "rustyline" @@ -5573,7 +5598,7 @@ version = "16.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62fd9ca5ebc709e8535e8ef7c658eb51457987e48c98ead2be482172accc408d" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "cfg-if", "clipboard-win", "fd-lock", @@ -5584,7 +5609,7 @@ dependencies = [ "nix", "radix_trie", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width 0.2.1", "utf8parse", "windows-sys 0.59.0", ] @@ -5625,6 +5650,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "schemars_derive" version = "0.8.22" @@ -5634,7 +5671,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5655,7 +5692,7 @@ version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", "core-foundation", "core-foundation-sys", "libc", @@ -5713,7 +5750,7 @@ checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5724,7 +5761,7 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5747,7 +5784,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5759,7 +5796,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5776,15 +5813,16 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b6f7f2fcb69f747921f79f3926bd1e203fce4fef62c268dd3abfb6d86029aa" +checksum = "bf65a400f8f66fb7b0552869ad70157166676db75ed8181f8104ea91cf9d0b42" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.9.0", + "schemars 0.9.0", "serde", "serde_derive", "serde_json", @@ -5794,14 +5832,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.12.0" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d00caa5193a3c8362ac2b73be6b9e768aa5a4b2f721d8f4b339600c3cb51f8e" +checksum = "81679d9ed988d5e9a5e6531dc3f2c28efbd639cbd1dfb628df08edea6004da77" dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5872,18 +5910,15 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" -version = "0.4.9" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "snap" @@ -5911,9 +5946,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" dependencies = [ "libc", "windows-sys 0.52.0", @@ -5921,9 +5956,9 @@ dependencies = [ [[package]] name = "sqllogictest" -version = "0.28.0" +version = "0.28.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b2f0b80fc250ed3fdd82fc88c0ada5ad62ee1ed5314ac5474acfa52082f518" +checksum = "9fcbf91368a8d6807093d94f274fa4d0978cd78a310fee1d20368c545a606f7a" dependencies = [ "async-trait", "educe", @@ -5963,7 +5998,7 @@ checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -5974,9 +6009,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "stacker" -version = "0.1.18" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d08feb8f695b465baed819b03c128dc23f57a694510ab1f06c77f763975685e" +checksum = "cddb07e32ddb770749da91081d8d0ac3a16f1a569a18b20348cd371f5dead06b" dependencies = [ "cc", "cfg-if", @@ -6017,7 +6052,7 @@ dependencies = [ "proc-macro2", "quote", "structmeta-derive", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6028,7 +6063,7 @@ checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6071,7 +6106,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6086,9 +6121,9 @@ dependencies = [ [[package]] name = "substrait" -version = "0.55.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3a359aeb711c1e1944c0c4178bbb2d679d39237ac5bfe28f7e0506e522e5ce6" +checksum = "13de2e20128f2a018dab1cfa30be83ae069219a65968c6f89df66ad124de2397" dependencies = [ "heck 0.5.0", "pbjson", @@ -6100,12 +6135,12 @@ dependencies = [ "prost-types", "protobuf-src", "regress", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", "serde_yaml", - "syn 2.0.101", + "syn 2.0.104", "typify", "walkdir", ] @@ -6129,9 +6164,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.101" +version = "2.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" dependencies = [ "proc-macro2", "quote", @@ -6155,7 +6190,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6191,9 +6226,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" dependencies = [ "fastrand", - "getrandom 0.3.1", + "getrandom 0.3.3", "once_cell", - "rustix 1.0.2", + "rustix 1.0.7", "windows-sys 0.59.0", ] @@ -6287,7 +6322,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6298,17 +6333,16 @@ checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", ] [[package]] @@ -6399,9 +6433,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.44.1" +version = "1.45.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f382da615b842244d4b8738c82ed1275e6c5dd90c459a30941cd07080b06c91a" +checksum = "75ef51a33ef1da925cea3e4eb122833cb377c61439ca401b770f54902b806779" dependencies = [ "backtrace", "bytes", @@ -6423,7 +6457,7 @@ checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6503,15 +6537,15 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.9" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" [[package]] name = "toml_edit" -version = "0.22.26" +version = "0.22.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" dependencies = [ "indexmap 2.9.0", "toml_datetime", @@ -6583,6 +6617,24 @@ dependencies = [ "tower-service", ] +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags 2.9.1", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + [[package]] name = "tower-layer" version = "0.3.3" @@ -6608,20 +6660,20 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "tracing-core" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" dependencies = [ "once_cell", "valuable", @@ -6675,19 +6727,9 @@ checksum = "e78122066b0cb818b8afd08f7ed22f7fdbc3e90815035726f0840d0d26c0747a" [[package]] name = "twox-hash" -version = "1.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" -dependencies = [ - "cfg-if", - "static_assertions", -] - -[[package]] -name = "twox-hash" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b17f197b3050ba473acf9181f7b1d3b66d1cf7356c6cc57886662276e65908" +checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" [[package]] name = "typed-arena" @@ -6712,7 +6754,7 @@ checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -6723,9 +6765,9 @@ checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" [[package]] name = "typify" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcc5bec3cdff70fd542e579aa2e52967833e543a25fae0d14579043d2e868a50" +checksum = "6c6c647a34e851cf0260ccc14687f17cdcb8302ff1a8a687a24b97ca0f82406f" dependencies = [ "typify-impl", "typify-macro", @@ -6733,38 +6775,38 @@ dependencies = [ [[package]] name = "typify-impl" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b52a67305054e1da6f3d99ad94875dcd0c7c49adbd17b4b64f0eefb7ae5bf8ab" +checksum = "741b7f1e2e1338c0bee5ad5a7d3a9bbd4e24c33765c08b7691810e68d879365d" dependencies = [ "heck 0.5.0", "log", "proc-macro2", "quote", "regress", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", - "syn 2.0.101", + "syn 2.0.104", "thiserror 2.0.12", "unicode-ident", ] [[package]] name = "typify-macro" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ff5799be156e4f635c348c6051d165e1c59997827155133351a8c4d333d9841" +checksum = "7560adf816a1e8dad7c63d8845ef6e31e673e39eab310d225636779230cbedeb" dependencies = [ "proc-macro2", "quote", - "schemars", + "schemars 0.8.22", "semver", "serde", "serde_json", "serde_tokenstream", - "syn 2.0.101", + "syn 2.0.104", "typify-impl", ] @@ -6809,9 +6851,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" [[package]] name = "unindent" @@ -6867,7 +6909,7 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ - "getrandom 0.3.1", + "getrandom 0.3.3", "js-sys", "serde", "wasm-bindgen", @@ -6921,9 +6963,9 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] name = "wasi" @@ -6962,7 +7004,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "wasm-bindgen-shared", ] @@ -6997,7 +7039,7 @@ checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7032,7 +7074,7 @@ checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] @@ -7086,7 +7128,7 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" dependencies = [ - "redox_syscall 0.5.8", + "redox_syscall 0.5.13", "wasite", "web-sys", ] @@ -7124,9 +7166,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.61.1" +version = "0.61.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5ee8f3d025738cb02bad7868bbb5f8a6327501e870bf51f1b455b0a2454a419" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" dependencies = [ "windows-collections", "windows-core", @@ -7136,95 +7178,85 @@ dependencies = [ ] [[package]] -name = "windows-core" -version = "0.52.0" +name = "windows-collections" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" dependencies = [ - "windows-targets 0.52.6", + "windows-core", ] [[package]] name = "windows-core" -version = "0.57.0" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2ed2439a290666cd67ecce2b0ffaad89c2a56b976b736e6ece670297897832d" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" dependencies = [ "windows-implement", "windows-interface", - "windows-result 0.1.2", - "windows-targets 0.52.6", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core", + "windows-link", + "windows-threading", ] [[package]] name = "windows-implement" version = "0.60.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9107ddc059d5b6fbfbffdfa7a7fe3e22a226def0b2608f72e9d552763d3e1ad7" +checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "windows-interface" version = "0.59.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29bee4b38ea3cde66011baa44dba677c432a78593e202392d1e9070cf2a7fca7" +checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "windows-link" -version = "0.1.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" [[package]] -name = "windows-registry" +name = "windows-numerics" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0" -dependencies = [ - "windows-result 0.2.0", - "windows-strings", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-registry" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4286ad90ddb45071efd1a66dfa43eb02dd0dfbae1545ad6cc3c51cf34d7e8ba3" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" dependencies = [ - "windows-result", - "windows-strings 0.3.1", - "windows-targets 0.53.0", + "windows-core", + "windows-link", ] [[package]] name = "windows-result" -version = "0.2.0" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" dependencies = [ "windows-link", ] -[[package]] -name = "windows-strings" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87fa48cc5d406560701792be122a10132491cff9d0aeb23583cc2dcafc847319" -dependencies = [ - "windows-result 0.2.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows-strings" version = "0.4.2" @@ -7252,6 +7284,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.2", +] + [[package]] name = "windows-targets" version = "0.52.6" @@ -7270,25 +7311,9 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" -dependencies = [ - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - -[[package]] -name = "windows-targets" -version = "0.53.0" +version = "0.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e4c7e8ceaaf9cb7d7507c974735728ab453b67ef8f18febdd7c11fe59dca8b" +checksum = "c66f69fcc9ce11da9966ddb31a40968cad001c5bedeb5c2b82ede4253ab48aef" dependencies = [ "windows_aarch64_gnullvm 0.53.0", "windows_aarch64_msvc 0.53.0", @@ -7321,12 +7346,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" @@ -7339,12 +7358,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.52.6" @@ -7369,12 +7382,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.52.6" @@ -7387,12 +7394,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" @@ -7405,12 +7406,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" @@ -7423,12 +7418,6 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" @@ -7443,9 +7432,9 @@ checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" [[package]] name = "winnow" -version = "0.7.2" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59690dea168f2198d1a3b0cac23b8063efcd11012f10ae4698f284808c8ef603" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" dependencies = [ "memchr", ] @@ -7456,7 +7445,7 @@ version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" dependencies = [ - "bitflags 2.8.0", + "bitflags 2.9.1", ] [[package]] @@ -7481,8 +7470,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d65cbf2f12c15564212d48f4e3dfb87923d25d611f2aed18f4cb23f0413d89e" dependencies = [ "libc", - "linux-raw-sys 0.4.15", - "rustix 0.38.44", + "rustix 1.0.7", ] [[package]] @@ -7520,49 +7508,28 @@ checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" -dependencies = [ - "byteorder", - "zerocopy-derive 0.7.35", -] - -[[package]] -name = "zerocopy" -version = "0.8.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79386d31a42a4996e3336b0919ddb90f81112af416270cff95b5f5af22b839c2" -dependencies = [ - "zerocopy-derive 0.8.18", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.100", + "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.18" +version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76331675d372f91bf8d17e13afbd5fe639200b73d01f0fc748bb059f9cca2db7" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" dependencies = [ "proc-macro2", "quote", - "syn 2.0.100", + "syn 2.0.104", ] [[package]] @@ -7582,7 +7549,7 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", "synstructure", ] @@ -7622,14 +7589,14 @@ checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn 2.0.104", ] [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" [[package]] name = "zstd" diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index 6f1899628ce62..6d8b56275d0e5 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -306,11 +306,12 @@ impl ExecutionPlan for DataSourceExec { fn with_node_id( self: Arc, _node_id: usize, - ) -> datafusion_common::Result>> { + ) -> Result>> { let mut new_plan = DataSourceExec::new(self.data_source.clone()); let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) } + Ok(Some(Arc::new(new_plan))) + } fn handle_child_pushdown_result( &self, child_pushdown_result: ChildPushdownResult, diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index a4156e3f8d1f5..ffa2417fc5b66 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -1886,7 +1886,7 @@ impl Expr { /// Check if the Expr is literal pub fn is_literal(&self) -> bool { match self { - Expr::Literal(_) => true, + Expr::Literal(_, _) => true, _ => false, } } diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 410c392fb5b22..cfa22c3b61426 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -1412,13 +1412,13 @@ fn infer_predicates_from_equalities(predicates: Vec) -> Result> { if let Expr::Column(col) = left.as_ref() { // Only add to map if right side is a literal - if matches!(right.as_ref(), Expr::Literal(_)) { + if matches!(right.as_ref(), Expr::Literal(_, _)) { equality_map.insert(col.clone(), *right.clone()); final_predicates.push(predicate.clone()); } } else if let Expr::Column(col) = right.as_ref() { // Only add to map if left side is a literal - if matches!(left.as_ref(), Expr::Literal(_)) { + if matches!(left.as_ref(), Expr::Literal(_, _)) { equality_map.insert(col.clone(), *right.clone()); final_predicates.push(predicate.clone()); } @@ -2308,7 +2308,7 @@ mod tests { plan, @r" Projection: test.a, test1.d - Cross Join: + Cross Join: Projection: test.a, test.b, test.c TableScan: test, full_filters=[test.a = Int32(1)] Projection: test1.d, test1.e, test1.f @@ -2338,7 +2338,7 @@ mod tests { plan, @r" Projection: test.a, test1.a - Cross Join: + Cross Join: Projection: test.a, test.b, test.c TableScan: test, full_filters=[test.a = Int32(1)] Projection: test1.a, test1.b, test1.c diff --git a/datafusion/optimizer/src/simplify_predicates.rs b/datafusion/optimizer/src/simplify_predicates.rs index 1516d3f46d91a..5ff7e42d95310 100644 --- a/datafusion/optimizer/src/simplify_predicates.rs +++ b/datafusion/optimizer/src/simplify_predicates.rs @@ -103,7 +103,7 @@ fn simplify_column_predicates(predicates: Vec) -> Result> { // If we have equality predicates, they're the most restrictive if !eq_predicates.is_empty() { if eq_predicates.len() > 1 { - result.push(Expr::Literal(ScalarValue::Boolean(Some(false)))); + result.push(Expr::Literal(ScalarValue::Boolean(Some(false)), None)); } else { result.push(eq_predicates[0].clone()); } @@ -150,11 +150,11 @@ fn find_most_restrictive_predicate( // Extract the literal value based on which side has it let mut scalar_value = None; if right.is_literal() { - if let Expr::Literal(scalar) = right.as_ref() { + if let Expr::Literal(scalar, _) = right.as_ref() { scalar_value = Some(scalar.clone()); } } else if left.is_literal() { - if let Expr::Literal(scalar) = left.as_ref() { + if let Expr::Literal(scalar, _) = left.as_ref() { scalar_value = Some(scalar.clone()); } } diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 347f82b6c9368..0650bfbcf14ec 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -42,6 +42,7 @@ use datafusion_physical_expr::utils::map_columns_before_projection; use datafusion_physical_expr::{ physical_exprs_equal, EquivalenceProperties, PhysicalExpr, PhysicalExprRef, }; +use datafusion_physical_expr_common::sort_expr::LexOrdering; use datafusion_physical_plan::aggregates::{ AggregateExec, AggregateMode, PhysicalGroupBy, }; @@ -1033,7 +1034,7 @@ fn remove_dist_changing_operators( /// " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2", /// " DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet", /// ``` -pub fn replace_order_preserving_variants( +fn replace_order_preserving_variants( mut context: DistributionContext, ordering_satisfied: bool, ) -> Result<(DistributionContext, Option)> { @@ -1056,9 +1057,8 @@ pub fn replace_order_preserving_variants( let child_plan = Arc::clone(&context.children[0].plan); if !ordering_satisfied { // It's safe to unwrap because `CoalescePartitionsExec` supports `fetch`. - context.plan = CoalescePartitionsExec::new(child_plan) - .with_fetch(fetch) - .unwrap(); + context.plan = + Arc::new(CoalescePartitionsExec::new(child_plan).with_fetch(fetch)); return Ok((context, None)); } context.plan = Arc::new(CoalescePartitionsExec::new(child_plan)); diff --git a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs index 670ac9ed0f638..caa536107d8da 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs @@ -36,7 +36,7 @@ use datafusion_physical_plan::execution_plan::EmissionType; use datafusion_physical_plan::repartition::RepartitionExec; use datafusion_physical_plan::sorts::sort_preserving_merge::SortPreservingMergeExec; use datafusion_physical_plan::tree_node::PlanContext; -use datafusion_physical_plan::{ExecutionPlan, ExecutionPlanProperties}; +use datafusion_physical_plan::ExecutionPlanProperties; use itertools::izip; diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs index 5d3f9eaa8fc2e..f674ee0b2064f 100644 --- a/datafusion/physical-plan/src/work_table.rs +++ b/datafusion/physical-plan/src/work_table.rs @@ -238,7 +238,7 @@ impl ExecutionPlan for WorkTableExec { new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } - + fn partition_statistics(&self, _partition: Option) -> Result { Ok(Statistics::new_unknown(&self.schema())) } From 6851d8ee631c909156e2dce10d386f411b74a025 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Fri, 4 Jul 2025 14:21:46 -0400 Subject: [PATCH 072/177] Add the missing equivalence info for filter pushdown --- datafusion/datasource/src/source.rs | 22 ++++++++++++++++++- datafusion/physical-plan/src/filter.rs | 4 +++- .../physical-plan/src/filter_pushdown.rs | 12 ++++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index 6d8b56275d0e5..9c32f2170071a 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -33,8 +33,11 @@ use crate::file_scan_config::FileScanConfig; use datafusion_common::config::ConfigOptions; use datafusion_common::{Constraints, Result, Statistics}; use datafusion_execution::{SendableRecordBatchStream, TaskContext}; -use datafusion_physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; +use datafusion_physical_expr::{ + conjunction, EquivalenceProperties, Partitioning, PhysicalExpr, +}; use datafusion_physical_expr_common::sort_expr::LexOrdering; +use datafusion_physical_plan::filter::collect_columns_from_predicate; use datafusion_physical_plan::filter_pushdown::{ ChildPushdownResult, FilterPushdownPropagation, }; @@ -328,6 +331,9 @@ impl ExecutionPlan for DataSourceExec { new_node.data_source = data_source; new_node.cache = Self::compute_properties(Arc::clone(&new_node.data_source)); + // Add the missing filters' equivalence info when filters pushdown is applied + let filter = conjunction(res.filters.collect_supported()); + new_node = new_node.add_filter_equivalence_info(filter)?; Ok(FilterPushdownPropagation { filters: res.filters, updated_node: Some(Arc::new(new_node)), @@ -374,6 +380,20 @@ impl DataSourceExec { self } + /// Add filters' equivalence info + fn add_filter_equivalence_info( + mut self, + filter: Arc, + ) -> Result { + let (equal_pairs, _) = collect_columns_from_predicate(&filter); + for (lhs, rhs) in equal_pairs { + self.cache + .eq_properties + .add_equal_conditions(&Arc::clone(lhs), &Arc::clone(rhs))? + } + Ok(self) + } + fn compute_properties(data_source: Arc) -> PlanProperties { PlanProperties::new( data_source.eq_properties(), diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 9d76daa2ea526..07aea16d06283 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -749,7 +749,9 @@ impl RecordBatchStream for FilterExecStream { } /// Return the equals Column-Pairs and Non-equals Column-Pairs -fn collect_columns_from_predicate(predicate: &Arc) -> EqualAndNonEqual { +pub fn collect_columns_from_predicate( + predicate: &Arc, +) -> EqualAndNonEqual { let mut eq_predicate_columns = Vec::::new(); let mut ne_predicate_columns = Vec::::new(); diff --git a/datafusion/physical-plan/src/filter_pushdown.rs b/datafusion/physical-plan/src/filter_pushdown.rs index 4e84fe36f98f3..9d4580be79d15 100644 --- a/datafusion/physical-plan/src/filter_pushdown.rs +++ b/datafusion/physical-plan/src/filter_pushdown.rs @@ -102,6 +102,18 @@ impl PredicateSupports { .collect() } + /// Collect supported filters into a Vec, without removing them from the original + /// [`PredicateSupport`]. + pub fn collect_supported(&self) -> Vec> { + self.0 + .iter() + .filter_map(|f| match f { + PredicateSupport::Supported(expr) => Some(Arc::clone(expr)), + PredicateSupport::Unsupported(_) => None, + }) + .collect() + } + /// Collect all filters into a Vec, without removing them from the original /// FilterPushdowns. pub fn collect_all(self) -> Vec> { From 054d193ebf4882642e069de48f413119efab1007 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Sat, 12 Jul 2025 16:29:34 +0800 Subject: [PATCH 073/177] 48.0.1 --- Cargo.lock | 78 +-- Cargo.toml | 68 +-- datafusion/common/src/config.rs | 4 +- .../core/src/execution/context/parquet.rs | 27 +- datafusion/datasource-parquet/src/source.rs | 13 +- datafusion/ffi/src/udwf/mod.rs | 68 ++- .../ffi/src/udwf/partition_evaluator_args.rs | 27 +- .../sqllogictest/test_files/explain_tree.slt | 453 ++++++++---------- .../test_files/information_schema.slt | 4 +- datafusion/sqllogictest/test_files/limit.slt | 2 +- .../test_files/parquet_filter_pushdown.slt | 126 ++++- .../test_files/parquet_statistics.slt | 12 +- .../sqllogictest/test_files/repartition.slt | 4 +- dev/changelog/48.0.1.md | 41 ++ dev/release/generate-changelog.py | 3 + docs/source/user-guide/configs.md | 4 +- docs/source/user-guide/sql/ddl.md | 8 +- 17 files changed, 558 insertions(+), 384 deletions(-) create mode 100644 dev/changelog/48.0.1.md diff --git a/Cargo.lock b/Cargo.lock index fe67ae9bea784..e7fb4ef136c7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1808,7 +1808,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "arrow-ipc", @@ -1878,7 +1878,7 @@ dependencies = [ [[package]] name = "datafusion-benchmarks" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "datafusion", @@ -1902,7 +1902,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -1926,7 +1926,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -1947,7 +1947,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "assert_cmd", @@ -1976,7 +1976,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.0" +version = "48.0.1" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2003,7 +2003,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.0" +version = "48.0.1" dependencies = [ "futures", "log", @@ -2012,7 +2012,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-compression", @@ -2047,7 +2047,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "48.0.0" +version = "48.0.1" dependencies = [ "apache-avro", "arrow", @@ -2072,7 +2072,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2095,7 +2095,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2118,7 +2118,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2148,11 +2148,11 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.0" +version = "48.0.1" [[package]] name = "datafusion-examples" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "arrow-flight", @@ -2182,7 +2182,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "chrono", @@ -2201,7 +2201,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "chrono", @@ -2223,7 +2223,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2234,7 +2234,7 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "48.0.0" +version = "48.0.1" dependencies = [ "abi_stable", "arrow", @@ -2255,7 +2255,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "arrow-buffer", @@ -2284,7 +2284,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.0" +version = "48.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2305,7 +2305,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.0" +version = "48.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2318,7 +2318,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "arrow-ord", @@ -2339,7 +2339,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2353,7 +2353,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2369,7 +2369,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.0" +version = "48.0.1" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2377,7 +2377,7 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.0" +version = "48.0.1" dependencies = [ "datafusion-expr", "quote", @@ -2386,7 +2386,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2412,7 +2412,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.0" +version = "48.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2437,7 +2437,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "48.0.0" +version = "48.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2449,7 +2449,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2468,7 +2468,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.0" +version = "48.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2504,7 +2504,7 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "chrono", @@ -2526,7 +2526,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2539,7 +2539,7 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2561,7 +2561,7 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "datafusion-catalog", @@ -2575,7 +2575,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "bigdecimal", @@ -2599,7 +2599,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "48.0.0" +version = "48.0.1" dependencies = [ "arrow", "async-trait", @@ -2632,7 +2632,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "48.0.0" +version = "48.0.1" dependencies = [ "async-recursion", "async-trait", @@ -2652,7 +2652,7 @@ dependencies = [ [[package]] name = "datafusion-wasmtest" -version = "48.0.0" +version = "48.0.1" dependencies = [ "chrono", "console_error_panic_hook", diff --git a/Cargo.toml b/Cargo.toml index 64483eeb93da3..366701bdae2ec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,7 @@ repository = "https://github.com/apache/datafusion" # Define Minimum Supported Rust Version (MSRV) rust-version = "1.82.0" # Define DataFusion version -version = "48.0.0" +version = "48.0.1" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can @@ -108,39 +108,39 @@ chrono = { version = "0.4.41", default-features = false } criterion = "0.5.1" ctor = "0.4.0" dashmap = "6.0.1" -datafusion = { path = "datafusion/core", version = "48.0.0", default-features = false } -datafusion-catalog = { path = "datafusion/catalog", version = "48.0.0" } -datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "48.0.0" } -datafusion-common = { path = "datafusion/common", version = "48.0.0", default-features = false } -datafusion-common-runtime = { path = "datafusion/common-runtime", version = "48.0.0" } -datafusion-datasource = { path = "datafusion/datasource", version = "48.0.0", default-features = false } -datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "48.0.0", default-features = false } -datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "48.0.0", default-features = false } -datafusion-datasource-json = { path = "datafusion/datasource-json", version = "48.0.0", default-features = false } -datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "48.0.0", default-features = false } -datafusion-doc = { path = "datafusion/doc", version = "48.0.0" } -datafusion-execution = { path = "datafusion/execution", version = "48.0.0" } -datafusion-expr = { path = "datafusion/expr", version = "48.0.0" } -datafusion-expr-common = { path = "datafusion/expr-common", version = "48.0.0" } -datafusion-ffi = { path = "datafusion/ffi", version = "48.0.0" } -datafusion-functions = { path = "datafusion/functions", version = "48.0.0" } -datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "48.0.0" } -datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "48.0.0" } -datafusion-functions-nested = { path = "datafusion/functions-nested", version = "48.0.0" } -datafusion-functions-table = { path = "datafusion/functions-table", version = "48.0.0" } -datafusion-functions-window = { path = "datafusion/functions-window", version = "48.0.0" } -datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "48.0.0" } -datafusion-macros = { path = "datafusion/macros", version = "48.0.0" } -datafusion-optimizer = { path = "datafusion/optimizer", version = "48.0.0", default-features = false } -datafusion-physical-expr = { path = "datafusion/physical-expr", version = "48.0.0", default-features = false } -datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "48.0.0", default-features = false } -datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "48.0.0" } -datafusion-physical-plan = { path = "datafusion/physical-plan", version = "48.0.0" } -datafusion-proto = { path = "datafusion/proto", version = "48.0.0" } -datafusion-proto-common = { path = "datafusion/proto-common", version = "48.0.0" } -datafusion-session = { path = "datafusion/session", version = "48.0.0" } -datafusion-spark = { path = "datafusion/spark", version = "48.0.0" } -datafusion-sql = { path = "datafusion/sql", version = "48.0.0" } +datafusion = { path = "datafusion/core", version = "48.0.1", default-features = false } +datafusion-catalog = { path = "datafusion/catalog", version = "48.0.1" } +datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "48.0.1" } +datafusion-common = { path = "datafusion/common", version = "48.0.1", default-features = false } +datafusion-common-runtime = { path = "datafusion/common-runtime", version = "48.0.1" } +datafusion-datasource = { path = "datafusion/datasource", version = "48.0.1", default-features = false } +datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "48.0.1", default-features = false } +datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "48.0.1", default-features = false } +datafusion-datasource-json = { path = "datafusion/datasource-json", version = "48.0.1", default-features = false } +datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "48.0.1", default-features = false } +datafusion-doc = { path = "datafusion/doc", version = "48.0.1" } +datafusion-execution = { path = "datafusion/execution", version = "48.0.1" } +datafusion-expr = { path = "datafusion/expr", version = "48.0.1" } +datafusion-expr-common = { path = "datafusion/expr-common", version = "48.0.1" } +datafusion-ffi = { path = "datafusion/ffi", version = "48.0.1" } +datafusion-functions = { path = "datafusion/functions", version = "48.0.1" } +datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "48.0.1" } +datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "48.0.1" } +datafusion-functions-nested = { path = "datafusion/functions-nested", version = "48.0.1" } +datafusion-functions-table = { path = "datafusion/functions-table", version = "48.0.1" } +datafusion-functions-window = { path = "datafusion/functions-window", version = "48.0.1" } +datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "48.0.1" } +datafusion-macros = { path = "datafusion/macros", version = "48.0.1" } +datafusion-optimizer = { path = "datafusion/optimizer", version = "48.0.1", default-features = false } +datafusion-physical-expr = { path = "datafusion/physical-expr", version = "48.0.1", default-features = false } +datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "48.0.1", default-features = false } +datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "48.0.1" } +datafusion-physical-plan = { path = "datafusion/physical-plan", version = "48.0.1" } +datafusion-proto = { path = "datafusion/proto", version = "48.0.1" } +datafusion-proto-common = { path = "datafusion/proto-common", version = "48.0.1" } +datafusion-session = { path = "datafusion/session", version = "48.0.1" } +datafusion-spark = { path = "datafusion/spark", version = "48.0.1" } +datafusion-sql = { path = "datafusion/sql", version = "48.0.1" } doc-comment = "0.3" env_logger = "0.11" futures = "0.3" diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 696113913f27a..0d34815a248f7 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -294,8 +294,8 @@ config_namespace! { /// Should DataFusion collect statistics when first creating a table. /// Has no effect after the table is created. Applies to the default - /// `ListingTableProvider` in DataFusion. Defaults to false. - pub collect_statistics: bool, default = false + /// `ListingTableProvider` in DataFusion. Defaults to true. + pub collect_statistics: bool, default = true /// Number of partitions for query execution. Increasing partitions can increase /// concurrency. diff --git a/datafusion/core/src/execution/context/parquet.rs b/datafusion/core/src/execution/context/parquet.rs index eea2b804770a3..2fb763bee495f 100644 --- a/datafusion/core/src/execution/context/parquet.rs +++ b/datafusion/core/src/execution/context/parquet.rs @@ -34,13 +34,12 @@ impl SessionContext { /// /// # Note: Statistics /// - /// NOTE: by default, statistics are not collected when reading the Parquet - /// files as this can slow down the initial DataFrame creation. However, - /// collecting statistics can greatly accelerate queries with certain - /// filters. + /// NOTE: by default, statistics are collected when reading the Parquet + /// files This can slow down the initial DataFrame creation while + /// greatly accelerating queries with certain filters. /// - /// To enable collect statistics, set the [config option] - /// `datafusion.execution.collect_statistics` to `true`. See + /// To disable statistics collection, set the [config option] + /// `datafusion.execution.collect_statistics` to `false`. See /// [`ConfigOptions`] and [`ExecutionOptions::collect_statistics`] for more /// details. /// @@ -171,28 +170,28 @@ mod tests { #[tokio::test] async fn register_parquet_respects_collect_statistics_config() -> Result<()> { - // The default is false + // The default is true let mut config = SessionConfig::new(); config.options_mut().explain.physical_plan_only = true; config.options_mut().explain.show_statistics = true; let content = explain_query_all_with_config(config).await?; - assert_contains!(content, "statistics=[Rows=Absent,"); + assert_contains!(content, "statistics=[Rows=Exact("); - // Explicitly set to false + // Explicitly set to true let mut config = SessionConfig::new(); config.options_mut().explain.physical_plan_only = true; config.options_mut().explain.show_statistics = true; - config.options_mut().execution.collect_statistics = false; + config.options_mut().execution.collect_statistics = true; let content = explain_query_all_with_config(config).await?; - assert_contains!(content, "statistics=[Rows=Absent,"); + assert_contains!(content, "statistics=[Rows=Exact("); - // Explicitly set to true + // Explicitly set to false let mut config = SessionConfig::new(); config.options_mut().explain.physical_plan_only = true; config.options_mut().explain.show_statistics = true; - config.options_mut().execution.collect_statistics = true; + config.options_mut().execution.collect_statistics = false; let content = explain_query_all_with_config(config).await?; - assert_contains!(content, "statistics=[Rows=Exact(10),"); + assert_contains!(content, "statistics=[Rows=Absent,"); Ok(()) } diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index 0412288d68758..69fcb35187243 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -344,9 +344,7 @@ impl ParquetSource { } /// If true, the predicate will be used during the parquet scan. - /// Defaults to false - /// - /// [`Expr`]: datafusion_expr::Expr + /// Defaults to false. pub fn with_pushdown_filters(mut self, pushdown_filters: bool) -> Self { self.table_parquet_options.global.pushdown_filters = pushdown_filters; self @@ -617,7 +615,13 @@ impl FileSource for ParquetSource { let Some(file_schema) = self.file_schema.clone() else { return Ok(FilterPushdownPropagation::unsupported(filters)); }; - // Can we push down the filters themselves into the scan or only use stats pruning? + // Determine if based on configs we should push filters down. + // If either the table / scan itself or the config has pushdown enabled, + // we will push down the filters. + // If both are disabled, we will not push down the filters. + // By default they are both disabled. + // Regardless of pushdown, we will update the predicate to include the filters + // because even if scan pushdown is disabled we can still use the filters for stats pruning. let config_pushdown_enabled = config.execution.parquet.pushdown_filters; let table_pushdown_enabled = self.pushdown_filters(); let pushdown_filters = table_pushdown_enabled || config_pushdown_enabled; @@ -646,6 +650,7 @@ impl FileSource for ParquetSource { None => conjunction(allowed_filters.iter().cloned()), }; source.predicate = Some(predicate); + source = source.with_pushdown_filters(pushdown_filters); let source = Arc::new(source); let filters = PredicateSupports::new( allowed_filters diff --git a/datafusion/ffi/src/udwf/mod.rs b/datafusion/ffi/src/udwf/mod.rs index aaa3f5c99253d..504bf7a411f1a 100644 --- a/datafusion/ffi/src/udwf/mod.rs +++ b/datafusion/ffi/src/udwf/mod.rs @@ -363,4 +363,70 @@ impl From<&FFI_SortOptions> for SortOptions { } #[cfg(test)] -mod tests {} +#[cfg(feature = "integration-tests")] +mod tests { + use crate::tests::create_record_batch; + use crate::udwf::{FFI_WindowUDF, ForeignWindowUDF}; + use arrow::array::{create_array, ArrayRef}; + use datafusion::functions_window::lead_lag::{lag_udwf, WindowShift}; + use datafusion::logical_expr::expr::Sort; + use datafusion::logical_expr::{col, ExprFunctionExt, WindowUDF, WindowUDFImpl}; + use datafusion::prelude::SessionContext; + use std::sync::Arc; + + fn create_test_foreign_udwf( + original_udwf: impl WindowUDFImpl + 'static, + ) -> datafusion::common::Result { + let original_udwf = Arc::new(WindowUDF::from(original_udwf)); + + let local_udwf: FFI_WindowUDF = Arc::clone(&original_udwf).into(); + + let foreign_udwf: ForeignWindowUDF = (&local_udwf).try_into()?; + Ok(foreign_udwf.into()) + } + + #[test] + fn test_round_trip_udwf() -> datafusion::common::Result<()> { + let original_udwf = lag_udwf(); + let original_name = original_udwf.name().to_owned(); + + // Convert to FFI format + let local_udwf: FFI_WindowUDF = Arc::clone(&original_udwf).into(); + + // Convert back to native format + let foreign_udwf: ForeignWindowUDF = (&local_udwf).try_into()?; + let foreign_udwf: WindowUDF = foreign_udwf.into(); + + assert_eq!(original_name, foreign_udwf.name()); + Ok(()) + } + + #[tokio::test] + async fn test_lag_udwf() -> datafusion::common::Result<()> { + let udwf = create_test_foreign_udwf(WindowShift::lag())?; + + let ctx = SessionContext::default(); + let df = ctx.read_batch(create_record_batch(-5, 5))?; + + let df = df.select(vec![ + col("a"), + udwf.call(vec![col("a")]) + .order_by(vec![Sort::new(col("a"), true, true)]) + .build() + .unwrap() + .alias("lag_a"), + ])?; + + df.clone().show().await?; + + let result = df.collect().await?; + let expected = + create_array!(Int32, [None, Some(-5), Some(-4), Some(-3), Some(-2)]) + as ArrayRef; + + assert_eq!(result.len(), 1); + assert_eq!(result[0].column(1), &expected); + + Ok(()) + } +} diff --git a/datafusion/ffi/src/udwf/partition_evaluator_args.rs b/datafusion/ffi/src/udwf/partition_evaluator_args.rs index e74d47aa1a161..dffeb23741b66 100644 --- a/datafusion/ffi/src/udwf/partition_evaluator_args.rs +++ b/datafusion/ffi/src/udwf/partition_evaluator_args.rs @@ -75,17 +75,24 @@ impl TryFrom> for FFI_PartitionEvaluatorArgs { }) .collect(); - let max_column = required_columns.keys().max().unwrap_or(&0).to_owned(); - let fields: Vec<_> = (0..max_column) - .map(|idx| match required_columns.get(&idx) { - Some((name, data_type)) => Field::new(*name, (*data_type).clone(), true), - None => Field::new( - format!("ffi_partition_evaluator_col_{idx}"), - DataType::Null, - true, - ), + let max_column = required_columns.keys().max(); + let fields: Vec<_> = max_column + .map(|max_column| { + (0..(max_column + 1)) + .map(|idx| match required_columns.get(&idx) { + Some((name, data_type)) => { + Field::new(*name, (*data_type).clone(), true) + } + None => Field::new( + format!("ffi_partition_evaluator_col_{idx}"), + DataType::Null, + true, + ), + }) + .collect() }) - .collect(); + .unwrap_or_default(); + let schema = Arc::new(Schema::new(fields)); let codec = DefaultPhysicalExtensionCodec {}; diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 15bf615765713..22183195c3df0 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -291,47 +291,40 @@ explain SELECT table1.string_col, table2.date_col FROM table1 JOIN table2 ON tab ---- physical_plan 01)┌───────────────────────────┐ -02)│ CoalesceBatchesExec │ +02)│ ProjectionExec │ 03)│ -------------------- │ -04)│ target_batch_size: │ -05)│ 8192 │ -06)└─────────────┬─────────────┘ -07)┌─────────────┴─────────────┐ -08)│ HashJoinExec │ -09)│ -------------------- │ -10)│ on: ├──────────────┐ -11)│ (int_col = int_col) │ │ -12)└─────────────┬─────────────┘ │ -13)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -14)│ CoalesceBatchesExec ││ CoalesceBatchesExec │ -15)│ -------------------- ││ -------------------- │ -16)│ target_batch_size: ││ target_batch_size: │ -17)│ 8192 ││ 8192 │ -18)└─────────────┬─────────────┘└─────────────┬─────────────┘ -19)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -20)│ RepartitionExec ││ RepartitionExec │ -21)│ -------------------- ││ -------------------- │ -22)│ partition_count(in->out): ││ partition_count(in->out): │ -23)│ 4 -> 4 ││ 4 -> 4 │ -24)│ ││ │ -25)│ partitioning_scheme: ││ partitioning_scheme: │ -26)│ Hash([int_col@0], 4) ││ Hash([int_col@0], 4) │ -27)└─────────────┬─────────────┘└─────────────┬─────────────┘ -28)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -29)│ RepartitionExec ││ RepartitionExec │ -30)│ -------------------- ││ -------------------- │ -31)│ partition_count(in->out): ││ partition_count(in->out): │ -32)│ 1 -> 4 ││ 1 -> 4 │ -33)│ ││ │ -34)│ partitioning_scheme: ││ partitioning_scheme: │ -35)│ RoundRobinBatch(4) ││ RoundRobinBatch(4) │ -36)└─────────────┬─────────────┘└─────────────┬─────────────┘ -37)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -38)│ DataSourceExec ││ DataSourceExec │ -39)│ -------------------- ││ -------------------- │ -40)│ files: 1 ││ files: 1 │ -41)│ format: csv ││ format: parquet │ -42)└───────────────────────────┘└───────────────────────────┘ +04)│ date_col: date_col │ +05)│ │ +06)│ string_col: │ +07)│ string_col │ +08)└─────────────┬─────────────┘ +09)┌─────────────┴─────────────┐ +10)│ CoalesceBatchesExec │ +11)│ -------------------- │ +12)│ target_batch_size: │ +13)│ 8192 │ +14)└─────────────┬─────────────┘ +15)┌─────────────┴─────────────┐ +16)│ HashJoinExec │ +17)│ -------------------- │ +18)│ on: ├──────────────┐ +19)│ (int_col = int_col) │ │ +20)└─────────────┬─────────────┘ │ +21)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +22)│ DataSourceExec ││ RepartitionExec │ +23)│ -------------------- ││ -------------------- │ +24)│ files: 1 ││ partition_count(in->out): │ +25)│ format: parquet ││ 1 -> 4 │ +26)│ ││ │ +27)│ ││ partitioning_scheme: │ +28)│ ││ RoundRobinBatch(4) │ +29)└───────────────────────────┘└─────────────┬─────────────┘ +30)-----------------------------┌─────────────┴─────────────┐ +31)-----------------------------│ DataSourceExec │ +32)-----------------------------│ -------------------- │ +33)-----------------------------│ files: 1 │ +34)-----------------------------│ format: csv │ +35)-----------------------------└───────────────────────────┘ # 3 Joins query TT @@ -365,48 +358,41 @@ physical_plan 19)│ (int_col = int_col) │ │ 20)└─────────────┬─────────────┘ │ 21)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -22)│ DataSourceExec ││ CoalesceBatchesExec │ +22)│ DataSourceExec ││ ProjectionExec │ 23)│ -------------------- ││ -------------------- │ -24)│ bytes: 1560 ││ target_batch_size: │ -25)│ format: memory ││ 8192 │ +24)│ bytes: 1560 ││ date_col: date_col │ +25)│ format: memory ││ int_col: int_col │ 26)│ rows: 1 ││ │ -27)└───────────────────────────┘└─────────────┬─────────────┘ -28)-----------------------------┌─────────────┴─────────────┐ -29)-----------------------------│ HashJoinExec │ -30)-----------------------------│ -------------------- │ -31)-----------------------------│ on: ├──────────────┐ -32)-----------------------------│ (int_col = int_col) │ │ -33)-----------------------------└─────────────┬─────────────┘ │ -34)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -35)-----------------------------│ CoalesceBatchesExec ││ CoalesceBatchesExec │ -36)-----------------------------│ -------------------- ││ -------------------- │ -37)-----------------------------│ target_batch_size: ││ target_batch_size: │ -38)-----------------------------│ 8192 ││ 8192 │ -39)-----------------------------└─────────────┬─────────────┘└─────────────┬─────────────┘ -40)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -41)-----------------------------│ RepartitionExec ││ RepartitionExec │ -42)-----------------------------│ -------------------- ││ -------------------- │ -43)-----------------------------│ partition_count(in->out): ││ partition_count(in->out): │ -44)-----------------------------│ 4 -> 4 ││ 4 -> 4 │ -45)-----------------------------│ ││ │ -46)-----------------------------│ partitioning_scheme: ││ partitioning_scheme: │ -47)-----------------------------│ Hash([int_col@0], 4) ││ Hash([int_col@0], 4) │ -48)-----------------------------└─────────────┬─────────────┘└─────────────┬─────────────┘ -49)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -50)-----------------------------│ RepartitionExec ││ RepartitionExec │ -51)-----------------------------│ -------------------- ││ -------------------- │ -52)-----------------------------│ partition_count(in->out): ││ partition_count(in->out): │ -53)-----------------------------│ 1 -> 4 ││ 1 -> 4 │ -54)-----------------------------│ ││ │ -55)-----------------------------│ partitioning_scheme: ││ partitioning_scheme: │ -56)-----------------------------│ RoundRobinBatch(4) ││ RoundRobinBatch(4) │ -57)-----------------------------└─────────────┬─────────────┘└─────────────┬─────────────┘ -58)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -59)-----------------------------│ DataSourceExec ││ DataSourceExec │ -60)-----------------------------│ -------------------- ││ -------------------- │ -61)-----------------------------│ files: 1 ││ files: 1 │ -62)-----------------------------│ format: csv ││ format: parquet │ -63)-----------------------------└───────────────────────────┘└───────────────────────────┘ +27)│ ││ string_col: │ +28)│ ││ string_col │ +29)└───────────────────────────┘└─────────────┬─────────────┘ +30)-----------------------------┌─────────────┴─────────────┐ +31)-----------------------------│ CoalesceBatchesExec │ +32)-----------------------------│ -------------------- │ +33)-----------------------------│ target_batch_size: │ +34)-----------------------------│ 8192 │ +35)-----------------------------└─────────────┬─────────────┘ +36)-----------------------------┌─────────────┴─────────────┐ +37)-----------------------------│ HashJoinExec │ +38)-----------------------------│ -------------------- │ +39)-----------------------------│ on: ├──────────────┐ +40)-----------------------------│ (int_col = int_col) │ │ +41)-----------------------------└─────────────┬─────────────┘ │ +42)-----------------------------┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +43)-----------------------------│ DataSourceExec ││ RepartitionExec │ +44)-----------------------------│ -------------------- ││ -------------------- │ +45)-----------------------------│ files: 1 ││ partition_count(in->out): │ +46)-----------------------------│ format: parquet ││ 1 -> 4 │ +47)-----------------------------│ ││ │ +48)-----------------------------│ ││ partitioning_scheme: │ +49)-----------------------------│ ││ RoundRobinBatch(4) │ +50)-----------------------------└───────────────────────────┘└─────────────┬─────────────┘ +51)----------------------------------------------------------┌─────────────┴─────────────┐ +52)----------------------------------------------------------│ DataSourceExec │ +53)----------------------------------------------------------│ -------------------- │ +54)----------------------------------------------------------│ files: 1 │ +55)----------------------------------------------------------│ format: csv │ +56)----------------------------------------------------------└───────────────────────────┘ # Long Filter (demonstrate what happens with wrapping) query TT @@ -1029,20 +1015,11 @@ physical_plan 11)│ bigint_col │ 12)└─────────────┬─────────────┘ 13)┌─────────────┴─────────────┐ -14)│ RepartitionExec │ +14)│ DataSourceExec │ 15)│ -------------------- │ -16)│ partition_count(in->out): │ -17)│ 1 -> 4 │ -18)│ │ -19)│ partitioning_scheme: │ -20)│ RoundRobinBatch(4) │ -21)└─────────────┬─────────────┘ -22)┌─────────────┴─────────────┐ -23)│ DataSourceExec │ -24)│ -------------------- │ -25)│ files: 1 │ -26)│ format: parquet │ -27)└───────────────────────────┘ +16)│ files: 1 │ +17)│ format: parquet │ +18)└───────────────────────────┘ # Query with projection on memory @@ -1186,69 +1163,64 @@ explain select * from table1 inner join table2 on table1.int_col = table2.int_co ---- physical_plan 01)┌───────────────────────────┐ -02)│ CoalesceBatchesExec │ +02)│ ProjectionExec │ 03)│ -------------------- │ -04)│ target_batch_size: │ -05)│ 8192 │ -06)└─────────────┬─────────────┘ -07)┌─────────────┴─────────────┐ -08)│ HashJoinExec │ -09)│ -------------------- │ -10)│ on: │ -11)│ (int_col = int_col), (CAST├──────────────┐ -12)│ (table1.string_col AS │ │ -13)│ Utf8View) = │ │ -14)│ string_col) │ │ -15)└─────────────┬─────────────┘ │ -16)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -17)│ CoalesceBatchesExec ││ CoalesceBatchesExec │ -18)│ -------------------- ││ -------------------- │ -19)│ target_batch_size: ││ target_batch_size: │ -20)│ 8192 ││ 8192 │ -21)└─────────────┬─────────────┘└─────────────┬─────────────┘ -22)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -23)│ RepartitionExec ││ RepartitionExec │ -24)│ -------------------- ││ -------------------- │ -25)│ partition_count(in->out): ││ partition_count(in->out): │ -26)│ 4 -> 4 ││ 4 -> 4 │ -27)│ ││ │ -28)│ partitioning_scheme: ││ partitioning_scheme: │ -29)│ Hash([int_col@0, CAST ││ Hash([int_col@0, │ -30)│ (table1.string_col ││ string_col@1], │ -31)│ AS Utf8View)@4], 4) ││ 4) │ -32)└─────────────┬─────────────┘└─────────────┬─────────────┘ -33)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -34)│ ProjectionExec ││ RepartitionExec │ -35)│ -------------------- ││ -------------------- │ -36)│ CAST(table1.string_col AS ││ partition_count(in->out): │ -37)│ Utf8View): ││ 1 -> 4 │ -38)│ CAST(string_col AS ││ │ -39)│ Utf8View) ││ partitioning_scheme: │ -40)│ ││ RoundRobinBatch(4) │ -41)│ bigint_col: ││ │ -42)│ bigint_col ││ │ -43)│ ││ │ -44)│ date_col: date_col ││ │ -45)│ int_col: int_col ││ │ -46)│ ││ │ -47)│ string_col: ││ │ -48)│ string_col ││ │ -49)└─────────────┬─────────────┘└─────────────┬─────────────┘ -50)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -51)│ RepartitionExec ││ DataSourceExec │ -52)│ -------------------- ││ -------------------- │ -53)│ partition_count(in->out): ││ files: 1 │ -54)│ 1 -> 4 ││ format: parquet │ -55)│ ││ │ -56)│ partitioning_scheme: ││ │ -57)│ RoundRobinBatch(4) ││ │ -58)└─────────────┬─────────────┘└───────────────────────────┘ -59)┌─────────────┴─────────────┐ -60)│ DataSourceExec │ -61)│ -------------------- │ -62)│ files: 1 │ -63)│ format: csv │ -64)└───────────────────────────┘ +04)│ bigint_col: │ +05)│ bigint_col │ +06)│ │ +07)│ date_col: date_col │ +08)│ int_col: int_col │ +09)│ │ +10)│ string_col: │ +11)│ string_col │ +12)└─────────────┬─────────────┘ +13)┌─────────────┴─────────────┐ +14)│ CoalesceBatchesExec │ +15)│ -------------------- │ +16)│ target_batch_size: │ +17)│ 8192 │ +18)└─────────────┬─────────────┘ +19)┌─────────────┴─────────────┐ +20)│ HashJoinExec │ +21)│ -------------------- │ +22)│ on: │ +23)│ (int_col = int_col), ├──────────────┐ +24)│ (string_col = CAST │ │ +25)│ (table1.string_col AS │ │ +26)│ Utf8View)) │ │ +27)└─────────────┬─────────────┘ │ +28)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +29)│ DataSourceExec ││ ProjectionExec │ +30)│ -------------------- ││ -------------------- │ +31)│ files: 1 ││ CAST(table1.string_col AS │ +32)│ format: parquet ││ Utf8View): │ +33)│ ││ CAST(string_col AS │ +34)│ ││ Utf8View) │ +35)│ ││ │ +36)│ ││ bigint_col: │ +37)│ ││ bigint_col │ +38)│ ││ │ +39)│ ││ date_col: date_col │ +40)│ ││ int_col: int_col │ +41)│ ││ │ +42)│ ││ string_col: │ +43)│ ││ string_col │ +44)└───────────────────────────┘└─────────────┬─────────────┘ +45)-----------------------------┌─────────────┴─────────────┐ +46)-----------------------------│ RepartitionExec │ +47)-----------------------------│ -------------------- │ +48)-----------------------------│ partition_count(in->out): │ +49)-----------------------------│ 1 -> 4 │ +50)-----------------------------│ │ +51)-----------------------------│ partitioning_scheme: │ +52)-----------------------------│ RoundRobinBatch(4) │ +53)-----------------------------└─────────────┬─────────────┘ +54)-----------------------------┌─────────────┴─────────────┐ +55)-----------------------------│ DataSourceExec │ +56)-----------------------------│ -------------------- │ +57)-----------------------------│ files: 1 │ +58)-----------------------------│ format: csv │ +59)-----------------------------└───────────────────────────┘ # Query with outer hash join. query TT @@ -1256,71 +1228,66 @@ explain select * from table1 left outer join table2 on table1.int_col = table2.i ---- physical_plan 01)┌───────────────────────────┐ -02)│ CoalesceBatchesExec │ +02)│ ProjectionExec │ 03)│ -------------------- │ -04)│ target_batch_size: │ -05)│ 8192 │ -06)└─────────────┬─────────────┘ -07)┌─────────────┴─────────────┐ -08)│ HashJoinExec │ -09)│ -------------------- │ -10)│ join_type: Left │ -11)│ │ -12)│ on: ├──────────────┐ -13)│ (int_col = int_col), (CAST│ │ -14)│ (table1.string_col AS │ │ -15)│ Utf8View) = │ │ -16)│ string_col) │ │ -17)└─────────────┬─────────────┘ │ -18)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -19)│ CoalesceBatchesExec ││ CoalesceBatchesExec │ -20)│ -------------------- ││ -------------------- │ -21)│ target_batch_size: ││ target_batch_size: │ -22)│ 8192 ││ 8192 │ -23)└─────────────┬─────────────┘└─────────────┬─────────────┘ -24)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -25)│ RepartitionExec ││ RepartitionExec │ -26)│ -------------------- ││ -------------------- │ -27)│ partition_count(in->out): ││ partition_count(in->out): │ -28)│ 4 -> 4 ││ 4 -> 4 │ -29)│ ││ │ -30)│ partitioning_scheme: ││ partitioning_scheme: │ -31)│ Hash([int_col@0, CAST ││ Hash([int_col@0, │ -32)│ (table1.string_col ││ string_col@1], │ -33)│ AS Utf8View)@4], 4) ││ 4) │ -34)└─────────────┬─────────────┘└─────────────┬─────────────┘ -35)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -36)│ ProjectionExec ││ RepartitionExec │ -37)│ -------------------- ││ -------------------- │ -38)│ CAST(table1.string_col AS ││ partition_count(in->out): │ -39)│ Utf8View): ││ 1 -> 4 │ -40)│ CAST(string_col AS ││ │ -41)│ Utf8View) ││ partitioning_scheme: │ -42)│ ││ RoundRobinBatch(4) │ -43)│ bigint_col: ││ │ -44)│ bigint_col ││ │ -45)│ ││ │ -46)│ date_col: date_col ││ │ -47)│ int_col: int_col ││ │ -48)│ ││ │ -49)│ string_col: ││ │ -50)│ string_col ││ │ -51)└─────────────┬─────────────┘└─────────────┬─────────────┘ -52)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -53)│ RepartitionExec ││ DataSourceExec │ -54)│ -------------------- ││ -------------------- │ -55)│ partition_count(in->out): ││ files: 1 │ -56)│ 1 -> 4 ││ format: parquet │ -57)│ ││ │ -58)│ partitioning_scheme: ││ │ -59)│ RoundRobinBatch(4) ││ │ -60)└─────────────┬─────────────┘└───────────────────────────┘ -61)┌─────────────┴─────────────┐ -62)│ DataSourceExec │ -63)│ -------------------- │ -64)│ files: 1 │ -65)│ format: csv │ -66)└───────────────────────────┘ +04)│ bigint_col: │ +05)│ bigint_col │ +06)│ │ +07)│ date_col: date_col │ +08)│ int_col: int_col │ +09)│ │ +10)│ string_col: │ +11)│ string_col │ +12)└─────────────┬─────────────┘ +13)┌─────────────┴─────────────┐ +14)│ CoalesceBatchesExec │ +15)│ -------------------- │ +16)│ target_batch_size: │ +17)│ 8192 │ +18)└─────────────┬─────────────┘ +19)┌─────────────┴─────────────┐ +20)│ HashJoinExec │ +21)│ -------------------- │ +22)│ join_type: Right │ +23)│ │ +24)│ on: ├──────────────┐ +25)│ (int_col = int_col), │ │ +26)│ (string_col = CAST │ │ +27)│ (table1.string_col AS │ │ +28)│ Utf8View)) │ │ +29)└─────────────┬─────────────┘ │ +30)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +31)│ DataSourceExec ││ ProjectionExec │ +32)│ -------------------- ││ -------------------- │ +33)│ files: 1 ││ CAST(table1.string_col AS │ +34)│ format: parquet ││ Utf8View): │ +35)│ ││ CAST(string_col AS │ +36)│ ││ Utf8View) │ +37)│ ││ │ +38)│ ││ bigint_col: │ +39)│ ││ bigint_col │ +40)│ ││ │ +41)│ ││ date_col: date_col │ +42)│ ││ int_col: int_col │ +43)│ ││ │ +44)│ ││ string_col: │ +45)│ ││ string_col │ +46)└───────────────────────────┘└─────────────┬─────────────┘ +47)-----------------------------┌─────────────┴─────────────┐ +48)-----------------------------│ RepartitionExec │ +49)-----------------------------│ -------------------- │ +50)-----------------------------│ partition_count(in->out): │ +51)-----------------------------│ 1 -> 4 │ +52)-----------------------------│ │ +53)-----------------------------│ partitioning_scheme: │ +54)-----------------------------│ RoundRobinBatch(4) │ +55)-----------------------------└─────────────┬─────────────┘ +56)-----------------------------┌─────────────┴─────────────┐ +57)-----------------------------│ DataSourceExec │ +58)-----------------------------│ -------------------- │ +59)-----------------------------│ files: 1 │ +60)-----------------------------│ format: csv │ +61)-----------------------------└───────────────────────────┘ # Query with nested loop join. query TT @@ -1339,35 +1306,8 @@ physical_plan 10)│ format: csv ││ │ 11)└───────────────────────────┘└─────────────┬─────────────┘ 12)-----------------------------┌─────────────┴─────────────┐ -13)-----------------------------│ AggregateExec │ -14)-----------------------------│ -------------------- │ -15)-----------------------------│ aggr: count(1) │ -16)-----------------------------│ mode: Final │ -17)-----------------------------└─────────────┬─────────────┘ -18)-----------------------------┌─────────────┴─────────────┐ -19)-----------------------------│ CoalescePartitionsExec │ -20)-----------------------------└─────────────┬─────────────┘ -21)-----------------------------┌─────────────┴─────────────┐ -22)-----------------------------│ AggregateExec │ -23)-----------------------------│ -------------------- │ -24)-----------------------------│ aggr: count(1) │ -25)-----------------------------│ mode: Partial │ -26)-----------------------------└─────────────┬─────────────┘ -27)-----------------------------┌─────────────┴─────────────┐ -28)-----------------------------│ RepartitionExec │ -29)-----------------------------│ -------------------- │ -30)-----------------------------│ partition_count(in->out): │ -31)-----------------------------│ 1 -> 4 │ -32)-----------------------------│ │ -33)-----------------------------│ partitioning_scheme: │ -34)-----------------------------│ RoundRobinBatch(4) │ -35)-----------------------------└─────────────┬─────────────┘ -36)-----------------------------┌─────────────┴─────────────┐ -37)-----------------------------│ DataSourceExec │ -38)-----------------------------│ -------------------- │ -39)-----------------------------│ files: 1 │ -40)-----------------------------│ format: parquet │ -41)-----------------------------└───────────────────────────┘ +13)-----------------------------│ PlaceholderRowExec │ +14)-----------------------------└───────────────────────────┘ # Query with cross join. query TT @@ -1378,20 +1318,11 @@ physical_plan 02)│ CrossJoinExec ├──────────────┐ 03)└─────────────┬─────────────┘ │ 04)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -05)│ DataSourceExec ││ RepartitionExec │ +05)│ DataSourceExec ││ DataSourceExec │ 06)│ -------------------- ││ -------------------- │ -07)│ files: 1 ││ partition_count(in->out): │ -08)│ format: csv ││ 1 -> 4 │ -09)│ ││ │ -10)│ ││ partitioning_scheme: │ -11)│ ││ RoundRobinBatch(4) │ -12)└───────────────────────────┘└─────────────┬─────────────┘ -13)-----------------------------┌─────────────┴─────────────┐ -14)-----------------------------│ DataSourceExec │ -15)-----------------------------│ -------------------- │ -16)-----------------------------│ files: 1 │ -17)-----------------------------│ format: parquet │ -18)-----------------------------└───────────────────────────┘ +07)│ files: 1 ││ files: 1 │ +08)│ format: csv ││ format: parquet │ +09)└───────────────────────────┘└───────────────────────────┘ # Query with sort merge join. diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 02a82eff5a3ad..9f39dbbd5ba25 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -216,7 +216,7 @@ datafusion.catalog.location NULL datafusion.catalog.newlines_in_values false datafusion.execution.batch_size 8192 datafusion.execution.coalesce_batches true -datafusion.execution.collect_statistics false +datafusion.execution.collect_statistics true datafusion.execution.enable_recursive_ctes true datafusion.execution.enforce_batch_size_in_joins false datafusion.execution.keep_partition_by_columns false @@ -326,7 +326,7 @@ datafusion.catalog.location NULL Location scanned to load tables for `default` s datafusion.catalog.newlines_in_values false Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. datafusion.execution.batch_size 8192 Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption datafusion.execution.coalesce_batches true When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting -datafusion.execution.collect_statistics false Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to false. +datafusion.execution.collect_statistics true Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. datafusion.execution.enable_recursive_ctes true Should DataFusion support recursive CTEs datafusion.execution.enforce_batch_size_in_joins false Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. datafusion.execution.keep_partition_by_columns false Should DataFusion keep the columns used for partition_by in the output RecordBatches diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 2f8944f462a18..1af14a52e2bc6 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -854,7 +854,7 @@ physical_plan 02)--SortExec: TopK(fetch=1000), expr=[part_key@1 ASC NULLS LAST], preserve_partitioning=[false] 03)----ProjectionExec: expr=[1 as foo, part_key@0 as part_key] 04)------CoalescePartitionsExec: fetch=1 -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-0.parquet:0..794], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:0..794], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..794]]}, projection=[part_key], limit=1, file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..265], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:265..530], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:530..794]]}, projection=[part_key], limit=1, file_type=parquet query I with selection as ( diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt index 1b6ae13fbe771..f4fb0e87c43b2 100644 --- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt +++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt @@ -75,6 +75,31 @@ NULL NULL NULL +query T +select a from t_pushdown where b > 2 ORDER BY a; +---- +baz +foo +NULL +NULL +NULL + +query TT +EXPLAIN select a from t where b > 2 ORDER BY a; +---- +logical_plan +01)Sort: t.a ASC NULLS LAST +02)--Projection: t.a +03)----Filter: t.b > Int32(2) +04)------TableScan: t projection=[a, b], partial_filters=[t.b > Int32(2)] +physical_plan +01)SortPreservingMergeExec: [a@0 ASC NULLS LAST] +02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----CoalesceBatchesExec: target_batch_size=8192 +04)------FilterExec: b@1 > 2, projection=[a@0] +05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a, b], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[] + query TT EXPLAIN select a from t_pushdown where b > 2 ORDER BY a; ---- @@ -88,6 +113,104 @@ physical_plan 02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true] 03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[] +# If we set the setting to `true` it override's the table's setting +statement ok +set datafusion.execution.parquet.pushdown_filters = true; + +query T +select a from t where b > 2 ORDER BY a; +---- +baz +foo +NULL +NULL +NULL + +query T +select a from t_pushdown where b > 2 ORDER BY a; +---- +baz +foo +NULL +NULL +NULL + +query TT +EXPLAIN select a from t where b > 2 ORDER BY a; +---- +logical_plan +01)Sort: t.a ASC NULLS LAST +02)--Projection: t.a +03)----Filter: t.b > Int32(2) +04)------TableScan: t projection=[a, b], partial_filters=[t.b > Int32(2)] +physical_plan +01)SortPreservingMergeExec: [a@0 ASC NULLS LAST] +02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[] + +query TT +EXPLAIN select a from t_pushdown where b > 2 ORDER BY a; +---- +logical_plan +01)Sort: t_pushdown.a ASC NULLS LAST +02)--Projection: t_pushdown.a +03)----Filter: t_pushdown.b > Int32(2) +04)------TableScan: t_pushdown projection=[a, b], partial_filters=[t_pushdown.b > Int32(2)] +physical_plan +01)SortPreservingMergeExec: [a@0 ASC NULLS LAST] +02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[] + +# If we reset the default the table created without pushdown goes back to disabling it +statement ok +set datafusion.execution.parquet.pushdown_filters = false; + +query T +select a from t where b > 2 ORDER BY a; +---- +baz +foo +NULL +NULL +NULL + +query T +select a from t_pushdown where b > 2 ORDER BY a; +---- +baz +foo +NULL +NULL +NULL + +query TT +EXPLAIN select a from t where b > 2 ORDER BY a; +---- +logical_plan +01)Sort: t.a ASC NULLS LAST +02)--Projection: t.a +03)----Filter: t.b > Int32(2) +04)------TableScan: t projection=[a, b], partial_filters=[t.b > Int32(2)] +physical_plan +01)SortPreservingMergeExec: [a@0 ASC NULLS LAST] +02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----CoalesceBatchesExec: target_batch_size=8192 +04)------FilterExec: b@1 > 2, projection=[a@0] +05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2 +06)----------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a, b], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[] + +query TT +EXPLAIN select a from t_pushdown where b > 2 ORDER BY a; +---- +logical_plan +01)Sort: t_pushdown.a ASC NULLS LAST +02)--Projection: t_pushdown.a +03)----Filter: t_pushdown.b > Int32(2) +04)------TableScan: t_pushdown projection=[a, b], partial_filters=[t_pushdown.b > Int32(2)] +physical_plan +01)SortPreservingMergeExec: [a@0 ASC NULLS LAST] +02)--SortExec: expr=[a@0 ASC NULLS LAST], preserve_partitioning=[true] +03)----DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_table/2.parquet]]}, projection=[a], file_type=parquet, predicate=b@1 > 2, pruning_predicate=b_null_count@1 != row_count@2 AND b_max@0 > 2, required_guarantees=[] # When filter pushdown *is* enabled, ParquetExec can filter exactly, # not just metadata, so we expect to see no FilterExec @@ -212,8 +335,7 @@ logical_plan physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 02)--FilterExec: val@0 != part@1 -03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=3 -04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_part_test/part=a/file.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_part_test/part=b/file.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_part_test/part=c/file.parquet]]}, projection=[val, part], file_type=parquet +03)----DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_part_test/part=a/file.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_part_test/part=b/file.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_filter_pushdown/parquet_part_test/part=c/file.parquet]]}, projection=[val, part], file_type=parquet # If we reference only a partition column it gets evaluted during the listing phase query TT diff --git a/datafusion/sqllogictest/test_files/parquet_statistics.slt b/datafusion/sqllogictest/test_files/parquet_statistics.slt index c707b9f5bbd54..efbe69bd856c2 100644 --- a/datafusion/sqllogictest/test_files/parquet_statistics.slt +++ b/datafusion/sqllogictest/test_files/parquet_statistics.slt @@ -46,7 +46,7 @@ statement ok set datafusion.explain.show_statistics = true; ###### -# By default, the statistics are not gathered +# By default, the statistics are gathered ###### # Recreate the table to pick up the current setting @@ -59,18 +59,18 @@ query TT EXPLAIN SELECT * FROM test_table WHERE column1 = 1; ---- physical_plan -01)CoalesceBatchesExec: target_batch_size=8192, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] -02)--FilterExec: column1@0 = 1, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)))]] -03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] +01)CoalesceBatchesExec: target_batch_size=8192, statistics=[Rows=Inexact(2), Bytes=Inexact(44), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0))]] +02)--FilterExec: column1@0 = 1, statistics=[Rows=Inexact(2), Bytes=Inexact(44), [(Col[0]: Min=Exact(Int64(1)) Max=Exact(Int64(1)) Null=Inexact(0))]] +03)----RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=2, statistics=[Rows=Inexact(5), Bytes=Inexact(173), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]] 04)------DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet_statistics/test_table/1.parquet]]}, projection=[column1], file_type=parquet, predicate=column1@0 = 1, pruning_predicate=column1_null_count@2 != row_count@3 AND column1_min@0 <= 1 AND 1 <= column1_max@1, required_guarantees=[column1 in (1)] -05), statistics=[Rows=Absent, Bytes=Absent, [(Col[0]:)]] +05), statistics=[Rows=Inexact(5), Bytes=Inexact(173), [(Col[0]: Min=Inexact(Int64(1)) Max=Inexact(Int64(4)) Null=Inexact(0))]] # cleanup statement ok DROP TABLE test_table; ###### -# When the setting is true, the statistics are gathered +# When the setting is true, statistics are gathered ###### statement ok diff --git a/datafusion/sqllogictest/test_files/repartition.slt b/datafusion/sqllogictest/test_files/repartition.slt index 70666346e2cab..29d20d10b6715 100644 --- a/datafusion/sqllogictest/test_files/repartition.slt +++ b/datafusion/sqllogictest/test_files/repartition.slt @@ -46,8 +46,8 @@ physical_plan 01)AggregateExec: mode=FinalPartitioned, gby=[column1@0 as column1], aggr=[sum(parquet_table.column2)] 02)--CoalesceBatchesExec: target_batch_size=8192 03)----RepartitionExec: partitioning=Hash([column1@0], 4), input_partitions=4 -04)------AggregateExec: mode=Partial, gby=[column1@0 as column1], aggr=[sum(parquet_table.column2)] -05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +05)--------AggregateExec: mode=Partial, gby=[column1@0 as column1], aggr=[sum(parquet_table.column2)] 06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/repartition/parquet_table/2.parquet]]}, projection=[column1, column2], file_type=parquet # disable round robin repartitioning diff --git a/dev/changelog/48.0.1.md b/dev/changelog/48.0.1.md new file mode 100644 index 0000000000000..dcd4cc9c15479 --- /dev/null +++ b/dev/changelog/48.0.1.md @@ -0,0 +1,41 @@ + + +# Apache DataFusion 48.0.1 Changelog + +This release consists of 3 commits from 2 contributors. See credits at the end of this changelog for more information. + +See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgrading.html) for information on how to upgrade from previous versions. + +**Bug Fixes:** + +- [branch-48] Set the default value of datafusion.execution.collect_statistics to true #16447 [#16659](https://github.com/apache/datafusion/pull/16659) (blaginin) +- [branch-48] Fix parquet filter_pushdown: respect parquet filter pushdown config i… [#16656](https://github.com/apache/datafusion/pull/16656) (alamb) +- [branch-48] fix: column indices in FFI partition evaluator (#16480) [#16657](https://github.com/apache/datafusion/pull/16657) (alamb) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 2 Andrew Lamb + 1 Dmitrii Blaginin +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/dev/release/generate-changelog.py b/dev/release/generate-changelog.py index 1349416bcaa59..830d329f73c4f 100755 --- a/dev/release/generate-changelog.py +++ b/dev/release/generate-changelog.py @@ -124,6 +124,9 @@ def generate_changelog(repo, repo_name, tag1, tag2, version): print(f"This release consists of {commit_count} commits from {contributor_count} contributors. " f"See credits at the end of this changelog for more information.\n") + print("See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgrading.html) " + "for information on how to upgrade from previous versions.\n") + print_pulls(repo_name, "Breaking changes", breaking) print_pulls(repo_name, "Performance related", performance) print_pulls(repo_name, "Implemented enhancements", enhancements) diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 6d65f54e228d1..05cc36651a1a8 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -47,7 +47,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | | datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | | datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | -| datafusion.execution.collect_statistics | false | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to false. | +| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. | | datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | | datafusion.execution.time_zone | +00:00 | The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour | | datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | @@ -70,7 +70,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 48.0.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 48.0.1 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | diff --git a/docs/source/user-guide/sql/ddl.md b/docs/source/user-guide/sql/ddl.md index ff8fa9bac0017..1d971594ada92 100644 --- a/docs/source/user-guide/sql/ddl.md +++ b/docs/source/user-guide/sql/ddl.md @@ -95,14 +95,14 @@ LOCATION '/mnt/nyctaxi/tripdata.parquet'; :::{note} Statistics -: By default, when a table is created, DataFusion will _NOT_ read the files +: By default, when a table is created, DataFusion will read the files to gather statistics, which can be expensive but can accelerate subsequent -queries substantially. If you want to gather statistics +queries substantially. If you don't want to gather statistics when creating a table, set the `datafusion.execution.collect_statistics` -configuration option to `true` before creating the table. For example: +configuration option to `false` before creating the table. For example: ```sql -SET datafusion.execution.collect_statistics = true; +SET datafusion.execution.collect_statistics = false; ``` See the [config settings docs](../configs.md) for more details. From 1ded6ef3db287adcee80e3ee4fed16d08b75655e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 19 Jul 2025 07:14:11 -0400 Subject: [PATCH 074/177] [branch-49] Update version to `49.0.0`, add changelog (#16822) * Update version to 49.0.0 * Add 49.0.0 changelog * prettier * Update changelog and config --- Cargo.lock | 80 +++---- Cargo.toml | 72 +++--- dev/changelog/49.0.0.md | 373 ++++++++++++++++++++++++++++++ docs/source/user-guide/configs.md | 2 +- 4 files changed, 450 insertions(+), 77 deletions(-) create mode 100644 dev/changelog/49.0.0.md diff --git a/Cargo.lock b/Cargo.lock index 2c98c5ce20ad2..a088005a0f197 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1818,7 +1818,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "arrow-ipc", @@ -1890,7 +1890,7 @@ dependencies = [ [[package]] name = "datafusion-benchmarks" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "datafusion", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -1938,7 +1938,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -1959,7 +1959,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "assert_cmd", @@ -1991,7 +1991,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "48.0.0" +version = "49.0.0" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2019,7 +2019,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "48.0.0" +version = "49.0.0" dependencies = [ "futures", "log", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-compression", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "48.0.0" +version = "49.0.0" dependencies = [ "apache-avro", "arrow", @@ -2088,7 +2088,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2111,7 +2111,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2134,7 +2134,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2166,11 +2166,11 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "48.0.0" +version = "49.0.0" [[package]] name = "datafusion-examples" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "arrow-flight", @@ -2201,7 +2201,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "chrono", @@ -2220,7 +2220,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "datafusion-common", @@ -2254,7 +2254,7 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "48.0.0" +version = "49.0.0" dependencies = [ "abi_stable", "arrow", @@ -2275,7 +2275,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "arrow-buffer", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "48.0.0" +version = "49.0.0" dependencies = [ "ahash 0.8.12", "arrow", @@ -2325,7 +2325,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "48.0.0" +version = "49.0.0" dependencies = [ "ahash 0.8.12", "arrow", @@ -2338,7 +2338,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "arrow-ord", @@ -2360,7 +2360,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "datafusion-common", @@ -2390,7 +2390,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "48.0.0" +version = "49.0.0" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2398,7 +2398,7 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "48.0.0" +version = "49.0.0" dependencies = [ "datafusion-expr", "quote", @@ -2407,7 +2407,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2434,7 +2434,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "48.0.0" +version = "49.0.0" dependencies = [ "ahash 0.8.12", "arrow", @@ -2459,7 +2459,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "48.0.0" +version = "49.0.0" dependencies = [ "ahash 0.8.12", "arrow", @@ -2471,7 +2471,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "datafusion-common", @@ -2492,7 +2492,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "48.0.0" +version = "49.0.0" dependencies = [ "ahash 0.8.12", "arrow", @@ -2528,7 +2528,7 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "chrono", @@ -2550,7 +2550,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "datafusion-common", @@ -2563,7 +2563,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "arrow-schema", @@ -2582,7 +2582,7 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "criterion", @@ -2620,7 +2620,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "bigdecimal", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "48.0.0" +version = "49.0.0" dependencies = [ "arrow", "async-trait", @@ -2677,7 +2677,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "48.0.0" +version = "49.0.0" dependencies = [ "async-recursion", "async-trait", @@ -2697,7 +2697,7 @@ dependencies = [ [[package]] name = "datafusion-wasmtest" -version = "48.0.0" +version = "49.0.0" dependencies = [ "chrono", "console_error_panic_hook", diff --git a/Cargo.toml b/Cargo.toml index 7f5c79ae3a572..11cd3c637a971 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ repository = "https://github.com/apache/datafusion" # Define Minimum Supported Rust Version (MSRV) rust-version = "1.85.1" # Define DataFusion version -version = "48.0.0" +version = "49.0.0" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can @@ -110,41 +110,41 @@ chrono = { version = "0.4.41", default-features = false } criterion = "0.5.1" ctor = "0.4.0" dashmap = "6.0.1" -datafusion = { path = "datafusion/core", version = "48.0.0", default-features = false } -datafusion-catalog = { path = "datafusion/catalog", version = "48.0.0" } -datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "48.0.0" } -datafusion-common = { path = "datafusion/common", version = "48.0.0", default-features = false } -datafusion-common-runtime = { path = "datafusion/common-runtime", version = "48.0.0" } -datafusion-datasource = { path = "datafusion/datasource", version = "48.0.0", default-features = false } -datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "48.0.0", default-features = false } -datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "48.0.0", default-features = false } -datafusion-datasource-json = { path = "datafusion/datasource-json", version = "48.0.0", default-features = false } -datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "48.0.0", default-features = false } -datafusion-doc = { path = "datafusion/doc", version = "48.0.0" } -datafusion-execution = { path = "datafusion/execution", version = "48.0.0" } -datafusion-expr = { path = "datafusion/expr", version = "48.0.0" } -datafusion-expr-common = { path = "datafusion/expr-common", version = "48.0.0" } -datafusion-ffi = { path = "datafusion/ffi", version = "48.0.0" } -datafusion-functions = { path = "datafusion/functions", version = "48.0.0" } -datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "48.0.0" } -datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "48.0.0" } -datafusion-functions-nested = { path = "datafusion/functions-nested", version = "48.0.0" } -datafusion-functions-table = { path = "datafusion/functions-table", version = "48.0.0" } -datafusion-functions-window = { path = "datafusion/functions-window", version = "48.0.0" } -datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "48.0.0" } -datafusion-macros = { path = "datafusion/macros", version = "48.0.0" } -datafusion-optimizer = { path = "datafusion/optimizer", version = "48.0.0", default-features = false } -datafusion-physical-expr = { path = "datafusion/physical-expr", version = "48.0.0", default-features = false } -datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "48.0.0", default-features = false } -datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "48.0.0" } -datafusion-physical-plan = { path = "datafusion/physical-plan", version = "48.0.0" } -datafusion-proto = { path = "datafusion/proto", version = "48.0.0" } -datafusion-proto-common = { path = "datafusion/proto-common", version = "48.0.0" } -datafusion-pruning = { path = "datafusion/pruning", version = "48.0.0" } -datafusion-session = { path = "datafusion/session", version = "48.0.0" } -datafusion-spark = { path = "datafusion/spark", version = "48.0.0" } -datafusion-sql = { path = "datafusion/sql", version = "48.0.0" } -datafusion-substrait = { path = "datafusion/substrait", version = "48.0.0" } +datafusion = { path = "datafusion/core", version = "49.0.0", default-features = false } +datafusion-catalog = { path = "datafusion/catalog", version = "49.0.0" } +datafusion-catalog-listing = { path = "datafusion/catalog-listing", version = "49.0.0" } +datafusion-common = { path = "datafusion/common", version = "49.0.0", default-features = false } +datafusion-common-runtime = { path = "datafusion/common-runtime", version = "49.0.0" } +datafusion-datasource = { path = "datafusion/datasource", version = "49.0.0", default-features = false } +datafusion-datasource-avro = { path = "datafusion/datasource-avro", version = "49.0.0", default-features = false } +datafusion-datasource-csv = { path = "datafusion/datasource-csv", version = "49.0.0", default-features = false } +datafusion-datasource-json = { path = "datafusion/datasource-json", version = "49.0.0", default-features = false } +datafusion-datasource-parquet = { path = "datafusion/datasource-parquet", version = "49.0.0", default-features = false } +datafusion-doc = { path = "datafusion/doc", version = "49.0.0" } +datafusion-execution = { path = "datafusion/execution", version = "49.0.0" } +datafusion-expr = { path = "datafusion/expr", version = "49.0.0" } +datafusion-expr-common = { path = "datafusion/expr-common", version = "49.0.0" } +datafusion-ffi = { path = "datafusion/ffi", version = "49.0.0" } +datafusion-functions = { path = "datafusion/functions", version = "49.0.0" } +datafusion-functions-aggregate = { path = "datafusion/functions-aggregate", version = "49.0.0" } +datafusion-functions-aggregate-common = { path = "datafusion/functions-aggregate-common", version = "49.0.0" } +datafusion-functions-nested = { path = "datafusion/functions-nested", version = "49.0.0" } +datafusion-functions-table = { path = "datafusion/functions-table", version = "49.0.0" } +datafusion-functions-window = { path = "datafusion/functions-window", version = "49.0.0" } +datafusion-functions-window-common = { path = "datafusion/functions-window-common", version = "49.0.0" } +datafusion-macros = { path = "datafusion/macros", version = "49.0.0" } +datafusion-optimizer = { path = "datafusion/optimizer", version = "49.0.0", default-features = false } +datafusion-physical-expr = { path = "datafusion/physical-expr", version = "49.0.0", default-features = false } +datafusion-physical-expr-common = { path = "datafusion/physical-expr-common", version = "49.0.0", default-features = false } +datafusion-physical-optimizer = { path = "datafusion/physical-optimizer", version = "49.0.0" } +datafusion-physical-plan = { path = "datafusion/physical-plan", version = "49.0.0" } +datafusion-proto = { path = "datafusion/proto", version = "49.0.0" } +datafusion-proto-common = { path = "datafusion/proto-common", version = "49.0.0" } +datafusion-pruning = { path = "datafusion/pruning", version = "49.0.0" } +datafusion-session = { path = "datafusion/session", version = "49.0.0" } +datafusion-spark = { path = "datafusion/spark", version = "49.0.0" } +datafusion-sql = { path = "datafusion/sql", version = "49.0.0" } +datafusion-substrait = { path = "datafusion/substrait", version = "49.0.0" } doc-comment = "0.3" env_logger = "0.11" futures = "0.3" diff --git a/dev/changelog/49.0.0.md b/dev/changelog/49.0.0.md new file mode 100644 index 0000000000000..c30bfaf3ea656 --- /dev/null +++ b/dev/changelog/49.0.0.md @@ -0,0 +1,373 @@ + + +# Apache DataFusion 49.0.0 Changelog + +This release consists of 251 commits from 71 contributors. See credits at the end of this changelog for more information. + +See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgrading.html) for information on how to upgrade from previous versions. + +**Breaking changes:** + +- feat: add metadata to literal expressions [#16170](https://github.com/apache/datafusion/pull/16170) (timsaucer) +- [MAJOR] Equivalence System Overhaul [#16217](https://github.com/apache/datafusion/pull/16217) (ozankabak) +- remove unused methods in SortExec [#16457](https://github.com/apache/datafusion/pull/16457) (adriangb) +- Move Pruning Logic to a Dedicated datafusion-pruning Crate for Improved Modularity [#16549](https://github.com/apache/datafusion/pull/16549) (kosiew) +- Fix type of ExecutionOptions::time_zone [#16569](https://github.com/apache/datafusion/pull/16569) (findepi) +- Convert Option> to Vec [#16615](https://github.com/apache/datafusion/pull/16615) (ViggoC) +- Refactor error handling to use boxed errors for DataFusionError variants [#16672](https://github.com/apache/datafusion/pull/16672) (kosiew) +- Reuse Rows allocation in RowCursorStream [#16647](https://github.com/apache/datafusion/pull/16647) (Dandandan) +- refactor: shrink `SchemaError` [#16653](https://github.com/apache/datafusion/pull/16653) (crepererum) +- Remove unused AggregateUDF struct [#16683](https://github.com/apache/datafusion/pull/16683) (ViggoC) + +**Performance related:** + +- Add late pruning of Parquet files based on file level statistics [#16014](https://github.com/apache/datafusion/pull/16014) (adriangb) +- Add fast paths for try_process_unnest [#16389](https://github.com/apache/datafusion/pull/16389) (simonvandel) +- Set the default value of `datafusion.execution.collect_statistics` to `true` [#16447](https://github.com/apache/datafusion/pull/16447) (AdamGS) +- Perf: Optimize CursorValues compare performance for StringViewArray (1.4X faster for sort-tpch Q11) [#16509](https://github.com/apache/datafusion/pull/16509) (zhuqi-lucas) +- Simplify predicates in `PushDownFilter` optimizer rule [#16362](https://github.com/apache/datafusion/pull/16362) (xudong963) +- optimize `ScalarValue::to_array_of_size` for structural types [#16706](https://github.com/apache/datafusion/pull/16706) (ding-young) +- Refactor filter pushdown APIs to enable joins to pass through filters [#16732](https://github.com/apache/datafusion/pull/16732) (adriangb) +- perf: Optimize hash joins with an empty build side [#16716](https://github.com/apache/datafusion/pull/16716) (nuno-faria) +- Per file filter evaluation [#15057](https://github.com/apache/datafusion/pull/15057) (adriangb) + +**Implemented enhancements:** + +- feat: Support defining custom MetricValues in PhysicalPlans [#16195](https://github.com/apache/datafusion/pull/16195) (sfluor) +- feat: Allow cancelling of grouping operations which are CPU bound [#16196](https://github.com/apache/datafusion/pull/16196) (zhuqi-lucas) +- feat: support FixedSizeList for array_has [#16333](https://github.com/apache/datafusion/pull/16333) (chenkovsky) +- feat: Support tpch and tpch10 benchmark for csv format [#16373](https://github.com/apache/datafusion/pull/16373) (zhuqi-lucas) +- feat: Support RightMark join for NestedLoop and Hash join [#16083](https://github.com/apache/datafusion/pull/16083) (jonathanc-n) +- feat: mapping sql Char/Text/String default to Utf8View [#16290](https://github.com/apache/datafusion/pull/16290) (zhuqi-lucas) +- feat: support fixed size list for array reverse [#16423](https://github.com/apache/datafusion/pull/16423) (chenkovsky) +- feat: add SchemaProvider::table_type(table_name: &str) [#16401](https://github.com/apache/datafusion/pull/16401) (epgif) +- feat: derive `Debug` and `Clone` for `ScalarFunctionArgs` [#16471](https://github.com/apache/datafusion/pull/16471) (crepererum) +- feat: support `map_entries` builtin function [#16557](https://github.com/apache/datafusion/pull/16557) (comphead) +- feat: add `array_min` scalar function and associated tests [#16574](https://github.com/apache/datafusion/pull/16574) (dharanad) +- feat: Finalize support for `RightMark` join + `Mark` join swap [#16488](https://github.com/apache/datafusion/pull/16488) (jonathanc-n) +- feat: Parquet modular encryption [#16351](https://github.com/apache/datafusion/pull/16351) (corwinjoy) +- feat: Support `u32` indices for `HashJoinExec` [#16434](https://github.com/apache/datafusion/pull/16434) (jonathanc-n) +- feat: expose intersect distinct/except distinct in dataframe api [#16578](https://github.com/apache/datafusion/pull/16578) (chenkovsky) +- feat: Add a configuration to make parquet encryption optional [#16649](https://github.com/apache/datafusion/pull/16649) (corwinjoy) + +**Fixed bugs:** + +- fix: preserve null_equals_null flag in eliminate_cross_join rule [#16356](https://github.com/apache/datafusion/pull/16356) (waynexia) +- fix: Fix SparkSha2 to be compliant with Spark response and add support for Int32 [#16350](https://github.com/apache/datafusion/pull/16350) (rishvin) +- fix: Fixed error handling for `generate_series/range` [#16391](https://github.com/apache/datafusion/pull/16391) (jonathanc-n) +- fix: Enable WASM compilation by making sqlparser's recursive-protection optional [#16418](https://github.com/apache/datafusion/pull/16418) (jonmmease) +- fix: create file for empty stream [#16342](https://github.com/apache/datafusion/pull/16342) (chenkovsky) +- fix: document and fix macro hygiene for `config_field!` [#16473](https://github.com/apache/datafusion/pull/16473) (crepererum) +- fix: make `with_new_state` a trait method for `ExecutionPlan` [#16469](https://github.com/apache/datafusion/pull/16469) (geoffreyclaude) +- fix: column indices in FFI partition evaluator [#16480](https://github.com/apache/datafusion/pull/16480) (timsaucer) +- fix: support within_group [#16538](https://github.com/apache/datafusion/pull/16538) (chenkovsky) +- fix: disallow specify both order_by and within_group [#16606](https://github.com/apache/datafusion/pull/16606) (watchingthewheelsgo) +- fix: format within_group error message [#16613](https://github.com/apache/datafusion/pull/16613) (watchingthewheelsgo) +- fix: reserved keywords in qualified column names [#16584](https://github.com/apache/datafusion/pull/16584) (crepererum) +- fix: support scalar function nested in get_field in Unparser [#16610](https://github.com/apache/datafusion/pull/16610) (chenkovsky) +- fix: sqllogictest runner label condition mismatch [#16633](https://github.com/apache/datafusion/pull/16633) (lliangyu-lin) +- fix: port arrow inline fast key fix to datafusion [#16698](https://github.com/apache/datafusion/pull/16698) (zhuqi-lucas) +- fix: try to lower plain reserved functions to columns as well [#16669](https://github.com/apache/datafusion/pull/16669) (crepererum) +- fix: Fix CI failing due to #16686 [#16718](https://github.com/apache/datafusion/pull/16718) (jonathanc-n) +- fix: return NULL if any of the param to make_date is NULL [#16759](https://github.com/apache/datafusion/pull/16759) (feniljain) +- fix: add `order_requirement` & `dist_requirement` to `OutputRequirementExec` display [#16726](https://github.com/apache/datafusion/pull/16726) (Loaki07) +- fix: support nullable columns in pre-sorted data sources [#16783](https://github.com/apache/datafusion/pull/16783) (crepererum) + +**Documentation updates:** + +- Minor: Add upgrade guide for `Expr::WindowFunction` [#16313](https://github.com/apache/datafusion/pull/16313) (alamb) +- Fix `array_position` on empty list [#16292](https://github.com/apache/datafusion/pull/16292) (Blizzara) +- Fix: mark "Spilling (to disk) Joins" as supported in features [#16343](https://github.com/apache/datafusion/pull/16343) (kosiew) +- Fix cp_solver doc formatting [#16352](https://github.com/apache/datafusion/pull/16352) (xudong963) +- docs: Expand `MemoryPool` docs with related structs [#16289](https://github.com/apache/datafusion/pull/16289) (2010YOUY01) +- Support datafusion-cli access to public S3 buckets that do not require authentication [#16300](https://github.com/apache/datafusion/pull/16300) (alamb) +- Document Table Constraint Enforcement Behavior in Custom Table Providers Guide [#16340](https://github.com/apache/datafusion/pull/16340) (kosiew) +- doc: Add SQL examples for SEMI + ANTI Joins [#16316](https://github.com/apache/datafusion/pull/16316) (jonathanc-n) +- [datafusion-spark] Example of using Spark compatible function library [#16384](https://github.com/apache/datafusion/pull/16384) (alamb) +- Add note in upgrade guide about changes to `Expr::Scalar` in 48.0.0 [#16360](https://github.com/apache/datafusion/pull/16360) (alamb) +- Update PMC management instructions to follow new ASF process [#16417](https://github.com/apache/datafusion/pull/16417) (alamb) +- Add design process section to the docs [#16397](https://github.com/apache/datafusion/pull/16397) (alamb) +- Unify Metadata Handing: use `FieldMetadata` in `Expr::Alias` and `ExprSchemable` [#16320](https://github.com/apache/datafusion/pull/16320) (alamb) +- TopK dynamic filter pushdown attempt 2 [#15770](https://github.com/apache/datafusion/pull/15770) (adriangb) +- Update Roadmap documentation [#16399](https://github.com/apache/datafusion/pull/16399) (alamb) +- doc: Add comments to clarify algorithm for `MarkJoin`s [#16436](https://github.com/apache/datafusion/pull/16436) (jonathanc-n) +- Add compression option to SpillManager [#16268](https://github.com/apache/datafusion/pull/16268) (ding-young) +- Redirect user defined function webpage [#16475](https://github.com/apache/datafusion/pull/16475) (alamb) +- Use Tokio's task budget consistently, better APIs to support task cancellation [#16398](https://github.com/apache/datafusion/pull/16398) (pepijnve) +- doc: upgrade guide for new compression option for spill files [#16472](https://github.com/apache/datafusion/pull/16472) (2010YOUY01) +- Introduce Async User Defined Functions [#14837](https://github.com/apache/datafusion/pull/14837) (goldmedal) +- Minor: Add more links to cooperative / scheduling docs [#16484](https://github.com/apache/datafusion/pull/16484) (alamb) +- doc: Document DESCRIBE comman in ddl.md [#16524](https://github.com/apache/datafusion/pull/16524) (krikera) +- Add more doc for physical filter pushdown [#16504](https://github.com/apache/datafusion/pull/16504) (xudong963) +- chore: fix CI failures on `ddl.md` [#16526](https://github.com/apache/datafusion/pull/16526) (comphead) +- Add some comments about adding new dependencies in datafusion-sql [#16543](https://github.com/apache/datafusion/pull/16543) (alamb) +- Add note for planning release in Upgrade Guides [#16534](https://github.com/apache/datafusion/pull/16534) (xudong963) +- Consolidate configuration sections in docs [#16544](https://github.com/apache/datafusion/pull/16544) (alamb) +- Minor: add clearer link to the main website from intro paragraph. [#16556](https://github.com/apache/datafusion/pull/16556) (alamb) +- Simplify AsyncScalarUdfImpl so it extends ScalarUdfImpl [#16523](https://github.com/apache/datafusion/pull/16523) (alamb) +- docs: Minor grammatical fixes for the scalar UDF docs [#16618](https://github.com/apache/datafusion/pull/16618) (ianthetechie) +- Implementation for regex_instr [#15928](https://github.com/apache/datafusion/pull/15928) (nirnayroy) +- Update Upgrade Guide for 48.0.1 [#16699](https://github.com/apache/datafusion/pull/16699) (alamb) +- ensure MemTable has at least one partition [#16754](https://github.com/apache/datafusion/pull/16754) (waynexia) +- Restore custom SchemaAdapter functionality for Parquet [#16791](https://github.com/apache/datafusion/pull/16791) (adriangb) +- Update `upgrading.md` for new unified config for sql string mapping to utf8view [#16809](https://github.com/apache/datafusion/pull/16809) (zhuqi-lucas) +- docs: Remove reference to forthcoming example (#16817) [#16818](https://github.com/apache/datafusion/pull/16818) (m09526) + +**Other:** + +- chore(deps): bump sqllogictest from 0.28.2 to 0.28.3 [#16286](https://github.com/apache/datafusion/pull/16286) (dependabot[bot]) +- chore(deps-dev): bump webpack-dev-server from 4.15.1 to 5.2.1 in /datafusion/wasmtest/datafusion-wasm-app [#16253](https://github.com/apache/datafusion/pull/16253) (dependabot[bot]) +- Improve DataFusion subcrate readme files [#16263](https://github.com/apache/datafusion/pull/16263) (alamb) +- Fix intermittent SQL logic test failure in limit.slt by adding ORDER BY clause [#16257](https://github.com/apache/datafusion/pull/16257) (kosiew) +- Extend benchmark comparison script with more detailed statistics [#16262](https://github.com/apache/datafusion/pull/16262) (pepijnve) +- chore(deps): bump flate2 from 1.1.1 to 1.1.2 [#16338](https://github.com/apache/datafusion/pull/16338) (dependabot[bot]) +- chore(deps): bump petgraph from 0.8.1 to 0.8.2 [#16337](https://github.com/apache/datafusion/pull/16337) (dependabot[bot]) +- chore(deps): bump substrait from 0.56.0 to 0.57.0 [#16143](https://github.com/apache/datafusion/pull/16143) (dependabot[bot]) +- Add test for ordering of predicate pushdown into parquet [#16169](https://github.com/apache/datafusion/pull/16169) (adriangb) +- Fix distinct count for DictionaryArray to correctly account for nulls in values array [#16258](https://github.com/apache/datafusion/pull/16258) (kosiew) +- Fix inconsistent schema projection in ListingTable even when schema is specified [#16305](https://github.com/apache/datafusion/pull/16305) (kosiew) +- tpch: move reading of SQL queries out of timed span. [#16357](https://github.com/apache/datafusion/pull/16357) (pepijnve) +- chore(deps): bump clap from 4.5.39 to 4.5.40 [#16354](https://github.com/apache/datafusion/pull/16354) (dependabot[bot]) +- chore(deps): bump syn from 2.0.101 to 2.0.102 [#16355](https://github.com/apache/datafusion/pull/16355) (dependabot[bot]) +- Encapsulate metadata for literals on to a `FieldMetadata` structure [#16317](https://github.com/apache/datafusion/pull/16317) (alamb) +- Add support `UInt64` and other integer data types for `to_hex` [#16335](https://github.com/apache/datafusion/pull/16335) (tlm365) +- Document `copy_array_data` function with example [#16361](https://github.com/apache/datafusion/pull/16361) (alamb) +- Fix array_agg memory over use [#16346](https://github.com/apache/datafusion/pull/16346) (gabotechs) +- Update publish command [#16377](https://github.com/apache/datafusion/pull/16377) (xudong963) +- Add more context to error message for datafusion-cli config failure [#16379](https://github.com/apache/datafusion/pull/16379) (alamb) +- Fix: datafusion-sqllogictest 48.0.0 can't be published [#16376](https://github.com/apache/datafusion/pull/16376) (xudong963) +- bug: remove busy-wait while sort is ongoing [#16322](https://github.com/apache/datafusion/pull/16322) (pepijnve) +- chore: refactor Substrait consumer's "rename_field" and implement the rest of types [#16345](https://github.com/apache/datafusion/pull/16345) (Blizzara) +- chore(deps): bump object_store from 0.12.1 to 0.12.2 [#16368](https://github.com/apache/datafusion/pull/16368) (dependabot[bot]) +- Disable `datafusion-cli` tests for hash_collision tests, fix extended CI [#16382](https://github.com/apache/datafusion/pull/16382) (alamb) +- Fix array_concat with NULL arrays [#16348](https://github.com/apache/datafusion/pull/16348) (alexanderbianchi) +- Minor: add testing case for add YieldStreamExec and polish docs [#16369](https://github.com/apache/datafusion/pull/16369) (zhuqi-lucas) +- chore(deps): bump aws-config from 1.6.3 to 1.8.0 [#16394](https://github.com/apache/datafusion/pull/16394) (dependabot[bot]) +- fix typo in test file name [#16403](https://github.com/apache/datafusion/pull/16403) (adriangb) +- Add topk_tpch benchmark [#16410](https://github.com/apache/datafusion/pull/16410) (Dandandan) +- Reduce some cloning [#16404](https://github.com/apache/datafusion/pull/16404) (simonvandel) +- chore(deps): bump syn from 2.0.102 to 2.0.103 [#16393](https://github.com/apache/datafusion/pull/16393) (dependabot[bot]) +- Simplify expressions passed to table functions [#16388](https://github.com/apache/datafusion/pull/16388) (simonvandel) +- Minor: Clean-up `bench.sh` usage message [#16416](https://github.com/apache/datafusion/pull/16416) (2010YOUY01) +- chore(deps): bump rust_decimal from 1.37.1 to 1.37.2 [#16422](https://github.com/apache/datafusion/pull/16422) (dependabot[bot]) +- Migrate core test to insta, part1 [#16324](https://github.com/apache/datafusion/pull/16324) (Chen-Yuan-Lai) +- chore(deps): bump mimalloc from 0.1.46 to 0.1.47 [#16426](https://github.com/apache/datafusion/pull/16426) (dependabot[bot]) +- chore(deps): bump libc from 0.2.172 to 0.2.173 [#16421](https://github.com/apache/datafusion/pull/16421) (dependabot[bot]) +- Use dedicated NullEquality enum instead of null_equals_null boolean [#16419](https://github.com/apache/datafusion/pull/16419) (tobixdev) +- chore: generate basic spark function tests [#16409](https://github.com/apache/datafusion/pull/16409) (shehabgamin) +- Fix CI Failure: replace false with NullEqualsNothing [#16437](https://github.com/apache/datafusion/pull/16437) (ding-young) +- chore(deps): bump bzip2 from 0.5.2 to 0.6.0 [#16441](https://github.com/apache/datafusion/pull/16441) (dependabot[bot]) +- chore(deps): bump libc from 0.2.173 to 0.2.174 [#16440](https://github.com/apache/datafusion/pull/16440) (dependabot[bot]) +- Remove redundant license-header-check CI job [#16451](https://github.com/apache/datafusion/pull/16451) (alamb) +- Remove unused feature in `physical-plan` and fix compilation error in benchmark [#16449](https://github.com/apache/datafusion/pull/16449) (AdamGS) +- Temporarily fix bug in dynamic top-k optimization [#16465](https://github.com/apache/datafusion/pull/16465) (AdamGS) +- Ignore `sort_query_fuzzer_runner` [#16462](https://github.com/apache/datafusion/pull/16462) (blaginin) +- Revert "Ignore `sort_query_fuzzer_runner` (#16462)" [#16470](https://github.com/apache/datafusion/pull/16470) (2010YOUY01) +- Reapply "Ignore `sort_query_fuzzer_runner` (#16462)" (#16470) [#16485](https://github.com/apache/datafusion/pull/16485) (alamb) +- Fix constant window for evaluate stateful [#16430](https://github.com/apache/datafusion/pull/16430) (suibianwanwank) +- Use UDTF name in logical plan table scan [#16468](https://github.com/apache/datafusion/pull/16468) (Jeadie) +- refactor reassign_predicate_columns to accept an &Schema instead of &Arc [#16499](https://github.com/apache/datafusion/pull/16499) (adriangb) +- re-enable `sort_query_fuzzer_runner` [#16491](https://github.com/apache/datafusion/pull/16491) (adriangb) +- Example for using a separate threadpool for CPU bound work (try 3) [#16331](https://github.com/apache/datafusion/pull/16331) (alamb) +- chore(deps): bump syn from 2.0.103 to 2.0.104 [#16507](https://github.com/apache/datafusion/pull/16507) (dependabot[bot]) +- use 'lit' as the field name for literal values [#16498](https://github.com/apache/datafusion/pull/16498) (adriangb) +- [datafusion-spark] Implement `factorical` function [#16125](https://github.com/apache/datafusion/pull/16125) (tlm365) +- Add DESC alias for DESCRIBE command. [#16514](https://github.com/apache/datafusion/pull/16514) (lucqui) +- Split clickbench query set into one file per query [#16476](https://github.com/apache/datafusion/pull/16476) (pepijnve) +- Support query filter on all benchmarks [#16477](https://github.com/apache/datafusion/pull/16477) (pepijnve) +- `TableProvider` to skip files in the folder which non relevant to selected reader [#16487](https://github.com/apache/datafusion/pull/16487) (comphead) +- Reuse `BaselineMetrics` in `UnnestMetrics` [#16497](https://github.com/apache/datafusion/pull/16497) (hendrikmakait) +- Fix array_has to return false for empty arrays instead of null [#16529](https://github.com/apache/datafusion/pull/16529) (kosiew) +- Minor: Add documentation to `AggregateWindowExpr::get_result_column` [#16479](https://github.com/apache/datafusion/pull/16479) (alamb) +- Fix WindowFrame::new with order_by [#16537](https://github.com/apache/datafusion/pull/16537) (findepi) +- chore(deps): bump object_store from 0.12.1 to 0.12.2 [#16548](https://github.com/apache/datafusion/pull/16548) (dependabot[bot]) +- chore(deps): bump mimalloc from 0.1.46 to 0.1.47 [#16547](https://github.com/apache/datafusion/pull/16547) (dependabot[bot]) +- Add support for Arrow Duration type in Substrait [#16503](https://github.com/apache/datafusion/pull/16503) (jkosh44) +- Allow unparser to override the alias name for the specific dialect [#16540](https://github.com/apache/datafusion/pull/16540) (goldmedal) +- Avoid clones when calling find_window_exprs [#16551](https://github.com/apache/datafusion/pull/16551) (findepi) +- Update `spilled_bytes` metric to reflect actual disk usage [#16535](https://github.com/apache/datafusion/pull/16535) (ding-young) +- adapt filter expressions to file schema during parquet scan [#16461](https://github.com/apache/datafusion/pull/16461) (adriangb) +- datafusion-cli: Use correct S3 region if it is not specified [#16502](https://github.com/apache/datafusion/pull/16502) (liamzwbao) +- Add nested struct casting support and integrate into SchemaAdapter [#16371](https://github.com/apache/datafusion/pull/16371) (kosiew) +- Improve err message grammar [#16566](https://github.com/apache/datafusion/pull/16566) (findepi) +- refactor: move PruningPredicate into its own module [#16587](https://github.com/apache/datafusion/pull/16587) (adriangb) +- chore(deps): bump indexmap from 2.9.0 to 2.10.0 [#16582](https://github.com/apache/datafusion/pull/16582) (dependabot[bot]) +- Skip re-pruning based on partition values and file level stats if there are no dynamic filters [#16424](https://github.com/apache/datafusion/pull/16424) (adriangb) +- Support timestamp and date arguments for `range` and `generate_series` table functions [#16552](https://github.com/apache/datafusion/pull/16552) (simonvandel) +- Fix normalization of columns in JOIN ... USING. [#16560](https://github.com/apache/datafusion/pull/16560) (brunal) +- Revert Finalize support for `RightMark` join + `Mark` join [#16597](https://github.com/apache/datafusion/pull/16597) (comphead) +- move min_batch/max_batch to functions-aggregate-common [#16593](https://github.com/apache/datafusion/pull/16593) (adriangb) +- Allow usage of table functions in relations [#16571](https://github.com/apache/datafusion/pull/16571) (osipovartem) +- Update to arrow/parquet 55.2.0 [#16575](https://github.com/apache/datafusion/pull/16575) (alamb) +- Improve field naming in first_value, last_value implementation [#16631](https://github.com/apache/datafusion/pull/16631) (findepi) +- Fix spurious failure in convert_batches test helper [#16627](https://github.com/apache/datafusion/pull/16627) (findepi) +- Aggregate UDF cleanup [#16628](https://github.com/apache/datafusion/pull/16628) (findepi) +- Avoid treating incomparable scalars as equal [#16624](https://github.com/apache/datafusion/pull/16624) (findepi) +- restore topk pre-filtering of batches and make sort query fuzzer less sensitive to expected non determinism [#16501](https://github.com/apache/datafusion/pull/16501) (alamb) +- Add support for Arrow Time types in Substrait [#16558](https://github.com/apache/datafusion/pull/16558) (jkosh44) +- chore(deps): bump substrait from 0.57.0 to 0.58.0 [#16640](https://github.com/apache/datafusion/pull/16640) (dependabot[bot]) +- Support explain tree format debug for benchmark debug [#16604](https://github.com/apache/datafusion/pull/16604) (zhuqi-lucas) +- Add microbenchmark for spilling with compression [#16512](https://github.com/apache/datafusion/pull/16512) (ding-young) +- Fix parquet filter_pushdown: respect parquet filter pushdown config in scan [#16646](https://github.com/apache/datafusion/pull/16646) (adriangb) +- chore(deps): bump aws-config from 1.8.0 to 1.8.1 [#16651](https://github.com/apache/datafusion/pull/16651) (dependabot[bot]) +- Migrate core test to insta, part 2 [#16617](https://github.com/apache/datafusion/pull/16617) (Chen-Yuan-Lai) +- Update all spark SLT files [#16637](https://github.com/apache/datafusion/pull/16637) (findepi) +- Add PhysicalExpr optimizer and cast unwrapping [#16530](https://github.com/apache/datafusion/pull/16530) (adriangb) +- benchmark: Support sort_tpch10 for benchmark [#16671](https://github.com/apache/datafusion/pull/16671) (zhuqi-lucas) +- chore(deps): bump tokio from 1.45.1 to 1.46.0 [#16666](https://github.com/apache/datafusion/pull/16666) (dependabot[bot]) +- Fix TopK Sort incorrectly pushed down past Join with anti join [#16641](https://github.com/apache/datafusion/pull/16641) (zhuqi-lucas) +- Improve error message when ScalarValue fails to cast array [#16670](https://github.com/apache/datafusion/pull/16670) (findepi) +- Add an example of embedding indexes inside a parquet file [#16395](https://github.com/apache/datafusion/pull/16395) (zhuqi-lucas) +- `datafusion-cli`: Refactor statement execution logic [#16634](https://github.com/apache/datafusion/pull/16634) (liamzwbao) +- Add SchemaAdapterFactory Support for ListingTable with Schema Evolution and Mapping [#16583](https://github.com/apache/datafusion/pull/16583) (kosiew) +- Perf: fast CursorValues compare for StringViewArray using inline*key*… [#16630](https://github.com/apache/datafusion/pull/16630) (zhuqi-lucas) +- Update to Rust 1.88 [#16663](https://github.com/apache/datafusion/pull/16663) (melroy12) +- Refactor StreamJoinMetrics to reuse BaselineMetrics [#16674](https://github.com/apache/datafusion/pull/16674) (Standing-Man) +- chore: refactor `BuildProbeJoinMetrics` to use `BaselineMetrics` [#16500](https://github.com/apache/datafusion/pull/16500) (Samyak2) +- Use compression type in CSV file suffices [#16609](https://github.com/apache/datafusion/pull/16609) (theirix) +- Clarify the generality of the embedded parquet index [#16692](https://github.com/apache/datafusion/pull/16692) (alamb) +- Refactor SortMergeJoinMetrics to reuse BaselineMetrics [#16675](https://github.com/apache/datafusion/pull/16675) (Standing-Man) +- Add support for Arrow Dictionary type in Substrait [#16608](https://github.com/apache/datafusion/pull/16608) (jkosh44) +- Fix duplicate field name error in Join::try_new_with_project_input during physical planning [#16454](https://github.com/apache/datafusion/pull/16454) (LiaCastaneda) +- chore(deps): bump tokio from 1.46.0 to 1.46.1 [#16700](https://github.com/apache/datafusion/pull/16700) (dependabot[bot]) +- Add reproducer for tpch Q16 deserialization bug [#16662](https://github.com/apache/datafusion/pull/16662) (NGA-TRAN) +- Minor: Update release instructions [#16701](https://github.com/apache/datafusion/pull/16701) (alamb) +- refactor filter pushdown APIs [#16642](https://github.com/apache/datafusion/pull/16642) (adriangb) +- Add comments to ClickBench queries about setting binary_as_string [#16605](https://github.com/apache/datafusion/pull/16605) (alamb) +- minor: improve display output for FFI execution plans [#16713](https://github.com/apache/datafusion/pull/16713) (timsaucer) +- Revert "fix: create file for empty stream" [#16682](https://github.com/apache/datafusion/pull/16682) (brunal) +- Add the missing equivalence info for filter pushdown [#16686](https://github.com/apache/datafusion/pull/16686) (liamzwbao) +- Fix sqllogictests test running compatibility (ignore `--test-threads`) [#16694](https://github.com/apache/datafusion/pull/16694) (mjgarton) +- Fix: Make `CopyTo` logical plan output schema consistent with physical schema [#16705](https://github.com/apache/datafusion/pull/16705) (bert-beyondloops) +- chore(devcontainer): use debian's `protobuf-compiler` package [#16687](https://github.com/apache/datafusion/pull/16687) (fvj) +- Add link to upgrade guide in changelog script [#16680](https://github.com/apache/datafusion/pull/16680) (alamb) +- Improve display format of BoundedWindowAggExec [#16645](https://github.com/apache/datafusion/pull/16645) (geetanshjuneja) +- Bump the MSRV due to transitive dependencies [#16728](https://github.com/apache/datafusion/pull/16728) (rtyler) +- Fix: optimize projections for unnest logical plan. [#16632](https://github.com/apache/datafusion/pull/16632) (bert-beyondloops) +- Use the `test-threads` option in sqllogictests [#16722](https://github.com/apache/datafusion/pull/16722) (mjgarton) +- chore(deps): bump clap from 4.5.40 to 4.5.41 [#16735](https://github.com/apache/datafusion/pull/16735) (dependabot[bot]) +- chore: make more clarity for internal errors [#16741](https://github.com/apache/datafusion/pull/16741) (comphead) +- Remove parquet_filter and parquet `sort` benchmarks [#16730](https://github.com/apache/datafusion/pull/16730) (alamb) +- Perform type coercion for corr aggregate function [#15776](https://github.com/apache/datafusion/pull/15776) (kumarlokesh) +- Improve dictionary null handling in hashing and expand aggregate test coverage for nulls [#16466](https://github.com/apache/datafusion/pull/16466) (kosiew) +- Improve Ci cache [#16709](https://github.com/apache/datafusion/pull/16709) (blaginin) +- Fix in list round trip in df proto [#16744](https://github.com/apache/datafusion/pull/16744) (XiangpengHao) +- chore: Make `GroupValues` and APIs on `PhysicalGroupBy` aggregation APIs public [#16733](https://github.com/apache/datafusion/pull/16733) (haohuaijin) +- Extend binary coercion rules to support Decimal arithmetic operations with integer(signed and unsigned) types [#16668](https://github.com/apache/datafusion/pull/16668) (jatin510) +- Support Type Coercion for NULL in Binary Arithmetic Expressions [#16761](https://github.com/apache/datafusion/pull/16761) (kosiew) +- chore(deps): bump chrono-tz from 0.10.3 to 0.10.4 [#16769](https://github.com/apache/datafusion/pull/16769) (dependabot[bot]) +- limit intermediate batch size in nested_loop_join [#16443](https://github.com/apache/datafusion/pull/16443) (UBarney) +- Add serialization/deserialization and round-trip tests for all tpc-h queries [#16742](https://github.com/apache/datafusion/pull/16742) (NGA-TRAN) +- Auto start testcontainers for `datafusion-cli` [#16644](https://github.com/apache/datafusion/pull/16644) (blaginin) +- Refactor BinaryTypeCoercer to Handle Null Coercion Early and Avoid Redundant Checks [#16768](https://github.com/apache/datafusion/pull/16768) (kosiew) +- Remove fixed version from MSRV check [#16786](https://github.com/apache/datafusion/pull/16786) (findepi) +- Add `clickbench_pushdown` benchmark [#16731](https://github.com/apache/datafusion/pull/16731) (alamb) +- add filter to handle backtrace [#16752](https://github.com/apache/datafusion/pull/16752) (geetanshjuneja) +- Support min/max aggregates for FixedSizeBinary type [#16765](https://github.com/apache/datafusion/pull/16765) (theirix) +- fix tests in page_pruning when filter pushdown is enabled by default [#16794](https://github.com/apache/datafusion/pull/16794) (XiangpengHao) +- Automatically split large single RecordBatches in `MemorySource` into smaller batches [#16734](https://github.com/apache/datafusion/pull/16734) (kosiew) +- CI: Fix slow join test [#16796](https://github.com/apache/datafusion/pull/16796) (2010YOUY01) +- Benchmark for char expression [#16743](https://github.com/apache/datafusion/pull/16743) (ajita-asthana) +- Add example of custom file schema casting rules [#16803](https://github.com/apache/datafusion/pull/16803) (adriangb) +- Fix discrepancy in Float64 to timestamp(9) casts for constants [#16639](https://github.com/apache/datafusion/pull/16639) (findepi) +- Fix: Preserve sorting for the COPY TO plan [#16785](https://github.com/apache/datafusion/pull/16785) (bert-beyondloops) +- chore(deps): bump object_store from 0.12.2 to 0.12.3 [#16807](https://github.com/apache/datafusion/pull/16807) (dependabot[bot]) +- Implement equals for stateful functions [#16781](https://github.com/apache/datafusion/pull/16781) (findepi) +- benchmark: Add parquet h2o support [#16804](https://github.com/apache/datafusion/pull/16804) (zhuqi-lucas) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 32 Andrew Lamb + 26 dependabot[bot] + 19 Adrian Garcia Badaracco + 14 kosiew + 13 Piotr Findeisen + 13 Qi Zhu + 7 Jonathan Chen + 6 Chen Chongchen + 6 Marco Neumann + 6 Pepijn Van Eeckhoudt + 6 xudong.w + 5 Oleks V + 5 Yongting You + 5 ding-young + 4 Simon Vandel Sillesen + 3 Adam Gutglick + 3 Bert Vermeiren + 3 Dmitrii Blaginin + 3 Joseph Koshakow + 3 Liam Bao + 3 Tim Saucer + 2 Alan Tang + 2 Arttu + 2 Bruno + 2 Corwin Joy + 2 Daniël Heres + 2 Geetansh Juneja + 2 Ian Lai + 2 Jax Liu + 2 Martin Garton + 2 Nga Tran + 2 Ruihang Xia + 2 Tai Le Manh + 2 ViggoC + 2 Xiangpeng Hao + 2 haiywu + 2 theirix + 1 Ajeeta Asthana + 1 Artem Osipov + 1 Dharan Aditya + 1 Gabriel + 1 Geoffrey Claude + 1 Hendrik Makait + 1 Huaijin + 1 Ian Wagner + 1 Jack Eadie + 1 Jagdish Parihar + 1 Jon Mease + 1 Julius von Froreich + 1 K + 1 Leon Lin + 1 Loakesh Indiran + 1 Lokesh + 1 Lucas Earl + 1 Lía Adriana + 1 Mehmet Ozan Kabak + 1 Melroy dsilva + 1 Nirnay Roy + 1 Nuno Faria + 1 R. Tyler Croy + 1 Rishab Joshi + 1 Sami Tabet + 1 Samyak Sarnayak + 1 Shehab Amin + 1 Tobias Schwarzinger + 1 UBarney + 1 alexanderbianchi + 1 epgif + 1 feniljain + 1 m09526 + 1 suibianwanwan +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index c618aa18c2318..9ac1c59caa800 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -70,7 +70,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 48.0.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 49.0.0 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | From 273d37a5968571900bfe9efa1ee89f97914da118 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Tue, 22 Jul 2025 03:55:35 -0700 Subject: [PATCH 075/177] chore: use `equals_datatype` for `BinaryExpr` (#16813) (#16847) * chore: use `equals_datatype` instead of direct type comparison for `BinaryExpr` * chore: use `equals_datatype` instead of direct type comparison for `BinaryExpr` (cherry picked from commit acff1b6bdd288a15755fda36d939b0fbdae144d2) --- .../physical-expr/src/expressions/binary.rs | 65 ++++++++++++++++++- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/binary.rs b/datafusion/physical-expr/src/expressions/binary.rs index 798e68a459ce6..eff948c6a0f43 100644 --- a/datafusion/physical-expr/src/expressions/binary.rs +++ b/datafusion/physical-expr/src/expressions/binary.rs @@ -387,8 +387,8 @@ impl PhysicalExpr for BinaryExpr { let input_schema = schema.as_ref(); if left_data_type.is_nested() { - if right_data_type != left_data_type { - return internal_err!("type mismatch"); + if !left_data_type.equals_datatype(&right_data_type) { + return internal_err!("Cannot evaluate binary expression because of type mismatch: left {}, right {} ", left_data_type, right_data_type); } return apply_cmp_for_nested(self.op, &lhs, &rhs); } @@ -5399,4 +5399,65 @@ mod tests { Interval::make(Some(false), Some(false)).unwrap() ); } + + #[test] + fn test_evaluate_nested_type() { + let batch_schema = Arc::new(Schema::new(vec![ + Field::new( + "a", + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), + true, + ), + Field::new( + "b", + DataType::List(Arc::new(Field::new_list_field(DataType::Int32, true))), + true, + ), + ])); + + let mut list_builder_a = ListBuilder::new(Int32Builder::new()); + + list_builder_a.append_value([Some(1)]); + list_builder_a.append_value([Some(2)]); + list_builder_a.append_value([]); + list_builder_a.append_value([None]); + + let list_array_a: ArrayRef = Arc::new(list_builder_a.finish()); + + let mut list_builder_b = ListBuilder::new(Int32Builder::new()); + + list_builder_b.append_value([Some(1)]); + list_builder_b.append_value([Some(2)]); + list_builder_b.append_value([]); + list_builder_b.append_value([None]); + + let list_array_b: ArrayRef = Arc::new(list_builder_b.finish()); + + let batch = + RecordBatch::try_new(batch_schema, vec![list_array_a, list_array_b]).unwrap(); + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "a", + DataType::List(Arc::new(Field::new("foo", DataType::Int32, true))), + true, + ), + Field::new( + "b", + DataType::List(Arc::new(Field::new("bar", DataType::Int32, true))), + true, + ), + ])); + + let a = Arc::new(Column::new("a", 0)) as _; + let b = Arc::new(Column::new("b", 1)) as _; + + let eq_expr = + binary_expr(Arc::clone(&a), Operator::Eq, Arc::clone(&b), &schema).unwrap(); + + let eq_result = eq_expr.evaluate(&batch).unwrap(); + let expected = + BooleanArray::from_iter(vec![Some(true), Some(true), Some(true), Some(true)]); + assert_eq!(eq_result.into_array(4).unwrap().as_boolean(), &expected); + } } From afb90999d0a1ab500f42a32251370f214f837d1e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 22 Jul 2025 08:14:50 -0400 Subject: [PATCH 076/177] [branch-49] Final Changelog Tweaks (#16852) --- dev/changelog/49.0.0.md | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/dev/changelog/49.0.0.md b/dev/changelog/49.0.0.md index c30bfaf3ea656..239c7c9dfc973 100644 --- a/dev/changelog/49.0.0.md +++ b/dev/changelog/49.0.0.md @@ -19,7 +19,7 @@ under the License. # Apache DataFusion 49.0.0 Changelog -This release consists of 251 commits from 71 contributors. See credits at the end of this changelog for more information. +This release consists of 253 commits from 71 contributors. See credits at the end of this changelog for more information. See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgrading.html) for information on how to upgrade from previous versions. @@ -35,6 +35,7 @@ See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgradi - Reuse Rows allocation in RowCursorStream [#16647](https://github.com/apache/datafusion/pull/16647) (Dandandan) - refactor: shrink `SchemaError` [#16653](https://github.com/apache/datafusion/pull/16653) (crepererum) - Remove unused AggregateUDF struct [#16683](https://github.com/apache/datafusion/pull/16683) (ViggoC) +- Bump the MSRV to `1.85.1` due to transitive dependencies (`aws-sdk`) [#16728](https://github.com/apache/datafusion/pull/16728) (rtyler) **Performance related:** @@ -89,6 +90,9 @@ See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgradi - fix: return NULL if any of the param to make_date is NULL [#16759](https://github.com/apache/datafusion/pull/16759) (feniljain) - fix: add `order_requirement` & `dist_requirement` to `OutputRequirementExec` display [#16726](https://github.com/apache/datafusion/pull/16726) (Loaki07) - fix: support nullable columns in pre-sorted data sources [#16783](https://github.com/apache/datafusion/pull/16783) (crepererum) +- fix: The inconsistency between scalar and array on the cast decimal to timestamp [#16539](https://github.com/apache/datafusion/pull/16539) (chenkovsky) +- fix: unit test for object_storage [#16824](https://github.com/apache/datafusion/pull/16824) (chenkovsky) +- fix(docs): Update broken links to `TableProvider` docs [#16830](https://github.com/apache/datafusion/pull/16830) (jcsherin) **Documentation updates:** @@ -129,6 +133,8 @@ See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgradi - Restore custom SchemaAdapter functionality for Parquet [#16791](https://github.com/apache/datafusion/pull/16791) (adriangb) - Update `upgrading.md` for new unified config for sql string mapping to utf8view [#16809](https://github.com/apache/datafusion/pull/16809) (zhuqi-lucas) - docs: Remove reference to forthcoming example (#16817) [#16818](https://github.com/apache/datafusion/pull/16818) (m09526) +- docs: Fix broken links [#16839](https://github.com/apache/datafusion/pull/16839) (2010YOUY01) +- Add note to upgrade guide about MSRV update [#16845](https://github.com/apache/datafusion/pull/16845) (alamb) **Other:** @@ -259,7 +265,6 @@ See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgradi - chore(devcontainer): use debian's `protobuf-compiler` package [#16687](https://github.com/apache/datafusion/pull/16687) (fvj) - Add link to upgrade guide in changelog script [#16680](https://github.com/apache/datafusion/pull/16680) (alamb) - Improve display format of BoundedWindowAggExec [#16645](https://github.com/apache/datafusion/pull/16645) (geetanshjuneja) -- Bump the MSRV due to transitive dependencies [#16728](https://github.com/apache/datafusion/pull/16728) (rtyler) - Fix: optimize projections for unnest logical plan. [#16632](https://github.com/apache/datafusion/pull/16632) (bert-beyondloops) - Use the `test-threads` option in sqllogictests [#16722](https://github.com/apache/datafusion/pull/16722) (mjgarton) - chore(deps): bump clap from 4.5.40 to 4.5.41 [#16735](https://github.com/apache/datafusion/pull/16735) (dependabot[bot]) @@ -291,13 +296,22 @@ See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgradi - chore(deps): bump object_store from 0.12.2 to 0.12.3 [#16807](https://github.com/apache/datafusion/pull/16807) (dependabot[bot]) - Implement equals for stateful functions [#16781](https://github.com/apache/datafusion/pull/16781) (findepi) - benchmark: Add parquet h2o support [#16804](https://github.com/apache/datafusion/pull/16804) (zhuqi-lucas) +- chore: use `equals_datatype` for `BinaryExpr` [#16813](https://github.com/apache/datafusion/pull/16813) (comphead) +- chore: add tests for out of bounds for NullArray [#16802](https://github.com/apache/datafusion/pull/16802) (comphead) +- Refactor binary.rs tests into modular submodules under `binary/tests` [#16782](https://github.com/apache/datafusion/pull/16782) (kosiew) +- cache generation of dictionary keys and null arrays for ScalarValue [#16789](https://github.com/apache/datafusion/pull/16789) (adriangb) +- refactor(examples): remove redundant call to create directory in `parquet_embedded_index.rs` [#16825](https://github.com/apache/datafusion/pull/16825) (jcsherin) +- Add benchmark for ByteViewGroupValueBuilder [#16826](https://github.com/apache/datafusion/pull/16826) (zhuqi-lucas) +- Simplify try cast expr evaluation [#16834](https://github.com/apache/datafusion/pull/16834) (lewiszlw) +- Fix flaky test case in joins.slt [#16849](https://github.com/apache/datafusion/pull/16849) (findepi) +- chore(deps): bump sysinfo from 0.35.2 to 0.36.1 [#16850](https://github.com/apache/datafusion/pull/16850) (dependabot[bot]) ## Credits Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. ``` - 32 Andrew Lamb + 33 Andrew Lamb 26 dependabot[bot] 19 Adrian Garcia Badaracco 14 kosiew @@ -306,9 +320,9 @@ Thank you to everyone who contributed to this release. Here is a breakdown of co 7 Jonathan Chen 6 Chen Chongchen 6 Marco Neumann + 6 Oleks V 6 Pepijn Van Eeckhoudt 6 xudong.w - 5 Oleks V 5 Yongting You 5 ding-young 4 Simon Vandel Sillesen From e4dd102938b6b40d95982e44e5b831681b01aaf4 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 4 Aug 2025 15:49:09 +0800 Subject: [PATCH 077/177] branch 49 --- datafusion/expr/src/expr.rs | 5 - datafusion/expr/src/literal.rs | 1 - datafusion/optimizer/src/push_down_filter.rs | 1 - .../simplify_expressions/expr_simplifier.rs | 2 +- .../optimizer/src/simplify_predicates.rs | 19 ++- .../physical-expr/src/expressions/in_list.rs | 4 - .../physical-expr/src/expressions/literal.rs | 1 - datafusion/physical-expr/src/planner.rs | 1 - .../physical-expr/src/window/aggregate.rs | 8 ++ .../src/window/sliding_aggregate.rs | 4 + .../physical-expr/src/window/window_expr.rs | 4 + .../src/enforce_distribution.rs | 119 ++++++------------ .../physical-plan/src/joins/hash_join.rs | 3 +- .../src/joins/sort_merge_join.rs | 3 +- .../src/joins/symmetric_hash_join.rs | 2 +- datafusion/physical-plan/src/sorts/sort.rs | 1 + 16 files changed, 66 insertions(+), 112 deletions(-) diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index c393eb9789ee1..0749ff0e98b71 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -284,13 +284,8 @@ pub enum Expr { Column(Column), /// A named reference to a variable in a registry. ScalarVariable(DataType, Vec), -<<<<<<< HEAD - /// A constant value along with associated metadata - Literal(ScalarValue, Option>), -======= /// A constant value along with associated [`FieldMetadata`]. Literal(ScalarValue, Option), ->>>>>>> upstream/branch-49 /// A binary expression such as "age > 21" BinaryExpr(BinaryExpr), /// LIKE expression diff --git a/datafusion/expr/src/literal.rs b/datafusion/expr/src/literal.rs index 877d80c45a3b1..c4bd43bc0a620 100644 --- a/datafusion/expr/src/literal.rs +++ b/datafusion/expr/src/literal.rs @@ -20,7 +20,6 @@ use crate::expr::FieldMetadata; use crate::Expr; use datafusion_common::ScalarValue; -use std::collections::HashMap; /// Create a literal expression pub fn lit(n: T) -> Expr { diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index 06830eb1c4019..63d6d7f5082c2 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -40,7 +40,6 @@ use datafusion_expr::{ }; use crate::optimizer::ApplyOrder; -use crate::simplify_expressions::simplify_predicates; use crate::utils::{has_all_column_refs, is_restrict_null_predicate}; use crate::{simplify_predicates::simplify_predicates, OptimizerConfig, OptimizerRule}; diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 8cac3d2235c53..9a3a8bcd23a7f 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -18,7 +18,7 @@ //! Expression simplification API use std::borrow::Cow; -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; use std::ops::Not; use arrow::{ diff --git a/datafusion/optimizer/src/simplify_predicates.rs b/datafusion/optimizer/src/simplify_predicates.rs index 5ff7e42d95310..aa563abc48fe9 100644 --- a/datafusion/optimizer/src/simplify_predicates.rs +++ b/datafusion/optimizer/src/simplify_predicates.rs @@ -43,8 +43,8 @@ pub(crate) fn simplify_predicates(predicates: Vec) -> Result> { { let left_col = extract_column_from_expr(left); let right_col = extract_column_from_expr(right); - let left_lit = left.is_literal(); - let right_lit = right.is_literal(); + let left_lit = left.as_literal().is_some(); + let right_lit = right.as_literal().is_some(); if let (Some(col), true) = (&left_col, right_lit) { column_predicates.entry(col.clone()).or_default().push(pred); } else if let (true, Some(col)) = (left_lit, &right_col) { @@ -80,7 +80,7 @@ fn simplify_column_predicates(predicates: Vec) -> Result> { for pred in predicates { match &pred { Expr::BinaryExpr(BinaryExpr { left: _, op, right }) => { - let right_is_literal = right.is_literal(); + let right_is_literal = right.as_literal().is_some(); match (op, right_is_literal) { (Operator::Gt, true) | (Operator::Lt, false) @@ -149,14 +149,11 @@ fn find_most_restrictive_predicate( if let Expr::BinaryExpr(BinaryExpr { left, op: _, right }) = pred { // Extract the literal value based on which side has it let mut scalar_value = None; - if right.is_literal() { - if let Expr::Literal(scalar, _) = right.as_ref() { - scalar_value = Some(scalar.clone()); - } - } else if left.is_literal() { - if let Expr::Literal(scalar, _) = left.as_ref() { - scalar_value = Some(scalar.clone()); - } + if let Some(scalar) = right.as_literal() { + scalar_value = Some(scalar.clone()); + } + if let Some(scalar) = left.as_literal() { + scalar_value = Some(scalar.clone()); } if let Some(scalar) = scalar_value { diff --git a/datafusion/physical-expr/src/expressions/in_list.rs b/datafusion/physical-expr/src/expressions/in_list.rs index 969862f7444b4..b6fe84ea51579 100644 --- a/datafusion/physical-expr/src/expressions/in_list.rs +++ b/datafusion/physical-expr/src/expressions/in_list.rs @@ -1451,11 +1451,7 @@ mod tests { let sql_string = fmt_sql(expr.as_ref()).to_string(); let display_string = expr.to_string(); assert_eq!(sql_string, "a IN (a, b)"); -<<<<<<< HEAD - assert_eq!(display_string, "Use a@0 IN (SET) ([Literal { value: Utf8(\"a\"), field: Field { name: \"a\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(\"b\"), field: Field { name: \"b\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }])"); -======= assert_eq!(display_string, "Use a@0 IN (SET) ([Literal { value: Utf8(\"a\"), field: Field { name: \"lit\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }, Literal { value: Utf8(\"b\"), field: Field { name: \"lit\", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} } }])"); ->>>>>>> upstream/branch-49 // Test: a NOT IN ('a', 'b') let list = vec![lit("a"), lit("b")]; diff --git a/datafusion/physical-expr/src/expressions/literal.rs b/datafusion/physical-expr/src/expressions/literal.rs index eb0ac652978a9..1a2ebf000f1df 100644 --- a/datafusion/physical-expr/src/expressions/literal.rs +++ b/datafusion/physical-expr/src/expressions/literal.rs @@ -18,7 +18,6 @@ //! Literal expressions for physical operations use std::any::Any; -use std::collections::HashMap; use std::hash::Hash; use std::sync::Arc; diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index f1d57948c025c..fbc19b1202ee8 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -15,7 +15,6 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; use std::sync::Arc; use crate::ScalarFunctionExpr; diff --git a/datafusion/physical-expr/src/window/aggregate.rs b/datafusion/physical-expr/src/window/aggregate.rs index 81fa325998832..6f0e7c963d144 100644 --- a/datafusion/physical-expr/src/window/aggregate.rs +++ b/datafusion/physical-expr/src/window/aggregate.rs @@ -46,6 +46,7 @@ pub struct PlainAggregateWindowExpr { partition_by: Vec>, order_by: Vec, window_frame: Arc, + is_constant_in_partition: bool, } impl PlainAggregateWindowExpr { @@ -56,11 +57,14 @@ impl PlainAggregateWindowExpr { order_by: &[PhysicalSortExpr], window_frame: Arc, ) -> Self { + let is_constant_in_partition = + Self::is_window_constant_in_partition(order_by, &window_frame); Self { aggregate, partition_by: partition_by.to_vec(), order_by: order_by.to_vec(), window_frame, + is_constant_in_partition, } } @@ -246,4 +250,8 @@ impl AggregateWindowExpr for PlainAggregateWindowExpr { accumulator.evaluate() } } + + fn is_constant_in_partition(&self) -> bool { + self.is_constant_in_partition + } } diff --git a/datafusion/physical-expr/src/window/sliding_aggregate.rs b/datafusion/physical-expr/src/window/sliding_aggregate.rs index 9f21d8a3f3173..33921a57a6ce0 100644 --- a/datafusion/physical-expr/src/window/sliding_aggregate.rs +++ b/datafusion/physical-expr/src/window/sliding_aggregate.rs @@ -217,4 +217,8 @@ impl AggregateWindowExpr for SlidingAggregateWindowExpr { accumulator.evaluate() } } + + fn is_constant_in_partition(&self) -> bool { + false + } } diff --git a/datafusion/physical-expr/src/window/window_expr.rs b/datafusion/physical-expr/src/window/window_expr.rs index 2517aecdfd03e..56566d1222006 100644 --- a/datafusion/physical-expr/src/window/window_expr.rs +++ b/datafusion/physical-expr/src/window/window_expr.rs @@ -187,6 +187,10 @@ pub trait AggregateWindowExpr: WindowExpr { accumulator: &mut Box, ) -> Result; + /// Indicates whether this window function always produces the same result + /// for all rows in the partition. + fn is_constant_in_partition(&self) -> bool; + /// Evaluates the window function against the batch. fn aggregate_evaluate(&self, batch: &RecordBatch) -> Result { let mut accumulator = self.get_accumulator()?; diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index c485cb0e47606..163a76956ef5c 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -42,7 +42,6 @@ use datafusion_physical_expr::utils::map_columns_before_projection; use datafusion_physical_expr::{ physical_exprs_equal, EquivalenceProperties, PhysicalExpr, PhysicalExprRef, }; -use datafusion_physical_expr_common::sort_expr::LexOrdering; use datafusion_physical_plan::aggregates::{ AggregateExec, AggregateMode, PhysicalGroupBy, }; @@ -937,7 +936,7 @@ fn add_hash_on_top( /// /// Updated node with an execution plan, where desired single /// distribution is satisfied by adding [`SortPreservingMergeExec`]. -fn add_spm_on_top( +fn add_merge_on_top( input: DistributionContext, fetch: &mut Option, ) -> DistributionContext { @@ -949,19 +948,10 @@ fn add_spm_on_top( // - Preserving ordering is not helpful in terms of satisfying ordering requirements // - Usage of order preserving variants is not desirable // (determined by flag `config.optimizer.bounded_order_preserving_variants`) - let should_preserve_ordering = input.plan.output_ordering().is_some(); - - let new_plan = if should_preserve_ordering { + let new_plan = if let Some(ordering) = input.plan.output_ordering() { Arc::new( - SortPreservingMergeExec::new( - input - .plan - .output_ordering() - .unwrap_or(&LexOrdering::default()) - .clone(), - Arc::clone(&input.plan), - ) - .with_fetch(fetch.take()), + SortPreservingMergeExec::new(ordering.clone(), Arc::clone(&input.plan)) + .with_fetch(fetch.take()), ) as _ } else { Arc::new(CoalescePartitionsExec::new(Arc::clone(&input.plan))) as _ @@ -992,8 +982,13 @@ fn add_spm_on_top( /// ``` fn remove_dist_changing_operators( mut distribution_context: DistributionContext, -) -> Result<(DistributionContext, Option)> { +) -> Result<( + DistributionContext, + Option, + Option>, +)> { let mut fetch = None; + let mut spm: Option> = None; while is_repartition(&distribution_context.plan) || is_coalesce_partitions(&distribution_context.plan) || is_sort_preserving_merge(&distribution_context.plan) @@ -1002,6 +997,7 @@ fn remove_dist_changing_operators( if let Some(child_fetch) = distribution_context.plan.fetch() { if fetch.is_none() { fetch = Some(child_fetch); + spm = Some(distribution_context.plan); } else { fetch = Some(fetch.unwrap().min(child_fetch)); } @@ -1013,7 +1009,7 @@ fn remove_dist_changing_operators( // Note that they will be re-inserted later on if necessary or helpful. } - Ok((distribution_context, fetch)) + Ok((distribution_context, fetch, spm)) } /// Updates the [`DistributionContext`] if preserving ordering while changing partitioning is not helpful or desirable. @@ -1224,6 +1220,7 @@ pub fn ensure_distribution( children, }, mut fetch, + spm, ) = remove_dist_changing_operators(dist_context)?; if let Some(exec) = plan.as_any().downcast_ref::() { @@ -1286,10 +1283,16 @@ pub fn ensure_distribution( } } - // Satisfy the distribution requirement if it is unmet. - match &requirement { - Distribution::SinglePartition => { - child = add_spm_on_top(child, &mut fetch); + // Satisfy the distribution requirement if it is unmet. + match &requirement { + Distribution::SinglePartition => { + child = add_merge_on_top(child, &mut fetch); + } + Distribution::HashPartitioned(exprs) => { + if add_roundrobin { + // Add round-robin repartitioning on top of the operator + // to increase parallelism. + child = add_roundrobin_on_top(child, target_partitions)?; } // When inserting hash is necessary to satisfy hash requirement, insert hash repartition. if hash_necessary { @@ -1320,7 +1323,8 @@ pub fn ensure_distribution( if (!ordering_satisfied || !order_preserving_variants_desirable) && child.data { - child = replace_order_preserving_variants(child)?; + let (replaced_child, fetch) = replace_order_preserving_variants(child, ordering_satisfied)?; + child = replaced_child; // If ordering requirements were satisfied before repartitioning, // make sure ordering requirements are still satisfied after. if ordering_satisfied { @@ -1328,10 +1332,7 @@ pub fn ensure_distribution( child = add_sort_above_with_check( child, sort_req, - plan.as_any() - .downcast_ref::() - .map(|output| output.fetch()) - .unwrap_or(None), + fetch, )?; } } @@ -1343,62 +1344,19 @@ pub fn ensure_distribution( // Operator requires specific distribution. Distribution::SinglePartition | Distribution::HashPartitioned(_) => { // Since there is no ordering requirement, preserving ordering is pointless - child = replace_order_preserving_variants(child)?; + child = replace_order_preserving_variants(child, false)?.0; } Distribution::UnspecifiedDistribution => { - if add_roundrobin { - // Add round-robin repartitioning on top of the operator - // to increase parallelism. - child = add_roundrobin_on_top(child, target_partitions)?; + // Since ordering is lost, trying to preserve ordering is pointless + if !maintains || plan.as_any().is::() { + child = replace_order_preserving_variants(child,false)?.0; } + } }; + } - // There is an ordering requirement of the operator: - if let Some(required_input_ordering) = required_input_ordering { - // Either: - // - Ordering requirement cannot be satisfied by preserving ordering through repartitions, or - // - using order preserving variant is not desirable. - let ordering_satisfied = child - .plan - .equivalence_properties() - .ordering_satisfy_requirement(&required_input_ordering); - if (!ordering_satisfied || !order_preserving_variants_desirable) - && child.data - { - let (replaced_child, fetch) = - replace_order_preserving_variants(child, ordering_satisfied)?; - child = replaced_child; - // If ordering requirements were satisfied before repartitioning, - // make sure ordering requirements are still satisfied after. - if ordering_satisfied { - // Make sure to satisfy ordering requirement: - child = add_sort_above_with_check( - child, - required_input_ordering.clone(), - fetch, - ); - } - } - // Stop tracking distribution changing operators - child.data = false; - } else { - // no ordering requirement - match requirement { - // Operator requires specific distribution. - Distribution::SinglePartition | Distribution::HashPartitioned(_) => { - // Since there is no ordering requirement, preserving ordering is pointless - child = replace_order_preserving_variants(child, false)?.0; - } - Distribution::UnspecifiedDistribution => { - // Since ordering is lost, trying to preserve ordering is pointless - if !maintains || plan.as_any().is::() { - child = replace_order_preserving_variants(child, false)?.0; - } - } - } - } - Ok(child) + Ok(child) }, ) .collect::>>()?; @@ -1447,15 +1405,8 @@ pub fn ensure_distribution( // It was removed by `remove_dist_changing_operators` // and we need to add it back. if fetch.is_some() { - let plan = Arc::new( - SortPreservingMergeExec::new( - plan.output_ordering() - .unwrap_or(&LexOrdering::default()) - .clone(), - plan, - ) - .with_fetch(fetch.take()), - ); + // It's safe to unwrap because `spm` is set only if `fetch` is set. + let plan = spm.unwrap().with_fetch(fetch.take()).unwrap(); optimized_distribution_ctx = DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); } diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index 3603b6eb34b7a..ac1493d09ce7e 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -916,12 +916,13 @@ impl ExecutionPlan for HashJoinExec { self.join_type(), self.projection.clone(), *self.partition_mode(), - self.null_equals_null, + self.null_equality, )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } + /// Tries to push `projection` down through `hash_join`. If possible, performs the /// pushdown and returns a new [`HashJoinExec`] as the top plan which has projections /// as its children. Otherwise, returns `None`. diff --git a/datafusion/physical-plan/src/joins/sort_merge_join.rs b/datafusion/physical-plan/src/joins/sort_merge_join.rs index 99f9c3e5ab2ca..e9e6904a035af 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join.rs @@ -554,12 +554,13 @@ impl ExecutionPlan for SortMergeJoinExec { self.filter.clone(), self.join_type(), self.sort_options.clone(), - self.null_equals_null, + self.null_equality, )?; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } + /// Tries to swap the projection with its input [`SortMergeJoinExec`]. If it can be done, /// it returns the new swapped version having the [`SortMergeJoinExec`] as the top plan. /// Otherwise, it returns None. diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index b6560106f6899..29c22e116f22e 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -484,7 +484,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { self.on.clone(), self.filter.clone(), self.join_type(), - self.null_equals_null, + self.null_equality, self.left_sort_exprs.clone(), self.right_sort_exprs.clone(), self.mode, diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index a014c3c70fe4b..5186293e59975 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1250,6 +1250,7 @@ impl ExecutionPlan for SortExec { preserve_partitioning: self.preserve_partitioning, cache: self.cache.clone().with_node_id(_node_id), common_sort_prefix: self.common_sort_prefix.clone(), + filter: self.filter.clone(), }; Ok(Some(Arc::new(new_plan))) } From 9cfb9cd013f33bcdae25360790da7101ee33266f Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 6 Aug 2025 16:32:21 -0400 Subject: [PATCH 078/177] remove warning from every file open (#16968) (#17059) this is too noisy and not helpful yet, we don't have a fully implemented alternative (cherry picked from commit 0183244a426e5e1fda28c7b6e6961f70f04c0744) Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> --- datafusion/datasource-parquet/src/source.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index 2251c5c20fe51..aeeece1e47f35 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -469,12 +469,6 @@ impl FileSource for ParquetSource { .file_column_projection_indices() .unwrap_or_else(|| (0..base_config.file_schema.fields().len()).collect()); - if self.schema_adapter_factory.is_some() { - log::warn!("The SchemaAdapter API will be removed from ParquetSource in a future release. \ - Use PhysicalExprAdapterFactory API instead. \ - See https://github.com/apache/datafusion/issues/16800 for discussion and https://datafusion.apache.org/library-user-guide/upgrading.html#datafusion-49-0-0 for upgrade instructions."); - } - let (expr_adapter_factory, schema_adapter_factory) = match ( base_config.expr_adapter_factory.as_ref(), self.schema_adapter_factory.as_ref(), From f6ec4c38d884d0ad5bfb66221fd4ce179af4e84b Mon Sep 17 00:00:00 2001 From: Pepijn Van Eeckhoudt Date: Thu, 7 Aug 2025 16:45:45 +0200 Subject: [PATCH 079/177] #16994 Ensure CooperativeExec#maintains_input_order returns a Vec of the correct size (#16995) (#17068) * #16994 Ensure CooperativeExec#maintains_input_order returns a Vec of the correct size * #16994 Extend default ExecutionPlan invariant checks Add checks that verify the length of the vectors returned by methods that need to return a value per child. (cherry picked from commit 2968331e4c4a8e3596afb2e56a3f0e9e4a864674) --- datafusion/physical-plan/src/coop.rs | 2 +- .../physical-plan/src/execution_plan.rs | 40 +++++++++++++++++-- datafusion/physical-plan/src/union.rs | 7 +++- datafusion/physical-plan/src/work_table.rs | 8 ---- 4 files changed, 42 insertions(+), 15 deletions(-) diff --git a/datafusion/physical-plan/src/coop.rs b/datafusion/physical-plan/src/coop.rs index be0afa07eac2c..89d5ba6f4da9b 100644 --- a/datafusion/physical-plan/src/coop.rs +++ b/datafusion/physical-plan/src/coop.rs @@ -254,7 +254,7 @@ impl ExecutionPlan for CooperativeExec { } fn maintains_input_order(&self) -> Vec { - self.input.maintains_input_order() + vec![true; self.children().len()] } fn children(&self) -> Vec<&Arc> { diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index 6d51bf195dc6f..3ed85b9267e31 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -48,7 +48,7 @@ use crate::stream::RecordBatchStreamAdapter; use arrow::array::{Array, RecordBatch}; use arrow::datatypes::SchemaRef; use datafusion_common::config::ConfigOptions; -use datafusion_common::{exec_err, Constraints, Result}; +use datafusion_common::{exec_err, Constraints, DataFusionError, Result}; use datafusion_common_runtime::JoinSet; use datafusion_execution::TaskContext; use datafusion_physical_expr::EquivalenceProperties; @@ -118,10 +118,11 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync { /// Returns an error if this individual node does not conform to its invariants. /// These invariants are typically only checked in debug mode. /// - /// A default set of invariants is provided in the default implementation. + /// A default set of invariants is provided in the [check_default_invariants] function. + /// The default implementation of `check_invariants` calls this function. /// Extension nodes can provide their own invariants. - fn check_invariants(&self, _check: InvariantLevel) -> Result<()> { - Ok(()) + fn check_invariants(&self, check: InvariantLevel) -> Result<()> { + check_default_invariants(self, check) } /// Specifies the data distribution requirements for all the @@ -1045,6 +1046,37 @@ impl PlanProperties { } } +macro_rules! check_len { + ($target:expr, $func_name:ident, $expected_len:expr) => { + let actual_len = $target.$func_name().len(); + if actual_len != $expected_len { + return internal_err!( + "{}::{} returned Vec with incorrect size: {} != {}", + $target.name(), + stringify!($func_name), + actual_len, + $expected_len + ); + } + }; +} + +/// Checks a set of invariants that apply to all ExecutionPlan implementations. +/// Returns an error if the given node does not conform. +pub fn check_default_invariants( + plan: &P, + _check: InvariantLevel, +) -> Result<(), DataFusionError> { + let children_len = plan.children().len(); + + check_len!(plan, maintains_input_order, children_len); + check_len!(plan, required_input_ordering, children_len); + check_len!(plan, required_input_distribution, children_len); + check_len!(plan, benefits_from_input_partitioning, children_len); + + Ok(()) +} + /// Indicate whether a data exchange is needed for the input of `plan`, which will be very helpful /// especially for the distributed engine to judge whether need to deal with shuffling. /// Currently, there are 3 kinds of execution plan which needs data exchange diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 73d7933e7c053..aca03c57b1b48 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -33,7 +33,8 @@ use super::{ SendableRecordBatchStream, Statistics, }; use crate::execution_plan::{ - boundedness_from_children, emission_type_from_children, InvariantLevel, + boundedness_from_children, check_default_invariants, emission_type_from_children, + InvariantLevel, }; use crate::metrics::BaselineMetrics; use crate::projection::{make_with_child, ProjectionExec}; @@ -176,7 +177,9 @@ impl ExecutionPlan for UnionExec { &self.cache } - fn check_invariants(&self, _check: InvariantLevel) -> Result<()> { + fn check_invariants(&self, check: InvariantLevel) -> Result<()> { + check_default_invariants(self, check)?; + (self.inputs().len() >= 2) .then_some(()) .ok_or(DataFusionError::Internal( diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs index 076e30ab902d4..40a22f94b81f6 100644 --- a/datafusion/physical-plan/src/work_table.rs +++ b/datafusion/physical-plan/src/work_table.rs @@ -174,14 +174,6 @@ impl ExecutionPlan for WorkTableExec { &self.cache } - fn maintains_input_order(&self) -> Vec { - vec![false] - } - - fn benefits_from_input_partitioning(&self) -> Vec { - vec![false] - } - fn children(&self) -> Vec<&Arc> { vec![] } From c7fbb3fe8a6dd7f06c14a9be00b4a8c9b2d946e9 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 8 Aug 2025 13:38:44 -0500 Subject: [PATCH 080/177] Add ExecutionPlan::reset_state (#17028) (#17096) * Add ExecutionPlan::reset_state * Update datafusion/sqllogictest/test_files/cte.slt * Add reference * fmt * add to upgrade guide * add explain plan, implement in more plans * fmt * only explain --------- Co-authored-by: Robert Ream --- .../src/expressions/dynamic_filters.rs | 8 ++ .../physical-plan/src/execution_plan.rs | 25 ++++++ .../physical-plan/src/joins/cross_join.rs | 12 +++ .../physical-plan/src/joins/hash_join.rs | 20 +++++ .../physical-plan/src/recursive_query.rs | 5 +- datafusion/physical-plan/src/sorts/sort.rs | 80 ++++++++++++++----- datafusion/sqllogictest/test_files/cte.slt | 55 +++++++++++++ 7 files changed, 180 insertions(+), 25 deletions(-) diff --git a/datafusion/physical-expr/src/expressions/dynamic_filters.rs b/datafusion/physical-expr/src/expressions/dynamic_filters.rs index ba30b916b9f87..ea10b1197b1d7 100644 --- a/datafusion/physical-expr/src/expressions/dynamic_filters.rs +++ b/datafusion/physical-expr/src/expressions/dynamic_filters.rs @@ -32,6 +32,10 @@ use datafusion_expr::ColumnarValue; use datafusion_physical_expr_common::physical_expr::{DynEq, DynHash}; /// A dynamic [`PhysicalExpr`] that can be updated by anyone with a reference to it. +/// +/// Any `ExecutionPlan` that uses this expression and holds a reference to it internally should probably also +/// implement `ExecutionPlan::reset_state` to remain compatible with recursive queries and other situations where +/// the same `ExecutionPlan` is reused with different data. #[derive(Debug)] pub struct DynamicFilterPhysicalExpr { /// The original children of this PhysicalExpr, if any. @@ -121,6 +125,10 @@ impl DynamicFilterPhysicalExpr { /// do not change* since those will be used to determine what columns need to read or projected /// when evaluating the expression. /// + /// Any `ExecutionPlan` that uses this expression and holds a reference to it internally should probably also + /// implement `ExecutionPlan::reset_state` to remain compatible with recursive queries and other situations where + /// the same `ExecutionPlan` is reused with different data. + /// /// [`collect_columns`]: crate::utils::collect_columns #[allow(dead_code)] // Only used in tests for now pub fn new( diff --git a/datafusion/physical-plan/src/execution_plan.rs b/datafusion/physical-plan/src/execution_plan.rs index 3ed85b9267e31..3f0facb24df4f 100644 --- a/datafusion/physical-plan/src/execution_plan.rs +++ b/datafusion/physical-plan/src/execution_plan.rs @@ -196,6 +196,31 @@ pub trait ExecutionPlan: Debug + DisplayAs + Send + Sync { children: Vec>, ) -> Result>; + /// Reset any internal state within this [`ExecutionPlan`]. + /// + /// This method is called when an [`ExecutionPlan`] needs to be re-executed, + /// such as in recursive queries. Unlike [`ExecutionPlan::with_new_children`], this method + /// ensures that any stateful components (e.g., [`DynamicFilterPhysicalExpr`]) + /// are reset to their initial state. + /// + /// The default implementation simply calls [`ExecutionPlan::with_new_children`] with the existing children, + /// effectively creating a new instance of the [`ExecutionPlan`] with the same children but without + /// necessarily resetting any internal state. Implementations that require resetting of some + /// internal state should override this method to provide the necessary logic. + /// + /// This method should *not* reset state recursively for children, as it is expected that + /// it will be called from within a walk of the execution plan tree so that it will be called on each child later + /// or was already called on each child. + /// + /// Note to implementers: unlike [`ExecutionPlan::with_new_children`] this method does not accept new children as an argument, + /// thus it is expected that any cached plan properties will remain valid after the reset. + /// + /// [`DynamicFilterPhysicalExpr`]: datafusion_physical_expr::expressions::DynamicFilterPhysicalExpr + fn reset_state(self: Arc) -> Result> { + let children = self.children().into_iter().cloned().collect(); + self.with_new_children(children) + } + /// If supported, attempt to increase the partitioning of this `ExecutionPlan` to /// produce `target_partitions` partitions. /// diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index a41e668ab4dab..b8ea6330a1e2e 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -270,6 +270,18 @@ impl ExecutionPlan for CrossJoinExec { ))) } + fn reset_state(self: Arc) -> Result> { + let new_exec = CrossJoinExec { + left: Arc::clone(&self.left), + right: Arc::clone(&self.right), + schema: Arc::clone(&self.schema), + left_fut: Default::default(), // reset the build side! + metrics: ExecutionPlanMetricsSet::default(), + cache: self.cache.clone(), + }; + Ok(Arc::new(new_exec)) + } + fn required_input_distribution(&self) -> Vec { vec![ Distribution::SinglePartition, diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index a7f28ede4408f..84ca7ce19f887 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -769,6 +769,26 @@ impl ExecutionPlan for HashJoinExec { )?)) } + fn reset_state(self: Arc) -> Result> { + // Reset the left_fut to allow re-execution + Ok(Arc::new(HashJoinExec { + left: Arc::clone(&self.left), + right: Arc::clone(&self.right), + on: self.on.clone(), + filter: self.filter.clone(), + join_type: self.join_type, + join_schema: Arc::clone(&self.join_schema), + left_fut: OnceAsync::default(), + random_state: self.random_state.clone(), + mode: self.mode, + metrics: ExecutionPlanMetricsSet::new(), + projection: self.projection.clone(), + column_indices: self.column_indices.clone(), + null_equality: self.null_equality, + cache: self.cache.clone(), + })) + } + fn execute( &self, partition: usize, diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs index 99b460dfcfdcd..700a9076fecf0 100644 --- a/datafusion/physical-plan/src/recursive_query.rs +++ b/datafusion/physical-plan/src/recursive_query.rs @@ -372,7 +372,7 @@ fn assign_work_table( } /// Some plans will change their internal states after execution, making them unable to be executed again. -/// This function uses `ExecutionPlan::with_new_children` to fork a new plan with initial states. +/// This function uses [`ExecutionPlan::reset_state`] to reset any internal state within the plan. /// /// An example is `CrossJoinExec`, which loads the left table into memory and stores it in the plan. /// However, if the data of the left table is derived from the work table, it will become outdated @@ -383,8 +383,7 @@ fn reset_plan_states(plan: Arc) -> Result() { Ok(Transformed::no(plan)) } else { - let new_plan = Arc::clone(&plan) - .with_new_children(plan.children().into_iter().cloned().collect())?; + let new_plan = Arc::clone(&plan).reset_state()?; Ok(Transformed::yes(new_plan)) } }) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index bb572c4315fb8..b82f1769d092f 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -899,6 +899,29 @@ impl SortExec { self } + /// Add or reset `self.filter` to a new `DynamicFilterPhysicalExpr`. + fn create_filter(&self) -> Arc { + let children = self + .expr + .iter() + .map(|sort_expr| Arc::clone(&sort_expr.expr)) + .collect::>(); + Arc::new(DynamicFilterPhysicalExpr::new(children, lit(true))) + } + + fn cloned(&self) -> Self { + SortExec { + input: Arc::clone(&self.input), + expr: self.expr.clone(), + metrics_set: self.metrics_set.clone(), + preserve_partitioning: self.preserve_partitioning, + common_sort_prefix: self.common_sort_prefix.clone(), + fetch: self.fetch, + cache: self.cache.clone(), + filter: self.filter.clone(), + } + } + /// Modify how many rows to include in the result /// /// If None, then all rows will be returned, in sorted order. @@ -920,25 +943,13 @@ impl SortExec { } let filter = fetch.is_some().then(|| { // If we already have a filter, keep it. Otherwise, create a new one. - self.filter.clone().unwrap_or_else(|| { - let children = self - .expr - .iter() - .map(|sort_expr| Arc::clone(&sort_expr.expr)) - .collect::>(); - Arc::new(DynamicFilterPhysicalExpr::new(children, lit(true))) - }) + self.filter.clone().unwrap_or_else(|| self.create_filter()) }); - SortExec { - input: Arc::clone(&self.input), - expr: self.expr.clone(), - metrics_set: self.metrics_set.clone(), - preserve_partitioning: self.preserve_partitioning, - common_sort_prefix: self.common_sort_prefix.clone(), - fetch, - cache, - filter, - } + let mut new_sort = self.cloned(); + new_sort.fetch = fetch; + new_sort.cache = cache; + new_sort.filter = filter; + new_sort } /// Input schema @@ -1110,10 +1121,35 @@ impl ExecutionPlan for SortExec { self: Arc, children: Vec>, ) -> Result> { - let mut new_sort = SortExec::new(self.expr.clone(), Arc::clone(&children[0])) - .with_fetch(self.fetch) - .with_preserve_partitioning(self.preserve_partitioning); - new_sort.filter = self.filter.clone(); + let mut new_sort = self.cloned(); + assert!( + children.len() == 1, + "SortExec should have exactly one child" + ); + new_sort.input = Arc::clone(&children[0]); + // Recompute the properties based on the new input since they may have changed + let (cache, sort_prefix) = Self::compute_properties( + &new_sort.input, + new_sort.expr.clone(), + new_sort.preserve_partitioning, + )?; + new_sort.cache = cache; + new_sort.common_sort_prefix = sort_prefix; + + Ok(Arc::new(new_sort)) + } + + fn reset_state(self: Arc) -> Result> { + let children = self.children().into_iter().cloned().collect(); + let new_sort = self.with_new_children(children)?; + let mut new_sort = new_sort + .as_any() + .downcast_ref::() + .expect("cloned 1 lines above this line, we know the type") + .clone(); + // Our dynamic filter and execution metrics are the state we need to reset. + new_sort.filter = Some(new_sort.create_filter()); + new_sort.metrics_set = ExecutionPlanMetricsSet::new(); Ok(Arc::new(new_sort)) } diff --git a/datafusion/sqllogictest/test_files/cte.slt b/datafusion/sqllogictest/test_files/cte.slt index 32320a06f4fb0..5f8fd1a0b5efd 100644 --- a/datafusion/sqllogictest/test_files/cte.slt +++ b/datafusion/sqllogictest/test_files/cte.slt @@ -996,6 +996,61 @@ physical_plan 08)----------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 09)------------WorkTableExec: name=numbers +# Test for issue #16998: SortExec shares DynamicFilterPhysicalExpr across multiple executions +query II +with recursive r as ( + select 0 as k, 0 as v + union all + ( + select * + from r + order by v + limit 1 + ) +) +select * +from r +limit 5; +---- +0 0 +0 0 +0 0 +0 0 +0 0 + +query TT +explain +with recursive r as ( + select 0 as k, 0 as v + union all + ( + select * + from r + order by v + limit 1 + ) +) +select * +from r +limit 5; +---- +logical_plan +01)SubqueryAlias: r +02)--Limit: skip=0, fetch=5 +03)----RecursiveQuery: is_distinct=false +04)------Projection: Int64(0) AS k, Int64(0) AS v +05)--------EmptyRelation +06)------Sort: r.v ASC NULLS LAST, fetch=1 +07)--------Projection: r.k, r.v +08)----------TableScan: r +physical_plan +01)GlobalLimitExec: skip=0, fetch=5 +02)--RecursiveQueryExec: name=r, is_distinct=false +03)----ProjectionExec: expr=[0 as k, 0 as v] +04)------PlaceholderRowExec +05)----SortExec: TopK(fetch=1), expr=[v@1 ASC NULLS LAST], preserve_partitioning=[false] +06)------WorkTableExec: name=r + statement count 0 set datafusion.execution.enable_recursive_ctes = false; From ee28aa7673db2234b87117d664559e5857ac8c38 Mon Sep 17 00:00:00 2001 From: Adam Gutglick Date: Tue, 12 Aug 2025 15:23:33 +0100 Subject: [PATCH 081/177] [branch-49] Backport #17129 to branch 49 (#17143) * Preserve equivalence properties during projection pushdown (#17129) * Adds parquet data diffs --------- Co-authored-by: Matthew Kim <38759997+friendlymatthew@users.noreply.github.com> --- datafusion/datasource/src/source.rs | 35 +++++++++++++++++- datafusion/sqllogictest/data/1.parquet | Bin 0 -> 1381 bytes datafusion/sqllogictest/data/2.parquet | Bin 0 -> 1403 bytes .../test_files/parquet_filter_pushdown.slt | 32 ++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 datafusion/sqllogictest/data/1.parquet create mode 100644 datafusion/sqllogictest/data/2.parquet diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index fde1944ae066a..3a7ff1ef09911 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -22,6 +22,7 @@ use std::fmt; use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use datafusion_physical_expr::equivalence::ProjectionMapping; use datafusion_physical_plan::execution_plan::{ Boundedness, EmissionType, SchedulingType, }; @@ -324,7 +325,39 @@ impl ExecutionPlan for DataSourceExec { &self, projection: &ProjectionExec, ) -> Result>> { - self.data_source.try_swapping_with_projection(projection) + match self.data_source.try_swapping_with_projection(projection)? { + Some(new_plan) => { + if let Some(new_data_source_exec) = + new_plan.as_any().downcast_ref::() + { + let projection_mapping = ProjectionMapping::try_new( + projection.expr().iter().cloned(), + &self.schema(), + )?; + + // Project the equivalence properties to the new schema + let projected_eq_properties = self + .cache + .eq_properties + .project(&projection_mapping, new_data_source_exec.schema()); + + let preserved_exec = DataSourceExec { + data_source: Arc::clone(&new_data_source_exec.data_source), + cache: PlanProperties::new( + projected_eq_properties, + new_data_source_exec.cache.partitioning.clone(), + new_data_source_exec.cache.emission_type, + new_data_source_exec.cache.boundedness, + ) + .with_scheduling_type(new_data_source_exec.cache.scheduling_type), + }; + Ok(Some(Arc::new(preserved_exec))) + } else { + Ok(Some(new_plan)) + } + } + None => Ok(None), + } } fn handle_child_pushdown_result( diff --git a/datafusion/sqllogictest/data/1.parquet b/datafusion/sqllogictest/data/1.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a04f669eaeaaeb9edd468f7c28c00ba526627007 GIT binary patch literal 1381 zcmb_c&ubGw6n?v1(;NaO;*7hnhf?TLq_j!3QYDC!);7kpHj6Z=2w|IS+dz^Hn>A67 zMFhc<#~%6zcoe*N_9A!?57Lu(^p8;Rz1jS*p|w3YWOv@Y@6GqV_hvR5!cH-bW!cp{ zQyE+Wn0`O^8%i!fasu#`Qeg56A{fMHFeJ_*EMne(>51gOM@m04B9XuhVLq}{%n@gk z$At(4K>76U>#Hj#h=}$vePErFR72X9?^RDA)yS{Q_b8c>P>r+eI!6do4Gtjb2Fi_L z5r4r_hY`x@xlO*#J}b1J-vPtqM=Go1ip+hDJ(fTeFgTx$Ilk|8%k9dZ+i+L}SZoUP zXy7{)w_K}ELEglDOhf0zcHsCyIjA*Uv>L7Y>x7v`5MkQGt8T0AJ!`nlpzJm~HQ#HJ z-DBXYVH#-*Ocpa1AQHz?`Z-vPtNc*qZ&YjDivGWwW6a=n!6E@)ah%fF3*bK^%;euS zcBlZU(Ryk|i<6>WD*UZt9jVrVY7SdJv^Y!;&SvPvD>0fH_=|DI`L7G?w#?e^!6`kH z$vgZ&vGz6V;~2H%_>~*wPjl4455}>y4-qztrp8q(%D-us>Cp9=r)^>68d3ALqK=Ojf6vSPGg;?yByc@11i7Zm0 z8Cy=FEht2`PWGdk&FyW?Z|*q7u$`bARVQ$Ep0sOTbE4z=a=w0ZHaVL#()dM%KiD4w I*uX#7KgL)qfdBvi literal 0 HcmV?d00001 diff --git a/datafusion/sqllogictest/data/2.parquet b/datafusion/sqllogictest/data/2.parquet new file mode 100644 index 0000000000000000000000000000000000000000..b5e29f81baf152a741dc310da3348bd072532029 GIT binary patch literal 1403 zcmb_c&rcIU6n;Bw+8!EgjWh0=JrEK$jS?tT43fq;1)F#_p-<$9Kn(YR+=BP@u^m?8W z3YP&AA7+0-$pJ`C0KQTROnycNjo37r#At|yjN6cyNdA4Kvgm{4;6N-n1`zQX-w24==Gr0Biy9fIA-`!WOx#yi@=UmM$8ENs$LPj*P`6A z9u*h+z1F^vSuuJ%!#OYDBgR9{dwi+Jb7Bi;E?L18aLxGx0r5aE<7y2bY3Yjysmy|q zUO^=E{IRkeAfD++Ug)n{jmqlNu2YvVtzEm=FDGbv)%NV&C!M<6n&>)hIp4Z7 VlblKFY5cJIKX^F?uz`QOzW^hSGfe;h literal 0 HcmV?d00001 diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt index 24e76a570c009..61f4d6fc12a3b 100644 --- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt +++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt @@ -528,3 +528,35 @@ query TT select val, part from t_pushdown where part = val AND part = 'a'; ---- a a + +statement ok +COPY ( + SELECT + '00000000000000000000000000000001' AS trace_id, + '2023-10-01 00:00:00'::timestamptz AS start_timestamp, + 'prod' as deployment_environment +) +TO 'data/1.parquet'; + +statement ok +COPY ( + SELECT + '00000000000000000000000000000002' AS trace_id, + '2024-10-01 00:00:00'::timestamptz AS start_timestamp, + 'staging' as deployment_environment +) +TO 'data/2.parquet'; + +statement ok +CREATE EXTERNAL TABLE t1 STORED AS PARQUET LOCATION 'data/'; + +statement ok +SET datafusion.execution.parquet.pushdown_filters = true; + +query T +SELECT deployment_environment +FROM t1 +WHERE trace_id = '00000000000000000000000000000002' +ORDER BY start_timestamp, trace_id; +---- +staging From 52e4ef8a1d68cb131e008c8ef6eef931628b33ec Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Wed, 13 Aug 2025 12:03:49 -0700 Subject: [PATCH 082/177] Pass the input schema to stats_projection for ProjectionExpr (#17123) (#17174) * Pass the input schema to stats_projection for ProjectionExpr * Adds a test * fmt * clippy --------- Co-authored-by: Haresh Khanna --- datafusion/physical-plan/src/projection.rs | 88 +++++++++++++++++++++- 1 file changed, 86 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index a29f4aeb4090b..4c0c103a37690 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -249,7 +249,7 @@ impl ExecutionPlan for ProjectionExec { Ok(stats_projection( input_stats, self.expr.iter().map(|(e, _)| Arc::clone(e)), - Arc::clone(&self.schema), + Arc::clone(&self.input.schema()), )) } @@ -1030,8 +1030,10 @@ mod tests { use crate::common::collect; use crate::test; + use crate::test::exec::StatisticsExec; - use arrow::datatypes::DataType; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_common::stats::{ColumnStatistics, Precision, Statistics}; use datafusion_common::ScalarValue; use datafusion_expr::Operator; @@ -1230,4 +1232,86 @@ mod tests { assert_eq!(result, expected); } + + #[test] + fn test_projection_statistics_uses_input_schema() { + let input_schema = Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + Field::new("c", DataType::Int32, false), + Field::new("d", DataType::Int32, false), + Field::new("e", DataType::Int32, false), + Field::new("f", DataType::Int32, false), + ]); + + let input_statistics = Statistics { + num_rows: Precision::Exact(10), + column_statistics: vec![ + ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int32(Some(1))), + max_value: Precision::Exact(ScalarValue::Int32(Some(100))), + ..Default::default() + }, + ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int32(Some(5))), + max_value: Precision::Exact(ScalarValue::Int32(Some(50))), + ..Default::default() + }, + ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int32(Some(10))), + max_value: Precision::Exact(ScalarValue::Int32(Some(40))), + ..Default::default() + }, + ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int32(Some(20))), + max_value: Precision::Exact(ScalarValue::Int32(Some(30))), + ..Default::default() + }, + ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int32(Some(21))), + max_value: Precision::Exact(ScalarValue::Int32(Some(29))), + ..Default::default() + }, + ColumnStatistics { + min_value: Precision::Exact(ScalarValue::Int32(Some(24))), + max_value: Precision::Exact(ScalarValue::Int32(Some(26))), + ..Default::default() + }, + ], + ..Default::default() + }; + + let input = Arc::new(StatisticsExec::new(input_statistics, input_schema)); + + // Create projection expressions that reference columns from the input schema and the length + // of output schema columns < input schema columns and hence if we use the last few columns + // from the input schema in the expressions here, bounds_check would fail on them if output + // schema is supplied to the partitions_statistics method. + let exprs: Vec<(Arc, String)> = vec![ + ( + Arc::new(Column::new("c", 2)) as Arc, + "c_renamed".to_string(), + ), + ( + Arc::new(BinaryExpr::new( + Arc::new(Column::new("e", 4)), + Operator::Plus, + Arc::new(Column::new("f", 5)), + )) as Arc, + "e_plus_f".to_string(), + ), + ]; + + let projection = ProjectionExec::try_new(exprs, input).unwrap(); + + let stats = projection.partition_statistics(None).unwrap(); + + assert_eq!(stats.num_rows, Precision::Exact(10)); + assert_eq!( + stats.column_statistics.len(), + 2, + "Expected 2 columns in projection statistics" + ); + assert!(stats.total_byte_size.is_exact().unwrap_or(false)); + } } From f05b1285e90d5fd16b4c832cb229996beda138be Mon Sep 17 00:00:00 2001 From: Nuno Faria Date: Thu, 14 Aug 2025 13:16:20 +0100 Subject: [PATCH 083/177] [branch-49] fix: string_agg not respecting ORDER BY (#17058) * fix: string_agg not respecting ORDER BY * Fix equality of parametrizable ArrayAgg function (#17065) The `ArrayAgg` struct is stateful, therefore it must implement `AggregateUDFImpl::equals` and `hash_value` functions. * Implement AggregateUDFImpl::equals and AggregateUDFImpl::hash_value for ArrayAgg * Implement alternative fix * Remove 'use std::any::Any' * Add sqllogictest for string_agg plan * Revert as_any to their original implementations --------- Co-authored-by: Piotr Findeisen Co-authored-by: Andrew Lamb --- .../functions-aggregate/src/string_agg.rs | 4 + .../sqllogictest/test_files/aggregate.slt | 100 ++++++++++++++++++ 2 files changed, 104 insertions(+) diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs index 56c5ee1aaa676..5bf9020cd16ad 100644 --- a/datafusion/functions-aggregate/src/string_agg.rs +++ b/datafusion/functions-aggregate/src/string_agg.rs @@ -178,6 +178,10 @@ impl AggregateUDFImpl for StringAgg { ))) } + fn reverse_expr(&self) -> datafusion_expr::ReversedUDAF { + datafusion_expr::ReversedUDAF::Reversed(string_agg_udaf()) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index bdf327c98248a..753820b6b6193 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -6028,6 +6028,106 @@ GROUP BY dummy ---- text1 + +# Test string_agg with ORDER BY clasuses (issue #17011) +statement ok +create table t (k varchar, v int); + +statement ok +insert into t values ('a', 2), ('b', 3), ('c', 1), ('d', null); + +query T +select string_agg(k, ',' order by k) from t; +---- +a,b,c,d + +query T +select string_agg(k, ',' order by k desc) from t; +---- +d,c,b,a + +query T +select string_agg(k, ',' order by v) from t; +---- +c,a,b,d + +query T +select string_agg(k, ',' order by v nulls first) from t; +---- +d,c,a,b + +query T +select string_agg(k, ',' order by v desc) from t; +---- +d,b,a,c + +query T +select string_agg(k, ',' order by v desc nulls last) from t; +---- +b,a,c,d + +query T +-- odd indexes should appear first, ties solved by v +select string_agg(k, ',' order by v % 2 == 0, v) from t; +---- +c,b,a,d + +query T +-- odd indexes should appear first, ties solved by v desc +select string_agg(k, ',' order by v % 2 == 0, v desc) from t; +---- +b,c,a,d + +query T +select string_agg(k, ',' order by + case + when k = 'a' then 3 + when k = 'b' then 0 + when k = 'c' then 2 + when k = 'd' then 1 + end) +from t; +---- +b,d,c,a + +query T +select string_agg(k, ',' order by + case + when k = 'a' then 3 + when k = 'b' then 0 + when k = 'c' then 2 + when k = 'd' then 1 + end desc) +from t; +---- +a,c,d,b + +query TT +explain select string_agg(k, ',' order by v) from t; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v ASC NULLS LAST]]] +02)--TableScan: t projection=[k, v] +physical_plan +01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v ASC NULLS LAST]] +02)--SortExec: expr=[v@1 ASC NULLS LAST], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + +query TT +explain select string_agg(k, ',' order by v desc) from t; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]]] +02)--TableScan: t projection=[k, v] +physical_plan +01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]] +02)--SortExec: expr=[v@1 DESC], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + +statement ok +drop table t; + + # Tests for aggregating with NaN values statement ok CREATE TABLE float_table ( From d1a6e9a6558300115d913a54bbea27e141156606 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 14 Aug 2025 07:07:36 -0700 Subject: [PATCH 084/177] [branch-49] Update version to 49.0.1 and add changelog (#17175) * Update to version 49.0.1 * Add changelog for 49.0.1 * Fix sqllogictests * update configs * Update with PR * prettier * Fix slt race condition * Tweak release notes --- Cargo.lock | 80 +++++++++--------- Cargo.toml | 2 +- datafusion/sqllogictest/data/1.parquet | Bin 1381 -> 0 bytes datafusion/sqllogictest/data/2.parquet | Bin 1403 -> 0 bytes datafusion/sqllogictest/test_files/joins.slt | 20 ++--- .../test_files/parquet_filter_pushdown.slt | 6 +- dev/changelog/49.0.1.md | 48 +++++++++++ docs/source/user-guide/configs.md | 2 +- 8 files changed, 103 insertions(+), 55 deletions(-) delete mode 100644 datafusion/sqllogictest/data/1.parquet delete mode 100644 datafusion/sqllogictest/data/2.parquet create mode 100644 dev/changelog/49.0.1.md diff --git a/Cargo.lock b/Cargo.lock index a088005a0f197..6706ed46c02dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1818,7 +1818,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "arrow-ipc", @@ -1890,7 +1890,7 @@ dependencies = [ [[package]] name = "datafusion-benchmarks" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "datafusion", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -1938,7 +1938,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -1959,7 +1959,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "assert_cmd", @@ -1991,7 +1991,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "49.0.0" +version = "49.0.1" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2019,7 +2019,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "49.0.0" +version = "49.0.1" dependencies = [ "futures", "log", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-compression", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "49.0.0" +version = "49.0.1" dependencies = [ "apache-avro", "arrow", @@ -2088,7 +2088,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2111,7 +2111,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2134,7 +2134,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2166,11 +2166,11 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "49.0.0" +version = "49.0.1" [[package]] name = "datafusion-examples" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "arrow-flight", @@ -2201,7 +2201,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "chrono", @@ -2220,7 +2220,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2254,7 +2254,7 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "49.0.0" +version = "49.0.1" dependencies = [ "abi_stable", "arrow", @@ -2275,7 +2275,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "arrow-buffer", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "49.0.0" +version = "49.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2325,7 +2325,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "49.0.0" +version = "49.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2338,7 +2338,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "arrow-ord", @@ -2360,7 +2360,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2390,7 +2390,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "49.0.0" +version = "49.0.1" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2398,7 +2398,7 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "49.0.0" +version = "49.0.1" dependencies = [ "datafusion-expr", "quote", @@ -2407,7 +2407,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2434,7 +2434,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "49.0.0" +version = "49.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2459,7 +2459,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "49.0.0" +version = "49.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2471,7 +2471,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2492,7 +2492,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "49.0.0" +version = "49.0.1" dependencies = [ "ahash 0.8.12", "arrow", @@ -2528,7 +2528,7 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "chrono", @@ -2550,7 +2550,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "datafusion-common", @@ -2563,7 +2563,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "arrow-schema", @@ -2582,7 +2582,7 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "criterion", @@ -2620,7 +2620,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "bigdecimal", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "49.0.0" +version = "49.0.1" dependencies = [ "arrow", "async-trait", @@ -2677,7 +2677,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "49.0.0" +version = "49.0.1" dependencies = [ "async-recursion", "async-trait", @@ -2697,7 +2697,7 @@ dependencies = [ [[package]] name = "datafusion-wasmtest" -version = "49.0.0" +version = "49.0.1" dependencies = [ "chrono", "console_error_panic_hook", diff --git a/Cargo.toml b/Cargo.toml index 11cd3c637a971..742e2b8a19f9b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ repository = "https://github.com/apache/datafusion" # Define Minimum Supported Rust Version (MSRV) rust-version = "1.85.1" # Define DataFusion version -version = "49.0.0" +version = "49.0.1" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can diff --git a/datafusion/sqllogictest/data/1.parquet b/datafusion/sqllogictest/data/1.parquet deleted file mode 100644 index a04f669eaeaaeb9edd468f7c28c00ba526627007..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1381 zcmb_c&ubGw6n?v1(;NaO;*7hnhf?TLq_j!3QYDC!);7kpHj6Z=2w|IS+dz^Hn>A67 zMFhc<#~%6zcoe*N_9A!?57Lu(^p8;Rz1jS*p|w3YWOv@Y@6GqV_hvR5!cH-bW!cp{ zQyE+Wn0`O^8%i!fasu#`Qeg56A{fMHFeJ_*EMne(>51gOM@m04B9XuhVLq}{%n@gk z$At(4K>76U>#Hj#h=}$vePErFR72X9?^RDA)yS{Q_b8c>P>r+eI!6do4Gtjb2Fi_L z5r4r_hY`x@xlO*#J}b1J-vPtqM=Go1ip+hDJ(fTeFgTx$Ilk|8%k9dZ+i+L}SZoUP zXy7{)w_K}ELEglDOhf0zcHsCyIjA*Uv>L7Y>x7v`5MkQGt8T0AJ!`nlpzJm~HQ#HJ z-DBXYVH#-*Ocpa1AQHz?`Z-vPtNc*qZ&YjDivGWwW6a=n!6E@)ah%fF3*bK^%;euS zcBlZU(Ryk|i<6>WD*UZt9jVrVY7SdJv^Y!;&SvPvD>0fH_=|DI`L7G?w#?e^!6`kH z$vgZ&vGz6V;~2H%_>~*wPjl4455}>y4-qztrp8q(%D-us>Cp9=r)^>68d3ALqK=Ojf6vSPGg;?yByc@11i7Zm0 z8Cy=FEht2`PWGdk&FyW?Z|*q7u$`bARVQ$Ep0sOTbE4z=a=w0ZHaVL#()dM%KiD4w I*uX#7KgL)qfdBvi diff --git a/datafusion/sqllogictest/data/2.parquet b/datafusion/sqllogictest/data/2.parquet deleted file mode 100644 index b5e29f81baf152a741dc310da3348bd072532029..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1403 zcmb_c&rcIU6n;Bw+8!EgjWh0=JrEK$jS?tT43fq;1)F#_p-<$9Kn(YR+=BP@u^m?8W z3YP&AA7+0-$pJ`C0KQTROnycNjo37r#At|yjN6cyNdA4Kvgm{4;6N-n1`zQX-w24==Gr0Biy9fIA-`!WOx#yi@=UmM$8ENs$LPj*P`6A z9u*h+z1F^vSuuJ%!#OYDBgR9{dwi+Jb7Bi;E?L18aLxGx0r5aE<7y2bY3Yjysmy|q zUO^=E{IRkeAfD++Ug)n{jmqlNu2YvVtzEm=FDGbv)%NV&C!M<6n&>)hIp4Z7 VlblKFY5cJIKX^F?uz`QOzW^hSGfe;h diff --git a/datafusion/sqllogictest/test_files/joins.slt b/datafusion/sqllogictest/test_files/joins.slt index 6c40a71fa6ef5..71251c8b1625b 100644 --- a/datafusion/sqllogictest/test_files/joins.slt +++ b/datafusion/sqllogictest/test_files/joins.slt @@ -4240,23 +4240,23 @@ set datafusion.execution.target_partitions = 1; # Note we use csv as MemoryExec does not support limit push down (so doesn't manifest # bugs if limits are improperly pushed down) query I -COPY (values (1), (2), (3), (4), (5)) TO 'test_files/scratch/limit/t1.csv' +COPY (values (1), (2), (3), (4), (5)) TO 'test_files/scratch/joins/t1.csv' STORED AS CSV ---- 5 # store t2 in different order so the top N rows are not the same as the top N rows of t1 query I -COPY (values (5), (4), (3), (2), (1)) TO 'test_files/scratch/limit/t2.csv' +COPY (values (5), (4), (3), (2), (1)) TO 'test_files/scratch/joins/t2.csv' STORED AS CSV ---- 5 statement ok -create external table t1(a int) stored as CSV location 'test_files/scratch/limit/t1.csv'; +create external table t1(a int) stored as CSV location 'test_files/scratch/joins/t1.csv'; statement ok -create external table t2(b int) stored as CSV location 'test_files/scratch/limit/t2.csv'; +create external table t2(b int) stored as CSV location 'test_files/scratch/joins/t2.csv'; ###### ## LEFT JOIN w/ LIMIT @@ -4288,8 +4288,8 @@ logical_plan physical_plan 01)CoalesceBatchesExec: target_batch_size=3, fetch=2 02)--HashJoinExec: mode=CollectLeft, join_type=Left, on=[(a@0, b@0)] -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/t1.csv]]}, projection=[a], limit=2, file_type=csv, has_header=true -04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/t2.csv]]}, projection=[b], file_type=csv, has_header=true +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], limit=2, file_type=csv, has_header=true +04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], file_type=csv, has_header=true ###### ## RIGHT JOIN w/ LIMIT @@ -4322,8 +4322,8 @@ logical_plan physical_plan 01)CoalesceBatchesExec: target_batch_size=3, fetch=2 02)--HashJoinExec: mode=CollectLeft, join_type=Right, on=[(a@0, b@0)] -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/t1.csv]]}, projection=[a], file_type=csv, has_header=true -04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/t2.csv]]}, projection=[b], limit=2, file_type=csv, has_header=true +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], file_type=csv, has_header=true +04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], limit=2, file_type=csv, has_header=true ###### ## FULL JOIN w/ LIMIT @@ -4359,8 +4359,8 @@ logical_plan physical_plan 01)CoalesceBatchesExec: target_batch_size=3, fetch=2 02)--HashJoinExec: mode=CollectLeft, join_type=Full, on=[(a@0, b@0)] -03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/t1.csv]]}, projection=[a], file_type=csv, has_header=true -04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/limit/t2.csv]]}, projection=[b], file_type=csv, has_header=true +03)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t1.csv]]}, projection=[a], file_type=csv, has_header=true +04)----DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/joins/t2.csv]]}, projection=[b], file_type=csv, has_header=true statement ok drop table t1; diff --git a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt index 61f4d6fc12a3b..9f6f81789e894 100644 --- a/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt +++ b/datafusion/sqllogictest/test_files/parquet_filter_pushdown.slt @@ -536,7 +536,7 @@ COPY ( '2023-10-01 00:00:00'::timestamptz AS start_timestamp, 'prod' as deployment_environment ) -TO 'data/1.parquet'; +TO 'test_files/scratch/parquet_filter_pushdown/data/1.parquet'; statement ok COPY ( @@ -545,10 +545,10 @@ COPY ( '2024-10-01 00:00:00'::timestamptz AS start_timestamp, 'staging' as deployment_environment ) -TO 'data/2.parquet'; +TO 'test_files/scratch/parquet_filter_pushdown/data/2.parquet'; statement ok -CREATE EXTERNAL TABLE t1 STORED AS PARQUET LOCATION 'data/'; +CREATE EXTERNAL TABLE t1 STORED AS PARQUET LOCATION 'test_files/scratch/parquet_filter_pushdown/data/'; statement ok SET datafusion.execution.parquet.pushdown_filters = true; diff --git a/dev/changelog/49.0.1.md b/dev/changelog/49.0.1.md new file mode 100644 index 0000000000000..06d7c1e2c77a6 --- /dev/null +++ b/dev/changelog/49.0.1.md @@ -0,0 +1,48 @@ + + +# Apache DataFusion 49.0.1 Changelog + +This release consists of 5 commits from 5 contributors. See credits at the end of this changelog for more information. + +See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgrading.html) for information on how to upgrade from previous versions. + +**Other:** + +- [branch-49] Final Changelog Tweaks [#16852](https://github.com/apache/datafusion/pull/16852) (alamb) +- [branch-49] remove warning from every file open [#17059](https://github.com/apache/datafusion/pull/17059) (mbutrovich) +- [branch-49] Backport PR #16995 to branch-49 [#17068](https://github.com/apache/datafusion/pull/17068) (pepijnve) +- [branch-49] Backport "Add ExecutionPlan::reset_state (apache#17028)" to v49 [#17096](https://github.com/apache/datafusion/pull/17096) (adriangb) +- [branch-49] Backport #17129 to branch 49 [#17143](https://github.com/apache/datafusion/pull/17143) (AdamGS) +- [branch-49] Backport Pass the input schema to stats_projection for ProjectionExpr (#17123) [#17174](https://github.com/apache/datafusion/pull/17174) (alamb) +- [branch-49] fix: string_agg not respecting ORDER BY [#17058](https://github.com/apache/datafusion/pull/17058) (nuno-faria) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 1 Adam Gutglick + 1 Adrian Garcia Badaracco + 1 Andrew Lamb + 1 Matt Butrovich + 1 Pepijn Van Eeckhoudt +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 9ac1c59caa800..dc950de01f1b1 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -70,7 +70,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 49.0.0 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 49.0.1 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | From 374fcecca38136b29c8e1c145875e9e4464efb31 Mon Sep 17 00:00:00 2001 From: Huaijin Date: Wed, 20 Aug 2025 18:55:36 +0800 Subject: [PATCH 085/177] cherry-pick inlist fix (#17254) --- datafusion/proto/src/physical_plan/mod.rs | 36 +++++++------------ .../tests/cases/roundtrip_physical_plan.rs | 33 +++++++++++++++++ 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 52e0b20db2c2e..6e7546737d72c 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -611,29 +611,6 @@ impl protobuf::PhysicalPlanNode { ) -> Result> { let input: Arc = into_physical_plan(&filter.input, registry, runtime, extension_codec)?; - let projection = if !filter.projection.is_empty() { - Some( - filter - .projection - .iter() - .map(|i| *i as usize) - .collect::>(), - ) - } else { - None - }; - - // Use the projected schema if projection is present, otherwise use the full schema - let predicate_schema = if let Some(ref proj_indices) = projection { - // Create projected schema for parsing the predicate - let projected_fields: Vec<_> = proj_indices - .iter() - .map(|&i| input.schema().field(i).clone()) - .collect(); - Arc::new(Schema::new(projected_fields)) - } else { - input.schema() - }; let predicate = filter .expr @@ -642,7 +619,7 @@ impl protobuf::PhysicalPlanNode { parse_physical_expr( expr, registry, - predicate_schema.as_ref(), + input.schema().as_ref(), extension_codec, ) }) @@ -653,6 +630,17 @@ impl protobuf::PhysicalPlanNode { ) })?; let filter_selectivity = filter.default_filter_selectivity.try_into(); + let projection = if !filter.projection.is_empty() { + Some( + filter + .projection + .iter() + .map(|i| *i as usize) + .collect::>(), + ) + } else { + None + }; let filter = FilterExec::try_new(predicate, input)?.with_projection(projection)?; match filter_selectivity { diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 2d27a21447b22..24816c24a5afb 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -1839,3 +1839,36 @@ async fn test_round_trip_tpch_queries() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_tpch_part_in_list_query_with_real_parquet_data() -> Result<()> { + use datafusion_common::test_util::datafusion_test_data; + + let ctx = SessionContext::new(); + + // Register the TPC-H part table using the local test data + let test_data = datafusion_test_data(); + let table_sql = format!( + "CREATE EXTERNAL TABLE part STORED AS PARQUET LOCATION '{test_data}/tpch_part_small.parquet'" +); + ctx.sql(&table_sql).await.map_err(|e| { + DataFusionError::External(format!("Failed to create part table: {e}").into()) + })?; + + // Test the exact problematic query + let sql = + "SELECT p_size FROM part WHERE p_size IN (14, 6, 5, 31) and p_partkey > 1000"; + + let logical_plan = ctx.sql(sql).await?.into_unoptimized_plan(); + let optimized_plan = ctx.state().optimize(&logical_plan)?; + let physical_plan = ctx.state().create_physical_plan(&optimized_plan).await?; + + // Serialize the physical plan - bug may happen here already but not necessarily manifests + let codec = DefaultPhysicalExtensionCodec {}; + let proto = PhysicalPlanNode::try_from_physical_plan(physical_plan.clone(), &codec)?; + + // This will fail with the bug, but should succeed when fixed + let _deserialized_plan = + proto.try_into_physical_plan(&ctx, ctx.runtime_env().as_ref(), &codec)?; + Ok(()) +} From 930608a7161f89163727ec98684c4ecaf2d4bf04 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 21 Aug 2025 16:37:08 +0800 Subject: [PATCH 086/177] fix check license header --- datafusion/physical-plan/src/node_id.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/physical-plan/src/node_id.rs b/datafusion/physical-plan/src/node_id.rs index 2a246db0a77b5..7b8d0281eb73b 100644 --- a/datafusion/physical-plan/src/node_id.rs +++ b/datafusion/physical-plan/src/node_id.rs @@ -14,6 +14,7 @@ // KIND, either express or implied. See the License for the // specific language governing permissions and limitations // under the License. + use std::sync::Arc; use crate::ExecutionPlan; From 66ae5885b255e5409a214be2e14441d66f062942 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 21 Aug 2025 17:16:07 +0800 Subject: [PATCH 087/177] fix cargo check: cargo check --profile ci --workspace --all-targets --features integration-tests --locked --- datafusion-examples/examples/planner_api.rs | 2 +- datafusion/core/tests/fuzz_cases/sort_fuzz.rs | 2 +- .../core/tests/physical_optimizer/enforce_distribution.rs | 5 +++-- datafusion/physical-optimizer/src/enforce_distribution.rs | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/datafusion-examples/examples/planner_api.rs b/datafusion-examples/examples/planner_api.rs index 3e718d71f1fbb..690066a78d7c2 100644 --- a/datafusion-examples/examples/planner_api.rs +++ b/datafusion-examples/examples/planner_api.rs @@ -17,7 +17,7 @@ use datafusion::error::Result; use datafusion::logical_expr::LogicalPlan; -use datafusion::physical_plan::displayable; +use datafusion::physical_plan::{displayable, ExecutionPlan}; use datafusion::physical_planner::DefaultPhysicalPlanner; use datafusion::prelude::*; diff --git a/datafusion/core/tests/fuzz_cases/sort_fuzz.rs b/datafusion/core/tests/fuzz_cases/sort_fuzz.rs index 72aab1acd1f2a..703b8715821a8 100644 --- a/datafusion/core/tests/fuzz_cases/sort_fuzz.rs +++ b/datafusion/core/tests/fuzz_cases/sort_fuzz.rs @@ -224,7 +224,7 @@ impl SortTest { /// Sort the input using SortExec and ensure the results are /// correct according to `Vec::sort` both with and without spilling async fn run(&self) -> (Vec>, Vec) { - let input = Arc::clone(self.input()); + let input = self.input.clone(); let first_batch = input .iter() .flat_map(|p| p.iter()) diff --git a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs index 4034800c30cba..d98f6f59ca34f 100644 --- a/datafusion/core/tests/physical_optimizer/enforce_distribution.rs +++ b/datafusion/core/tests/physical_optimizer/enforce_distribution.rs @@ -3603,10 +3603,11 @@ fn test_replace_order_preserving_variants_with_fetch() -> Result<()> { ); // Apply the function - let result = replace_order_preserving_variants(dist_context)?; + let result = replace_order_preserving_variants(dist_context, false)?; // Verify the plan was transformed to CoalescePartitionsExec result + .0 .plan .as_any() .downcast_ref::() @@ -3614,7 +3615,7 @@ fn test_replace_order_preserving_variants_with_fetch() -> Result<()> { // Verify fetch was preserved assert_eq!( - result.plan.fetch(), + result.0.plan.fetch(), Some(5), "Fetch value was not preserved after transformation" ); diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 0650bfbcf14ec..77c7af76ccbef 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -1034,7 +1034,7 @@ fn remove_dist_changing_operators( /// " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2", /// " DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet", /// ``` -fn replace_order_preserving_variants( +pub fn replace_order_preserving_variants( mut context: DistributionContext, ordering_satisfied: bool, ) -> Result<(DistributionContext, Option)> { From 292641c851047443463dd20e9b4d1c62bb200329 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Thu, 21 Aug 2025 17:53:30 +0800 Subject: [PATCH 088/177] fix cargo example --- datafusion-examples/examples/planner_api.rs | 28 +-------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/datafusion-examples/examples/planner_api.rs b/datafusion-examples/examples/planner_api.rs index 690066a78d7c2..55aec7b0108a4 100644 --- a/datafusion-examples/examples/planner_api.rs +++ b/datafusion-examples/examples/planner_api.rs @@ -17,7 +17,7 @@ use datafusion::error::Result; use datafusion::logical_expr::LogicalPlan; -use datafusion::physical_plan::{displayable, ExecutionPlan}; +use datafusion::physical_plan::displayable; use datafusion::physical_planner::DefaultPhysicalPlanner; use datafusion::prelude::*; @@ -80,35 +80,9 @@ async fn to_physical_plan_in_one_api_demo( displayable(physical_plan.as_ref()).indent(false) ); - let traversal = extract_node_ids_from_execution_plan_tree(physical_plan.as_ref()); - let expected_traversal = vec![ - Some(0), - Some(1), - Some(2), - Some(3), - Some(4), - Some(5), - Some(6), - Some(7), - Some(8), - Some(9), - ]; - assert_eq!(expected_traversal, traversal); Ok(()) } -fn extract_node_ids_from_execution_plan_tree( - physical_plan: &dyn ExecutionPlan, -) -> Vec> { - let mut traversed_nodes: Vec> = vec![]; - for child in physical_plan.children() { - let node_ids = extract_node_ids_from_execution_plan_tree(child.as_ref()); - traversed_nodes.extend(node_ids); - } - traversed_nodes.push(physical_plan.properties().node_id()); - traversed_nodes -} - /// Converts a logical plan into a physical plan by utilizing the analyzer, /// optimizer, and query planner APIs separately. This flavor gives more /// control over the planning process. From a6068c27ca2bf91a3d289e4e5d42beb6600f0154 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 21 Aug 2025 12:18:59 -0400 Subject: [PATCH 089/177] FFI_RecordBatchStream was causing a memory leak (#17190) (#17270) Co-authored-by: Andrew Lamb --- datafusion/ffi/src/record_batch_stream.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/datafusion/ffi/src/record_batch_stream.rs b/datafusion/ffi/src/record_batch_stream.rs index 78d65a816fcc2..6c2282df88dd0 100644 --- a/datafusion/ffi/src/record_batch_stream.rs +++ b/datafusion/ffi/src/record_batch_stream.rs @@ -57,6 +57,9 @@ pub struct FFI_RecordBatchStream { /// Return the schema of the record batch pub schema: unsafe extern "C" fn(stream: &Self) -> WrappedSchema, + /// Release the memory of the private data when it is no longer being used. + pub release: unsafe extern "C" fn(arg: &mut Self), + /// Internal data. This is only to be accessed by the provider of the plan. /// The foreign library should never attempt to access this data. pub private_data: *mut c_void, @@ -82,6 +85,7 @@ impl FFI_RecordBatchStream { FFI_RecordBatchStream { poll_next: poll_next_fn_wrapper, schema: schema_fn_wrapper, + release: release_fn_wrapper, private_data, } } @@ -96,6 +100,12 @@ unsafe extern "C" fn schema_fn_wrapper(stream: &FFI_RecordBatchStream) -> Wrappe (*stream).schema().into() } +unsafe extern "C" fn release_fn_wrapper(provider: &mut FFI_RecordBatchStream) { + let private_data = + Box::from_raw(provider.private_data as *mut RecordBatchStreamPrivateData); + drop(private_data); +} + fn record_batch_to_wrapped_array( record_batch: RecordBatch, ) -> RResult { @@ -197,6 +207,12 @@ impl Stream for FFI_RecordBatchStream { } } +impl Drop for FFI_RecordBatchStream { + fn drop(&mut self) { + unsafe { (self.release)(self) } + } +} + #[cfg(test)] mod tests { use std::sync::Arc; From 0d04475ea87a56d327ab772b2b35a94c0faf5bf6 Mon Sep 17 00:00:00 2001 From: Oleks V Date: Thu, 21 Aug 2025 12:54:09 -0700 Subject: [PATCH 090/177] fix: align `array_has` null buffer for scalar (#17272) (#17274) * fix: align `array_has` null buffer for scalar (#17272) * fix: align `array_has` null buffer for scalar * merge --- datafusion/physical-expr-common/src/datum.rs | 21 ++++++++++++++++++-- datafusion/sqllogictest/test_files/array.slt | 15 ++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr-common/src/datum.rs b/datafusion/physical-expr-common/src/datum.rs index 233deff758c7b..7084bc440e86b 100644 --- a/datafusion/physical-expr-common/src/datum.rs +++ b/datafusion/physical-expr-common/src/datum.rs @@ -154,9 +154,26 @@ pub fn compare_op_for_nested( if matches!(op, Operator::IsDistinctFrom | Operator::IsNotDistinctFrom) { Ok(BooleanArray::new(values, None)) } else { - // If one of the side is NULL, we returns NULL + // If one of the side is NULL, we return NULL // i.e. NULL eq NULL -> NULL - let nulls = NullBuffer::union(l.nulls(), r.nulls()); + // For nested comparisons, we need to ensure the null buffer matches the result length + let nulls = match (is_l_scalar, is_r_scalar) { + (false, false) | (true, true) => NullBuffer::union(l.nulls(), r.nulls()), + (true, false) => { + // When left is null-scalar and right is array, expand left nulls to match result length + match l.nulls().filter(|nulls| !nulls.is_valid(0)) { + Some(_) => Some(NullBuffer::new_null(len)), // Left scalar is null + None => r.nulls().cloned(), // Left scalar is non-null + } + } + (false, true) => { + // When right is null-scalar and left is array, expand right nulls to match result length + match r.nulls().filter(|nulls| !nulls.is_valid(0)) { + Some(_) => Some(NullBuffer::new_null(len)), // Right scalar is null + None => l.nulls().cloned(), // Right scalar is non-null + } + } + }; Ok(BooleanArray::new(values, nulls)) } } diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index a3d9c3e1d9c1f..14261a6e627c9 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -8065,6 +8065,18 @@ FixedSizeList(Field { name: "item", data_type: Int32, nullable: true, dict_id: 0 statement error create table varying_fixed_size_col_table (a int[3]) as values ([1,2,3]), ([4,5]); +statement ok +COPY (select [[true, false], [false, true]] a, [false, true] b union select [[null, null]], null) to 'test_files/scratch/array/array_has/single_file.parquet' stored as parquet; + +statement ok +CREATE EXTERNAL TABLE array_has STORED AS PARQUET location 'test_files/scratch/array/array_has/single_file.parquet'; + +query B +select array_contains(a, b) from array_has order by 1 nulls last; +---- +true +NULL + ### Delete tables statement ok @@ -8243,3 +8255,6 @@ drop table values_all_empty; statement ok drop table fixed_size_col_table; + +statement ok +drop table array_has; From f43df3f2ae3aafb347996c58e852cc378807095b Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 21 Aug 2025 14:49:49 -0700 Subject: [PATCH 091/177] [branch-49] Prepare `49.0.2` version and changelog (#17277) * Update versio to 49.0.2 * Add changelog * update configuration docs --- Cargo.lock | 80 +++++++++++++++---------------- Cargo.toml | 2 +- dev/changelog/49.0.2.md | 45 +++++++++++++++++ docs/source/user-guide/configs.md | 2 +- 4 files changed, 87 insertions(+), 42 deletions(-) create mode 100644 dev/changelog/49.0.2.md diff --git a/Cargo.lock b/Cargo.lock index 6706ed46c02dd..8ffdb8c6403c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1818,7 +1818,7 @@ dependencies = [ [[package]] name = "datafusion" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "arrow-ipc", @@ -1890,7 +1890,7 @@ dependencies = [ [[package]] name = "datafusion-benchmarks" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "datafusion", @@ -1914,7 +1914,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -1938,7 +1938,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -1959,7 +1959,7 @@ dependencies = [ [[package]] name = "datafusion-cli" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "assert_cmd", @@ -1991,7 +1991,7 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "49.0.1" +version = "49.0.2" dependencies = [ "ahash 0.8.12", "apache-avro", @@ -2019,7 +2019,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "49.0.1" +version = "49.0.2" dependencies = [ "futures", "log", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-compression", @@ -2063,7 +2063,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-avro" -version = "49.0.1" +version = "49.0.2" dependencies = [ "apache-avro", "arrow", @@ -2088,7 +2088,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2111,7 +2111,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2134,7 +2134,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2166,11 +2166,11 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "49.0.1" +version = "49.0.2" [[package]] name = "datafusion-examples" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "arrow-flight", @@ -2201,7 +2201,7 @@ dependencies = [ [[package]] name = "datafusion-execution" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "chrono", @@ -2220,7 +2220,7 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2243,7 +2243,7 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "datafusion-common", @@ -2254,7 +2254,7 @@ dependencies = [ [[package]] name = "datafusion-ffi" -version = "49.0.1" +version = "49.0.2" dependencies = [ "abi_stable", "arrow", @@ -2275,7 +2275,7 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "arrow-buffer", @@ -2304,7 +2304,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "49.0.1" +version = "49.0.2" dependencies = [ "ahash 0.8.12", "arrow", @@ -2325,7 +2325,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "49.0.1" +version = "49.0.2" dependencies = [ "ahash 0.8.12", "arrow", @@ -2338,7 +2338,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "arrow-ord", @@ -2360,7 +2360,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2374,7 +2374,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "datafusion-common", @@ -2390,7 +2390,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "49.0.1" +version = "49.0.2" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2398,7 +2398,7 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "49.0.1" +version = "49.0.2" dependencies = [ "datafusion-expr", "quote", @@ -2407,7 +2407,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2434,7 +2434,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "49.0.1" +version = "49.0.2" dependencies = [ "ahash 0.8.12", "arrow", @@ -2459,7 +2459,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "49.0.1" +version = "49.0.2" dependencies = [ "ahash 0.8.12", "arrow", @@ -2471,7 +2471,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "datafusion-common", @@ -2492,7 +2492,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "49.0.1" +version = "49.0.2" dependencies = [ "ahash 0.8.12", "arrow", @@ -2528,7 +2528,7 @@ dependencies = [ [[package]] name = "datafusion-proto" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "chrono", @@ -2550,7 +2550,7 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "datafusion-common", @@ -2563,7 +2563,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "arrow-schema", @@ -2582,7 +2582,7 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "datafusion-spark" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "criterion", @@ -2620,7 +2620,7 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "bigdecimal", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "datafusion-sqllogictest" -version = "49.0.1" +version = "49.0.2" dependencies = [ "arrow", "async-trait", @@ -2677,7 +2677,7 @@ dependencies = [ [[package]] name = "datafusion-substrait" -version = "49.0.1" +version = "49.0.2" dependencies = [ "async-recursion", "async-trait", @@ -2697,7 +2697,7 @@ dependencies = [ [[package]] name = "datafusion-wasmtest" -version = "49.0.1" +version = "49.0.2" dependencies = [ "chrono", "console_error_panic_hook", diff --git a/Cargo.toml b/Cargo.toml index 742e2b8a19f9b..601d11f12dd81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ repository = "https://github.com/apache/datafusion" # Define Minimum Supported Rust Version (MSRV) rust-version = "1.85.1" # Define DataFusion version -version = "49.0.1" +version = "49.0.2" [workspace.dependencies] # We turn off default-features for some dependencies here so the workspaces which inherit them can diff --git a/dev/changelog/49.0.2.md b/dev/changelog/49.0.2.md new file mode 100644 index 0000000000000..7e6fc3e7eb487 --- /dev/null +++ b/dev/changelog/49.0.2.md @@ -0,0 +1,45 @@ + + +# Apache DataFusion 49.0.2 Changelog + +This release consists of 3 commits from 3 contributors. See credits at the end of this changelog for more information. + +See the [upgrade guide](https://datafusion.apache.org/library-user-guide/upgrading.html) for information on how to upgrade from previous versions. + +**Fixed bugs:** + +- fix: align `array_has` null buffer for scalar (#17272) [#17274](https://github.com/apache/datafusion/pull/17274) (comphead) + +**Other:** + +- [branch-49] Backport fix: deserialization error for FilterExec (predicates with inlist) [#17254](https://github.com/apache/datafusion/pull/17254) (haohuaijin) +- [branch-49] FFI_RecordBatchStream was causing a memory leak (#17190) [#17270](https://github.com/apache/datafusion/pull/17270) (timsaucer) + +## Credits + +Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor. + +``` + 1 Huaijin + 1 Oleks V + 1 Tim Saucer +``` + +Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index dc950de01f1b1..d453cb0684daf 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -70,7 +70,7 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | | datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 49.0.1 | (writing) Sets "created by" property | +| datafusion.execution.parquet.created_by | datafusion version 49.0.2 | (writing) Sets "created by" property | | datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | | datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | | datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | From 25058de27fe70bafa61514e445ea5cda2d9b3661 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 22 Aug 2025 15:25:19 +0800 Subject: [PATCH 092/177] fix cargo check --profile ci --no-default-features -p datafusion-proto --- datafusion/proto/Cargo.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion/proto/Cargo.toml b/datafusion/proto/Cargo.toml index e9a8b83dc4f91..a1eeabdf87f4a 100644 --- a/datafusion/proto/Cargo.toml +++ b/datafusion/proto/Cargo.toml @@ -46,7 +46,7 @@ avro = ["datafusion/avro", "datafusion-common/avro"] [dependencies] arrow = { workspace = true } chrono = { workspace = true } -datafusion = { workspace = true, default-features = false } +datafusion = { workspace = true, default-features = true } datafusion-common = { workspace = true, default-features = true } datafusion-expr = { workspace = true } datafusion-proto-common = { workspace = true } @@ -56,7 +56,6 @@ prost = { workspace = true } serde = { version = "1.0", optional = true } serde_json = { workspace = true, optional = true } [dev-dependencies] -datafusion = { workspace = true, default-features = true } datafusion-functions = { workspace = true, default-features = true } datafusion-functions-aggregate = { workspace = true } datafusion-functions-window-common = { workspace = true } From c46f7a9e4c5fd7c3bdf0d86141d2282af97896bf Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 22 Aug 2025 15:34:35 +0800 Subject: [PATCH 093/177] fix cargo doc --- datafusion/physical-plan/src/aggregates/group_values/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rs index 6c0584bdf5c25..1e4c7558bda39 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! [`GroupValues`] trait for storing and interning group keys +//! `GroupValues` trait for storing and interning group keys use arrow::array::types::{ Date32Type, Date64Type, Decimal128Type, Time32MillisecondType, Time32SecondType, @@ -84,7 +84,7 @@ mod null_builder; /// Each distinct group in a hash aggregation is identified by a unique group id /// (usize) which is assigned by instances of this trait. Group ids are /// continuous without gaps, starting from 0. -pub trait GroupValues: Send { +pub(crate) trait GroupValues: Send { /// Calculates the group id for each input row of `cols`, assigning new /// group ids as necessary. /// @@ -127,7 +127,7 @@ pub trait GroupValues: Send { /// /// [`GroupColumn`]: crate::aggregates::group_values::multi_group_by::GroupColumn /// -pub fn new_group_values( +pub(crate) fn new_group_values( schema: SchemaRef, group_ordering: &GroupOrdering, ) -> Result> { From deaf2e2d6dc1ac8b0c293494a865546f1682c949 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 22 Aug 2025 17:18:21 +0800 Subject: [PATCH 094/177] fix ut:custom_sources_cases::statistics::sql_limit(with_node_id of CoalescePartitionsExec missed fetch) --- .github/workflows/audit.yml | 2 +- datafusion/physical-plan/src/coalesce_partitions.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 491fa27c2a56a..ce41442f3c167 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -44,4 +44,4 @@ jobs: - name: Run audit check # Ignored until https://github.com/apache/datafusion/issues/15571 # ignored py03 warning until arrow 55 upgrade - run: cargo audit --ignore RUSTSEC-2024-0370 --ignore RUSTSEC-2025-0020 + run: cargo audit --ignore RUSTSEC-2024-0370 --ignore RUSTSEC-2025-0020 --ignore RUSTSEC-2025-0047 diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index d7beffb19faac..7daf0d753d15f 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -217,6 +217,7 @@ impl ExecutionPlan for CoalescePartitionsExec { _node_id: usize, ) -> Result>> { let mut new_plan = CoalescePartitionsExec::new(Arc::clone(self.input())); + new_plan.fetch = self.fetch; let new_props = new_plan.cache.clone().with_node_id(_node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) From f1b1bd83193554174a0f6eadcce782e51d181d15 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 22 Aug 2025 17:54:24 +0800 Subject: [PATCH 095/177] fix ut: test_no_pushdown_through_aggregates & test_plan_with_order_preserving_variants_preserves_fetch --- .../core/tests/physical_optimizer/filter_pushdown/mod.rs | 2 +- .../replace_with_order_preserving_variants.rs | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs index a28933d97bcd1..040b99f5ffdc1 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs @@ -276,7 +276,7 @@ fn test_no_pushdown_through_aggregates() { Ok: - FilterExec: b@1 = bar - CoalesceBatchesExec: target_batch_size=100 - - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt] + - AggregateExec: mode=Final, gby=[a@0 as a, b@1 as b], aggr=[cnt], ordering_mode=PartiallySorted([0]) - CoalesceBatchesExec: target_batch_size=10 - DataSourceExec: file_groups={1 group: [[test.paqruet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=a@0 = foo " diff --git a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs index caa536107d8da..9769e2e0366f7 100644 --- a/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs +++ b/datafusion/physical-optimizer/src/enforce_sorting/replace_with_order_preserving_variants.rs @@ -139,11 +139,6 @@ pub fn plan_with_order_preserving_variants( return Ok(sort_input); } else if is_coalesce_partitions(&sort_input.plan) && is_spm_better { let child = &sort_input.children[0].plan; - let mut fetch = fetch; - if let Some(coalesce_fetch) = sort_input.plan.fetch() { - // Get the min fetch between the `fetch` and the coalesce's fetch: - fetch = Some(coalesce_fetch.min(fetch.unwrap_or(usize::MAX))) - }; if let Some(ordering) = child.output_ordering() { let mut fetch = fetch; if let Some(coalesce_fetch) = sort_input.plan.fetch() { From 7dd5e6e7dff05047281e4a6cbdc31d093a4932d0 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 22 Aug 2025 18:27:52 +0800 Subject: [PATCH 096/177] fix format --- datafusion/optimizer/src/push_down_filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index cfa22c3b61426..f701fcf861e0d 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -2308,7 +2308,7 @@ mod tests { plan, @r" Projection: test.a, test1.d - Cross Join: + Cross Join: Projection: test.a, test.b, test.c TableScan: test, full_filters=[test.a = Int32(1)] Projection: test1.d, test1.e, test1.f @@ -2338,7 +2338,7 @@ mod tests { plan, @r" Projection: test.a, test1.a - Cross Join: + Cross Join: Projection: test.a, test.b, test.c TableScan: test, full_filters=[test.a = Int32(1)] Projection: test1.a, test1.b, test1.c From 2eca4c0d67ce39550d7004624c6d443f43bee25f Mon Sep 17 00:00:00 2001 From: xudong963 Date: Fri, 22 Aug 2025 19:03:23 +0800 Subject: [PATCH 097/177] fix roundtrip_test --- datafusion/physical-plan/src/empty.rs | 10 ++++++++++ .../proto/tests/cases/roundtrip_physical_plan.rs | 14 +++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs index 36634fbe6d7e9..eae1aaff1c492 100644 --- a/datafusion/physical-plan/src/empty.rs +++ b/datafusion/physical-plan/src/empty.rs @@ -173,6 +173,16 @@ impl ExecutionPlan for EmptyExec { None, )) } + + fn with_node_id( + self: Arc, + _node_id: usize, + ) -> Result>> { + let mut new_plan = EmptyExec::new(self.schema.clone()); + let new_props = new_plan.cache.clone().with_node_id(_node_id); + new_plan.cache = new_props; + Ok(Some(Arc::new(new_plan))) + } } #[cfg(test)] diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 7d56bb6c5db1b..b3827dae51b24 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -31,6 +31,9 @@ use arrow::csv::WriterBuilder; use arrow::datatypes::{Fields, TimeUnit}; use datafusion::physical_expr::aggregate::AggregateExprBuilder; use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; +use datafusion::physical_plan::node_id::{ + annotate_node_id_for_execution_plan, NodeIdAnnotator, +}; use datafusion_expr::dml::InsertOp; use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf; use datafusion_functions_aggregate::array_agg::array_agg_udaf; @@ -133,13 +136,22 @@ fn roundtrip_test_and_return( ctx: &SessionContext, codec: &dyn PhysicalExtensionCodec, ) -> Result> { + let mut annotator = NodeIdAnnotator::new(); + let exec_plan = annotate_node_id_for_execution_plan(&exec_plan, &mut annotator)?; let proto: protobuf::PhysicalPlanNode = protobuf::PhysicalPlanNode::try_from_physical_plan(exec_plan.clone(), codec) .expect("to proto"); let runtime = ctx.runtime_env(); - let result_exec_plan: Arc = proto + let mut result_exec_plan: Arc = proto .try_into_physical_plan(ctx, runtime.deref(), codec) .expect("from proto"); + + // Re-annotate the deserialized plan with node IDs to match the original plan structure + // This ensures that the roundtrip preserves the node_id values for comparison + let mut annotator = NodeIdAnnotator::new(); + result_exec_plan = + annotate_node_id_for_execution_plan(&result_exec_plan, &mut annotator)?; + assert_eq!(format!("{exec_plan:?}"), format!("{result_exec_plan:?}")); Ok(result_exec_plan) } From 8baa05db257c9a9e07c148269faf3388d39d49df Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 12:03:01 +0800 Subject: [PATCH 098/177] schema_force_view_types to true --- datafusion/common/src/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 0d34815a248f7..883d2b60a8976 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -457,7 +457,7 @@ config_namespace! { /// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, /// and `Binary/BinaryLarge` with `BinaryView`. - pub schema_force_view_types: bool, default = false + pub schema_force_view_types: bool, default = true /// (reading) If true, parquet reader will read columns of /// `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. From 9b2fbbbcaca511a184dd331b679f62c9721e8bd8 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 12:40:32 +0800 Subject: [PATCH 099/177] use utfview8 --- datafusion/core/tests/parquet/page_pruning.rs | 2 +- .../sqllogictest/test_files/describe.slt | 4 +- .../test_files/information_schema.slt | 4 +- datafusion/sqllogictest/test_files/map.slt | 2 +- .../sqllogictest/test_files/parquet.slt | 46 +++++++++---------- .../test_files/simplify_predicates.slt | 4 +- 6 files changed, 31 insertions(+), 31 deletions(-) diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rs index 12e3436550c57..9da879a32f6b5 100644 --- a/datafusion/core/tests/parquet/page_pruning.rs +++ b/datafusion/core/tests/parquet/page_pruning.rs @@ -160,7 +160,7 @@ async fn page_index_filter_one_col() { // 5.create filter date_string_col == "01/01/09"`; // Note this test doesn't apply type coercion so the literal must match the actual view type - let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8("01/01/09"))); + let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8view("01/01/09"))); let parquet_exec = get_parquet_exec(&state, filter).await; let mut results = parquet_exec.execute(0, task_ctx.clone()).unwrap(); let batch = results.next().await.unwrap().unwrap(); diff --git a/datafusion/sqllogictest/test_files/describe.slt b/datafusion/sqllogictest/test_files/describe.slt index 077e8e6474d1f..e4cb30628eec5 100644 --- a/datafusion/sqllogictest/test_files/describe.slt +++ b/datafusion/sqllogictest/test_files/describe.slt @@ -81,8 +81,8 @@ int_col Int32 YES bigint_col Int64 YES float_col Float32 YES double_col Float64 YES -date_string_col Utf8 YES -string_col Utf8 YES +date_string_col Utf8View YES +string_col Utf8View YES timestamp_col Timestamp(Nanosecond, None) YES year Int32 YES month Int32 YES diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 9f39dbbd5ba25..2ce64ffc68365 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -249,7 +249,7 @@ datafusion.execution.parquet.metadata_size_hint NULL datafusion.execution.parquet.pruning true datafusion.execution.parquet.pushdown_filters false datafusion.execution.parquet.reorder_filters false -datafusion.execution.parquet.schema_force_view_types false +datafusion.execution.parquet.schema_force_view_types true datafusion.execution.parquet.skip_arrow_metadata false datafusion.execution.parquet.skip_metadata true datafusion.execution.parquet.statistics_enabled page @@ -359,7 +359,7 @@ datafusion.execution.parquet.metadata_size_hint NULL (reading) If specified, the datafusion.execution.parquet.pruning true (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file datafusion.execution.parquet.pushdown_filters false (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". datafusion.execution.parquet.reorder_filters false (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query -datafusion.execution.parquet.schema_force_view_types false (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. +datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. datafusion.execution.parquet.skip_arrow_metadata false (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata datafusion.execution.parquet.statistics_enabled page (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt index 8546bdbdd0673..42a4ba6218016 100644 --- a/datafusion/sqllogictest/test_files/map.slt +++ b/datafusion/sqllogictest/test_files/map.slt @@ -45,7 +45,7 @@ describe data; ---- ints Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO strings Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO -timestamp Utf8 NO +timestamp Utf8View NO query ??T SELECT * FROM data ORDER by ints['bytes'] DESC LIMIT 10; diff --git a/datafusion/sqllogictest/test_files/parquet.slt b/datafusion/sqllogictest/test_files/parquet.slt index e5b3f740e2912..abc6fdab3c8a0 100644 --- a/datafusion/sqllogictest/test_files/parquet.slt +++ b/datafusion/sqllogictest/test_files/parquet.slt @@ -384,15 +384,15 @@ select arrow_typeof(binaryview_col), binaryview_col FROM binary_as_string_default; ---- -Binary 616161 Binary 616161 Binary 616161 -Binary 626262 Binary 626262 Binary 626262 -Binary 636363 Binary 636363 Binary 636363 -Binary 646464 Binary 646464 Binary 646464 -Binary 656565 Binary 656565 Binary 656565 -Binary 666666 Binary 666666 Binary 666666 -Binary 676767 Binary 676767 Binary 676767 -Binary 686868 Binary 686868 Binary 686868 -Binary 696969 Binary 696969 Binary 696969 +BinaryView 616161 BinaryView 616161 BinaryView 616161 +BinaryView 626262 BinaryView 626262 BinaryView 626262 +BinaryView 636363 BinaryView 636363 BinaryView 636363 +BinaryView 646464 BinaryView 646464 BinaryView 646464 +BinaryView 656565 BinaryView 656565 BinaryView 656565 +BinaryView 666666 BinaryView 666666 BinaryView 666666 +BinaryView 676767 BinaryView 676767 BinaryView 676767 +BinaryView 686868 BinaryView 686868 BinaryView 686868 +BinaryView 696969 BinaryView 696969 BinaryView 696969 # Run an explain plan to show the cast happens in the plan (a CAST is needed for the predicates) query TT @@ -405,11 +405,11 @@ EXPLAIN binaryview_col LIKE '%a%'; ---- logical_plan -01)Filter: CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8) LIKE Utf8("%a%") -02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8) LIKE Utf8("%a%")] +01)Filter: CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%") +02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 -02)--FilterExec: CAST(binary_col@0 AS Utf8) LIKE %a% AND CAST(largebinary_col@1 AS Utf8) LIKE %a% AND CAST(binaryview_col@2 AS Utf8) LIKE %a% +02)--FilterExec: CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% 03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/binary_as_string.parquet]]}, projection=[binary_col, largebinary_col, binaryview_col], file_type=parquet, predicate=CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% @@ -432,15 +432,15 @@ select arrow_typeof(binaryview_col), binaryview_col FROM binary_as_string_option; ---- -Utf8 aaa Utf8 aaa Utf8 aaa -Utf8 bbb Utf8 bbb Utf8 bbb -Utf8 ccc Utf8 ccc Utf8 ccc -Utf8 ddd Utf8 ddd Utf8 ddd -Utf8 eee Utf8 eee Utf8 eee -Utf8 fff Utf8 fff Utf8 fff -Utf8 ggg Utf8 ggg Utf8 ggg -Utf8 hhh Utf8 hhh Utf8 hhh -Utf8 iii Utf8 iii Utf8 iii +Utf8View aaa Utf8View aaa Utf8View aaa +Utf8View bbb Utf8View bbb Utf8View bbb +Utf8View ccc Utf8View ccc Utf8View ccc +Utf8View ddd Utf8View ddd Utf8View ddd +Utf8View eee Utf8View eee Utf8View eee +Utf8View fff Utf8View fff Utf8View fff +Utf8View ggg Utf8View ggg Utf8View ggg +Utf8View hhh Utf8View hhh Utf8View hhh +Utf8View iii Utf8View iii Utf8View iii # Run an explain plan to show the cast happens in the plan (there should be no casts) query TT @@ -453,8 +453,8 @@ EXPLAIN binaryview_col LIKE '%a%'; ---- logical_plan -01)Filter: binary_as_string_option.binary_col LIKE Utf8("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8("%a%") -02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8("%a%"), binary_as_string_option.largebinary_col LIKE Utf8("%a%"), binary_as_string_option.binaryview_col LIKE Utf8("%a%")] +01)Filter: binary_as_string_option.binary_col LIKE Utf8View("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8View("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8View("%a%") +02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8View("%a%"), binary_as_string_option.largebinary_col LIKE Utf8View("%a%"), binary_as_string_option.binaryview_col LIKE Utf8View("%a%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 02)--FilterExec: binary_col@0 LIKE %a% AND largebinary_col@1 LIKE %a% AND binaryview_col@2 LIKE %a% diff --git a/datafusion/sqllogictest/test_files/simplify_predicates.slt b/datafusion/sqllogictest/test_files/simplify_predicates.slt index 6d3eefdfc8213..cef78d97bb46c 100644 --- a/datafusion/sqllogictest/test_files/simplify_predicates.slt +++ b/datafusion/sqllogictest/test_files/simplify_predicates.slt @@ -84,7 +84,7 @@ query TT EXPLAIN SELECT * FROM test_data WHERE str_col > 'apple' AND str_col > 'banana'; ---- logical_plan -01)Filter: test_data.str_col > Utf8("banana") +01)Filter: test_data.str_col > Utf8View("banana") 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] # date_col > '2023-01-01' AND date_col > '2023-02-01' should simplify to date_col > '2023-02-01' @@ -120,7 +120,7 @@ WHERE int_col > 5 AND float_col BETWEEN 1 AND 100; ---- logical_plan -01)Filter: test_data.str_col LIKE Utf8("A%") AND test_data.float_col >= Float32(1) AND test_data.float_col <= Float32(100) AND test_data.int_col > Int32(10) +01)Filter: test_data.str_col LIKE Utf8View("A%") AND test_data.float_col >= Float32(1) AND test_data.float_col <= Float32(100) AND test_data.int_col > Int32(10) 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] statement ok From 63c2ebc6667d5de4b5214496fd2bea421ddc8e69 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 12:56:36 +0800 Subject: [PATCH 100/177] schema_force_view_types to false(try true after df49) --- datafusion/common/src/config.rs | 2 +- .../sqllogictest/test_files/describe.slt | 4 +- .../sqllogictest/test_files/explain.slt | 12 +- .../sqllogictest/test_files/explain_tree.slt | 108 ++++++------------ .../test_files/information_schema.slt | 4 +- datafusion/sqllogictest/test_files/limit.slt | 2 +- .../test_files/listing_table_statistics.slt | 2 +- datafusion/sqllogictest/test_files/map.slt | 2 +- .../sqllogictest/test_files/parquet.slt | 52 ++++----- 9 files changed, 76 insertions(+), 112 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 883d2b60a8976..0d34815a248f7 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -457,7 +457,7 @@ config_namespace! { /// (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, /// and `Binary/BinaryLarge` with `BinaryView`. - pub schema_force_view_types: bool, default = true + pub schema_force_view_types: bool, default = false /// (reading) If true, parquet reader will read columns of /// `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. diff --git a/datafusion/sqllogictest/test_files/describe.slt b/datafusion/sqllogictest/test_files/describe.slt index e4cb30628eec5..077e8e6474d1f 100644 --- a/datafusion/sqllogictest/test_files/describe.slt +++ b/datafusion/sqllogictest/test_files/describe.slt @@ -81,8 +81,8 @@ int_col Int32 YES bigint_col Int64 YES float_col Float32 YES double_col Float64 YES -date_string_col Utf8View YES -string_col Utf8View YES +date_string_col Utf8 YES +string_col Utf8 YES timestamp_col Timestamp(Nanosecond, None) YES year Int32 YES month Int32 YES diff --git a/datafusion/sqllogictest/test_files/explain.slt b/datafusion/sqllogictest/test_files/explain.slt index fff82ca1e5927..235f95eb4595d 100644 --- a/datafusion/sqllogictest/test_files/explain.slt +++ b/datafusion/sqllogictest/test_files/explain.slt @@ -297,8 +297,8 @@ initial_physical_plan 01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] initial_physical_plan_with_schema -01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] physical_plan after OutputRequirements 01)OutputRequirementExec, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] 02)--GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] @@ -321,7 +321,7 @@ physical_plan after LimitPushdown DataSourceExec: file_groups={1 group: [[WORKSP physical_plan after ProjectionPushdown SAME TEXT AS ABOVE physical_plan after SanityCheckPlan SAME TEXT AS ABOVE physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] -physical_plan_with_schema DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +physical_plan_with_schema DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] statement ok @@ -338,8 +338,8 @@ initial_physical_plan_with_stats 01)GlobalLimitExec: skip=0, fetch=10, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] initial_physical_plan_with_schema -01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +01)GlobalLimitExec: skip=0, fetch=10, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] physical_plan after OutputRequirements 01)OutputRequirementExec 02)--GlobalLimitExec: skip=0, fetch=10 @@ -363,7 +363,7 @@ physical_plan after ProjectionPushdown SAME TEXT AS ABOVE physical_plan after SanityCheckPlan SAME TEXT AS ABOVE physical_plan DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet physical_plan_with_stats DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, statistics=[Rows=Exact(8), Bytes=Exact(671), [(Col[0]:),(Col[1]:),(Col[2]:),(Col[3]:),(Col[4]:),(Col[5]:),(Col[6]:),(Col[7]:),(Col[8]:),(Col[9]:),(Col[10]:)]] -physical_plan_with_schema DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:BinaryView;N, string_col:BinaryView;N, timestamp_col:Timestamp(Nanosecond, None);N] +physical_plan_with_schema DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/parquet-testing/data/alltypes_plain.parquet]]}, projection=[id, bool_col, tinyint_col, smallint_col, int_col, bigint_col, float_col, double_col, date_string_col, string_col, timestamp_col], limit=10, file_type=parquet, schema=[id:Int32;N, bool_col:Boolean;N, tinyint_col:Int32;N, smallint_col:Int32;N, int_col:Int32;N, bigint_col:Int64;N, float_col:Float32;N, double_col:Float64;N, date_string_col:Binary;N, string_col:Binary;N, timestamp_col:Timestamp(Nanosecond, None);N] statement ok diff --git a/datafusion/sqllogictest/test_files/explain_tree.slt b/datafusion/sqllogictest/test_files/explain_tree.slt index 22183195c3df0..9d404fac603ca 100644 --- a/datafusion/sqllogictest/test_files/explain_tree.slt +++ b/datafusion/sqllogictest/test_files/explain_tree.slt @@ -1185,42 +1185,24 @@ physical_plan 21)│ -------------------- │ 22)│ on: │ 23)│ (int_col = int_col), ├──────────────┐ -24)│ (string_col = CAST │ │ -25)│ (table1.string_col AS │ │ -26)│ Utf8View)) │ │ -27)└─────────────┬─────────────┘ │ -28)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -29)│ DataSourceExec ││ ProjectionExec │ -30)│ -------------------- ││ -------------------- │ -31)│ files: 1 ││ CAST(table1.string_col AS │ -32)│ format: parquet ││ Utf8View): │ -33)│ ││ CAST(string_col AS │ -34)│ ││ Utf8View) │ -35)│ ││ │ -36)│ ││ bigint_col: │ -37)│ ││ bigint_col │ -38)│ ││ │ -39)│ ││ date_col: date_col │ -40)│ ││ int_col: int_col │ -41)│ ││ │ -42)│ ││ string_col: │ -43)│ ││ string_col │ -44)└───────────────────────────┘└─────────────┬─────────────┘ -45)-----------------------------┌─────────────┴─────────────┐ -46)-----------------------------│ RepartitionExec │ -47)-----------------------------│ -------------------- │ -48)-----------------------------│ partition_count(in->out): │ -49)-----------------------------│ 1 -> 4 │ -50)-----------------------------│ │ -51)-----------------------------│ partitioning_scheme: │ -52)-----------------------------│ RoundRobinBatch(4) │ -53)-----------------------------└─────────────┬─────────────┘ -54)-----------------------------┌─────────────┴─────────────┐ -55)-----------------------------│ DataSourceExec │ -56)-----------------------------│ -------------------- │ -57)-----------------------------│ files: 1 │ -58)-----------------------------│ format: csv │ -59)-----------------------------└───────────────────────────┘ +24)│ (string_col = │ │ +25)│ string_col) │ │ +26)└─────────────┬─────────────┘ │ +27)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +28)│ DataSourceExec ││ RepartitionExec │ +29)│ -------------------- ││ -------------------- │ +30)│ files: 1 ││ partition_count(in->out): │ +31)│ format: parquet ││ 1 -> 4 │ +32)│ ││ │ +33)│ ││ partitioning_scheme: │ +34)│ ││ RoundRobinBatch(4) │ +35)└───────────────────────────┘└─────────────┬─────────────┘ +36)-----------------------------┌─────────────┴─────────────┐ +37)-----------------------------│ DataSourceExec │ +38)-----------------------------│ -------------------- │ +39)-----------------------------│ files: 1 │ +40)-----------------------------│ format: csv │ +41)-----------------------------└───────────────────────────┘ # Query with outer hash join. query TT @@ -1252,42 +1234,24 @@ physical_plan 23)│ │ 24)│ on: ├──────────────┐ 25)│ (int_col = int_col), │ │ -26)│ (string_col = CAST │ │ -27)│ (table1.string_col AS │ │ -28)│ Utf8View)) │ │ -29)└─────────────┬─────────────┘ │ -30)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -31)│ DataSourceExec ││ ProjectionExec │ -32)│ -------------------- ││ -------------------- │ -33)│ files: 1 ││ CAST(table1.string_col AS │ -34)│ format: parquet ││ Utf8View): │ -35)│ ││ CAST(string_col AS │ -36)│ ││ Utf8View) │ -37)│ ││ │ -38)│ ││ bigint_col: │ -39)│ ││ bigint_col │ -40)│ ││ │ -41)│ ││ date_col: date_col │ -42)│ ││ int_col: int_col │ -43)│ ││ │ -44)│ ││ string_col: │ -45)│ ││ string_col │ -46)└───────────────────────────┘└─────────────┬─────────────┘ -47)-----------------------------┌─────────────┴─────────────┐ -48)-----------------------------│ RepartitionExec │ -49)-----------------------------│ -------------------- │ -50)-----------------------------│ partition_count(in->out): │ -51)-----------------------------│ 1 -> 4 │ -52)-----------------------------│ │ -53)-----------------------------│ partitioning_scheme: │ -54)-----------------------------│ RoundRobinBatch(4) │ -55)-----------------------------└─────────────┬─────────────┘ -56)-----------------------------┌─────────────┴─────────────┐ -57)-----------------------------│ DataSourceExec │ -58)-----------------------------│ -------------------- │ -59)-----------------------------│ files: 1 │ -60)-----------------------------│ format: csv │ -61)-----------------------------└───────────────────────────┘ +26)│ (string_col = │ │ +27)│ string_col) │ │ +28)└─────────────┬─────────────┘ │ +29)┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +30)│ DataSourceExec ││ RepartitionExec │ +31)│ -------------------- ││ -------------------- │ +32)│ files: 1 ││ partition_count(in->out): │ +33)│ format: parquet ││ 1 -> 4 │ +34)│ ││ │ +35)│ ││ partitioning_scheme: │ +36)│ ││ RoundRobinBatch(4) │ +37)└───────────────────────────┘└─────────────┬─────────────┘ +38)-----------------------------┌─────────────┴─────────────┐ +39)-----------------------------│ DataSourceExec │ +40)-----------------------------│ -------------------- │ +41)-----------------------------│ files: 1 │ +42)-----------------------------│ format: csv │ +43)-----------------------------└───────────────────────────┘ # Query with nested loop join. query TT diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 2ce64ffc68365..9f39dbbd5ba25 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -249,7 +249,7 @@ datafusion.execution.parquet.metadata_size_hint NULL datafusion.execution.parquet.pruning true datafusion.execution.parquet.pushdown_filters false datafusion.execution.parquet.reorder_filters false -datafusion.execution.parquet.schema_force_view_types true +datafusion.execution.parquet.schema_force_view_types false datafusion.execution.parquet.skip_arrow_metadata false datafusion.execution.parquet.skip_metadata true datafusion.execution.parquet.statistics_enabled page @@ -359,7 +359,7 @@ datafusion.execution.parquet.metadata_size_hint NULL (reading) If specified, the datafusion.execution.parquet.pruning true (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file datafusion.execution.parquet.pushdown_filters false (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". datafusion.execution.parquet.reorder_filters false (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query -datafusion.execution.parquet.schema_force_view_types true (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. +datafusion.execution.parquet.schema_force_view_types false (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. datafusion.execution.parquet.skip_arrow_metadata false (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to datafusion.execution.parquet.skip_metadata true (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata datafusion.execution.parquet.statistics_enabled page (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 1af14a52e2bc6..894ba424eddcc 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -854,7 +854,7 @@ physical_plan 02)--SortExec: TopK(fetch=1000), expr=[part_key@1 ASC NULLS LAST], preserve_partitioning=[false] 03)----ProjectionExec: expr=[1 as foo, part_key@0 as part_key] 04)------CoalescePartitionsExec: fetch=1 -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..265], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:265..530], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:530..794]]}, projection=[part_key], limit=1, file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:0..265], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:265..530], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:530..794]]}, projection=[part_key], limit=1, file_type=parquet query I with selection as ( diff --git a/datafusion/sqllogictest/test_files/listing_table_statistics.slt b/datafusion/sqllogictest/test_files/listing_table_statistics.slt index 890d1f2e9250e..d3af6b321c7e6 100644 --- a/datafusion/sqllogictest/test_files/listing_table_statistics.slt +++ b/datafusion/sqllogictest/test_files/listing_table_statistics.slt @@ -35,7 +35,7 @@ query TT explain format indent select * from t; ---- logical_plan TableScan: t projection=[int_col, str_col] -physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/table/2.parquet]]}, projection=[int_col, str_col], file_type=parquet, statistics=[Rows=Exact(4), Bytes=Exact(288), [(Col[0]: Min=Exact(Int64(-1)) Max=Exact(Int64(3)) Null=Exact(0)),(Col[1]: Min=Exact(Utf8View("a")) Max=Exact(Utf8View("d")) Null=Exact(0))]] +physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/table/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/table/2.parquet]]}, projection=[int_col, str_col], file_type=parquet, statistics=[Rows=Exact(4), Bytes=Exact(288), [(Col[0]: Min=Exact(Int64(-1)) Max=Exact(Int64(3)) Null=Exact(0)),(Col[1]: Min=Exact(Utf8("a")) Max=Exact(Utf8("d")) Null=Exact(0))]] statement ok drop table t; diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt index 42a4ba6218016..8546bdbdd0673 100644 --- a/datafusion/sqllogictest/test_files/map.slt +++ b/datafusion/sqllogictest/test_files/map.slt @@ -45,7 +45,7 @@ describe data; ---- ints Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO strings Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO -timestamp Utf8View NO +timestamp Utf8 NO query ??T SELECT * FROM data ORDER by ints['bytes'] DESC LIMIT 10; diff --git a/datafusion/sqllogictest/test_files/parquet.slt b/datafusion/sqllogictest/test_files/parquet.slt index abc6fdab3c8a0..3e640d7d6b9a8 100644 --- a/datafusion/sqllogictest/test_files/parquet.slt +++ b/datafusion/sqllogictest/test_files/parquet.slt @@ -384,15 +384,15 @@ select arrow_typeof(binaryview_col), binaryview_col FROM binary_as_string_default; ---- -BinaryView 616161 BinaryView 616161 BinaryView 616161 -BinaryView 626262 BinaryView 626262 BinaryView 626262 -BinaryView 636363 BinaryView 636363 BinaryView 636363 -BinaryView 646464 BinaryView 646464 BinaryView 646464 -BinaryView 656565 BinaryView 656565 BinaryView 656565 -BinaryView 666666 BinaryView 666666 BinaryView 666666 -BinaryView 676767 BinaryView 676767 BinaryView 676767 -BinaryView 686868 BinaryView 686868 BinaryView 686868 -BinaryView 696969 BinaryView 696969 BinaryView 696969 +Binary 616161 LargeBinary 616161 BinaryView 616161 +Binary 626262 LargeBinary 626262 BinaryView 626262 +Binary 636363 LargeBinary 636363 BinaryView 636363 +Binary 646464 LargeBinary 646464 BinaryView 646464 +Binary 656565 LargeBinary 656565 BinaryView 656565 +Binary 666666 LargeBinary 666666 BinaryView 666666 +Binary 676767 LargeBinary 676767 BinaryView 676767 +Binary 686868 LargeBinary 686868 BinaryView 686868 +Binary 696969 LargeBinary 696969 BinaryView 696969 # Run an explain plan to show the cast happens in the plan (a CAST is needed for the predicates) query TT @@ -405,13 +405,13 @@ EXPLAIN binaryview_col LIKE '%a%'; ---- logical_plan -01)Filter: CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%") -02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.largebinary_col AS Utf8View) LIKE Utf8View("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%")] +01)Filter: CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%") AND CAST(binary_as_string_default.largebinary_col AS LargeUtf8) LIKE LargeUtf8("%a%") AND CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%") +02)--TableScan: binary_as_string_default projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[CAST(binary_as_string_default.binary_col AS Utf8) LIKE Utf8("%a%"), CAST(binary_as_string_default.largebinary_col AS LargeUtf8) LIKE LargeUtf8("%a%"), CAST(binary_as_string_default.binaryview_col AS Utf8View) LIKE Utf8View("%a%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 -02)--FilterExec: CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% +02)--FilterExec: CAST(binary_col@0 AS Utf8) LIKE %a% AND CAST(largebinary_col@1 AS LargeUtf8) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% 03)----RepartitionExec: partitioning=RoundRobinBatch(2), input_partitions=1 -04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/binary_as_string.parquet]]}, projection=[binary_col, largebinary_col, binaryview_col], file_type=parquet, predicate=CAST(binary_col@0 AS Utf8View) LIKE %a% AND CAST(largebinary_col@1 AS Utf8View) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/binary_as_string.parquet]]}, projection=[binary_col, largebinary_col, binaryview_col], file_type=parquet, predicate=CAST(binary_col@0 AS Utf8) LIKE %a% AND CAST(largebinary_col@1 AS LargeUtf8) LIKE %a% AND CAST(binaryview_col@2 AS Utf8View) LIKE %a% statement ok @@ -432,15 +432,15 @@ select arrow_typeof(binaryview_col), binaryview_col FROM binary_as_string_option; ---- -Utf8View aaa Utf8View aaa Utf8View aaa -Utf8View bbb Utf8View bbb Utf8View bbb -Utf8View ccc Utf8View ccc Utf8View ccc -Utf8View ddd Utf8View ddd Utf8View ddd -Utf8View eee Utf8View eee Utf8View eee -Utf8View fff Utf8View fff Utf8View fff -Utf8View ggg Utf8View ggg Utf8View ggg -Utf8View hhh Utf8View hhh Utf8View hhh -Utf8View iii Utf8View iii Utf8View iii +Utf8 aaa LargeUtf8 aaa Utf8View aaa +Utf8 bbb LargeUtf8 bbb Utf8View bbb +Utf8 ccc LargeUtf8 ccc Utf8View ccc +Utf8 ddd LargeUtf8 ddd Utf8View ddd +Utf8 eee LargeUtf8 eee Utf8View eee +Utf8 fff LargeUtf8 fff Utf8View fff +Utf8 ggg LargeUtf8 ggg Utf8View ggg +Utf8 hhh LargeUtf8 hhh Utf8View hhh +Utf8 iii LargeUtf8 iii Utf8View iii # Run an explain plan to show the cast happens in the plan (there should be no casts) query TT @@ -453,8 +453,8 @@ EXPLAIN binaryview_col LIKE '%a%'; ---- logical_plan -01)Filter: binary_as_string_option.binary_col LIKE Utf8View("%a%") AND binary_as_string_option.largebinary_col LIKE Utf8View("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8View("%a%") -02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8View("%a%"), binary_as_string_option.largebinary_col LIKE Utf8View("%a%"), binary_as_string_option.binaryview_col LIKE Utf8View("%a%")] +01)Filter: binary_as_string_option.binary_col LIKE Utf8("%a%") AND binary_as_string_option.largebinary_col LIKE LargeUtf8("%a%") AND binary_as_string_option.binaryview_col LIKE Utf8View("%a%") +02)--TableScan: binary_as_string_option projection=[binary_col, largebinary_col, binaryview_col], partial_filters=[binary_as_string_option.binary_col LIKE Utf8("%a%"), binary_as_string_option.largebinary_col LIKE LargeUtf8("%a%"), binary_as_string_option.binaryview_col LIKE Utf8View("%a%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 02)--FilterExec: binary_col@0 LIKE %a% AND largebinary_col@1 LIKE %a% AND binaryview_col@2 LIKE %a% @@ -619,8 +619,8 @@ query TT explain select * from foo where starts_with(column1, 'f'); ---- logical_plan -01)Filter: foo.column1 LIKE Utf8View("f%") -02)--TableScan: foo projection=[column1], partial_filters=[foo.column1 LIKE Utf8View("f%")] +01)Filter: foo.column1 LIKE Utf8("f%") +02)--TableScan: foo projection=[column1], partial_filters=[foo.column1 LIKE Utf8("f%")] physical_plan 01)CoalesceBatchesExec: target_batch_size=8192 02)--FilterExec: column1@0 LIKE f% From ed718c015eb4fdf1a06724622adf3d441d42bc67 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 13:07:12 +0800 Subject: [PATCH 101/177] fix page_index_filter_one_col and remove an example of proto --- datafusion/core/tests/parquet/page_pruning.rs | 3 ++- datafusion/proto/src/lib.rs | 23 ------------------- datafusion/sqllogictest/test_files/limit.slt | 2 +- 3 files changed, 3 insertions(+), 25 deletions(-) diff --git a/datafusion/core/tests/parquet/page_pruning.rs b/datafusion/core/tests/parquet/page_pruning.rs index 9da879a32f6b5..fef8fd1d401e5 100644 --- a/datafusion/core/tests/parquet/page_pruning.rs +++ b/datafusion/core/tests/parquet/page_pruning.rs @@ -160,7 +160,8 @@ async fn page_index_filter_one_col() { // 5.create filter date_string_col == "01/01/09"`; // Note this test doesn't apply type coercion so the literal must match the actual view type - let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8view("01/01/09"))); + // xudong: use new_utf8, because schema_force_view_types was changed to false now. + let filter = col("date_string_col").eq(lit(ScalarValue::new_utf8("01/01/09"))); let parquet_exec = get_parquet_exec(&state, filter).await; let mut results = parquet_exec.execute(0, task_ctx.clone()).unwrap(); let batch = results.next().await.unwrap().unwrap(); diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rs index 2df162f21e3a3..f7da0f1f2f741 100644 --- a/datafusion/proto/src/lib.rs +++ b/datafusion/proto/src/lib.rs @@ -97,29 +97,6 @@ //! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip)); //! # Ok(()) //! # } -//! ``` -//! # Example: Serializing [`ExecutionPlan`]s -//! -//! ``` -//! # use datafusion::prelude::*; -//! # use datafusion_common::Result; -//! # use datafusion_proto::bytes::{physical_plan_from_bytes,physical_plan_to_bytes}; -//! # #[tokio::main] -//! # async fn main() -> Result<()>{ -//! // Create a plan that scans table 't' -//! let ctx = SessionContext::new(); -//! ctx.register_csv("t1", "tests/testdata/test.csv", CsvReadOptions::default()).await?; -//! let physical_plan = ctx.table("t1").await?.create_physical_plan().await?; -//! -//! // Convert the plan into bytes (for sending over the network, etc.) -//! let bytes = physical_plan_to_bytes(physical_plan.clone())?; -//! -//! // Decode bytes from somewhere (over network, etc.) back to ExecutionPlan -//! let physical_round_trip = physical_plan_from_bytes(&bytes, &ctx)?; -//! assert_eq!(format!("{:?}", physical_plan), format!("{:?}", physical_round_trip)); -//! # Ok(()) -//! # } -//! ``` pub mod bytes; pub mod common; pub mod generated; diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 894ba424eddcc..1af14a52e2bc6 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -854,7 +854,7 @@ physical_plan 02)--SortExec: TopK(fetch=1000), expr=[part_key@1 ASC NULLS LAST], preserve_partitioning=[false] 03)----ProjectionExec: expr=[1 as foo, part_key@0 as part_key] 04)------CoalescePartitionsExec: fetch=1 -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:0..265], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:265..530], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:530..794]]}, projection=[part_key], limit=1, file_type=parquet +05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..265], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:265..530], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:530..794]]}, projection=[part_key], limit=1, file_type=parquet query I with selection as ( From 0bb16fa06e20f2552d19b41738d37b8241e148e3 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 13:23:20 +0800 Subject: [PATCH 102/177] fix configs.md --- docs/source/user-guide/configs.md | 109 +++++++++++++++++++++++++++++- 1 file changed, 108 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 05cc36651a1a8..75184fcd00e75 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -56,7 +56,114 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.execution.parquet.metadata_size_hint | NULL | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer | | datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | | datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | -| datafusion.execution.parquet.schema_force_view_types | true | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. | +| datafusion.execution.parquet.schema_force_view_types | false | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. | +| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. | +| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | +| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | +| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | +| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in bytes | +| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | +| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | +| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), lzo, brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | +| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | +| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | +| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | +| datafusion.execution.parquet.created_by | datafusion version 48.0.1 | (writing) Sets "created by" property | +| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | +| datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | +| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | +| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | +| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | +| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | +| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | +| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | +| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | +| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | +| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | +| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | +| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | +| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption | +| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | +| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | +| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | +| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | +| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | +| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | +| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | +| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. | +| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | +| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | +| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | +| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | +| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | +| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_file_min_size | 10485760 | Minimum total files size in bytes to perform file scan repartitioning. | +| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | +| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | +| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | +| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | +| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | +| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | +| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | +| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan | +| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | +| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | +| datafusion.optimizer.hash_join_single_partition_threshold | 1048576 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | +| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | +| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | +| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | +| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | +| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | +| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | +| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | +| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | +| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | +| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | +| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | +| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | +| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB and Databricks. | +| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | +| datafusion.sql_parser.map_varchar_to_utf8view | true | If true, `VARCHAR` is mapped to `Utf8View` during SQL planning. If false, `VARCHAR` is mapped to `Utf8` during SQL planning. Default is false. | +| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | +| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | +| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | +| datafusion.format.null | | Format string for nulls | +| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | +| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | +| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | +| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | +| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | +| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | +| datafusion.format.types_info | false | Show types in visual representation batches | +| key | default | description | +| ----- | --------- | ------------- | +| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | +| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | +| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | +| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | +| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | +| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | +| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | +| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | +| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | +| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | +| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. | +| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | +| datafusion.execution.time_zone | +00:00 | The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour | +| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | +| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | +| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata | +| datafusion.execution.parquet.metadata_size_hint | NULL | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer | +| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | +| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | +| datafusion.execution.parquet.schema_force_view_types | false | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. | | datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. | | datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | | datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | From 09ff8f7b91136b55684f302c7b67692fb530e0a9 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 13:41:48 +0800 Subject: [PATCH 103/177] fix clippy --- datafusion/datasource/src/sink.rs | 4 +-- datafusion/datasource/src/source.rs | 6 ++-- datafusion/expr/src/expr.rs | 5 +--- .../optimizer/src/simplify_predicates.rs | 18 ++++++------ .../src/enforce_distribution.rs | 29 +++++++++---------- .../physical-plan/src/aggregates/mod.rs | 4 +-- datafusion/physical-plan/src/analyze.rs | 4 +-- .../physical-plan/src/coalesce_batches.rs | 4 +-- .../physical-plan/src/coalesce_partitions.rs | 4 +-- datafusion/physical-plan/src/empty.rs | 6 ++-- datafusion/physical-plan/src/filter.rs | 4 +-- .../physical-plan/src/joins/cross_join.rs | 4 +-- .../physical-plan/src/joins/hash_join.rs | 4 +-- .../src/joins/sort_merge_join.rs | 4 +-- .../src/joins/symmetric_hash_join.rs | 4 +-- datafusion/physical-plan/src/limit.rs | 4 +-- .../physical-plan/src/placeholder_row.rs | 4 +-- datafusion/physical-plan/src/projection.rs | 4 +-- .../physical-plan/src/recursive_query.rs | 4 +-- .../physical-plan/src/repartition/mod.rs | 4 +-- .../physical-plan/src/sorts/partial_sort.rs | 4 +-- datafusion/physical-plan/src/sorts/sort.rs | 4 +-- .../src/sorts/sort_preserving_merge.rs | 4 +-- datafusion/physical-plan/src/streaming.rs | 4 +-- datafusion/physical-plan/src/union.rs | 4 +-- datafusion/physical-plan/src/unnest.rs | 4 +-- datafusion/physical-plan/src/values.rs | 4 +-- .../src/windows/window_agg_exec.rs | 4 +-- datafusion/physical-plan/src/work_table.rs | 4 +-- 29 files changed, 77 insertions(+), 83 deletions(-) diff --git a/datafusion/datasource/src/sink.rs b/datafusion/datasource/src/sink.rs index faec74b98c5d7..7b84ffe1ff1b8 100644 --- a/datafusion/datasource/src/sink.rs +++ b/datafusion/datasource/src/sink.rs @@ -251,14 +251,14 @@ impl ExecutionPlan for DataSinkExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = DataSinkExec::new( Arc::clone(self.input()), Arc::clone(&self.sink), self.sort_order.clone(), ); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index 9c32f2170071a..521ad9d91573c 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -308,10 +308,10 @@ impl ExecutionPlan for DataSourceExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { - let mut new_plan = DataSourceExec::new(self.data_source.clone()); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let mut new_plan = DataSourceExec::new(Arc::clone(&self.data_source)); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index ffa2417fc5b66..bbfdf7d870555 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -1885,10 +1885,7 @@ impl Expr { /// Check if the Expr is literal pub fn is_literal(&self) -> bool { - match self { - Expr::Literal(_, _) => true, - _ => false, - } + matches!(self, Expr::Literal(_, _)) } } diff --git a/datafusion/optimizer/src/simplify_predicates.rs b/datafusion/optimizer/src/simplify_predicates.rs index 5ff7e42d95310..198e630203977 100644 --- a/datafusion/optimizer/src/simplify_predicates.rs +++ b/datafusion/optimizer/src/simplify_predicates.rs @@ -31,16 +31,16 @@ pub(crate) fn simplify_predicates(predicates: Vec) -> Result> { for pred in predicates { match &pred { - Expr::BinaryExpr(BinaryExpr { left, op, right }) - if matches!( - op, + Expr::BinaryExpr(BinaryExpr { + left, + op: Operator::Gt - | Operator::GtEq - | Operator::Lt - | Operator::LtEq - | Operator::Eq - ) => - { + | Operator::GtEq + | Operator::Lt + | Operator::LtEq + | Operator::Eq, + right, + }) => { let left_col = extract_column_from_expr(left); let right_col = extract_column_from_expr(right); let left_lit = left.is_literal(); diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 77c7af76ccbef..8f8c7274cf78e 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -951,17 +951,16 @@ fn add_spm_on_top( // (determined by flag `config.optimizer.bounded_order_preserving_variants`) let should_preserve_ordering = input.plan.output_ordering().is_some(); + let ordering = input + .plan + .output_ordering() + .cloned() + .unwrap_or_else(LexOrdering::default); + let new_plan = if should_preserve_ordering { Arc::new( - SortPreservingMergeExec::new( - input - .plan - .output_ordering() - .unwrap_or(&LexOrdering::default()) - .clone(), - Arc::clone(&input.plan), - ) - .with_fetch(fetch.take()), + SortPreservingMergeExec::new(ordering, Arc::clone(&input.plan)) + .with_fetch(fetch.take()), ) as _ } else { Arc::new(CoalescePartitionsExec::new(Arc::clone(&input.plan))) as _ @@ -1405,14 +1404,12 @@ pub fn ensure_distribution( // It was removed by `remove_dist_changing_operators` // and we need to add it back. if fetch.is_some() { + let ordering = plan + .output_ordering() + .cloned() + .unwrap_or_else(LexOrdering::default); let plan = Arc::new( - SortPreservingMergeExec::new( - plan.output_ordering() - .unwrap_or(&LexOrdering::default()) - .clone(), - plan, - ) - .with_fetch(fetch.take()), + SortPreservingMergeExec::new(ordering, plan).with_fetch(fetch.take()), ); optimized_distribution_ctx = DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 98901be23b90c..8082527bfd8d3 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1023,7 +1023,7 @@ impl ExecutionPlan for AggregateExec { } fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = AggregateExec { mode: self.mode, @@ -1040,7 +1040,7 @@ impl ExecutionPlan for AggregateExec { metrics: self.metrics.clone(), }; - let new_props: PlanProperties = new_plan.cache.clone().with_node_id(_node_id); + let new_props: PlanProperties = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs index 791a5f8cb2d9c..c67430b467b69 100644 --- a/datafusion/physical-plan/src/analyze.rs +++ b/datafusion/physical-plan/src/analyze.rs @@ -213,7 +213,7 @@ impl ExecutionPlan for AnalyzeExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = AnalyzeExec::new( self.verbose, @@ -221,7 +221,7 @@ impl ExecutionPlan for AnalyzeExec { Arc::clone(self.input()), Arc::clone(&self.schema), ); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/coalesce_batches.rs b/datafusion/physical-plan/src/coalesce_batches.rs index 088d3a2850bbb..7d57ece88e867 100644 --- a/datafusion/physical-plan/src/coalesce_batches.rs +++ b/datafusion/physical-plan/src/coalesce_batches.rs @@ -228,12 +228,12 @@ impl ExecutionPlan for CoalesceBatchesExec { } fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = CoalesceBatchesExec::new(Arc::clone(self.input()), self.target_batch_size) .with_fetch(self.fetch()); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index 7daf0d753d15f..fcff76d54b1a2 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -214,11 +214,11 @@ impl ExecutionPlan for CoalescePartitionsExec { } fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = CoalescePartitionsExec::new(Arc::clone(self.input())); new_plan.fetch = self.fetch; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/empty.rs b/datafusion/physical-plan/src/empty.rs index eae1aaff1c492..b005e5e1ef9bc 100644 --- a/datafusion/physical-plan/src/empty.rs +++ b/datafusion/physical-plan/src/empty.rs @@ -176,10 +176,10 @@ impl ExecutionPlan for EmptyExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { - let mut new_plan = EmptyExec::new(self.schema.clone()); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let mut new_plan = EmptyExec::new(Arc::clone(&self.schema)); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 07aea16d06283..60b48d542efd1 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -431,12 +431,12 @@ impl ExecutionPlan for FilterExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = FilterExec::try_new(Arc::clone(&self.predicate), Arc::clone(self.input()))? .with_projection(self.projection.clone())?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 3ceea2e8833b3..3159ef1d3699f 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -350,11 +350,11 @@ impl ExecutionPlan for CrossJoinExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = CrossJoinExec::new(Arc::clone(&self.left), Arc::clone(&self.right)); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/joins/hash_join.rs b/datafusion/physical-plan/src/joins/hash_join.rs index f018de2e0a6d8..c868bbb6ef1bf 100644 --- a/datafusion/physical-plan/src/joins/hash_join.rs +++ b/datafusion/physical-plan/src/joins/hash_join.rs @@ -906,7 +906,7 @@ impl ExecutionPlan for HashJoinExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = HashJoinExec::try_new( Arc::clone(&self.left), @@ -918,7 +918,7 @@ impl ExecutionPlan for HashJoinExec { *self.partition_mode(), self.null_equals_null, )?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/joins/sort_merge_join.rs b/datafusion/physical-plan/src/joins/sort_merge_join.rs index 34915a96ba213..d1f22ad7542bf 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join.rs @@ -528,7 +528,7 @@ impl ExecutionPlan for SortMergeJoinExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = SortMergeJoinExec::try_new( Arc::clone(&self.left), @@ -539,7 +539,7 @@ impl ExecutionPlan for SortMergeJoinExec { self.sort_options.clone(), self.null_equals_null, )?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index 82b099b22999f..21926d099996a 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -478,7 +478,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = SymmetricHashJoinExec::try_new( Arc::clone(&self.left), @@ -491,7 +491,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { self.right_sort_exprs.clone(), self.mode, )?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/limit.rs b/datafusion/physical-plan/src/limit.rs index 34aca9a74a951..c1410677330a5 100644 --- a/datafusion/physical-plan/src/limit.rs +++ b/datafusion/physical-plan/src/limit.rs @@ -212,11 +212,11 @@ impl ExecutionPlan for GlobalLimitExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = GlobalLimitExec::new(Arc::clone(self.input()), self.skip, self.fetch); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/placeholder_row.rs b/datafusion/physical-plan/src/placeholder_row.rs index 98acadb0bce60..45f4d5e1437c4 100644 --- a/datafusion/physical-plan/src/placeholder_row.rs +++ b/datafusion/physical-plan/src/placeholder_row.rs @@ -185,10 +185,10 @@ impl ExecutionPlan for PlaceholderRowExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = PlaceholderRowExec::new(Arc::clone(&self.schema)); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index 7149bf031683d..a917d5547cc8f 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -265,11 +265,11 @@ impl ExecutionPlan for ProjectionExec { } fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = ProjectionExec::try_new(self.expr.clone(), Arc::clone(self.input()))?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/recursive_query.rs b/datafusion/physical-plan/src/recursive_query.rs index 9dde7dc91b6e6..516863dbe78d5 100644 --- a/datafusion/physical-plan/src/recursive_query.rs +++ b/datafusion/physical-plan/src/recursive_query.rs @@ -209,7 +209,7 @@ impl ExecutionPlan for RecursiveQueryExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = RecursiveQueryExec::try_new( self.name.clone(), @@ -217,7 +217,7 @@ impl ExecutionPlan for RecursiveQueryExec { Arc::clone(&self.recursive_term), self.is_distinct, )?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/repartition/mod.rs b/datafusion/physical-plan/src/repartition/mod.rs index 1102c9897e193..84a11839d0286 100644 --- a/datafusion/physical-plan/src/repartition/mod.rs +++ b/datafusion/physical-plan/src/repartition/mod.rs @@ -766,7 +766,7 @@ impl ExecutionPlan for RepartitionExec { } fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = RepartitionExec { input: Arc::clone(&self.input), @@ -775,7 +775,7 @@ impl ExecutionPlan for RepartitionExec { preserve_order: self.preserve_order, cache: self.cache.clone(), }; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/sorts/partial_sort.rs b/datafusion/physical-plan/src/sorts/partial_sort.rs index 0cb6180667c0f..60b27fac3fa37 100644 --- a/datafusion/physical-plan/src/sorts/partial_sort.rs +++ b/datafusion/physical-plan/src/sorts/partial_sort.rs @@ -327,7 +327,7 @@ impl ExecutionPlan for PartialSortExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = PartialSortExec { expr: self.expr.clone(), @@ -338,7 +338,7 @@ impl ExecutionPlan for PartialSortExec { fetch: self.fetch, cache: self.cache.clone(), }; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 5df18c1526ae1..9eef517aa6d6a 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1208,7 +1208,7 @@ impl ExecutionPlan for SortExec { } fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let new_plan = SortExec { input: Arc::clone(self.input()), @@ -1216,7 +1216,7 @@ impl ExecutionPlan for SortExec { fetch: self.fetch, metrics_set: self.metrics_set.clone(), preserve_partitioning: self.preserve_partitioning, - cache: self.cache.clone().with_node_id(_node_id), + cache: self.cache.clone().with_node_id(node_id), common_sort_prefix: self.common_sort_prefix.clone(), }; Ok(Some(Arc::new(new_plan))) diff --git a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs index 7b2af16641796..519acbafe9846 100644 --- a/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs +++ b/datafusion/physical-plan/src/sorts/sort_preserving_merge.rs @@ -352,12 +352,12 @@ impl ExecutionPlan for SortPreservingMergeExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = SortPreservingMergeExec::new(self.expr.clone(), Arc::clone(self.input())) .with_fetch(self.fetch()); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/streaming.rs b/datafusion/physical-plan/src/streaming.rs index 0dead076d5eec..7a614ccb34984 100644 --- a/datafusion/physical-plan/src/streaming.rs +++ b/datafusion/physical-plan/src/streaming.rs @@ -352,7 +352,7 @@ impl ExecutionPlan for StreamingTableExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = StreamingTableExec { partitions: self.partitions.clone(), @@ -364,7 +364,7 @@ impl ExecutionPlan for StreamingTableExec { cache: self.cache.clone(), metrics: self.metrics.clone(), }; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 35a06c8d25381..591e28e4c3220 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -300,10 +300,10 @@ impl ExecutionPlan for UnionExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = UnionExec::new(self.inputs.clone()); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index 548ab4f10f874..460b52d925d56 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -202,7 +202,7 @@ impl ExecutionPlan for UnnestExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = UnnestExec::new( Arc::clone(self.input()), @@ -211,7 +211,7 @@ impl ExecutionPlan for UnnestExec { Arc::clone(&self.schema), self.options.clone(), ); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/values.rs b/datafusion/physical-plan/src/values.rs index 388b6110e4d8c..e5dc3b1162338 100644 --- a/datafusion/physical-plan/src/values.rs +++ b/datafusion/physical-plan/src/values.rs @@ -231,13 +231,13 @@ impl ExecutionPlan for ValuesExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = ValuesExec::try_new_from_batches( Arc::clone(&self.schema), self.data.clone(), )?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/windows/window_agg_exec.rs b/datafusion/physical-plan/src/windows/window_agg_exec.rs index dd6c510b968d9..f5a14e1344deb 100644 --- a/datafusion/physical-plan/src/windows/window_agg_exec.rs +++ b/datafusion/physical-plan/src/windows/window_agg_exec.rs @@ -302,14 +302,14 @@ impl ExecutionPlan for WindowAggExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = WindowAggExec::try_new( self.window_expr.clone(), Arc::clone(self.input()), self.can_repartition, )?; - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } diff --git a/datafusion/physical-plan/src/work_table.rs b/datafusion/physical-plan/src/work_table.rs index f674ee0b2064f..ffec2f82a2a89 100644 --- a/datafusion/physical-plan/src/work_table.rs +++ b/datafusion/physical-plan/src/work_table.rs @@ -230,11 +230,11 @@ impl ExecutionPlan for WorkTableExec { fn with_node_id( self: Arc, - _node_id: usize, + node_id: usize, ) -> Result>> { let mut new_plan = WorkTableExec::new(self.name.clone(), Arc::clone(&self.schema)); - let new_props = new_plan.cache.clone().with_node_id(_node_id); + let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) } From 1545f2dd47b2d05bdc83a67dab132314e8a9bdd0 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 13:45:39 +0800 Subject: [PATCH 104/177] update configs.md --- docs/source/user-guide/configs.md | 107 ------------------------------ 1 file changed, 107 deletions(-) diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 75184fcd00e75..2727d52679018 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -142,110 +142,3 @@ Environment variables are read during `SessionConfig` initialisation so they mus | datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | | datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | | datafusion.format.types_info | false | Show types in visual representation batches | -| key | default | description | -| ----- | --------- | ------------- | -| datafusion.catalog.create_default_catalog_and_schema | true | Whether the default catalog and schema should be created automatically. | -| datafusion.catalog.default_catalog | datafusion | The default catalog name - this impacts what SQL queries use if not specified | -| datafusion.catalog.default_schema | public | The default schema name - this impacts what SQL queries use if not specified | -| datafusion.catalog.information_schema | false | Should DataFusion provide access to `information_schema` virtual tables for displaying schema information | -| datafusion.catalog.location | NULL | Location scanned to load tables for `default` schema | -| datafusion.catalog.format | NULL | Type of `TableProvider` to use when loading `default` schema | -| datafusion.catalog.has_header | true | Default value for `format.has_header` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. | -| datafusion.catalog.newlines_in_values | false | Specifies whether newlines in (quoted) CSV values are supported. This is the default value for `format.newlines_in_values` for `CREATE EXTERNAL TABLE` if not specified explicitly in the statement. Parsing newlines in quoted values may be affected by execution behaviour such as parallel file scanning. Setting this to `true` ensures that newlines in values are parsed successfully, which may reduce performance. | -| datafusion.execution.batch_size | 8192 | Default batch size while creating new batches, it's especially useful for buffer-in-memory batches since creating tiny batches would result in too much metadata memory consumption | -| datafusion.execution.coalesce_batches | true | When set to true, record batches will be examined between each operator and small batches will be coalesced into larger batches. This is helpful when there are highly selective filters or joins that could produce tiny output batches. The target batch size is determined by the configuration setting | -| datafusion.execution.collect_statistics | true | Should DataFusion collect statistics when first creating a table. Has no effect after the table is created. Applies to the default `ListingTableProvider` in DataFusion. Defaults to true. | -| datafusion.execution.target_partitions | 0 | Number of partitions for query execution. Increasing partitions can increase concurrency. Defaults to the number of CPU cores on the system | -| datafusion.execution.time_zone | +00:00 | The default time zone Some functions, e.g. `EXTRACT(HOUR from SOME_TIME)`, shift the underlying datetime according to this time zone, and then extract the hour | -| datafusion.execution.parquet.enable_page_index | true | (reading) If true, reads the Parquet data page level metadata (the Page Index), if present, to reduce the I/O and number of rows decoded. | -| datafusion.execution.parquet.pruning | true | (reading) If true, the parquet reader attempts to skip entire row groups based on the predicate in the query and the metadata (min/max values) stored in the parquet file | -| datafusion.execution.parquet.skip_metadata | true | (reading) If true, the parquet reader skip the optional embedded metadata that may be in the file Schema. This setting can help avoid schema conflicts when querying multiple parquet files with schemas containing compatible types but different metadata | -| datafusion.execution.parquet.metadata_size_hint | NULL | (reading) If specified, the parquet reader will try and fetch the last `size_hint` bytes of the parquet file optimistically. If not specified, two reads are required: One read to fetch the 8-byte parquet footer and another to fetch the metadata length encoded in the footer | -| datafusion.execution.parquet.pushdown_filters | false | (reading) If true, filter expressions are be applied during the parquet decoding operation to reduce the number of rows decoded. This optimization is sometimes called "late materialization". | -| datafusion.execution.parquet.reorder_filters | false | (reading) If true, filter expressions evaluated during the parquet decoding operation will be reordered heuristically to minimize the cost of evaluation. If false, the filters are applied in the same order as written in the query | -| datafusion.execution.parquet.schema_force_view_types | false | (reading) If true, parquet reader will read columns of `Utf8/Utf8Large` with `Utf8View`, and `Binary/BinaryLarge` with `BinaryView`. | -| datafusion.execution.parquet.binary_as_string | false | (reading) If true, parquet reader will read columns of `Binary/LargeBinary` with `Utf8`, and `BinaryView` with `Utf8View`. Parquet files generated by some legacy writers do not correctly set the UTF8 flag for strings, causing string columns to be loaded as BLOB instead. | -| datafusion.execution.parquet.coerce_int96 | NULL | (reading) If true, parquet reader will read columns of physical type int96 as originating from a different resolution than nanosecond. This is useful for reading data from systems like Spark which stores microsecond resolution timestamps in an int96 allowing it to write values with a larger date range than 64-bit timestamps with nanosecond resolution. | -| datafusion.execution.parquet.bloom_filter_on_read | true | (reading) Use any available bloom filters when reading parquet files | -| datafusion.execution.parquet.data_pagesize_limit | 1048576 | (writing) Sets best effort maximum size of data page in bytes | -| datafusion.execution.parquet.write_batch_size | 1024 | (writing) Sets write_batch_size in bytes | -| datafusion.execution.parquet.writer_version | 1.0 | (writing) Sets parquet writer version valid values are "1.0" and "2.0" | -| datafusion.execution.parquet.skip_arrow_metadata | false | (writing) Skip encoding the embedded arrow metadata in the KV_meta This is analogous to the `ArrowWriterOptions::with_skip_arrow_metadata`. Refer to | -| datafusion.execution.parquet.compression | zstd(3) | (writing) Sets default parquet compression codec. Valid values are: uncompressed, snappy, gzip(level), lzo, brotli(level), lz4, zstd(level), and lz4_raw. These values are not case sensitive. If NULL, uses default parquet writer setting Note that this default setting is not the same as the default parquet writer setting. | -| datafusion.execution.parquet.dictionary_enabled | true | (writing) Sets if dictionary encoding is enabled. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.dictionary_page_size_limit | 1048576 | (writing) Sets best effort maximum dictionary page size, in bytes | -| datafusion.execution.parquet.statistics_enabled | page | (writing) Sets if statistics are enabled for any column Valid values are: "none", "chunk", and "page" These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.max_statistics_size | 4096 | (writing) Sets max statistics size for any column. If NULL, uses default parquet writer setting max_statistics_size is deprecated, currently it is not being used | -| datafusion.execution.parquet.max_row_group_size | 1048576 | (writing) Target maximum number of rows in each row group (defaults to 1M rows). Writing larger row groups requires more memory to write, but can get better compression and be faster to read. | -| datafusion.execution.parquet.created_by | datafusion version 48.0.1 | (writing) Sets "created by" property | -| datafusion.execution.parquet.column_index_truncate_length | 64 | (writing) Sets column index truncate length | -| datafusion.execution.parquet.statistics_truncate_length | NULL | (writing) Sets statictics truncate length. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.data_page_row_count_limit | 20000 | (writing) Sets best effort maximum number of rows in data page | -| datafusion.execution.parquet.encoding | NULL | (writing) Sets default encoding for any column. Valid values are: plain, plain_dictionary, rle, bit_packed, delta_binary_packed, delta_length_byte_array, delta_byte_array, rle_dictionary, and byte_stream_split. These values are not case sensitive. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_on_write | false | (writing) Write bloom filters for all columns when creating parquet files | -| datafusion.execution.parquet.bloom_filter_fpp | NULL | (writing) Sets bloom filter false positive probability. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.bloom_filter_ndv | NULL | (writing) Sets bloom filter number of distinct values. If NULL, uses default parquet writer setting | -| datafusion.execution.parquet.allow_single_file_parallelism | true | (writing) Controls whether DataFusion will attempt to speed up writing parquet files by serializing them in parallel. Each column in each row group in each output file are serialized in parallel leveraging a maximum possible core count of n_files*n_row_groups*n_columns. | -| datafusion.execution.parquet.maximum_parallel_row_group_writers | 1 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.parquet.maximum_buffered_record_batches_per_stream | 2 | (writing) By default parallel parquet writer is tuned for minimum memory usage in a streaming execution plan. You may see a performance benefit when writing large parquet files by increasing maximum_parallel_row_group_writers and maximum_buffered_record_batches_per_stream if your system has idle cores and can tolerate additional memory usage. Boosting these values is likely worthwhile when writing out already in-memory data, such as from a cached data frame. | -| datafusion.execution.planning_concurrency | 0 | Fan-out during initial physical planning. This is mostly use to plan `UNION` children in parallel. Defaults to the number of CPU cores on the system | -| datafusion.execution.skip_physical_aggregate_schema_check | false | When set to true, skips verifying that the schema produced by planning the input of `LogicalPlan::Aggregate` exactly matches the schema of the input plan. When set to false, if the schema does not match exactly (including nullability and metadata), a planning error will be raised. This is used to workaround bugs in the planner that are now caught by the new schema verification step. | -| datafusion.execution.sort_spill_reservation_bytes | 10485760 | Specifies the reserved memory for each spillable sort operation to facilitate an in-memory merge. When a sort operation spills to disk, the in-memory data must be sorted and merged before being written to a file. This setting reserves a specific amount of memory for that in-memory sort/merge process. Note: This setting is irrelevant if the sort operation cannot spill (i.e., if there's no `DiskManager` configured). | -| datafusion.execution.sort_in_place_threshold_bytes | 1048576 | When sorting, below what size should data be concatenated and sorted in a single RecordBatch rather than sorted in batches and merged. | -| datafusion.execution.meta_fetch_concurrency | 32 | Number of files to read in parallel when inferring schema and statistics | -| datafusion.execution.minimum_parallel_output_files | 4 | Guarantees a minimum level of output files running in parallel. RecordBatches will be distributed in round robin fashion to each parallel writer. Each writer is closed and a new file opened once soft_max_rows_per_output_file is reached. | -| datafusion.execution.soft_max_rows_per_output_file | 50000000 | Target number of rows in output files when writing multiple. This is a soft max, so it can be exceeded slightly. There also will be one file smaller than the limit if the total number of rows written is not roughly divisible by the soft max | -| datafusion.execution.max_buffered_batches_per_output_file | 2 | This is the maximum number of RecordBatches buffered for each output file being worked. Higher values can potentially give faster write performance at the cost of higher peak memory consumption | -| datafusion.execution.listing_table_ignore_subdirectory | true | Should sub directories be ignored when scanning directories for data files. Defaults to true (ignores subdirectories), consistent with Hive. Note that this setting does not affect reading partitioned tables (e.g. `/table/year=2021/month=01/data.parquet`). | -| datafusion.execution.enable_recursive_ctes | true | Should DataFusion support recursive CTEs | -| datafusion.execution.split_file_groups_by_statistics | false | Attempt to eliminate sorts by packing & sorting files with non-overlapping statistics into the same file groups. Currently experimental | -| datafusion.execution.keep_partition_by_columns | false | Should DataFusion keep the columns used for partition_by in the output RecordBatches | -| datafusion.execution.skip_partial_aggregation_probe_ratio_threshold | 0.8 | Aggregation ratio (number of distinct groups / number of input rows) threshold for skipping partial aggregation. If the value is greater then partial aggregation will skip aggregation for further input | -| datafusion.execution.skip_partial_aggregation_probe_rows_threshold | 100000 | Number of input rows partial aggregation partition should process, before aggregation ratio check and trying to switch to skipping aggregation mode | -| datafusion.execution.use_row_number_estimates_to_optimize_partitioning | false | Should DataFusion use row number estimates at the input to decide whether increasing parallelism is beneficial or not. By default, only exact row numbers (not estimates) are used for this decision. Setting this flag to `true` will likely produce better plans. if the source of statistics is accurate. We plan to make this the default in the future. | -| datafusion.execution.enforce_batch_size_in_joins | false | Should DataFusion enforce batch size in joins or not. By default, DataFusion will not enforce batch size in joins. Enforcing batch size in joins can reduce memory usage when joining large tables with a highly-selective join filter, but is also slightly slower. | -| datafusion.execution.objectstore_writer_buffer_size | 10485760 | Size (bytes) of data buffer DataFusion uses when writing output files. This affects the size of the data chunks that are uploaded to remote object stores (e.g. AWS S3). If very large (>= 100 GiB) output files are being written, it may be necessary to increase this size to avoid errors from the remote end point. | -| datafusion.optimizer.enable_distinct_aggregation_soft_limit | true | When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. | -| datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | -| datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | -| datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | -| datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_file_min_size | 10485760 | Minimum total files size in bytes to perform file scan repartitioning. | -| datafusion.optimizer.repartition_joins | true | Should DataFusion repartition data using the join keys to execute joins in parallel using the provided `target_partitions` level | -| datafusion.optimizer.allow_symmetric_joins_without_pruning | true | Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. | -| datafusion.optimizer.repartition_file_scans | true | When set to `true`, datasource partitions will be repartitioned to achieve maximum parallelism. This applies to both in-memory partitions and FileSource's file groups (1 group is 1 partition). For FileSources, only Parquet and CSV formats are currently supported. If set to `true` for a FileSource, all files will be repartitioned evenly (i.e., a single large file might be partitioned into smaller chunks) for parallel scanning. If set to `false` for a FileSource, different files will be read in parallel, but repartitioning won't happen within a single file. If set to `true` for an in-memory source, all memtable's partitions will have their batches repartitioned evenly to the desired number of `target_partitions`. Repartitioning can change the total number of partitions and batches per partition, but does not slice the initial record tables provided to the MemTable on creation. | -| datafusion.optimizer.repartition_windows | true | Should DataFusion repartition data using the partitions keys to execute window functions in parallel using the provided `target_partitions` level | -| datafusion.optimizer.repartition_sorts | true | Should DataFusion execute sorts in a per-partition fashion and merge afterwards instead of coalescing first and sorting globally. With this flag is enabled, plans in the form below `text "SortExec: [a@0 ASC]", " CoalescePartitionsExec", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` would turn into the plan below which performs better in multithreaded environments `text "SortPreservingMergeExec: [a@0 ASC]", " SortExec: [a@0 ASC]", " RepartitionExec: partitioning=RoundRobinBatch(8), input_partitions=1", ` | -| datafusion.optimizer.prefer_existing_sort | false | When true, DataFusion will opportunistically remove sorts when the data is already sorted, (i.e. setting `preserve_order` to true on `RepartitionExec` and using `SortPreservingMergeExec`) When false, DataFusion will maximize plan parallelism using `RepartitionExec` even if this requires subsequently resorting data using a `SortExec`. | -| datafusion.optimizer.skip_failed_rules | false | When set to true, the logical plan optimizer will produce warning messages if any optimization rules produce errors and then proceed to the next rule. When set to false, any rules that produce errors will cause the query to fail | -| datafusion.optimizer.max_passes | 3 | Number of times that the optimizer will attempt to optimize the plan | -| datafusion.optimizer.top_down_join_key_reordering | true | When set to true, the physical plan optimizer will run a top down process to reorder the join keys | -| datafusion.optimizer.prefer_hash_join | true | When set to true, the physical plan optimizer will prefer HashJoin over SortMergeJoin. HashJoin can work more efficiently than SortMergeJoin but consumes more memory | -| datafusion.optimizer.hash_join_single_partition_threshold | 1048576 | The maximum estimated size in bytes for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.hash_join_single_partition_threshold_rows | 131072 | The maximum estimated size in rows for one input side of a HashJoin will be collected into a single partition | -| datafusion.optimizer.default_filter_selectivity | 20 | The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). | -| datafusion.optimizer.prefer_existing_union | false | When set to true, the optimizer will not attempt to convert Union to Interleave | -| datafusion.optimizer.expand_views_at_output | false | When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. | -| datafusion.explain.logical_plan_only | false | When set to true, the explain statement will only print logical plans | -| datafusion.explain.physical_plan_only | false | When set to true, the explain statement will only print physical plans | -| datafusion.explain.show_statistics | false | When set to true, the explain statement will print operator statistics for physical plans | -| datafusion.explain.show_sizes | true | When set to true, the explain statement will print the partition sizes | -| datafusion.explain.show_schema | false | When set to true, the explain statement will print schema information | -| datafusion.explain.format | indent | Display format of explain. Default is "indent". When set to "tree", it will print the plan in a tree-rendered format. | -| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | -| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | -| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | -| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB and Databricks. | -| datafusion.sql_parser.support_varchar_with_length | true | If true, permit lengths for `VARCHAR` such as `VARCHAR(20)`, but ignore the length. If false, error if a `VARCHAR` with a length is specified. The Arrow type system does not have a notion of maximum string length and thus DataFusion can not enforce such limits. | -| datafusion.sql_parser.map_varchar_to_utf8view | true | If true, `VARCHAR` is mapped to `Utf8View` during SQL planning. If false, `VARCHAR` is mapped to `Utf8` during SQL planning. Default is false. | -| datafusion.sql_parser.collect_spans | false | When set to true, the source locations relative to the original SQL query (i.e. [`Span`](https://docs.rs/sqlparser/latest/sqlparser/tokenizer/struct.Span.html)) will be collected and recorded in the logical plan nodes. | -| datafusion.sql_parser.recursion_limit | 50 | Specifies the recursion depth limit when parsing complex SQL Queries | -| datafusion.format.safe | true | If set to `true` any formatting errors will be written to the output instead of being converted into a [`std::fmt::Error`] | -| datafusion.format.null | | Format string for nulls | -| datafusion.format.date_format | %Y-%m-%d | Date format for date arrays | -| datafusion.format.datetime_format | %Y-%m-%dT%H:%M:%S%.f | Format for DateTime arrays | -| datafusion.format.timestamp_format | %Y-%m-%dT%H:%M:%S%.f | Timestamp format for timestamp arrays | -| datafusion.format.timestamp_tz_format | NULL | Timestamp format for timestamp with timezone arrays. When `None`, ISO 8601 format is used. | -| datafusion.format.time_format | %H:%M:%S%.f | Time format for time arrays | -| datafusion.format.duration_format | pretty | Duration format. Can be either `"pretty"` or `"ISO8601"` | -| datafusion.format.types_info | false | Show types in visual representation batches | From ca5b0fb74a66e71f358572c7cdf77e5e931f4f0b Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 14:20:40 +0800 Subject: [PATCH 105/177] fix flaky test limit.test --- datafusion/sqllogictest/test_files/limit.slt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 1af14a52e2bc6..2e09af4a20eea 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -734,7 +734,7 @@ explain select * from testSubQueryLimit as t1 join (select * from testSubQueryLi ---- logical_plan 01)Limit: skip=0, fetch=10 -02)--Cross Join: +02)--Cross Join: 03)----SubqueryAlias: t1 04)------Limit: skip=0, fetch=10 05)--------TableScan: testsubquerylimit projection=[a, b], fetch=10 @@ -759,7 +759,7 @@ explain select * from testSubQueryLimit as t1 join (select * from testSubQueryLi ---- logical_plan 01)Limit: skip=0, fetch=2 -02)--Cross Join: +02)--Cross Join: 03)----SubqueryAlias: t1 04)------Limit: skip=0, fetch=2 05)--------TableScan: testsubquerylimit projection=[a, b], fetch=2 @@ -830,6 +830,9 @@ CREATE EXTERNAL TABLE test_limit_with_partitions STORED AS PARQUET LOCATION 'test_files/scratch/parquet/test_limit_with_partitions/'; +statement ok +set datafusion.explain.logical_plan_only = true; + query TT explain with selection as ( @@ -849,12 +852,9 @@ logical_plan 04)------SubqueryAlias: selection 05)--------Limit: skip=0, fetch=1 06)----------TableScan: test_limit_with_partitions projection=[part_key], fetch=1 -physical_plan -01)ProjectionExec: expr=[foo@0 as foo] -02)--SortExec: TopK(fetch=1000), expr=[part_key@1 ASC NULLS LAST], preserve_partitioning=[false] -03)----ProjectionExec: expr=[1 as foo, part_key@0 as part_key] -04)------CoalescePartitionsExec: fetch=1 -05)--------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..265], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:265..530], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:530..794]]}, projection=[part_key], limit=1, file_type=parquet + +statement ok +set datafusion.explain.logical_plan_only = false; query I with selection as ( From d8c3e03f58dbe20d2b0c35523840dcec3ab29719 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Wed, 25 Jun 2025 09:51:04 +0800 Subject: [PATCH 106/177] Simplify predicates in `PushDownFilter` optimizer rule (#16362) * Simplify predicates in filter * add slt test * Use BtreeMap to make tests stable * process edge coner * add doc for simplify_predicates.rs * add as_literal to make code neat * reorgnize file * reduce clone call --- Cargo.lock | 12 +- datafusion/expr/src/expr.rs | 10 +- datafusion/optimizer/src/lib.rs | 1 - datafusion/optimizer/src/push_down_filter.rs | 6 +- .../optimizer/src/simplify_expressions/mod.rs | 2 + .../simplify_predicates.rs | 247 ++++++++++++++++++ .../optimizer/src/simplify_predicates.rs | 194 -------------- .../test_files/simplify_predicates.slt | 30 ++- 8 files changed, 292 insertions(+), 210 deletions(-) create mode 100644 datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs delete mode 100644 datafusion/optimizer/src/simplify_predicates.rs diff --git a/Cargo.lock b/Cargo.lock index e7fb4ef136c7a..41d372f71723b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3959,9 +3959,9 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libmimalloc-sys" -version = "0.1.43" +version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88cd67e9de251c1781dbe2f641a1a3ad66eaae831b8a2c38fbdc5ddae16d4d" +checksum = "ec9d6fac27761dabcd4ee73571cdb06b7022dc99089acbe5435691edffaac0f4" dependencies = [ "cc", "libc", @@ -4092,9 +4092,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.47" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1791cbe101e95af5764f06f20f6760521f7158f69dbf9d6baf941ee1bf6bc40" +checksum = "995942f432bbb4822a7e9c3faa87a695185b0d09273ba85f097b54f4e458f2af" dependencies = [ "libmimalloc-sys", ] @@ -4320,9 +4320,9 @@ dependencies = [ [[package]] name = "object_store" -version = "0.12.2" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7781f96d79ed0f961a7021424ab01840efbda64ae7a505aaea195efc91eaaec4" +checksum = "d94ac16b433c0ccf75326388c893d2835ab7457ea35ab8ba5d745c053ef5fa16" dependencies = [ "async-trait", "base64 0.22.1", diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index bbfdf7d870555..25b2ddfc00474 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -1883,9 +1883,13 @@ impl Expr { } } - /// Check if the Expr is literal - pub fn is_literal(&self) -> bool { - matches!(self, Expr::Literal(_, _)) + /// Check if the Expr is literal and get the literal value if it is. + pub fn as_literal(&self) -> Option<&ScalarValue> { + if let Expr::Literal(lit, _) = self { + Some(lit) + } else { + None + } } } diff --git a/datafusion/optimizer/src/lib.rs b/datafusion/optimizer/src/lib.rs index ea14cf114030a..280010e3d92c0 100644 --- a/datafusion/optimizer/src/lib.rs +++ b/datafusion/optimizer/src/lib.rs @@ -61,7 +61,6 @@ pub mod push_down_limit; pub mod replace_distinct_aggregate; pub mod scalar_subquery_to_join; pub mod simplify_expressions; -mod simplify_predicates; pub mod single_distinct_to_groupby; pub mod utils; diff --git a/datafusion/optimizer/src/push_down_filter.rs b/datafusion/optimizer/src/push_down_filter.rs index f701fcf861e0d..b1aa0c09249ff 100644 --- a/datafusion/optimizer/src/push_down_filter.rs +++ b/datafusion/optimizer/src/push_down_filter.rs @@ -41,7 +41,7 @@ use datafusion_expr::{ use crate::optimizer::ApplyOrder; use crate::utils::{has_all_column_refs, is_restrict_null_predicate}; -use crate::{simplify_predicates::simplify_predicates, OptimizerConfig, OptimizerRule}; +use crate::{simplify_expressions::simplify_predicates, OptimizerConfig, OptimizerRule}; /// Optimizer rule for pushing (moving) filter expressions down in a plan so /// they are applied as early as possible. @@ -783,7 +783,9 @@ impl OptimizerRule for PushDownFilter { let new_predicates = simplify_predicates(predicate)?; if old_predicate_len != new_predicates.len() { let Some(new_predicate) = conjunction(new_predicates) else { - return plan_err!("at least one expression exists"); + // new_predicates is empty - remove the filter entirely + // Return the child plan without the filter + return Ok(Transformed::yes(Arc::unwrap_or_clone(filter.input))); }; filter.predicate = new_predicate; } diff --git a/datafusion/optimizer/src/simplify_expressions/mod.rs b/datafusion/optimizer/src/simplify_expressions/mod.rs index 5fbee02e3909e..7ae38eec9a3ad 100644 --- a/datafusion/optimizer/src/simplify_expressions/mod.rs +++ b/datafusion/optimizer/src/simplify_expressions/mod.rs @@ -23,6 +23,7 @@ mod guarantees; mod inlist_simplifier; mod regex; pub mod simplify_exprs; +mod simplify_predicates; mod unwrap_cast; mod utils; @@ -31,6 +32,7 @@ pub use datafusion_expr::simplify::{SimplifyContext, SimplifyInfo}; pub use expr_simplifier::*; pub use simplify_exprs::*; +pub use simplify_predicates::simplify_predicates; // Export for test in datafusion/core/tests/optimizer_integration.rs pub use guarantees::GuaranteeRewriter; diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs new file mode 100644 index 0000000000000..32b2315e15d58 --- /dev/null +++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs @@ -0,0 +1,247 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Simplifies predicates by reducing redundant or overlapping conditions. +//! +//! This module provides functionality to optimize logical predicates used in query planning +//! by eliminating redundant conditions, thus reducing the number of predicates to evaluate. +//! Unlike the simplifier in `simplify_expressions/simplify_exprs.rs`, which focuses on +//! general expression simplification (e.g., constant folding and algebraic simplifications), +//! this module specifically targets predicate optimization by handling containment relationships. +//! For example, it can simplify `x > 5 AND x > 6` to just `x > 6`, as the latter condition +//! encompasses the former, resulting in fewer checks during query execution. + +use datafusion_common::{Column, Result, ScalarValue}; +use datafusion_expr::{BinaryExpr, Cast, Expr, Operator}; +use std::collections::BTreeMap; + +/// Simplifies a list of predicates by removing redundancies. +/// +/// This function takes a vector of predicate expressions and groups them by the column they reference. +/// Predicates that reference a single column and are comparison operations (e.g., >, >=, <, <=, =) +/// are analyzed to remove redundant conditions. For instance, `x > 5 AND x > 6` is simplified to +/// `x > 6`. Other predicates that do not fit this pattern are retained as-is. +/// +/// # Arguments +/// * `predicates` - A vector of `Expr` representing the predicates to simplify. +/// +/// # Returns +/// A `Result` containing a vector of simplified `Expr` predicates. +pub fn simplify_predicates(predicates: Vec) -> Result> { + // Early return for simple cases + if predicates.len() <= 1 { + return Ok(predicates); + } + + // Group predicates by their column reference + let mut column_predicates: BTreeMap> = BTreeMap::new(); + let mut other_predicates = Vec::new(); + + for pred in predicates { + match &pred { + Expr::BinaryExpr(BinaryExpr { + left, + op: + Operator::Gt + | Operator::GtEq + | Operator::Lt + | Operator::LtEq + | Operator::Eq, + right, + }) => { + let left_col = extract_column_from_expr(left); + let right_col = extract_column_from_expr(right); + if let (Some(col), Some(_)) = (&left_col, right.as_literal()) { + column_predicates.entry(col.clone()).or_default().push(pred); + } else if let (Some(_), Some(col)) = (left.as_literal(), &right_col) { + column_predicates.entry(col.clone()).or_default().push(pred); + } else { + other_predicates.push(pred); + } + } + _ => other_predicates.push(pred), + } + } + + // Process each column's predicates to remove redundancies + let mut result = other_predicates; + for (_, preds) in column_predicates { + let simplified = simplify_column_predicates(preds)?; + result.extend(simplified); + } + + Ok(result) +} + +/// Simplifies predicates related to a single column. +/// +/// This function processes a list of predicates that all reference the same column and +/// simplifies them based on their operators. It groups predicates into greater-than (>, >=), +/// less-than (<, <=), and equality (=) categories, then selects the most restrictive condition +/// in each category to reduce redundancy. For example, among `x > 5` and `x > 6`, only `x > 6` +/// is retained as it is more restrictive. +/// +/// # Arguments +/// * `predicates` - A vector of `Expr` representing predicates for a single column. +/// +/// # Returns +/// A `Result` containing a vector of simplified `Expr` predicates for the column. +fn simplify_column_predicates(predicates: Vec) -> Result> { + if predicates.len() <= 1 { + return Ok(predicates); + } + + // Group by operator type, but combining similar operators + let mut greater_predicates = Vec::new(); // Combines > and >= + let mut less_predicates = Vec::new(); // Combines < and <= + let mut eq_predicates = Vec::new(); + + for pred in predicates { + match &pred { + Expr::BinaryExpr(BinaryExpr { left: _, op, right }) => { + match (op, right.as_literal().is_some()) { + (Operator::Gt, true) + | (Operator::Lt, false) + | (Operator::GtEq, true) + | (Operator::LtEq, false) => greater_predicates.push(pred), + (Operator::Lt, true) + | (Operator::Gt, false) + | (Operator::LtEq, true) + | (Operator::GtEq, false) => less_predicates.push(pred), + (Operator::Eq, _) => eq_predicates.push(pred), + _ => unreachable!("Unexpected operator: {}", op), + } + } + _ => unreachable!("Unexpected predicate {}", pred.to_string()), + } + } + + let mut result = Vec::new(); + + if !eq_predicates.is_empty() { + // If there are many equality predicates, we can only keep one if they are all the same + if eq_predicates.len() == 1 + || eq_predicates.iter().all(|e| e == &eq_predicates[0]) + { + result.push(eq_predicates.pop().unwrap()); + } else { + // If they are not the same, add a false predicate + result.push(Expr::Literal(ScalarValue::Boolean(Some(false)), None)); + } + } + + // Handle all greater-than-style predicates (keep the most restrictive - highest value) + if !greater_predicates.is_empty() { + if let Some(most_restrictive) = + find_most_restrictive_predicate(&greater_predicates, true)? + { + result.push(most_restrictive); + } else { + result.extend(greater_predicates); + } + } + + // Handle all less-than-style predicates (keep the most restrictive - lowest value) + if !less_predicates.is_empty() { + if let Some(most_restrictive) = + find_most_restrictive_predicate(&less_predicates, false)? + { + result.push(most_restrictive); + } else { + result.extend(less_predicates); + } + } + + Ok(result) +} + +/// Finds the most restrictive predicate from a list based on literal values. +/// +/// This function iterates through a list of predicates to identify the most restrictive one +/// by comparing their literal values. For greater-than predicates, the highest value is most +/// restrictive, while for less-than predicates, the lowest value is most restrictive. +/// +/// # Arguments +/// * `predicates` - A slice of `Expr` representing predicates to compare. +/// * `find_greater` - A boolean indicating whether to find the highest value (true for >, >=) +/// or the lowest value (false for <, <=). +/// +/// # Returns +/// A `Result` containing an `Option` with the most restrictive predicate, if any. +fn find_most_restrictive_predicate( + predicates: &[Expr], + find_greater: bool, +) -> Result> { + if predicates.is_empty() { + return Ok(None); + } + + let mut most_restrictive_idx = 0; + let mut best_value: Option<&ScalarValue> = None; + + for (idx, pred) in predicates.iter().enumerate() { + if let Expr::BinaryExpr(BinaryExpr { left, op: _, right }) = pred { + // Extract the literal value based on which side has it + let scalar_value = match (right.as_literal(), left.as_literal()) { + (Some(scalar), _) => Some(scalar), + (_, Some(scalar)) => Some(scalar), + _ => None, + }; + + if let Some(scalar) = scalar_value { + if let Some(current_best) = best_value { + if let Some(comparison) = scalar.partial_cmp(current_best) { + let is_better = if find_greater { + comparison == std::cmp::Ordering::Greater + } else { + comparison == std::cmp::Ordering::Less + }; + + if is_better { + best_value = Some(scalar); + most_restrictive_idx = idx; + } + } + } else { + best_value = Some(scalar); + most_restrictive_idx = idx; + } + } + } + } + + Ok(Some(predicates[most_restrictive_idx].clone())) +} + +/// Extracts a column reference from an expression, if present. +/// +/// This function checks if the given expression is a column reference or contains one, +/// such as within a cast operation. It returns the `Column` if found. +/// +/// # Arguments +/// * `expr` - A reference to an `Expr` to inspect for a column reference. +/// +/// # Returns +/// An `Option` containing the column reference if found, otherwise `None`. +fn extract_column_from_expr(expr: &Expr) -> Option { + match expr { + Expr::Column(col) => Some(col.clone()), + // Handle cases where the column might be wrapped in a cast or other operation + Expr::Cast(Cast { expr, .. }) => extract_column_from_expr(expr), + _ => None, + } +} diff --git a/datafusion/optimizer/src/simplify_predicates.rs b/datafusion/optimizer/src/simplify_predicates.rs deleted file mode 100644 index 198e630203977..0000000000000 --- a/datafusion/optimizer/src/simplify_predicates.rs +++ /dev/null @@ -1,194 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion_common::{Column, Result, ScalarValue}; -use datafusion_expr::{BinaryExpr, Cast, Expr, Operator}; -use std::collections::BTreeMap; - -pub(crate) fn simplify_predicates(predicates: Vec) -> Result> { - // Early return for simple cases - if predicates.len() <= 1 { - return Ok(predicates); - } - - // Group predicates by their column reference - let mut column_predicates: BTreeMap> = BTreeMap::new(); - let mut other_predicates = Vec::new(); - - for pred in predicates { - match &pred { - Expr::BinaryExpr(BinaryExpr { - left, - op: - Operator::Gt - | Operator::GtEq - | Operator::Lt - | Operator::LtEq - | Operator::Eq, - right, - }) => { - let left_col = extract_column_from_expr(left); - let right_col = extract_column_from_expr(right); - let left_lit = left.is_literal(); - let right_lit = right.is_literal(); - if let (Some(col), true) = (&left_col, right_lit) { - column_predicates.entry(col.clone()).or_default().push(pred); - } else if let (true, Some(col)) = (left_lit, &right_col) { - column_predicates.entry(col.clone()).or_default().push(pred); - } else { - other_predicates.push(pred); - } - } - _ => other_predicates.push(pred), - } - } - - // Process each column's predicates to remove redundancies - let mut result = other_predicates; - for (_, preds) in column_predicates { - let simplified = simplify_column_predicates(preds)?; - result.extend(simplified); - } - - Ok(result) -} - -fn simplify_column_predicates(predicates: Vec) -> Result> { - if predicates.len() <= 1 { - return Ok(predicates); - } - - // Group by operator type, but combining similar operators - let mut greater_predicates = Vec::new(); // Combines > and >= - let mut less_predicates = Vec::new(); // Combines < and <= - let mut eq_predicates = Vec::new(); - - for pred in predicates { - match &pred { - Expr::BinaryExpr(BinaryExpr { left: _, op, right }) => { - let right_is_literal = right.is_literal(); - match (op, right_is_literal) { - (Operator::Gt, true) - | (Operator::Lt, false) - | (Operator::GtEq, true) - | (Operator::LtEq, false) => greater_predicates.push(pred), - (Operator::Lt, true) - | (Operator::Gt, false) - | (Operator::LtEq, true) - | (Operator::GtEq, false) => less_predicates.push(pred), - (Operator::Eq, _) => eq_predicates.push(pred), - _ => unreachable!("Unexpected operator: {}", op), - } - } - _ => unreachable!("Unexpected predicate {}", pred.to_string()), - } - } - - let mut result = Vec::new(); - - // If we have equality predicates, they're the most restrictive - if !eq_predicates.is_empty() { - if eq_predicates.len() > 1 { - result.push(Expr::Literal(ScalarValue::Boolean(Some(false)), None)); - } else { - result.push(eq_predicates[0].clone()); - } - } else { - // Handle all greater-than-style predicates (keep the most restrictive - highest value) - if !greater_predicates.is_empty() { - if let Some(most_restrictive) = - find_most_restrictive_predicate(&greater_predicates, true)? - { - result.push(most_restrictive); - } else { - result.extend(greater_predicates); - } - } - - // Handle all less-than-style predicates (keep the most restrictive - lowest value) - if !less_predicates.is_empty() { - if let Some(most_restrictive) = - find_most_restrictive_predicate(&less_predicates, false)? - { - result.push(most_restrictive); - } else { - result.extend(less_predicates); - } - } - } - - Ok(result) -} - -fn find_most_restrictive_predicate( - predicates: &[Expr], - find_greater: bool, -) -> Result> { - if predicates.is_empty() { - return Ok(None); - } - - let mut most_restrictive = predicates[0].clone(); - let mut best_value: Option = None; - - for pred in predicates { - if let Expr::BinaryExpr(BinaryExpr { left, op: _, right }) = pred { - // Extract the literal value based on which side has it - let mut scalar_value = None; - if right.is_literal() { - if let Expr::Literal(scalar, _) = right.as_ref() { - scalar_value = Some(scalar.clone()); - } - } else if left.is_literal() { - if let Expr::Literal(scalar, _) = left.as_ref() { - scalar_value = Some(scalar.clone()); - } - } - - if let Some(scalar) = scalar_value { - if let Some(current_best) = &best_value { - if let Some(comparison) = scalar.partial_cmp(current_best) { - let is_better = if find_greater { - comparison == std::cmp::Ordering::Greater - } else { - comparison == std::cmp::Ordering::Less - }; - - if is_better { - best_value = Some(scalar); - most_restrictive = pred.clone(); - } - } - } else { - best_value = Some(scalar); - most_restrictive = pred.clone(); - } - } - } - } - - Ok(Some(most_restrictive)) -} - -fn extract_column_from_expr(expr: &Expr) -> Option { - match expr { - Expr::Column(col) => Some(col.clone()), - // Handle cases where the column might be wrapped in a cast or other operation - Expr::Cast(Cast { expr, .. }) => extract_column_from_expr(expr), - _ => None, - } -} diff --git a/datafusion/sqllogictest/test_files/simplify_predicates.slt b/datafusion/sqllogictest/test_files/simplify_predicates.slt index cef78d97bb46c..0dd551d96d0ce 100644 --- a/datafusion/sqllogictest/test_files/simplify_predicates.slt +++ b/datafusion/sqllogictest/test_files/simplify_predicates.slt @@ -70,13 +70,35 @@ logical_plan 01)Filter: test_data.float_col < Float32(8) AND test_data.int_col > Int32(6) 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] +# x = 7 AND x = 7 should simplify to x = 7 +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col = 7 AND int_col = 7; +---- +logical_plan +01)Filter: test_data.int_col = Int32(7) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] + +# x = 7 AND x = 6 should simplify to false +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col = 7 AND int_col = 6; +---- +logical_plan EmptyRelation + +# TODO: x = 7 AND x < 2 should simplify to false +query TT +EXPLAIN SELECT * FROM test_data WHERE int_col = 7 AND int_col < 2; +---- +logical_plan +01)Filter: test_data.int_col = Int32(7) AND test_data.int_col < Int32(2) +02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] -# x = 7 AND x > 5 should simplify to x = 7 + +# TODO: x = 7 AND x > 5 should simplify to x = 7 query TT EXPLAIN SELECT * FROM test_data WHERE int_col = 7 AND int_col > 5; ---- logical_plan -01)Filter: test_data.int_col = Int32(7) +01)Filter: test_data.int_col = Int32(7) AND test_data.int_col > Int32(5) 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] # str_col > 'apple' AND str_col > 'banana' should simplify to str_col > 'banana' @@ -148,7 +170,7 @@ logical_plan 07)------Filter: test_data2.value < Int32(50) AND test_data2.id > Int32(10) 08)--------TableScan: test_data2 projection=[id, value] -# Case 13: Handling negated predicates +# Handling negated predicates # NOT (x < 10) AND NOT (x < 5) should simplify to NOT (x < 10) query TT EXPLAIN SELECT * FROM test_data WHERE NOT (int_col < 10) AND NOT (int_col < 5); @@ -198,7 +220,7 @@ logical_plan 01)Filter: (test_data.int_col > Int32(5) OR test_data.float_col < Float32(10)) AND (test_data.int_col > Int32(6) OR test_data.float_col < Float32(8)) 02)--TableScan: test_data projection=[int_col, float_col, str_col, date_col, bool_col] -# Case 20: Combination of AND and OR with simplifiable predicates +# Combination of AND and OR with simplifiable predicates query TT EXPLAIN SELECT * FROM test_data WHERE (int_col > 5 AND int_col > 6) From 209988285b9def1040e20d61e74cb04e070ffaa1 Mon Sep 17 00:00:00 2001 From: kosiew Date: Sat, 7 Jun 2025 02:03:09 +0800 Subject: [PATCH 107/177] Fix intermittent SQL logic test failure in limit.slt by adding ORDER BY clause (#16257) * Add order by clause to limit query for consistent results * test: update explain plan --- datafusion/sqllogictest/test_files/limit.slt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 2e09af4a20eea..6f13570773555 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -830,14 +830,12 @@ CREATE EXTERNAL TABLE test_limit_with_partitions STORED AS PARQUET LOCATION 'test_files/scratch/parquet/test_limit_with_partitions/'; -statement ok -set datafusion.explain.logical_plan_only = true; - query TT explain with selection as ( select * from test_limit_with_partitions + order by part_key limit 1 ) select 1 as foo @@ -850,16 +848,19 @@ logical_plan 02)--Sort: selection.part_key ASC NULLS LAST, fetch=1000 03)----Projection: Int64(1) AS foo, selection.part_key 04)------SubqueryAlias: selection -05)--------Limit: skip=0, fetch=1 -06)----------TableScan: test_limit_with_partitions projection=[part_key], fetch=1 - -statement ok -set datafusion.explain.logical_plan_only = false; +05)--------Sort: test_limit_with_partitions.part_key ASC NULLS LAST, fetch=1 +06)----------TableScan: test_limit_with_partitions projection=[part_key] +physical_plan +01)ProjectionExec: expr=[1 as foo] +02)--SortPreservingMergeExec: [part_key@0 ASC NULLS LAST], fetch=1 +03)----SortExec: TopK(fetch=1), expr=[part_key@0 ASC NULLS LAST], preserve_partitioning=[true] +04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-0.parquet:0..794], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:0..794], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..794]]}, projection=[part_key], file_type=parquet query I with selection as ( select * from test_limit_with_partitions + order by part_key limit 1 ) select 1 as foo From ff8418c8ecbf74a02a709c8ca2ca23b72cb9a843 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 15:11:54 +0800 Subject: [PATCH 108/177] fix limit.rs --- datafusion/sqllogictest/test_files/limit.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/limit.slt b/datafusion/sqllogictest/test_files/limit.slt index 6f13570773555..a6bafd703de85 100644 --- a/datafusion/sqllogictest/test_files/limit.slt +++ b/datafusion/sqllogictest/test_files/limit.slt @@ -854,7 +854,7 @@ physical_plan 01)ProjectionExec: expr=[1 as foo] 02)--SortPreservingMergeExec: [part_key@0 ASC NULLS LAST], fetch=1 03)----SortExec: TopK(fetch=1), expr=[part_key@0 ASC NULLS LAST], preserve_partitioning=[true] -04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-0.parquet:0..794], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet:0..794], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet:0..794]]}, projection=[part_key], file_type=parquet +04)------DataSourceExec: file_groups={3 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-0.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/parquet/test_limit_with_partitions/part-2.parquet]]}, projection=[part_key], file_type=parquet query I with selection as ( From 2c7836ad071996dc3fbd6b60040a36b9ea439edd Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 15:26:19 +0800 Subject: [PATCH 109/177] fix tpch q19 --- datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part index ace2081eb18fe..3000165b4c181 100644 --- a/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part +++ b/datafusion/sqllogictest/test_files/tpch/plans/q19.slt.part @@ -59,7 +59,7 @@ logical_plan 03)----Projection: lineitem.l_extendedprice, lineitem.l_discount 04)------Inner Join: lineitem.l_partkey = part.p_partkey Filter: part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND lineitem.l_quantity >= Decimal128(Some(100),15,2) AND lineitem.l_quantity <= Decimal128(Some(1100),15,2) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND lineitem.l_quantity >= Decimal128(Some(1000),15,2) AND lineitem.l_quantity <= Decimal128(Some(2000),15,2) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND lineitem.l_quantity >= Decimal128(Some(2000),15,2) AND lineitem.l_quantity <= Decimal128(Some(3000),15,2) AND part.p_size <= Int32(15) 05)--------Projection: lineitem.l_partkey, lineitem.l_quantity, lineitem.l_extendedprice, lineitem.l_discount -06)----------Filter: (lineitem.l_quantity >= Decimal128(Some(100),15,2) AND lineitem.l_quantity <= Decimal128(Some(1100),15,2) OR lineitem.l_quantity >= Decimal128(Some(1000),15,2) AND lineitem.l_quantity <= Decimal128(Some(2000),15,2) OR lineitem.l_quantity >= Decimal128(Some(2000),15,2) AND lineitem.l_quantity <= Decimal128(Some(3000),15,2)) AND (lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG")) AND lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON") +06)----------Filter: lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON") AND (lineitem.l_quantity >= Decimal128(Some(100),15,2) AND lineitem.l_quantity <= Decimal128(Some(1100),15,2) OR lineitem.l_quantity >= Decimal128(Some(1000),15,2) AND lineitem.l_quantity <= Decimal128(Some(2000),15,2) OR lineitem.l_quantity >= Decimal128(Some(2000),15,2) AND lineitem.l_quantity <= Decimal128(Some(3000),15,2)) AND (lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG")) 07)------------TableScan: lineitem projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], partial_filters=[lineitem.l_shipmode = Utf8View("AIR") OR lineitem.l_shipmode = Utf8View("AIR REG"), lineitem.l_shipinstruct = Utf8View("DELIVER IN PERSON"), lineitem.l_quantity >= Decimal128(Some(100),15,2) AND lineitem.l_quantity <= Decimal128(Some(1100),15,2) OR lineitem.l_quantity >= Decimal128(Some(1000),15,2) AND lineitem.l_quantity <= Decimal128(Some(2000),15,2) OR lineitem.l_quantity >= Decimal128(Some(2000),15,2) AND lineitem.l_quantity <= Decimal128(Some(3000),15,2)] 08)--------Filter: (part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND part.p_size <= Int32(15)) AND part.p_size >= Int32(1) 09)----------TableScan: part projection=[p_partkey, p_brand, p_size, p_container], partial_filters=[part.p_size >= Int32(1), part.p_brand = Utf8View("Brand#12") AND part.p_container IN ([Utf8View("SM CASE"), Utf8View("SM BOX"), Utf8View("SM PACK"), Utf8View("SM PKG")]) AND part.p_size <= Int32(5) OR part.p_brand = Utf8View("Brand#23") AND part.p_container IN ([Utf8View("MED BAG"), Utf8View("MED BOX"), Utf8View("MED PKG"), Utf8View("MED PACK")]) AND part.p_size <= Int32(10) OR part.p_brand = Utf8View("Brand#34") AND part.p_container IN ([Utf8View("LG CASE"), Utf8View("LG BOX"), Utf8View("LG PACK"), Utf8View("LG PKG")]) AND part.p_size <= Int32(15)] @@ -73,7 +73,7 @@ physical_plan 07)------------CoalesceBatchesExec: target_batch_size=8192 08)--------------RepartitionExec: partitioning=Hash([l_partkey@0], 4), input_partitions=4 09)----------------CoalesceBatchesExec: target_batch_size=8192 -10)------------------FilterExec: (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG) AND l_shipinstruct@4 = DELIVER IN PERSON, projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] +10)------------------FilterExec: l_shipinstruct@4 = DELIVER IN PERSON AND (l_quantity@1 >= Some(100),15,2 AND l_quantity@1 <= Some(1100),15,2 OR l_quantity@1 >= Some(1000),15,2 AND l_quantity@1 <= Some(2000),15,2 OR l_quantity@1 >= Some(2000),15,2 AND l_quantity@1 <= Some(3000),15,2) AND (l_shipmode@5 = AIR OR l_shipmode@5 = AIR REG), projection=[l_partkey@0, l_quantity@1, l_extendedprice@2, l_discount@3] 11)--------------------DataSourceExec: file_groups={4 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:0..18561749], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:18561749..37123498], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:37123498..55685247], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/tpch/data/lineitem.tbl:55685247..74246996]]}, projection=[l_partkey, l_quantity, l_extendedprice, l_discount, l_shipinstruct, l_shipmode], file_type=csv, has_header=false 12)------------CoalesceBatchesExec: target_batch_size=8192 13)--------------RepartitionExec: partitioning=Hash([p_partkey@0], 4), input_partitions=4 From 9191f3922175d2945da60b1c63a2e68ab36a896f Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 16:15:09 +0800 Subject: [PATCH 110/177] public GroupValues & new_group_values --- .../src/aggregates/group_values/mod.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rs index 1e4c7558bda39..f2f489b7223c3 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -//! `GroupValues` trait for storing and interning group keys +//! [`GroupValues`] trait for storing and interning group keys use arrow::array::types::{ Date32Type, Date64Type, Decimal128Type, Time32MillisecondType, Time32SecondType, @@ -28,7 +28,7 @@ use datafusion_common::Result; use datafusion_expr::EmitTo; -pub(crate) mod multi_group_by; +pub mod multi_group_by; mod row; mod single_group_by; @@ -84,7 +84,7 @@ mod null_builder; /// Each distinct group in a hash aggregation is identified by a unique group id /// (usize) which is assigned by instances of this trait. Group ids are /// continuous without gaps, starting from 0. -pub(crate) trait GroupValues: Send { +pub trait GroupValues: Send { /// Calculates the group id for each input row of `cols`, assigning new /// group ids as necessary. /// @@ -119,15 +119,17 @@ pub(crate) trait GroupValues: Send { /// - If group by single column, and type of this column has /// the specific [`GroupValues`] implementation, such implementation /// will be chosen. -/// +/// /// - If group by multiple columns, and all column types have the specific -/// [`GroupColumn`] implementations, [`GroupValuesColumn`] will be chosen. +/// `GroupColumn` implementations, `GroupValuesColumn` will be chosen. /// -/// - Otherwise, the general implementation [`GroupValuesRows`] will be chosen. +/// - Otherwise, the general implementation `GroupValuesRows` will be chosen. /// -/// [`GroupColumn`]: crate::aggregates::group_values::multi_group_by::GroupColumn +/// `GroupColumn`: crate::aggregates::group_values::multi_group_by::GroupColumn +/// `GroupValuesColumn`: crate::aggregates::group_values::multi_group_by::GroupValuesColumn +/// `GroupValuesRows`: crate::aggregates::group_values::row::GroupValuesRows /// -pub(crate) fn new_group_values( +pub fn new_group_values( schema: SchemaRef, group_ordering: &GroupOrdering, ) -> Result> { From d358db4587210671bc1a7d0241d884327e7be0d4 Mon Sep 17 00:00:00 2001 From: xudong963 Date: Mon, 25 Aug 2025 16:35:50 +0800 Subject: [PATCH 111/177] fix clippy --- datafusion/physical-plan/src/aggregates/group_values/mod.rs | 2 +- .../src/aggregates/group_values/multi_group_by/mod.rs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/aggregates/group_values/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/mod.rs index f2f489b7223c3..c64be0de1e83f 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/mod.rs @@ -119,7 +119,7 @@ pub trait GroupValues: Send { /// - If group by single column, and type of this column has /// the specific [`GroupValues`] implementation, such implementation /// will be chosen. -/// +/// /// - If group by multiple columns, and all column types have the specific /// `GroupColumn` implementations, `GroupValuesColumn` will be chosen. /// diff --git a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs index 2ac0389454dec..9b547a45e8b08 100644 --- a/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs +++ b/datafusion/physical-plan/src/aggregates/group_values/multi_group_by/mod.rs @@ -91,6 +91,11 @@ pub trait GroupColumn: Send + Sync { /// Returns the number of rows stored in this builder fn len(&self) -> usize; + /// true if len == 0 + fn is_empty(&self) -> bool { + self.len() == 0 + } + /// Returns the number of bytes used by this [`GroupColumn`] fn size(&self) -> usize; From cefa63a42e616819cc0dd60ee39cdb6aa8792a9a Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 11:16:28 +0800 Subject: [PATCH 112/177] fix fetch with new order lex --- .../src/enforce_distribution.rs | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 86bdec312a24d..1a1bea02c7c16 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -940,28 +940,21 @@ fn add_merge_on_top( input: DistributionContext, fetch: &mut Option, ) -> DistributionContext { - // Add SortPreservingMerge only when partition count is larger than 1. + // Apply only when the partition count is larger than one. if input.plan.output_partitioning().partition_count() > 1 { // When there is an existing ordering, we preserve ordering // when decreasing partitions. This will be un-done in the future // if any of the following conditions is true // - Preserving ordering is not helpful in terms of satisfying ordering requirements // - Usage of order preserving variants is not desirable - // (determined by flag `config.optimizer.bounded_order_preserving_variants`) - let should_preserve_ordering = input.plan.output_ordering().is_some(); - - let ordering = input - .plan - .output_ordering() - .cloned() - .unwrap_or_else(LexOrdering::default); - - let new_plan = if should_preserve_ordering { - Arc::new( - SortPreservingMergeExec::new(ordering, Arc::clone(&input.plan)) - .with_fetch(fetch.take()), - ) as _ + // (determined by flag `config.optimizer.prefer_existing_sort`) + let new_plan = if let Some(req) = input.plan.output_ordering() { + Arc::new(SortPreservingMergeExec::new( + req.clone(), + Arc::clone(&input.plan), + ).with_fetch(*fetch)) as _ } else { + // If there is no input order, we can simply coalesce partitions: Arc::new(CoalescePartitionsExec::new(Arc::clone(&input.plan))) as _ }; From 1f47d469005cf56a6ae7b6c74ce12a2d8fb40552 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 11:19:02 +0800 Subject: [PATCH 113/177] fix fetch add back with new lex order --- .../src/enforce_distribution.rs | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 1a1bea02c7c16..c43f98c999c8f 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -949,10 +949,10 @@ fn add_merge_on_top( // - Usage of order preserving variants is not desirable // (determined by flag `config.optimizer.prefer_existing_sort`) let new_plan = if let Some(req) = input.plan.output_ordering() { - Arc::new(SortPreservingMergeExec::new( - req.clone(), - Arc::clone(&input.plan), - ).with_fetch(*fetch)) as _ + Arc::new( + SortPreservingMergeExec::new(req.clone(), Arc::clone(&input.plan)) + .with_fetch(*fetch), + ) as _ } else { // If there is no input order, we can simply coalesce partitions: Arc::new(CoalescePartitionsExec::new(Arc::clone(&input.plan))) as _ @@ -1406,13 +1406,8 @@ pub fn ensure_distribution( // It was removed by `remove_dist_changing_operators` // and we need to add it back. if fetch.is_some() { - let ordering = plan - .output_ordering() - .cloned() - .unwrap_or_else(LexOrdering::default); - let plan = Arc::new( - SortPreservingMergeExec::new(ordering, plan).with_fetch(fetch.take()), - ); + // It's safe to unwrap because `spm` is set only if `fetch` is set. + let plan = spm.unwrap().with_fetch(fetch.take()).unwrap(); optimized_distribution_ctx = DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); } From 95aadb9b9935b7aff147efa4ea56ae2d34550236 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 11:38:41 +0800 Subject: [PATCH 114/177] fix clippy --- datafusion/physical-optimizer/src/enforce_distribution.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 8f8c7274cf78e..40998783cce4e 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -989,6 +989,7 @@ fn add_spm_on_top( /// ```text /// "DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet", /// ``` +#[allow(clippy::type_complexity)] fn remove_dist_changing_operators( mut distribution_context: DistributionContext, ) -> Result<(DistributionContext, Option)> { @@ -1033,6 +1034,7 @@ fn remove_dist_changing_operators( /// " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2", /// " DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet", /// ``` +#[allow(clippy::type_complexity)] pub fn replace_order_preserving_variants( mut context: DistributionContext, ordering_satisfied: bool, From 70a3c94742a594cb7490b8d9ba4248990a24ec1d Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 11:42:28 +0800 Subject: [PATCH 115/177] fix clippy --- datafusion/physical-optimizer/src/enforce_distribution.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index c43f98c999c8f..0d458add31305 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -1031,6 +1031,7 @@ fn remove_dist_changing_operators( /// " RepartitionExec: partitioning=RoundRobinBatch(10), input_partitions=2", /// " DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet", /// ``` +#[allow(clippy::type_complexity)] pub fn replace_order_preserving_variants( mut context: DistributionContext, ordering_satisfied: bool, From a93e81e3713841b744811e360619d120052b5da2 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 11:51:21 +0800 Subject: [PATCH 116/177] add order needed --- .../physical-optimizer/src/enforce_distribution.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 0d458add31305..2f7c4e75038d3 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -1407,8 +1407,13 @@ pub fn ensure_distribution( // It was removed by `remove_dist_changing_operators` // and we need to add it back. if fetch.is_some() { - // It's safe to unwrap because `spm` is set only if `fetch` is set. - let plan = spm.unwrap().with_fetch(fetch.take()).unwrap(); + // We can make sure that `plan` has an ordering because + // `SortPreservingMergeExec` requires ordering to be constructed. + // If there is no ordering, `SortPreservingMergeExec::new` will panic + let ordering = plan.output_ordering().cloned().unwrap(); + let plan = Arc::new( + SortPreservingMergeExec::new(ordering, plan).with_fetch(fetch.take()), + ); optimized_distribution_ctx = DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); } From 6a3d4f8cc2d0f7250eafe4539d39a3b1ff0944e7 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 12:17:13 +0800 Subject: [PATCH 117/177] fix --- datafusion/physical-optimizer/src/enforce_distribution.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 2f7c4e75038d3..740235b68ee25 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -981,6 +981,7 @@ fn add_merge_on_top( /// ```text /// "DataSourceExec: file_groups={2 groups: \[\[x], \[y]]}, projection=\[a, b, c, d, e], output_ordering=\[a@0 ASC], file_type=parquet", /// ``` +#[allow(clippy::type_complexity)] fn remove_dist_changing_operators( mut distribution_context: DistributionContext, ) -> Result<( @@ -1222,7 +1223,7 @@ pub fn ensure_distribution( children, }, mut fetch, - spm, + _spm, ) = remove_dist_changing_operators(dist_context)?; if let Some(exec) = plan.as_any().downcast_ref::() { From 91e2904da4da233b3d56901abacdecd4287260f5 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Wed, 3 Sep 2025 15:29:23 +0800 Subject: [PATCH 118/177] fix auth check and port upstream fix: https://github.com/apache/datafusion/pull/17355 --- Cargo.lock | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9dc0a5b639888..9a6be2dc5b6af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4248,12 +4248,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -4453,12 +4452,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "4.2.1" @@ -6737,9 +6730,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "nu-ansi-term", "sharded-slab", From b571c3b5c5dcef7885785b636b0dd9aba4a4a898 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 11:50:54 +0800 Subject: [PATCH 119/177] Support csv truncte for datafusion --- datafusion/common/src/config.rs | 13 ++ .../core/src/datasource/file_format/csv.rs | 141 +++++++++++++++++- datafusion/datasource-csv/src/file_format.rs | 3 +- datafusion/proto-common/src/from_proto/mod.rs | 1 + .../proto-common/src/generated/prost.rs | 2 + datafusion/proto-common/src/to_proto/mod.rs | 1 + .../src/generated/datafusion_proto_common.rs | 2 + .../proto/src/logical_plan/file_formats.rs | 6 + 8 files changed, 167 insertions(+), 2 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 0d34815a248f7..2ae3aa27e3201 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -1946,6 +1946,10 @@ config_namespace! { // The input regex for Nulls when loading CSVs. pub null_regex: Option, default = None pub comment: Option, default = None + // Whether to allow truncated rows when parsing. + // By default this is set to false and will error if the CSV rows have different lengths. + // When set to true then it will allow records with less than the expected number of columns + pub truncated_rows: Option, default = None } } @@ -2038,6 +2042,15 @@ impl CsvOptions { self } + /// Whether to allow truncated rows when parsing. + /// By default this is set to false and will error if the CSV rows have different lengths. + /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. + /// If the record’s schema is not nullable, then it will still return an error. + pub fn with_truncated_rows(mut self, allow: bool) -> Self { + self.truncated_rows = Some(allow); + self + } + /// The delimiter character. pub fn delimiter(&self) -> u8 { self.delimiter diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index efec07abbca05..7eec55d70ffa0 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -47,7 +47,7 @@ mod tests { use datafusion_physical_plan::{collect, ExecutionPlan}; use arrow::array::{ - BooleanArray, Float64Array, Int32Array, RecordBatch, StringArray, + Array, BooleanArray, Float64Array, Int32Array, RecordBatch, StringArray, }; use arrow::compute::concat_batches; use arrow::csv::ReaderBuilder; @@ -55,6 +55,7 @@ mod tests { use async_trait::async_trait; use bytes::Bytes; use chrono::DateTime; + use datafusion_common::config::CsvOptions; use futures::stream::BoxStream; use futures::StreamExt; use insta::assert_snapshot; @@ -1174,4 +1175,142 @@ mod tests { .build_decoder(); DecoderDeserializer::new(CsvDecoder::new(decoder)) } + + fn csv_deserializer_with_truncated( + batch_size: usize, + schema: &Arc, + ) -> impl BatchDeserializer { + // using Arrow's ReaderBuilder and enabling truncated_rows + let decoder = ReaderBuilder::new(schema.clone()) + .with_batch_size(batch_size) + .with_truncated_rows(true) // <- enable runtime truncated_rows + .build_decoder(); + DecoderDeserializer::new(CsvDecoder::new(decoder)) + } + + #[tokio::test] + async fn infer_schema_with_truncated_rows_true() -> Result<()> { + let session_ctx = SessionContext::new(); + let state = session_ctx.state(); + + // CSV: header has 3 columns, but first data row has only 2 columns, second row has 3 + let csv_data = Bytes::from("a,b,c\n1,2\n3,4,5\n"); + let variable_object_store = Arc::new(VariableStream::new(csv_data, 1)); + let object_meta = ObjectMeta { + location: Path::parse("/")?, + last_modified: DateTime::default(), + size: u64::MAX, + e_tag: None, + version: None, + }; + + // Construct CsvFormat and enable truncated_rows via CsvOptions + let csv_options = CsvOptions::default().with_truncated_rows(true); + let csv_format = CsvFormat::default() + .with_has_header(true) + .with_options(csv_options) + .with_schema_infer_max_rec(10); + + let inferred_schema = csv_format + .infer_schema( + &state, + &(variable_object_store.clone() as Arc), + &[object_meta], + ) + .await?; + + // header has 3 columns; inferred schema should also have 3 + assert_eq!(inferred_schema.fields().len(), 3); + + // inferred columns should be nullable + for f in inferred_schema.fields() { + assert!(f.is_nullable()); + } + + Ok(()) + } + #[test] + fn test_decoder_truncated_rows_runtime() -> Result<()> { + // Synchronous test: Decoder API used here is synchronous + let schema = csv_schema(); // helper already defined in file + + // Construct a decoder that enables truncated_rows at runtime + let mut deserializer = csv_deserializer_with_truncated(10, &schema); + + // Provide two rows: first row complete, second row missing last column + let input = Bytes::from("0,0.0,true,0-string\n1,1.0,true\n"); + deserializer.digest(input); + + // Finish and collect output + deserializer.finish(); + + let output = deserializer.next()?; + match output { + DeserializerOutput::RecordBatch(batch) => { + // ensure at least two rows present + assert!(batch.num_rows() >= 2); + // column 4 (index 3) should be a StringArray where second row is NULL + let col4 = batch + .column(3) + .as_any() + .downcast_ref::() + .expect("column 4 should be StringArray"); + + // first row present, second row should be null + assert!(!col4.is_null(0)); + assert!(col4.is_null(1)); + } + other => panic!("expected RecordBatch but got {:?}", other), + } + Ok(()) + } + + #[tokio::test] + async fn infer_schema_truncated_rows_false_error() -> Result<()> { + let session_ctx = SessionContext::new(); + let state = session_ctx.state(); + + // CSV: header has 4 cols, first data row has 3 cols -> truncated at end + let csv_data = Bytes::from("id,a,b,c\n1,foo,bar\n2,foo,bar,baz\n"); + let variable_object_store = Arc::new(VariableStream::new(csv_data, 1)); + let object_meta = ObjectMeta { + location: Path::parse("/")?, + last_modified: DateTime::default(), + size: u64::MAX, + e_tag: None, + version: None, + }; + + // CsvFormat without enabling truncated_rows (default behavior = false) + let csv_format = CsvFormat::default() + .with_has_header(true) + .with_schema_infer_max_rec(10); + + let res = csv_format + .infer_schema( + &state, + &(variable_object_store.clone() as Arc), + &[object_meta], + ) + .await; + + // Expect an error due to unequal lengths / incorrect number of fields + assert!( + res.is_err(), + "expected infer_schema to error on truncated rows when disabled" + ); + + // Optional: check message contains indicative text (two known possibilities) + if let Err(err) = res { + let msg = format!("{err}"); + assert!( + msg.contains("Encountered unequal lengths") + || msg.contains("incorrect number of fields"), + "unexpected error message: {}", + msg + ); + } + + Ok(()) + } } diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index c9cd09bf676b7..fe86b48dc13bc 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -505,7 +505,8 @@ impl CsvFormat { .unwrap_or_else(|| state.config_options().catalog.has_header), ) .with_delimiter(self.options.delimiter) - .with_quote(self.options.quote); + .with_quote(self.options.quote) + .with_truncated_rows(self.options.truncated_rows.unwrap_or(false)); if let Some(null_regex) = &self.options.null_regex { let regex = Regex::new(null_regex.as_str()) diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index bd969db316872..39cf65070554e 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -900,6 +900,7 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions { null_regex: (!proto_opts.null_regex.is_empty()) .then(|| proto_opts.null_regex.clone()), comment: proto_opts.comment.first().copied(), + truncated_rows: proto_opts.truncated_rows.first().map(|h| *h != 0), }) } } diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs index a55714f190c57..16c045d9a8cfe 100644 --- a/datafusion/proto-common/src/generated/prost.rs +++ b/datafusion/proto-common/src/generated/prost.rs @@ -604,6 +604,8 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "18")] + pub truncated_rows : ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs index b6cbe5759cfcc..1a973982270b7 100644 --- a/datafusion/proto-common/src/to_proto/mod.rs +++ b/datafusion/proto-common/src/to_proto/mod.rs @@ -934,6 +934,7 @@ impl TryFrom<&CsvOptions> for protobuf::CsvOptions { null_value: opts.null_value.clone().unwrap_or_default(), null_regex: opts.null_regex.clone().unwrap_or_default(), comment: opts.comment.map_or_else(Vec::new, |h| vec![h]), + truncated_rows: opts.truncated_rows.map_or_else(Vec::new, |h| vec![h as u8]), }) } } diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rs index a55714f190c57..16c045d9a8cfe 100644 --- a/datafusion/proto/src/generated/datafusion_proto_common.rs +++ b/datafusion/proto/src/generated/datafusion_proto_common.rs @@ -604,6 +604,8 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "18")] + pub truncated_rows : ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rs index d3f6511ec98fa..55ef8bd79f382 100644 --- a/datafusion/proto/src/logical_plan/file_formats.rs +++ b/datafusion/proto/src/logical_plan/file_formats.rs @@ -72,6 +72,7 @@ impl CsvOptionsProto { newlines_in_values: options .newlines_in_values .map_or(vec![], |v| vec![v as u8]), + truncated_rows: options.truncated_rows.map_or(vec![], |v| vec![v as u8]), } } else { CsvOptionsProto::default() @@ -157,6 +158,11 @@ impl From<&CsvOptionsProto> for CsvOptions { } else { Some(proto.newlines_in_values[0] != 0) }, + truncated_rows: if proto.truncated_rows.is_empty() { + None + } else { + Some(proto.truncated_rows[0] != 0) + }, } } } From 1a2f8dc98d9791ee24476f3fef83d9ce5f68aa82 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 13:31:45 +0800 Subject: [PATCH 120/177] Addressed in latest PR --- .../physical-optimizer/src/enforce_distribution.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 740235b68ee25..42c08ce8f437e 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -951,7 +951,7 @@ fn add_merge_on_top( let new_plan = if let Some(req) = input.plan.output_ordering() { Arc::new( SortPreservingMergeExec::new(req.clone(), Arc::clone(&input.plan)) - .with_fetch(*fetch), + .with_fetch(fetch.take()), ) as _ } else { // If there is no input order, we can simply coalesce partitions: @@ -1223,7 +1223,7 @@ pub fn ensure_distribution( children, }, mut fetch, - _spm, + spm, ) = remove_dist_changing_operators(dist_context)?; if let Some(exec) = plan.as_any().downcast_ref::() { @@ -1408,13 +1408,8 @@ pub fn ensure_distribution( // It was removed by `remove_dist_changing_operators` // and we need to add it back. if fetch.is_some() { - // We can make sure that `plan` has an ordering because - // `SortPreservingMergeExec` requires ordering to be constructed. - // If there is no ordering, `SortPreservingMergeExec::new` will panic - let ordering = plan.output_ordering().cloned().unwrap(); - let plan = Arc::new( - SortPreservingMergeExec::new(ordering, plan).with_fetch(fetch.take()), - ); + // It's safe to unwrap because `spm` is set only if `fetch` is set. + let plan = spm.unwrap().with_fetch(fetch.take()).unwrap(); optimized_distribution_ctx = DistributionContext::new(plan, data, vec![optimized_distribution_ctx]); } From 63c54eaf17018951bca836d6c5a25db3bddcfe53 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 20:35:23 +0800 Subject: [PATCH 121/177] add generated field to proto --- datafusion/proto-common/src/generated/prost.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs index 16c045d9a8cfe..a55714f190c57 100644 --- a/datafusion/proto-common/src/generated/prost.rs +++ b/datafusion/proto-common/src/generated/prost.rs @@ -604,8 +604,6 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, - #[prost(bytes = "vec", tag = "18")] - pub truncated_rows : ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] From b7f98284fedd0ff5d0fba86ad54b61b81ff04dd1 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 20:42:47 +0800 Subject: [PATCH 122/177] generate proto --- datafusion/proto-common/src/generated/prost.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs index a55714f190c57..323f73923272b 100644 --- a/datafusion/proto-common/src/generated/prost.rs +++ b/datafusion/proto-common/src/generated/prost.rs @@ -604,6 +604,8 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + #[prost(bytes = "vec", tag = "18")] + pub truncated_rows: ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] From d0b757beffd1355df98ee1a532ff6377ad71b167 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 20:51:37 +0800 Subject: [PATCH 123/177] add proto message and generated. --- .../proto/datafusion_common.proto | 1 + .../proto-common/src/generated/pbjson.rs | 22 +++++++++++++++++++ .../proto-common/src/generated/prost.rs | 1 + 3 files changed, 24 insertions(+) diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.proto index 35f41155fa050..c8d49ce882212 100644 --- a/datafusion/proto-common/proto/datafusion_common.proto +++ b/datafusion/proto-common/proto/datafusion_common.proto @@ -424,6 +424,7 @@ message CsvOptions { bytes double_quote = 15; // Indicates if quotes are doubled bytes newlines_in_values = 16; // Indicates if newlines are supported in values bytes terminator = 17; // Optional terminator character as a byte + bytes truncated_rows = 18; // Indicates if truncated rows are allowed } // Options controlling CSV format diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rs index 1ac35742c73a4..14233b634fd2f 100644 --- a/datafusion/proto-common/src/generated/pbjson.rs +++ b/datafusion/proto-common/src/generated/pbjson.rs @@ -1566,6 +1566,9 @@ impl serde::Serialize for CsvOptions { if !self.terminator.is_empty() { len += 1; } + if !self.truncated_rows.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion_common.CsvOptions", len)?; if !self.has_header.is_empty() { #[allow(clippy::needless_borrow)] @@ -1638,6 +1641,11 @@ impl serde::Serialize for CsvOptions { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("terminator", pbjson::private::base64::encode(&self.terminator).as_str())?; } + if !self.truncated_rows.is_empty() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("truncatedRows", pbjson::private::base64::encode(&self.truncated_rows).as_str())?; + } struct_ser.end() } } @@ -1676,6 +1684,8 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { "newlines_in_values", "newlinesInValues", "terminator", + "truncated_rows", + "truncatedRows", ]; #[allow(clippy::enum_variant_names)] @@ -1697,6 +1707,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { DoubleQuote, NewlinesInValues, Terminator, + TruncatedRows, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -1735,6 +1746,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { "doubleQuote" | "double_quote" => Ok(GeneratedField::DoubleQuote), "newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues), "terminator" => Ok(GeneratedField::Terminator), + "truncatedRows" | "truncated_rows" => Ok(GeneratedField::TruncatedRows), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -1771,6 +1783,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { let mut double_quote__ = None; let mut newlines_in_values__ = None; let mut terminator__ = None; + let mut truncated_rows__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::HasHeader => { @@ -1893,6 +1906,14 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) ; } + GeneratedField::TruncatedRows => { + if truncated_rows__.is_some() { + return Err(serde::de::Error::duplicate_field("truncatedRows")); + } + truncated_rows__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } } } Ok(CsvOptions { @@ -1913,6 +1934,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { double_quote: double_quote__.unwrap_or_default(), newlines_in_values: newlines_in_values__.unwrap_or_default(), terminator: terminator__.unwrap_or_default(), + truncated_rows: truncated_rows__.unwrap_or_default(), }) } } diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs index 323f73923272b..96dadba835fa2 100644 --- a/datafusion/proto-common/src/generated/prost.rs +++ b/datafusion/proto-common/src/generated/prost.rs @@ -604,6 +604,7 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + /// Indicates if truncated rows are allowed #[prost(bytes = "vec", tag = "18")] pub truncated_rows: ::prost::alloc::vec::Vec, } From 5b9219d77537f2ae45724557899d53741aa6cd14 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 20:54:59 +0800 Subject: [PATCH 124/177] fix --- datafusion/proto/src/generated/datafusion_proto_common.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rs index 16c045d9a8cfe..96dadba835fa2 100644 --- a/datafusion/proto/src/generated/datafusion_proto_common.rs +++ b/datafusion/proto/src/generated/datafusion_proto_common.rs @@ -604,8 +604,9 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + /// Indicates if truncated rows are allowed #[prost(bytes = "vec", tag = "18")] - pub truncated_rows : ::prost::alloc::vec::Vec, + pub truncated_rows: ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] From 5aa43e51c77374f2cbedc20f2892981a130dde82 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 4 Sep 2025 21:11:35 +0800 Subject: [PATCH 125/177] fix clippy --- datafusion/core/src/datasource/file_format/csv.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 7eec55d70ffa0..444d4e6e0cd82 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -1260,7 +1260,7 @@ mod tests { assert!(!col4.is_null(0)); assert!(col4.is_null(1)); } - other => panic!("expected RecordBatch but got {:?}", other), + other => panic!("expected RecordBatch but got {other:?}"), } Ok(()) } @@ -1306,8 +1306,7 @@ mod tests { assert!( msg.contains("Encountered unequal lengths") || msg.contains("incorrect number of fields"), - "unexpected error message: {}", - msg + "unexpected error message: {msg}", ); } From cae4095e02f7c0acf2e2ba1f326767ca135123e8 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Fri, 5 Sep 2025 11:13:44 +0800 Subject: [PATCH 126/177] X-1035 Part-2: support csv scan to read truncted rows --- .../core/src/datasource/file_format/csv.rs | 40 +++++++++++++++++++ .../src/datasource/file_format/options.rs | 19 ++++++++- datafusion/datasource-csv/src/file_format.rs | 9 ++++- datafusion/datasource-csv/src/source.rs | 17 +++++++- datafusion/proto/proto/datafusion.proto | 1 + datafusion/proto/src/generated/pbjson.rs | 18 +++++++++ datafusion/proto/src/generated/prost.rs | 2 + datafusion/proto/src/physical_plan/mod.rs | 1 + 8 files changed, 104 insertions(+), 3 deletions(-) diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 444d4e6e0cd82..e119cea5f4f6b 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -1312,4 +1312,44 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_read_csv_truncated_rows_via_tempfile() -> Result<()> { + use std::io::Write; + + // create a SessionContext + let ctx = SessionContext::new(); + + // Create a temp file with a .csv suffix so the reader accepts it + let mut tmp = tempfile::Builder::new().suffix(".csv").tempfile()?; // ensures path ends with .csv + // CSV has header "a,b,c". First data row is truncated (only "1,2"), second row is complete. + write!(tmp, "a,b,c\n1,2\n3,4,5\n")?; + let path = tmp.path().to_str().unwrap().to_string(); + + // Build CsvReadOptions: header present, enable truncated_rows. + // (Use the exact builder method your crate exposes: `truncated_rows(true)` here, + // if the method name differs in your codebase use the appropriate one.) + let options = CsvReadOptions::default().truncated_rows(true); + + println!("options: {}, path: {path}", options.truncated_rows); + + // Call the API under test + let df = ctx.read_csv(&path, options).await?; + + // Collect the results and combine batches so we can inspect columns + let batches = df.collect().await?; + let combined = concat_batches(&batches[0].schema(), &batches)?; + + // Column 'c' is the 3rd column (index 2). The first data row was truncated -> should be NULL. + let col_c = combined.column(2); + assert!( + col_c.is_null(0), + "expected first row column 'c' to be NULL due to truncated row" + ); + + // Also ensure we read at least one row + assert!(combined.num_rows() >= 2); + + Ok(()) + } } diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 9aaf1cf598113..a197366130db3 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -91,6 +91,11 @@ pub struct CsvReadOptions<'a> { pub file_sort_order: Vec>, /// Optional regex to match null values pub null_regex: Option, + /// Whether to allow truncated rows when parsing. + /// By default this is set to false and will error if the CSV rows have different lengths. + /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. + /// If the record’s schema is not nullable, then it will still return an error. + pub truncated_rows: bool, } impl Default for CsvReadOptions<'_> { @@ -117,6 +122,7 @@ impl<'a> CsvReadOptions<'a> { file_sort_order: vec![], comment: None, null_regex: None, + truncated_rows: false, } } @@ -223,6 +229,16 @@ impl<'a> CsvReadOptions<'a> { self.null_regex = null_regex; self } + + /// Configure whether to allow truncated rows when parsing. + /// By default this is set to false and will error if the CSV rows have different lengths + /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. + /// If the record’s schema is not nullable, then it will still return an error. + /// See https://docs.rs/arrow/latest/arrow/csv/struct.ReaderBuilder.html#method.with_allow_truncated_rows + pub fn truncated_rows(mut self, truncated_rows: bool) -> Self { + self.truncated_rows = truncated_rows; + self + } } /// Options that control the reading of Parquet files. @@ -546,7 +562,8 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_newlines_in_values(self.newlines_in_values) .with_schema_infer_max_rec(self.schema_infer_max_records) .with_file_compression_type(self.file_compression_type.to_owned()) - .with_null_regex(self.null_regex.clone()); + .with_null_regex(self.null_regex.clone()) + .with_truncated_rows(self.truncated_rows); ListingOptions::new(Arc::new(file_format)) .with_file_extension(self.file_extension) diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index fe86b48dc13bc..f4ae99bb964bf 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -222,6 +222,11 @@ impl CsvFormat { self } + pub fn with_truncated_rows(mut self, truncated_rows: bool) -> Self { + self.options.truncated_rows = Some(truncated_rows); + self + } + /// Set the regex to use for null values in the CSV reader. /// - default to treat empty values as null. pub fn with_null_regex(mut self, null_regex: Option) -> Self { @@ -422,11 +427,13 @@ impl FileFormat for CsvFormat { .with_file_compression_type(self.options.compression.into()) .with_newlines_in_values(newlines_in_values); + let truncated_rows = self.options.truncated_rows.unwrap_or(false); let source = Arc::new( CsvSource::new(has_header, self.options.delimiter, self.options.quote) .with_escape(self.options.escape) .with_terminator(self.options.terminator) - .with_comment(self.options.comment), + .with_comment(self.options.comment) + .with_truncate_rows(truncated_rows), ); let config = conf_builder.with_source(source).build(); diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 3af1f2b345ba8..8b494689fb7e1 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -93,6 +93,7 @@ pub struct CsvSource { metrics: ExecutionPlanMetricsSet, projected_statistics: Option, schema_adapter_factory: Option>, + truncate_rows: bool, } impl CsvSource { @@ -110,6 +111,11 @@ impl CsvSource { pub fn has_header(&self) -> bool { self.has_header } + + // true if rows length support truncate + pub fn truncate_rows(&self) -> bool { + self.truncate_rows + } /// A column delimiter pub fn delimiter(&self) -> u8 { self.delimiter @@ -155,6 +161,13 @@ impl CsvSource { conf.comment = comment; conf } + + /// Whether to support truncate rows when read csv file + pub fn with_truncate_rows(&self, truncate_rows: bool) -> Self { + let mut conf = self.clone(); + conf.truncate_rows = truncate_rows; + conf + } } impl CsvSource { @@ -174,7 +187,8 @@ impl CsvSource { .expect("Batch size must be set before initializing builder"), ) .with_header(self.has_header) - .with_quote(self.quote); + .with_quote(self.quote) + .with_truncated_rows(self.truncate_rows); if let Some(terminator) = self.terminator { builder = builder.with_terminator(terminator); } @@ -335,6 +349,7 @@ impl FileOpener for CsvOpener { let config = CsvSource { has_header: csv_has_header, + truncate_rows: self.config.truncate_rows, ..(*self.config).clone() }; diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index 4c8b6c588d949..591e54ab49fd3 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -1023,6 +1023,7 @@ message CsvScanExecNode { string comment = 6; } bool newlines_in_values = 7; + bool truncate_rows = 8; } message JsonScanExecNode { diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 932422944508d..5cf096244ef60 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -3680,6 +3680,9 @@ impl serde::Serialize for CsvScanExecNode { if self.newlines_in_values { len += 1; } + if self.truncate_rows { + len += 1; + } if self.optional_escape.is_some() { len += 1; } @@ -3702,6 +3705,9 @@ impl serde::Serialize for CsvScanExecNode { if self.newlines_in_values { struct_ser.serialize_field("newlinesInValues", &self.newlines_in_values)?; } + if self.truncate_rows { + struct_ser.serialize_field("truncateRows", &self.truncate_rows)?; + } if let Some(v) = self.optional_escape.as_ref() { match v { csv_scan_exec_node::OptionalEscape::Escape(v) => { @@ -3734,6 +3740,8 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { "quote", "newlines_in_values", "newlinesInValues", + "truncate_rows", + "truncateRows", "escape", "comment", ]; @@ -3745,6 +3753,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { Delimiter, Quote, NewlinesInValues, + TruncateRows, Escape, Comment, } @@ -3773,6 +3782,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { "delimiter" => Ok(GeneratedField::Delimiter), "quote" => Ok(GeneratedField::Quote), "newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues), + "truncateRows" | "truncate_rows" => Ok(GeneratedField::TruncateRows), "escape" => Ok(GeneratedField::Escape), "comment" => Ok(GeneratedField::Comment), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), @@ -3799,6 +3809,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { let mut delimiter__ = None; let mut quote__ = None; let mut newlines_in_values__ = None; + let mut truncate_rows__ = None; let mut optional_escape__ = None; let mut optional_comment__ = None; while let Some(k) = map_.next_key()? { @@ -3833,6 +3844,12 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { } newlines_in_values__ = Some(map_.next_value()?); } + GeneratedField::TruncateRows => { + if truncate_rows__.is_some() { + return Err(serde::de::Error::duplicate_field("truncateRows")); + } + truncate_rows__ = Some(map_.next_value()?); + } GeneratedField::Escape => { if optional_escape__.is_some() { return Err(serde::de::Error::duplicate_field("escape")); @@ -3853,6 +3870,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { delimiter: delimiter__.unwrap_or_default(), quote: quote__.unwrap_or_default(), newlines_in_values: newlines_in_values__.unwrap_or_default(), + truncate_rows: truncate_rows__.unwrap_or_default(), optional_escape: optional_escape__, optional_comment: optional_comment__, }) diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index c2f4e93cef6ae..9c3ac34c437da 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -1543,6 +1543,8 @@ pub struct CsvScanExecNode { pub quote: ::prost::alloc::string::String, #[prost(bool, tag = "7")] pub newlines_in_values: bool, + #[prost(bool, tag = "8")] + pub truncate_rows: bool, #[prost(oneof = "csv_scan_exec_node::OptionalEscape", tags = "5")] pub optional_escape: ::core::option::Option, #[prost(oneof = "csv_scan_exec_node::OptionalComment", tags = "6")] diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 7a85a2a8efbd0..7a2c296bae7b2 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -2286,6 +2286,7 @@ impl protobuf::PhysicalPlanNode { None }, newlines_in_values: maybe_csv.newlines_in_values(), + truncate_rows: csv_config.truncate_rows(), }, )), })); From 86c8754b23fc0bdd982863961a0342c8335f3867 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Fri, 5 Sep 2025 11:29:47 +0800 Subject: [PATCH 127/177] fix CI --- datafusion/core/src/datasource/file_format/options.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index a197366130db3..6b30b5bcc954a 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -234,7 +234,6 @@ impl<'a> CsvReadOptions<'a> { /// By default this is set to false and will error if the CSV rows have different lengths /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. /// If the record’s schema is not nullable, then it will still return an error. - /// See https://docs.rs/arrow/latest/arrow/csv/struct.ReaderBuilder.html#method.with_allow_truncated_rows pub fn truncated_rows(mut self, truncated_rows: bool) -> Self { self.truncated_rows = truncated_rows; self From 253e49ce8ae192f2abbe8012bb5828834ecf2815 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Fri, 5 Sep 2025 11:35:55 +0800 Subject: [PATCH 128/177] add csvfmt with --- datafusion/datasource-csv/src/file_format.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index f4ae99bb964bf..e6ce981f7ada4 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -296,6 +296,13 @@ impl CsvFormat { self } + /// Set whether rows should be truncated to the column width + /// - defaults to false + pub fn with_truncate_rows(mut self, truncate_rows: bool) -> Self { + self.options.truncated_rows = Some(truncate_rows); + self + } + /// The delimiter character. pub fn delimiter(&self) -> u8 { self.options.delimiter From 9e7141f36ddbb0ad9c79ed3c95e332f495c1b078 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Mon, 8 Sep 2025 09:04:17 -0700 Subject: [PATCH 129/177] fix: Implement AggregateUDFImpl::reverse_expr for StringAgg (#17165) (#17473) * fix: Implement AggregateUDFImpl::reverse_expr for StringAgg * Add a test with two invocations of aggregateion --------- Co-authored-by: Nuno Faria --- .../functions-aggregate/src/string_agg.rs | 4 ++ .../sqllogictest/test_files/aggregate.slt | 53 ++++++++++++++++++- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs index a3a040da3ff7b..3986984b26304 100644 --- a/datafusion/functions-aggregate/src/string_agg.rs +++ b/datafusion/functions-aggregate/src/string_agg.rs @@ -178,6 +178,10 @@ impl AggregateUDFImpl for StringAgg { ))) } + fn reverse_expr(&self) -> datafusion_expr::ReversedUDAF { + datafusion_expr::ReversedUDAF::Reversed(string_agg_udaf()) + } + fn documentation(&self) -> Option<&Documentation> { self.doc() } diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index 35b2a6c03b399..caf8d637ec45e 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -6203,6 +6203,58 @@ from t; ---- a,c,d,b +# Test explain / reverse_expr for string_agg +query TT +explain select string_agg(k, ',' order by v) from t; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v ASC NULLS LAST]]] +02)--TableScan: t projection=[k, v] +physical_plan +01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v ASC NULLS LAST]] +02)--SortExec: expr=[v@1 ASC NULLS LAST], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + +query T +select string_agg(k, ',' order by v) from t; +---- +c,a,b,d + +query TT +explain select string_agg(k, ',' order by v desc) from t; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]]] +02)--TableScan: t projection=[k, v] +physical_plan +01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]] +02)--SortExec: expr=[v@1 DESC], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + +query T +select string_agg(k, ',' order by v desc) from t; +---- +d,b,a,c + +# Call string_agg with both ASC and DESC orderings, and expect only one sort +# (because the aggregate can handle reversed inputs) +query TT +explain select string_agg(k, ',' order by v asc), string_agg(k, ',' order by v desc) from t; +---- +logical_plan +01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v ASC NULLS LAST], string_agg(t.k, Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]]] +02)--TableScan: t projection=[k, v] +physical_plan +01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v ASC NULLS LAST], string_agg(t.k,Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]] +02)--SortExec: expr=[v@1 ASC NULLS LAST], preserve_partitioning=[false] +03)----DataSourceExec: partitions=1, partition_sizes=[1] + +query TT +select string_agg(k, ',' order by v asc), string_agg(k, ',' order by v desc) from t; +---- +c,a,b,d d,b,a,c + + statement ok drop table t; @@ -7444,4 +7496,3 @@ NULL NULL statement ok drop table distinct_avg; - From 10343c18292fc3a9ca62b59c8bb530169fe79b82 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 13 Sep 2025 01:17:34 -0700 Subject: [PATCH 130/177] Revert #17295 (Support from-first SQL syntax) (#17520) (#17544) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add failing test * Fix regression in SELECT FROM syntax with WHERE clause When using 'SELECT FROM table WHERE condition', the query should create an empty projection (no columns) while still filtering rows. This was broken by PR #17295 which added FROM-first syntax support. The issue was that both 'FROM table' and 'SELECT FROM table' resulted in empty projection lists, making them indistinguishable. The fix checks for the presence of a WHERE clause to differentiate: - 'FROM table' (no WHERE) -> add wildcard projection (all columns) - 'SELECT FROM table WHERE ...' -> keep empty projection Also updates the test expectation to correctly show the empty Projection node in the query plan. Fixes #17513 * Revert * Fix regression: SELECT FROM syntax should return empty projection Removes automatic wildcard projection for empty projections, fixing the regression where `SELECT FROM table` incorrectly returned all columns instead of empty projection. Note: This temporarily breaks FROM-first syntax. A proper fix would require distinguishing between `FROM table` and `SELECT FROM table` at the parser level. Fixes #17513 🤖 Generated with [Claude Code](https://claude.ai/code) * add a better regression test * remove comment * fmt * Update datafusion/sqllogictest/test_files/projection.slt * Update datafusion/core/tests/sql/select.rs * revert docs * fmt --------- Co-authored-by: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Co-authored-by: Claude Co-authored-by: Oleks V --- datafusion/core/tests/sql/select.rs | 25 +++++++++ datafusion/sql/src/select.rs | 8 --- .../sqllogictest/test_files/from-first.slt | 55 ------------------- .../sqllogictest/test_files/projection.slt | 28 ++++++++++ docs/source/user-guide/sql/select.md | 14 ----- 5 files changed, 53 insertions(+), 77 deletions(-) delete mode 100644 datafusion/sqllogictest/test_files/from-first.slt diff --git a/datafusion/core/tests/sql/select.rs b/datafusion/core/tests/sql/select.rs index 0e1210ebb8424..1978c189c4f8d 100644 --- a/datafusion/core/tests/sql/select.rs +++ b/datafusion/core/tests/sql/select.rs @@ -344,3 +344,28 @@ async fn test_version_function() { assert_eq!(version.value(0), expected_version); } + +/// Regression test for https://github.com/apache/datafusion/issues/17513 +/// See https://github.com/apache/datafusion/pull/17520 +#[tokio::test] +async fn test_select_no_projection() -> Result<()> { + let tmp_dir = TempDir::new()?; + // `create_ctx_with_partition` creates 10 rows per partition and we chose 1 partition + let ctx = create_ctx_with_partition(&tmp_dir, 1).await?; + + let results = ctx.sql("SELECT FROM test").await?.collect().await?; + // We should get all of the rows, just without any columns + let total_rows: usize = results.iter().map(|b| b.num_rows()).sum(); + assert_eq!(total_rows, 10); + // Check that none of the batches have any columns + for batch in &results { + assert_eq!(batch.num_columns(), 0); + } + // Sanity check the output, should be just empty columns + assert_snapshot!(batches_to_sort_string(&results), @r" + ++ + ++ + ++ + "); + Ok(()) +} diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 1a90e5e09b778..54c94003b716d 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -665,14 +665,6 @@ impl SqlToRel<'_, S> { let mut prepared_select_exprs = vec![]; let mut error_builder = DataFusionErrorBuilder::new(); - // Handle the case where no projection is specified but we have a valid FROM clause - // In this case, implicitly add a wildcard projection (SELECT *) - let projection = if projection.is_empty() && !empty_from { - vec![SelectItem::Wildcard(WildcardAdditionalOptions::default())] - } else { - projection - }; - for expr in projection { match self.sql_select_to_rex(expr, plan, empty_from, planner_context) { Ok(expr) => prepared_select_exprs.push(expr), diff --git a/datafusion/sqllogictest/test_files/from-first.slt b/datafusion/sqllogictest/test_files/from-first.slt deleted file mode 100644 index c4a305e85ea77..0000000000000 --- a/datafusion/sqllogictest/test_files/from-first.slt +++ /dev/null @@ -1,55 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -query I -FROM range(2) ----- -0 -1 - -query I -FROM range(2) -SELECT * ----- -0 -1 - -query I -FROM (SELECT * FROM range(2)) ----- -0 -1 - -query I -FROM (FROM range(2)) ----- -0 -1 - -query I -FROM range(2) -SELECT 1 ----- -1 -1 - -query I -FROM range(2) as r -SELECT r.value ----- -0 -1 diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.slt index 0f0cbac1fa323..97ebe2340dc27 100644 --- a/datafusion/sqllogictest/test_files/projection.slt +++ b/datafusion/sqllogictest/test_files/projection.slt @@ -252,3 +252,31 @@ physical_plan statement ok drop table t; + +# Regression test for +# https://github.com/apache/datafusion/issues/17513 + +query I +COPY (select 1 as a, 2 as b) +TO 'test_files/scratch/projection/17513.parquet' +STORED AS PARQUET; +---- +1 + +statement ok +create external table t1 stored as parquet location 'test_files/scratch/projection/17513.parquet'; + +query TT +explain format indent +select from t1 where t1.a > 1; +---- +logical_plan +01)Projection: +02)--Filter: t1.a > Int64(1) +03)----TableScan: t1 projection=[a], partial_filters=[t1.a > Int64(1)] +physical_plan +01)ProjectionExec: expr=[] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----FilterExec: a@0 > 1 +04)------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1 +05)--------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/projection/17513.parquet]]}, projection=[a], file_type=parquet, predicate=a@0 > 1, pruning_predicate=a_null_count@1 != row_count@2 AND a_max@0 > 1, required_guarantees=[] diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.md index eb8bca7a75ef0..39163cf492a4a 100644 --- a/docs/source/user-guide/sql/select.md +++ b/docs/source/user-guide/sql/select.md @@ -75,20 +75,6 @@ Example: SELECT t.a FROM table AS t ``` -The `FROM` clause can also come before the `SELECT` clause. -Example: - -```sql -FROM table AS t -SELECT t.a -``` - -If the `SELECT` clause is omitted, the `FROM` clause will return all columns from the table. - -```sql -FROM table -``` - ## WHERE clause Example: From a0fc6422e84baeebb6d9d335f1e46dc8713378b8 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Tue, 9 Sep 2025 10:20:36 +0800 Subject: [PATCH 131/177] Support csv truncated rows in datafusion (#17465) (cherry picked from commit 7b16d6be1fbbec5e9001baa22046b733e6db5514) --- datafusion/common/src/config.rs | 13 ++ .../core/src/datasource/file_format/csv.rs | 179 +++++++++++++++++- .../src/datasource/file_format/options.rs | 18 +- datafusion/datasource-csv/src/file_format.rs | 19 +- datafusion/datasource-csv/src/source.rs | 17 +- .../proto/datafusion_common.proto | 1 + datafusion/proto-common/src/from_proto/mod.rs | 1 + .../proto-common/src/generated/pbjson.rs | 22 +++ .../proto-common/src/generated/prost.rs | 3 + datafusion/proto-common/src/to_proto/mod.rs | 1 + datafusion/proto/proto/datafusion.proto | 1 + .../src/generated/datafusion_proto_common.rs | 3 + datafusion/proto/src/generated/pbjson.rs | 18 ++ datafusion/proto/src/generated/prost.rs | 2 + .../proto/src/logical_plan/file_formats.rs | 6 + datafusion/proto/src/physical_plan/mod.rs | 1 + 16 files changed, 300 insertions(+), 5 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index cdd8e72a06cc9..326536e71b559 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -2521,6 +2521,10 @@ config_namespace! { // The input regex for Nulls when loading CSVs. pub null_regex: Option, default = None pub comment: Option, default = None + // Whether to allow truncated rows when parsing. + // By default this is set to false and will error if the CSV rows have different lengths. + // When set to true then it will allow records with less than the expected number of columns + pub truncated_rows: Option, default = None } } @@ -2613,6 +2617,15 @@ impl CsvOptions { self } + /// Whether to allow truncated rows when parsing. + /// By default this is set to false and will error if the CSV rows have different lengths. + /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. + /// If the record’s schema is not nullable, then it will still return an error. + pub fn with_truncated_rows(mut self, allow: bool) -> Self { + self.truncated_rows = Some(allow); + self + } + /// The delimiter character. pub fn delimiter(&self) -> u8 { self.delimiter diff --git a/datafusion/core/src/datasource/file_format/csv.rs b/datafusion/core/src/datasource/file_format/csv.rs index 23ba9e6ec8736..6c4897f711c5c 100644 --- a/datafusion/core/src/datasource/file_format/csv.rs +++ b/datafusion/core/src/datasource/file_format/csv.rs @@ -48,7 +48,7 @@ mod tests { use datafusion_physical_plan::{collect, ExecutionPlan}; use arrow::array::{ - BooleanArray, Float64Array, Int32Array, RecordBatch, StringArray, + Array, BooleanArray, Float64Array, Int32Array, RecordBatch, StringArray, }; use arrow::compute::concat_batches; use arrow::csv::ReaderBuilder; @@ -1256,4 +1256,181 @@ mod tests { .build_decoder(); DecoderDeserializer::new(CsvDecoder::new(decoder)) } + + fn csv_deserializer_with_truncated( + batch_size: usize, + schema: &Arc, + ) -> impl BatchDeserializer { + // using Arrow's ReaderBuilder and enabling truncated_rows + let decoder = ReaderBuilder::new(schema.clone()) + .with_batch_size(batch_size) + .with_truncated_rows(true) // <- enable runtime truncated_rows + .build_decoder(); + DecoderDeserializer::new(CsvDecoder::new(decoder)) + } + + #[tokio::test] + async fn infer_schema_with_truncated_rows_true() -> Result<()> { + let session_ctx = SessionContext::new(); + let state = session_ctx.state(); + + // CSV: header has 3 columns, but first data row has only 2 columns, second row has 3 + let csv_data = Bytes::from("a,b,c\n1,2\n3,4,5\n"); + let variable_object_store = Arc::new(VariableStream::new(csv_data, 1)); + let object_meta = ObjectMeta { + location: Path::parse("/")?, + last_modified: DateTime::default(), + size: u64::MAX, + e_tag: None, + version: None, + }; + + // Construct CsvFormat and enable truncated_rows via CsvOptions + let csv_options = CsvOptions::default().with_truncated_rows(true); + let csv_format = CsvFormat::default() + .with_has_header(true) + .with_options(csv_options) + .with_schema_infer_max_rec(10); + + let inferred_schema = csv_format + .infer_schema( + &state, + &(variable_object_store.clone() as Arc), + &[object_meta], + ) + .await?; + + // header has 3 columns; inferred schema should also have 3 + assert_eq!(inferred_schema.fields().len(), 3); + + // inferred columns should be nullable + for f in inferred_schema.fields() { + assert!(f.is_nullable()); + } + + Ok(()) + } + #[test] + fn test_decoder_truncated_rows_runtime() -> Result<()> { + // Synchronous test: Decoder API used here is synchronous + let schema = csv_schema(); // helper already defined in file + + // Construct a decoder that enables truncated_rows at runtime + let mut deserializer = csv_deserializer_with_truncated(10, &schema); + + // Provide two rows: first row complete, second row missing last column + let input = Bytes::from("0,0.0,true,0-string\n1,1.0,true\n"); + deserializer.digest(input); + + // Finish and collect output + deserializer.finish(); + + let output = deserializer.next()?; + match output { + DeserializerOutput::RecordBatch(batch) => { + // ensure at least two rows present + assert!(batch.num_rows() >= 2); + // column 4 (index 3) should be a StringArray where second row is NULL + let col4 = batch + .column(3) + .as_any() + .downcast_ref::() + .expect("column 4 should be StringArray"); + + // first row present, second row should be null + assert!(!col4.is_null(0)); + assert!(col4.is_null(1)); + } + other => panic!("expected RecordBatch but got {other:?}"), + } + Ok(()) + } + + #[tokio::test] + async fn infer_schema_truncated_rows_false_error() -> Result<()> { + let session_ctx = SessionContext::new(); + let state = session_ctx.state(); + + // CSV: header has 4 cols, first data row has 3 cols -> truncated at end + let csv_data = Bytes::from("id,a,b,c\n1,foo,bar\n2,foo,bar,baz\n"); + let variable_object_store = Arc::new(VariableStream::new(csv_data, 1)); + let object_meta = ObjectMeta { + location: Path::parse("/")?, + last_modified: DateTime::default(), + size: u64::MAX, + e_tag: None, + version: None, + }; + + // CsvFormat without enabling truncated_rows (default behavior = false) + let csv_format = CsvFormat::default() + .with_has_header(true) + .with_schema_infer_max_rec(10); + + let res = csv_format + .infer_schema( + &state, + &(variable_object_store.clone() as Arc), + &[object_meta], + ) + .await; + + // Expect an error due to unequal lengths / incorrect number of fields + assert!( + res.is_err(), + "expected infer_schema to error on truncated rows when disabled" + ); + + // Optional: check message contains indicative text (two known possibilities) + if let Err(err) = res { + let msg = format!("{err}"); + assert!( + msg.contains("Encountered unequal lengths") + || msg.contains("incorrect number of fields"), + "unexpected error message: {msg}", + ); + } + + Ok(()) + } + + #[tokio::test] + async fn test_read_csv_truncated_rows_via_tempfile() -> Result<()> { + use std::io::Write; + + // create a SessionContext + let ctx = SessionContext::new(); + + // Create a temp file with a .csv suffix so the reader accepts it + let mut tmp = tempfile::Builder::new().suffix(".csv").tempfile()?; // ensures path ends with .csv + // CSV has header "a,b,c". First data row is truncated (only "1,2"), second row is complete. + write!(tmp, "a,b,c\n1,2\n3,4,5\n")?; + let path = tmp.path().to_str().unwrap().to_string(); + + // Build CsvReadOptions: header present, enable truncated_rows. + // (Use the exact builder method your crate exposes: `truncated_rows(true)` here, + // if the method name differs in your codebase use the appropriate one.) + let options = CsvReadOptions::default().truncated_rows(true); + + println!("options: {}, path: {path}", options.truncated_rows); + + // Call the API under test + let df = ctx.read_csv(&path, options).await?; + + // Collect the results and combine batches so we can inspect columns + let batches = df.collect().await?; + let combined = concat_batches(&batches[0].schema(), &batches)?; + + // Column 'c' is the 3rd column (index 2). The first data row was truncated -> should be NULL. + let col_c = combined.column(2); + assert!( + col_c.is_null(0), + "expected first row column 'c' to be NULL due to truncated row" + ); + + // Also ensure we read at least one row + assert!(combined.num_rows() >= 2); + + Ok(()) + } } diff --git a/datafusion/core/src/datasource/file_format/options.rs b/datafusion/core/src/datasource/file_format/options.rs index 02b792823a827..8c1bb02ef0737 100644 --- a/datafusion/core/src/datasource/file_format/options.rs +++ b/datafusion/core/src/datasource/file_format/options.rs @@ -91,6 +91,11 @@ pub struct CsvReadOptions<'a> { pub file_sort_order: Vec>, /// Optional regex to match null values pub null_regex: Option, + /// Whether to allow truncated rows when parsing. + /// By default this is set to false and will error if the CSV rows have different lengths. + /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. + /// If the record’s schema is not nullable, then it will still return an error. + pub truncated_rows: bool, } impl Default for CsvReadOptions<'_> { @@ -117,6 +122,7 @@ impl<'a> CsvReadOptions<'a> { file_sort_order: vec![], comment: None, null_regex: None, + truncated_rows: false, } } @@ -223,6 +229,15 @@ impl<'a> CsvReadOptions<'a> { self.null_regex = null_regex; self } + + /// Configure whether to allow truncated rows when parsing. + /// By default this is set to false and will error if the CSV rows have different lengths + /// When set to true then it will allow records with less than the expected number of columns and fill the missing columns with nulls. + /// If the record’s schema is not nullable, then it will still return an error. + pub fn truncated_rows(mut self, truncated_rows: bool) -> Self { + self.truncated_rows = truncated_rows; + self + } } /// Options that control the reading of Parquet files. @@ -558,7 +573,8 @@ impl ReadOptions<'_> for CsvReadOptions<'_> { .with_newlines_in_values(self.newlines_in_values) .with_schema_infer_max_rec(self.schema_infer_max_records) .with_file_compression_type(self.file_compression_type.to_owned()) - .with_null_regex(self.null_regex.clone()); + .with_null_regex(self.null_regex.clone()) + .with_truncated_rows(self.truncated_rows); ListingOptions::new(Arc::new(file_format)) .with_file_extension(self.file_extension) diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index 4eeb431584ba7..e09ac3af7c661 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -222,6 +222,11 @@ impl CsvFormat { self } + pub fn with_truncated_rows(mut self, truncated_rows: bool) -> Self { + self.options.truncated_rows = Some(truncated_rows); + self + } + /// Set the regex to use for null values in the CSV reader. /// - default to treat empty values as null. pub fn with_null_regex(mut self, null_regex: Option) -> Self { @@ -291,6 +296,13 @@ impl CsvFormat { self } + /// Set whether rows should be truncated to the column width + /// - defaults to false + pub fn with_truncate_rows(mut self, truncate_rows: bool) -> Self { + self.options.truncated_rows = Some(truncate_rows); + self + } + /// The delimiter character. pub fn delimiter(&self) -> u8 { self.options.delimiter @@ -426,11 +438,13 @@ impl FileFormat for CsvFormat { .with_file_compression_type(self.options.compression.into()) .with_newlines_in_values(newlines_in_values); + let truncated_rows = self.options.truncated_rows.unwrap_or(false); let source = Arc::new( CsvSource::new(has_header, self.options.delimiter, self.options.quote) .with_escape(self.options.escape) .with_terminator(self.options.terminator) - .with_comment(self.options.comment), + .with_comment(self.options.comment) + .with_truncate_rows(truncated_rows), ); let config = conf_builder.with_source(source).build(); @@ -509,7 +523,8 @@ impl CsvFormat { .unwrap_or_else(|| state.config_options().catalog.has_header), ) .with_delimiter(self.options.delimiter) - .with_quote(self.options.quote); + .with_quote(self.options.quote) + .with_truncated_rows(self.options.truncated_rows.unwrap_or(false)); if let Some(null_regex) = &self.options.null_regex { let regex = Regex::new(null_regex.as_str()) diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 8b95d9ba91ff2..e3c2b398c1b6e 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -94,6 +94,7 @@ pub struct CsvSource { metrics: ExecutionPlanMetricsSet, projected_statistics: Option, schema_adapter_factory: Option>, + truncate_rows: bool, } impl CsvSource { @@ -111,6 +112,11 @@ impl CsvSource { pub fn has_header(&self) -> bool { self.has_header } + + // true if rows length support truncate + pub fn truncate_rows(&self) -> bool { + self.truncate_rows + } /// A column delimiter pub fn delimiter(&self) -> u8 { self.delimiter @@ -156,6 +162,13 @@ impl CsvSource { conf.comment = comment; conf } + + /// Whether to support truncate rows when read csv file + pub fn with_truncate_rows(&self, truncate_rows: bool) -> Self { + let mut conf = self.clone(); + conf.truncate_rows = truncate_rows; + conf + } } impl CsvSource { @@ -175,7 +188,8 @@ impl CsvSource { .expect("Batch size must be set before initializing builder"), ) .with_header(self.has_header) - .with_quote(self.quote); + .with_quote(self.quote) + .with_truncated_rows(self.truncate_rows); if let Some(terminator) = self.terminator { builder = builder.with_terminator(terminator); } @@ -340,6 +354,7 @@ impl FileOpener for CsvOpener { let config = CsvSource { has_header: csv_has_header, + truncate_rows: self.config.truncate_rows, ..(*self.config).clone() }; diff --git a/datafusion/proto-common/proto/datafusion_common.proto b/datafusion/proto-common/proto/datafusion_common.proto index f5c79cf3d9a43..d89f73269c3d7 100644 --- a/datafusion/proto-common/proto/datafusion_common.proto +++ b/datafusion/proto-common/proto/datafusion_common.proto @@ -460,6 +460,7 @@ message CsvOptions { bytes double_quote = 15; // Indicates if quotes are doubled bytes newlines_in_values = 16; // Indicates if newlines are supported in values bytes terminator = 17; // Optional terminator character as a byte + bytes truncated_rows = 18; // Indicates if truncated rows are allowed } // Options controlling CSV format diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index c5242d0176e62..bbfd0dfd2ad2e 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -917,6 +917,7 @@ impl TryFrom<&protobuf::CsvOptions> for CsvOptions { null_regex: (!proto_opts.null_regex.is_empty()) .then(|| proto_opts.null_regex.clone()), comment: proto_opts.comment.first().copied(), + truncated_rows: proto_opts.truncated_rows.first().map(|h| *h != 0), }) } } diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rs index 48782ff1d93af..d0c699106f260 100644 --- a/datafusion/proto-common/src/generated/pbjson.rs +++ b/datafusion/proto-common/src/generated/pbjson.rs @@ -1663,6 +1663,9 @@ impl serde::Serialize for CsvOptions { if !self.terminator.is_empty() { len += 1; } + if !self.truncated_rows.is_empty() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion_common.CsvOptions", len)?; if !self.has_header.is_empty() { #[allow(clippy::needless_borrow)] @@ -1735,6 +1738,11 @@ impl serde::Serialize for CsvOptions { #[allow(clippy::needless_borrows_for_generic_args)] struct_ser.serialize_field("terminator", pbjson::private::base64::encode(&self.terminator).as_str())?; } + if !self.truncated_rows.is_empty() { + #[allow(clippy::needless_borrow)] + #[allow(clippy::needless_borrows_for_generic_args)] + struct_ser.serialize_field("truncatedRows", pbjson::private::base64::encode(&self.truncated_rows).as_str())?; + } struct_ser.end() } } @@ -1773,6 +1781,8 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { "newlines_in_values", "newlinesInValues", "terminator", + "truncated_rows", + "truncatedRows", ]; #[allow(clippy::enum_variant_names)] @@ -1794,6 +1804,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { DoubleQuote, NewlinesInValues, Terminator, + TruncatedRows, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -1832,6 +1843,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { "doubleQuote" | "double_quote" => Ok(GeneratedField::DoubleQuote), "newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues), "terminator" => Ok(GeneratedField::Terminator), + "truncatedRows" | "truncated_rows" => Ok(GeneratedField::TruncatedRows), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -1868,6 +1880,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { let mut double_quote__ = None; let mut newlines_in_values__ = None; let mut terminator__ = None; + let mut truncated_rows__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::HasHeader => { @@ -1990,6 +2003,14 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) ; } + GeneratedField::TruncatedRows => { + if truncated_rows__.is_some() { + return Err(serde::de::Error::duplicate_field("truncatedRows")); + } + truncated_rows__ = + Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) + ; + } } } Ok(CsvOptions { @@ -2010,6 +2031,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { double_quote: double_quote__.unwrap_or_default(), newlines_in_values: newlines_in_values__.unwrap_or_default(), terminator: terminator__.unwrap_or_default(), + truncated_rows: truncated_rows__.unwrap_or_default(), }) } } diff --git a/datafusion/proto-common/src/generated/prost.rs b/datafusion/proto-common/src/generated/prost.rs index aa23cea57470c..f09eef67867bb 100644 --- a/datafusion/proto-common/src/generated/prost.rs +++ b/datafusion/proto-common/src/generated/prost.rs @@ -646,6 +646,9 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + /// Indicates if truncated rows are allowed + #[prost(bytes = "vec", tag = "18")] + pub truncated_rows: ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] diff --git a/datafusion/proto-common/src/to_proto/mod.rs b/datafusion/proto-common/src/to_proto/mod.rs index c064270657332..2902a9ce54df3 100644 --- a/datafusion/proto-common/src/to_proto/mod.rs +++ b/datafusion/proto-common/src/to_proto/mod.rs @@ -934,6 +934,7 @@ impl TryFrom<&CsvOptions> for protobuf::CsvOptions { null_value: opts.null_value.clone().unwrap_or_default(), null_regex: opts.null_regex.clone().unwrap_or_default(), comment: opts.comment.map_or_else(Vec::new, |h| vec![h]), + truncated_rows: opts.truncated_rows.map_or_else(Vec::new, |h| vec![h as u8]), }) } } diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index bb985e6ea0265..4f411a4a93323 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -1032,6 +1032,7 @@ message CsvScanExecNode { string comment = 6; } bool newlines_in_values = 7; + bool truncate_rows = 8; } message JsonScanExecNode { diff --git a/datafusion/proto/src/generated/datafusion_proto_common.rs b/datafusion/proto/src/generated/datafusion_proto_common.rs index aa23cea57470c..f09eef67867bb 100644 --- a/datafusion/proto/src/generated/datafusion_proto_common.rs +++ b/datafusion/proto/src/generated/datafusion_proto_common.rs @@ -646,6 +646,9 @@ pub struct CsvOptions { /// Optional terminator character as a byte #[prost(bytes = "vec", tag = "17")] pub terminator: ::prost::alloc::vec::Vec, + /// Indicates if truncated rows are allowed + #[prost(bytes = "vec", tag = "18")] + pub truncated_rows: ::prost::alloc::vec::Vec, } /// Options controlling CSV format #[derive(Clone, Copy, PartialEq, ::prost::Message)] diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 2ddf063ee8782..ff7519aa5df29 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -3771,6 +3771,9 @@ impl serde::Serialize for CsvScanExecNode { if self.newlines_in_values { len += 1; } + if self.truncate_rows { + len += 1; + } if self.optional_escape.is_some() { len += 1; } @@ -3793,6 +3796,9 @@ impl serde::Serialize for CsvScanExecNode { if self.newlines_in_values { struct_ser.serialize_field("newlinesInValues", &self.newlines_in_values)?; } + if self.truncate_rows { + struct_ser.serialize_field("truncateRows", &self.truncate_rows)?; + } if let Some(v) = self.optional_escape.as_ref() { match v { csv_scan_exec_node::OptionalEscape::Escape(v) => { @@ -3825,6 +3831,8 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { "quote", "newlines_in_values", "newlinesInValues", + "truncate_rows", + "truncateRows", "escape", "comment", ]; @@ -3836,6 +3844,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { Delimiter, Quote, NewlinesInValues, + TruncateRows, Escape, Comment, } @@ -3864,6 +3873,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { "delimiter" => Ok(GeneratedField::Delimiter), "quote" => Ok(GeneratedField::Quote), "newlinesInValues" | "newlines_in_values" => Ok(GeneratedField::NewlinesInValues), + "truncateRows" | "truncate_rows" => Ok(GeneratedField::TruncateRows), "escape" => Ok(GeneratedField::Escape), "comment" => Ok(GeneratedField::Comment), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), @@ -3890,6 +3900,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { let mut delimiter__ = None; let mut quote__ = None; let mut newlines_in_values__ = None; + let mut truncate_rows__ = None; let mut optional_escape__ = None; let mut optional_comment__ = None; while let Some(k) = map_.next_key()? { @@ -3924,6 +3935,12 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { } newlines_in_values__ = Some(map_.next_value()?); } + GeneratedField::TruncateRows => { + if truncate_rows__.is_some() { + return Err(serde::de::Error::duplicate_field("truncateRows")); + } + truncate_rows__ = Some(map_.next_value()?); + } GeneratedField::Escape => { if optional_escape__.is_some() { return Err(serde::de::Error::duplicate_field("escape")); @@ -3944,6 +3961,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { delimiter: delimiter__.unwrap_or_default(), quote: quote__.unwrap_or_default(), newlines_in_values: newlines_in_values__.unwrap_or_default(), + truncate_rows: truncate_rows__.unwrap_or_default(), optional_escape: optional_escape__, optional_comment: optional_comment__, }) diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index 69f7542e48c93..ffb73086650f3 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -1564,6 +1564,8 @@ pub struct CsvScanExecNode { pub quote: ::prost::alloc::string::String, #[prost(bool, tag = "7")] pub newlines_in_values: bool, + #[prost(bool, tag = "8")] + pub truncate_rows: bool, #[prost(oneof = "csv_scan_exec_node::OptionalEscape", tags = "5")] pub optional_escape: ::core::option::Option, #[prost(oneof = "csv_scan_exec_node::OptionalComment", tags = "6")] diff --git a/datafusion/proto/src/logical_plan/file_formats.rs b/datafusion/proto/src/logical_plan/file_formats.rs index 654607bd733da..492795855cf6e 100644 --- a/datafusion/proto/src/logical_plan/file_formats.rs +++ b/datafusion/proto/src/logical_plan/file_formats.rs @@ -72,6 +72,7 @@ impl CsvOptionsProto { newlines_in_values: options .newlines_in_values .map_or(vec![], |v| vec![v as u8]), + truncated_rows: options.truncated_rows.map_or(vec![], |v| vec![v as u8]), } } else { CsvOptionsProto::default() @@ -157,6 +158,11 @@ impl From<&CsvOptionsProto> for CsvOptions { } else { Some(proto.newlines_in_values[0] != 0) }, + truncated_rows: if proto.truncated_rows.is_empty() { + None + } else { + Some(proto.truncated_rows[0] != 0) + }, } } } diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 8e38b0d1bf5b4..e577de5b1d0e0 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -2619,6 +2619,7 @@ impl protobuf::PhysicalPlanNode { None }, newlines_in_values: maybe_csv.newlines_in_values(), + truncate_rows: csv_config.truncate_rows(), }, )), })); From 8588da4725669f3bd7b640844ff09964c9ed12df Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Tue, 16 Sep 2025 19:50:13 +0800 Subject: [PATCH 132/177] fix clippy --- Cargo.lock | 40 ------------------- datafusion/datasource/src/source.rs | 1 - .../src/enforce_distribution.rs | 1 - .../physical-plan/src/joins/hash_join/exec.rs | 20 ---------- .../tests/cases/roundtrip_physical_plan.rs | 2 +- 5 files changed, 1 insertion(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bf54c75b610c..25bcaf68cb84a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1518,7 +1518,6 @@ dependencies = [ "encode_unicode", "libc", "once_cell", - "unicode-width 0.2.1", "windows-sys 0.59.0", ] @@ -5350,26 +5349,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "ref-cast" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", -] - [[package]] name = "regex" version = "1.11.2" @@ -6740,24 +6719,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "tower-http" -version = "0.6.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" -dependencies = [ - "bitflags 2.9.1", - "bytes", - "futures-util", - "http 1.3.1", - "http-body 1.0.1", - "iri-string", - "pin-project-lite", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "tower-layer" version = "0.3.3" @@ -7244,7 +7205,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" dependencies = [ "libredox", - "redox_syscall 0.5.13", "wasite", "web-sys", ] diff --git a/datafusion/datasource/src/source.rs b/datafusion/datasource/src/source.rs index 26b2a7b08aacc..702253da41d01 100644 --- a/datafusion/datasource/src/source.rs +++ b/datafusion/datasource/src/source.rs @@ -22,7 +22,6 @@ use std::fmt; use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use datafusion_physical_expr::equivalence::ProjectionMapping; use datafusion_physical_plan::execution_plan::{ Boundedness, EmissionType, SchedulingType, }; diff --git a/datafusion/physical-optimizer/src/enforce_distribution.rs b/datafusion/physical-optimizer/src/enforce_distribution.rs index 2da2c16bc68f8..34c4f52824e8b 100644 --- a/datafusion/physical-optimizer/src/enforce_distribution.rs +++ b/datafusion/physical-optimizer/src/enforce_distribution.rs @@ -945,7 +945,6 @@ fn add_hash_on_top( /// /// # Returns /// - /// Updated node with an execution plan, where desired single /// distribution is satisfied by adding [`SortPreservingMergeExec`]. fn add_merge_on_top( diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 54e0e7179e71c..8b6767e8583d7 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -882,26 +882,6 @@ impl ExecutionPlan for HashJoinExec { })) } - fn reset_state(self: Arc) -> Result> { - // Reset the left_fut to allow re-execution - Ok(Arc::new(HashJoinExec { - left: Arc::clone(&self.left), - right: Arc::clone(&self.right), - on: self.on.clone(), - filter: self.filter.clone(), - join_type: self.join_type, - join_schema: Arc::clone(&self.join_schema), - left_fut: OnceAsync::default(), - random_state: self.random_state.clone(), - mode: self.mode, - metrics: ExecutionPlanMetricsSet::new(), - projection: self.projection.clone(), - column_indices: self.column_indices.clone(), - null_equality: self.null_equality, - cache: self.cache.clone(), - })) - } - fn execute( &self, partition: usize, diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index c4ac38fb7e053..8f50e5ed28cc6 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -144,7 +144,7 @@ fn roundtrip_test_and_return( protobuf::PhysicalPlanNode::try_from_physical_plan(exec_plan.clone(), codec) .expect("to proto"); let runtime = ctx.runtime_env(); - let mut result_exec_plan: Arc = proto + let result_exec_plan: Arc = proto .try_into_physical_plan(ctx, runtime.deref(), codec) .expect("from proto"); From 238d58b8b014df3fb2f40c47e055e0a9f3541f8d Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Tue, 16 Sep 2025 20:39:47 +0800 Subject: [PATCH 133/177] fix test and fmt --- datafusion/core/src/datasource/file_format/parquet.rs | 4 ++-- datafusion/proto/src/physical_plan/mod.rs | 2 -- .../sqllogictest/test_files/listing_table_statistics.slt | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datafusion/core/src/datasource/file_format/parquet.rs b/datafusion/core/src/datasource/file_format/parquet.rs index 088c4408fff57..8200c0627b8c0 100644 --- a/datafusion/core/src/datasource/file_format/parquet.rs +++ b/datafusion/core/src/datasource/file_format/parquet.rs @@ -581,11 +581,11 @@ mod tests { assert_eq!(string_truncation_stats.null_count, Precision::Exact(2)); assert_eq!( string_truncation_stats.max_value, - Precision::Inexact(ScalarValue::Utf8View(Some("b".repeat(63) + "c"))) + Precision::Inexact(Utf8(Some("b".repeat(63) + "c"))) ); assert_eq!( string_truncation_stats.min_value, - Precision::Inexact(ScalarValue::Utf8View(Some("a".repeat(64)))) + Precision::Inexact(Utf8(Some("a".repeat(64)))) ); Ok(()) diff --git a/datafusion/proto/src/physical_plan/mod.rs b/datafusion/proto/src/physical_plan/mod.rs index 7125e3d030f58..e577de5b1d0e0 100644 --- a/datafusion/proto/src/physical_plan/mod.rs +++ b/datafusion/proto/src/physical_plan/mod.rs @@ -616,7 +616,6 @@ impl protobuf::PhysicalPlanNode { ) })?; - let filter_selectivity = filter.default_filter_selectivity.try_into(); let projection = if !filter.projection.is_empty() { Some( @@ -630,7 +629,6 @@ impl protobuf::PhysicalPlanNode { None }; - let filter = FilterExec::try_new(predicate, input)?.with_projection(projection)?; match filter_selectivity { diff --git a/datafusion/sqllogictest/test_files/listing_table_statistics.slt b/datafusion/sqllogictest/test_files/listing_table_statistics.slt index 37daf551c2c39..0b3b6106bbdc9 100644 --- a/datafusion/sqllogictest/test_files/listing_table_statistics.slt +++ b/datafusion/sqllogictest/test_files/listing_table_statistics.slt @@ -35,7 +35,7 @@ query TT explain format indent select * from t; ---- logical_plan TableScan: t projection=[int_col, str_col] -physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/2.parquet]]}, projection=[int_col, str_col], file_type=parquet, statistics=[Rows=Exact(4), Bytes=Exact(212), [(Col[0]: Min=Exact(Int64(-1)) Max=Exact(Int64(3)) Null=Exact(0)),(Col[1]: Min=Exact(Utf8View("a")) Max=Exact(Utf8View("d")) Null=Exact(0))]] +physical_plan DataSourceExec: file_groups={2 groups: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/1.parquet], [WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/listing_table_statistics/2.parquet]]}, projection=[int_col, str_col], file_type=parquet, statistics=[Rows=Exact(4), Bytes=Exact(212), [(Col[0]: Min=Exact(Int64(-1)) Max=Exact(Int64(3)) Null=Exact(0)),(Col[1]: Min=Exact(Utf8("a")) Max=Exact(Utf8("d")) Null=Exact(0))]] statement ok drop table t; From 891202abf962e1507d58d3100671ab11d7f4708f Mon Sep 17 00:00:00 2001 From: Rohan Krishnaswamy <47869999+rkrishn7@users.noreply.github.com> Date: Tue, 16 Sep 2025 10:31:42 -0700 Subject: [PATCH 134/177] fix: ignore non-existent columns when adding filter equivalence info in `FileScanConfig` (#17546) (#17600) --- datafusion/datasource/src/file_scan_config.rs | 73 ++++++++++++++++++- datafusion/datasource/src/test_util.rs | 12 +++ 2 files changed, 83 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs index 82d7057a98608..4e2235eae8fec 100644 --- a/datafusion/datasource/src/file_scan_config.rs +++ b/datafusion/datasource/src/file_scan_config.rs @@ -590,7 +590,11 @@ impl DataSource for FileScanConfig { // Note that this will *ignore* any non-projected columns: these don't factor into ordering / equivalence. match reassign_predicate_columns(filter, &schema, true) { Ok(filter) => { - match Self::add_filter_equivalence_info(filter, &mut eq_properties) { + match Self::add_filter_equivalence_info( + filter, + &mut eq_properties, + &schema, + ) { Ok(()) => {} Err(e) => { warn!("Failed to add filter equivalence info: {e}"); @@ -758,9 +762,24 @@ impl FileScanConfig { fn add_filter_equivalence_info( filter: Arc, eq_properties: &mut EquivalenceProperties, + schema: &Schema, ) -> Result<()> { + macro_rules! ignore_dangling_col { + ($col:expr) => { + if let Some(col) = $col.as_any().downcast_ref::() { + if schema.index_of(col.name()).is_err() { + continue; + } + } + }; + } + let (equal_pairs, _) = collect_columns_from_predicate(&filter); for (lhs, rhs) in equal_pairs { + // Ignore any binary expressions that reference non-existent columns in the current schema + // (e.g. due to unnecessary projections being removed) + ignore_dangling_col!(lhs); + ignore_dangling_col!(rhs); eq_properties.add_equal_conditions(Arc::clone(lhs), Arc::clone(rhs))? } Ok(()) @@ -1449,6 +1468,7 @@ pub fn wrap_partition_value_in_dict(val: ScalarValue) -> ScalarValue { #[cfg(test)] mod tests { use super::*; + use crate::test_util::col; use crate::{ generate_test_files, test_util::MockSource, tests::aggr_test_schema, verify_sort_integrity, @@ -1457,8 +1477,9 @@ mod tests { use arrow::array::{Int32Array, RecordBatch}; use datafusion_common::stats::Precision; use datafusion_common::{assert_batches_eq, internal_err}; - use datafusion_expr::SortExpr; + use datafusion_expr::{Operator, SortExpr}; use datafusion_physical_expr::create_physical_sort_expr; + use datafusion_physical_expr::expressions::{BinaryExpr, Literal}; /// Returns the column names on the schema pub fn columns(schema: &Schema) -> Vec { @@ -2214,6 +2235,54 @@ mod tests { assert_eq!(config.output_ordering.len(), 1); } + #[test] + fn equivalence_properties_after_schema_change() { + let file_schema = aggr_test_schema(); + let object_store_url = ObjectStoreUrl::parse("test:///").unwrap(); + // Create a file source with a filter + let file_source: Arc = + Arc::new(MockSource::default().with_filter(Arc::new(BinaryExpr::new( + col("c2", &file_schema).unwrap(), + Operator::Eq, + Arc::new(Literal::new(ScalarValue::Int32(Some(10)))), + )))); + + let config = FileScanConfigBuilder::new( + object_store_url.clone(), + Arc::clone(&file_schema), + Arc::clone(&file_source), + ) + .with_projection(Some(vec![0, 1, 2])) + .build(); + + // Simulate projection being updated. Since the filter has already been pushed down, + // the new projection won't include the filtered column. + let data_source = config + .try_swapping_with_projection(&[ProjectionExpr::new( + col("c3", &file_schema).unwrap(), + "c3".to_string(), + )]) + .unwrap() + .unwrap(); + + // Gather the equivalence properties from the new data source. There should + // be no equivalence class for column c2 since it was removed by the projection. + let eq_properties = data_source.eq_properties(); + let eq_group = eq_properties.eq_group(); + + for class in eq_group.iter() { + for expr in class.iter() { + if let Some(col) = expr.as_any().downcast_ref::() { + assert_ne!( + col.name(), + "c2", + "c2 should not be present in any equivalence class" + ); + } + } + } + } + #[test] fn test_file_scan_config_builder_defaults() { let file_schema = aggr_test_schema(); diff --git a/datafusion/datasource/src/test_util.rs b/datafusion/datasource/src/test_util.rs index e4a5114aa073e..f0aff1fa62b70 100644 --- a/datafusion/datasource/src/test_util.rs +++ b/datafusion/datasource/src/test_util.rs @@ -34,6 +34,14 @@ pub(crate) struct MockSource { metrics: ExecutionPlanMetricsSet, projected_statistics: Option, schema_adapter_factory: Option>, + filter: Option>, +} + +impl MockSource { + pub fn with_filter(mut self, filter: Arc) -> Self { + self.filter = Some(filter); + self + } } impl FileSource for MockSource { @@ -50,6 +58,10 @@ impl FileSource for MockSource { self } + fn filter(&self) -> Option> { + self.filter.clone() + } + fn with_batch_size(&self, _batch_size: usize) -> Arc { Arc::new(Self { ..self.clone() }) } From e16c24f9a9a59c8044943ee2814fc2d62c79c103 Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Wed, 17 Sep 2025 11:27:04 +0800 Subject: [PATCH 135/177] fix proto test --- datafusion/proto/tests/cases/roundtrip_physical_plan.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 8f50e5ed28cc6..b7620dc88ff69 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -144,10 +144,16 @@ fn roundtrip_test_and_return( protobuf::PhysicalPlanNode::try_from_physical_plan(exec_plan.clone(), codec) .expect("to proto"); let runtime = ctx.runtime_env(); - let result_exec_plan: Arc = proto + let mut result_exec_plan: Arc = proto .try_into_physical_plan(ctx, runtime.deref(), codec) .expect("from proto"); + // Qi: workaround for NodeId not being serialized/deserialized, + // otherwise the assert_eq! below will fail + let mut annotator2 = NodeIdAnnotator::new(); + result_exec_plan = + annotate_node_id_for_execution_plan(&result_exec_plan, &mut annotator2)?; + pretty_assertions::assert_eq!( format!("{exec_plan:?}"), format!("{result_exec_plan:?}") From acd9ddfe8f7d10fe554a0b96348d7cee108bb9ca Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Wed, 17 Sep 2025 15:07:09 +0800 Subject: [PATCH 136/177] remove unused file --- .../optimizer/src/simplify_predicates.rs | 191 ------------------ 1 file changed, 191 deletions(-) delete mode 100644 datafusion/optimizer/src/simplify_predicates.rs diff --git a/datafusion/optimizer/src/simplify_predicates.rs b/datafusion/optimizer/src/simplify_predicates.rs deleted file mode 100644 index aa563abc48fe9..0000000000000 --- a/datafusion/optimizer/src/simplify_predicates.rs +++ /dev/null @@ -1,191 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use datafusion_common::{Column, Result, ScalarValue}; -use datafusion_expr::{BinaryExpr, Cast, Expr, Operator}; -use std::collections::BTreeMap; - -pub(crate) fn simplify_predicates(predicates: Vec) -> Result> { - // Early return for simple cases - if predicates.len() <= 1 { - return Ok(predicates); - } - - // Group predicates by their column reference - let mut column_predicates: BTreeMap> = BTreeMap::new(); - let mut other_predicates = Vec::new(); - - for pred in predicates { - match &pred { - Expr::BinaryExpr(BinaryExpr { left, op, right }) - if matches!( - op, - Operator::Gt - | Operator::GtEq - | Operator::Lt - | Operator::LtEq - | Operator::Eq - ) => - { - let left_col = extract_column_from_expr(left); - let right_col = extract_column_from_expr(right); - let left_lit = left.as_literal().is_some(); - let right_lit = right.as_literal().is_some(); - if let (Some(col), true) = (&left_col, right_lit) { - column_predicates.entry(col.clone()).or_default().push(pred); - } else if let (true, Some(col)) = (left_lit, &right_col) { - column_predicates.entry(col.clone()).or_default().push(pred); - } else { - other_predicates.push(pred); - } - } - _ => other_predicates.push(pred), - } - } - - // Process each column's predicates to remove redundancies - let mut result = other_predicates; - for (_, preds) in column_predicates { - let simplified = simplify_column_predicates(preds)?; - result.extend(simplified); - } - - Ok(result) -} - -fn simplify_column_predicates(predicates: Vec) -> Result> { - if predicates.len() <= 1 { - return Ok(predicates); - } - - // Group by operator type, but combining similar operators - let mut greater_predicates = Vec::new(); // Combines > and >= - let mut less_predicates = Vec::new(); // Combines < and <= - let mut eq_predicates = Vec::new(); - - for pred in predicates { - match &pred { - Expr::BinaryExpr(BinaryExpr { left: _, op, right }) => { - let right_is_literal = right.as_literal().is_some(); - match (op, right_is_literal) { - (Operator::Gt, true) - | (Operator::Lt, false) - | (Operator::GtEq, true) - | (Operator::LtEq, false) => greater_predicates.push(pred), - (Operator::Lt, true) - | (Operator::Gt, false) - | (Operator::LtEq, true) - | (Operator::GtEq, false) => less_predicates.push(pred), - (Operator::Eq, _) => eq_predicates.push(pred), - _ => unreachable!("Unexpected operator: {}", op), - } - } - _ => unreachable!("Unexpected predicate {}", pred.to_string()), - } - } - - let mut result = Vec::new(); - - // If we have equality predicates, they're the most restrictive - if !eq_predicates.is_empty() { - if eq_predicates.len() > 1 { - result.push(Expr::Literal(ScalarValue::Boolean(Some(false)), None)); - } else { - result.push(eq_predicates[0].clone()); - } - } else { - // Handle all greater-than-style predicates (keep the most restrictive - highest value) - if !greater_predicates.is_empty() { - if let Some(most_restrictive) = - find_most_restrictive_predicate(&greater_predicates, true)? - { - result.push(most_restrictive); - } else { - result.extend(greater_predicates); - } - } - - // Handle all less-than-style predicates (keep the most restrictive - lowest value) - if !less_predicates.is_empty() { - if let Some(most_restrictive) = - find_most_restrictive_predicate(&less_predicates, false)? - { - result.push(most_restrictive); - } else { - result.extend(less_predicates); - } - } - } - - Ok(result) -} - -fn find_most_restrictive_predicate( - predicates: &[Expr], - find_greater: bool, -) -> Result> { - if predicates.is_empty() { - return Ok(None); - } - - let mut most_restrictive = predicates[0].clone(); - let mut best_value: Option = None; - - for pred in predicates { - if let Expr::BinaryExpr(BinaryExpr { left, op: _, right }) = pred { - // Extract the literal value based on which side has it - let mut scalar_value = None; - if let Some(scalar) = right.as_literal() { - scalar_value = Some(scalar.clone()); - } - if let Some(scalar) = left.as_literal() { - scalar_value = Some(scalar.clone()); - } - - if let Some(scalar) = scalar_value { - if let Some(current_best) = &best_value { - if let Some(comparison) = scalar.partial_cmp(current_best) { - let is_better = if find_greater { - comparison == std::cmp::Ordering::Greater - } else { - comparison == std::cmp::Ordering::Less - }; - - if is_better { - best_value = Some(scalar); - most_restrictive = pred.clone(); - } - } - } else { - best_value = Some(scalar); - most_restrictive = pred.clone(); - } - } - } - } - - Ok(Some(most_restrictive)) -} - -fn extract_column_from_expr(expr: &Expr) -> Option { - match expr { - Expr::Column(col) => Some(col.clone()), - // Handle cases where the column might be wrapped in a cast or other operation - Expr::Cast(Cast { expr, .. }) => extract_column_from_expr(expr), - _ => None, - } -} From 1a31b798fd0c45e2c7b83e2b9ce1cbc670ed66d3 Mon Sep 17 00:00:00 2001 From: Shehab Amin <11789402+shehabgamin@users.noreply.github.com> Date: Wed, 17 Sep 2025 18:37:37 +0000 Subject: [PATCH 137/177] fix: Ensure the CachedParquetFileReader respects the metadata prefetch hint (#17302) (#17613) Co-authored-by: Nuno Faria --- datafusion/datasource-parquet/src/reader.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rs index 9d2c52f721ba9..d0c008ad35cf7 100644 --- a/datafusion/datasource-parquet/src/reader.rs +++ b/datafusion/datasource-parquet/src/reader.rs @@ -209,6 +209,7 @@ impl ParquetFileReaderFactory for CachedParquetFileReaderFactory { file_metrics, file_meta, metadata_cache: Arc::clone(&self.metadata_cache), + metadata_size_hint, })) } } @@ -222,6 +223,7 @@ pub struct CachedParquetFileReader { pub inner: ParquetObjectReader, file_meta: FileMeta, metadata_cache: Arc, + metadata_size_hint: Option, } impl AsyncFileReader for CachedParquetFileReader { @@ -261,11 +263,10 @@ impl AsyncFileReader for CachedParquetFileReader { #[cfg(not(feature = "parquet_encryption"))] let file_decryption_properties = None; - // TODO there should be metadata prefetch hint here - // https://github.com/apache/datafusion/issues/17279 DFParquetMetadata::new(&self.store, &file_meta.object_meta) .with_decryption_properties(file_decryption_properties) .with_file_metadata_cache(Some(Arc::clone(&metadata_cache))) + .with_metadata_size_hint(self.metadata_size_hint) .fetch_metadata() .await .map_err(|e| { From 5b191f8c94469bf608565fadceaa21dc603a7a56 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Mon, 22 Sep 2025 19:53:32 +0800 Subject: [PATCH 138/177] fix: Partial AggregateMode will generate duplicate field names which will fail DFSchema construct (#17706) * fix: Partial AggregateMode will generate duplicate field names which will fail DFSchema construct * Update datafusion/common/src/dfschema.rs Co-authored-by: Andrew Lamb * fmt --------- Co-authored-by: Andrew Lamb (cherry picked from commit 52690c64b64e56d1aba5b2daee7be701b343576d) --- datafusion/common/src/dfschema.rs | 6 +- datafusion/core/tests/dataframe/mod.rs | 115 ++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index d3dda2888214d..f9e3b2cee40d9 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -913,7 +913,11 @@ impl TryFrom for DFSchema { field_qualifiers: vec![None; field_count], functional_dependencies: FunctionalDependencies::empty(), }; - dfschema.check_names()?; + // Without checking names, because schema here may have duplicate field names. + // For example, Partial AggregateMode will generate duplicate field names from + // state_fields. + // See + // dfschema.check_names()?; Ok(dfschema) } } diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index a563459f42a11..a8c244a34733b 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -32,6 +32,7 @@ use arrow::datatypes::{ }; use arrow::error::ArrowError; use arrow::util::pretty::pretty_format_batches; +use arrow_schema::{SortOptions, TimeUnit}; use datafusion::{assert_batches_eq, dataframe}; use datafusion_functions_aggregate::count::{count_all, count_all_window}; use datafusion_functions_aggregate::expr_fn::{ @@ -64,8 +65,8 @@ use datafusion::test_util::{ use datafusion_catalog::TableProvider; use datafusion_common::test_util::{batches_to_sort_string, batches_to_string}; use datafusion_common::{ - assert_contains, Constraint, Constraints, DataFusionError, ParamValues, ScalarValue, - TableReference, UnnestOptions, + assert_contains, Constraint, Constraints, DFSchema, DataFusionError, ParamValues, + ScalarValue, TableReference, UnnestOptions, }; use datafusion_common_runtime::SpawnedTask; use datafusion_datasource::file_format::format_as_file_type; @@ -79,10 +80,16 @@ use datafusion_expr::{ LogicalPlanBuilder, ScalarFunctionImplementation, SortExpr, WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, }; +use datafusion_physical_expr::aggregate::AggregateExprBuilder; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::Partitioning; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; -use datafusion_physical_plan::{displayable, ExecutionPlanProperties}; +use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; +use datafusion_physical_plan::aggregates::{ + AggregateExec, AggregateMode, PhysicalGroupBy, +}; +use datafusion_physical_plan::empty::EmptyExec; +use datafusion_physical_plan::{displayable, ExecutionPlan, ExecutionPlanProperties}; // Get string representation of the plan async fn physical_plan_to_string(df: &DataFrame) -> String { @@ -6322,3 +6329,105 @@ async fn test_copy_to_preserves_order() -> Result<()> { ); Ok(()) } + +#[tokio::test] +async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> { + let ctx = SessionContext::new(); + + // Simple schema with just the fields we need + let file_schema = Arc::new(Schema::new(vec![ + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + true, + ), + Field::new("ticker", DataType::Utf8, true), + Field::new("value", DataType::Float64, true), + Field::new("date", DataType::Utf8, false), + ])); + + let df_schema = DFSchema::try_from(file_schema.clone())?; + + let timestamp = col("timestamp"); + let value = col("value"); + let ticker = col("ticker"); + let date = col("date"); + + let mock_exec = Arc::new(EmptyExec::new(file_schema.clone())); + + // Build first_value aggregate + let first_value = Arc::new( + AggregateExprBuilder::new( + datafusion_functions_aggregate::first_last::first_value_udaf(), + vec![ctx.create_physical_expr(value.clone(), &df_schema)?], + ) + .alias("first_value(value)") + .order_by(vec![PhysicalSortExpr::new( + ctx.create_physical_expr(timestamp.clone(), &df_schema)?, + SortOptions::new(false, false), + )]) + .schema(file_schema.clone()) + .build() + .expect("Failed to build first_value"), + ); + + // Build last_value aggregate + let last_value = Arc::new( + AggregateExprBuilder::new( + datafusion_functions_aggregate::first_last::last_value_udaf(), + vec![ctx.create_physical_expr(value.clone(), &df_schema)?], + ) + .alias("last_value(value)") + .order_by(vec![PhysicalSortExpr::new( + ctx.create_physical_expr(timestamp.clone(), &df_schema)?, + SortOptions::new(false, false), + )]) + .schema(file_schema.clone()) + .build() + .expect("Failed to build last_value"), + ); + + let partial_agg = AggregateExec::try_new( + AggregateMode::Partial, + PhysicalGroupBy::new_single(vec![ + ( + ctx.create_physical_expr(date.clone(), &df_schema)?, + "date".to_string(), + ), + ( + ctx.create_physical_expr(ticker.clone(), &df_schema)?, + "ticker".to_string(), + ), + ]), + vec![first_value, last_value], + vec![None, None], + mock_exec, + file_schema, + ) + .expect("Failed to build partial agg"); + + // Assert that the schema field names match the expected names + let expected_field_names = vec![ + "date", + "ticker", + "first_value(value)[first_value]", + "timestamp@0", + "is_set", + "last_value(value)[last_value]", + "timestamp@0", + "is_set", + ]; + + let binding = partial_agg.schema(); + let actual_field_names: Vec<_> = binding.fields().iter().map(|f| f.name()).collect(); + assert_eq!(actual_field_names, expected_field_names); + + // Ensure that DFSchema::try_from does not fail + let partial_agg_exec_schema = DFSchema::try_from(partial_agg.schema()); + assert!( + partial_agg_exec_schema.is_ok(), + "Expected get AggregateExec schema to succeed with duplicate state fields" + ); + + Ok(()) +} From 4840c8aa8f685f14bc09dd4d605d235f2ddac302 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Mon, 22 Sep 2025 21:29:02 +0800 Subject: [PATCH 139/177] fix: Partial AggregateMode will generate duplicate field names which will fail DFSchema construct (#17706) (#17717) * fix: Partial AggregateMode will generate duplicate field names which will fail DFSchema construct * Update datafusion/common/src/dfschema.rs * fmt --------- (cherry picked from commit 52690c64b64e56d1aba5b2daee7be701b343576d) Co-authored-by: Andrew Lamb --- datafusion/common/src/dfschema.rs | 6 +- datafusion/core/tests/dataframe/mod.rs | 115 ++++++++++++++++++++++++- 2 files changed, 117 insertions(+), 4 deletions(-) diff --git a/datafusion/common/src/dfschema.rs b/datafusion/common/src/dfschema.rs index d3dda2888214d..f9e3b2cee40d9 100644 --- a/datafusion/common/src/dfschema.rs +++ b/datafusion/common/src/dfschema.rs @@ -913,7 +913,11 @@ impl TryFrom for DFSchema { field_qualifiers: vec![None; field_count], functional_dependencies: FunctionalDependencies::empty(), }; - dfschema.check_names()?; + // Without checking names, because schema here may have duplicate field names. + // For example, Partial AggregateMode will generate duplicate field names from + // state_fields. + // See + // dfschema.check_names()?; Ok(dfschema) } } diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index a563459f42a11..a8c244a34733b 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -32,6 +32,7 @@ use arrow::datatypes::{ }; use arrow::error::ArrowError; use arrow::util::pretty::pretty_format_batches; +use arrow_schema::{SortOptions, TimeUnit}; use datafusion::{assert_batches_eq, dataframe}; use datafusion_functions_aggregate::count::{count_all, count_all_window}; use datafusion_functions_aggregate::expr_fn::{ @@ -64,8 +65,8 @@ use datafusion::test_util::{ use datafusion_catalog::TableProvider; use datafusion_common::test_util::{batches_to_sort_string, batches_to_string}; use datafusion_common::{ - assert_contains, Constraint, Constraints, DataFusionError, ParamValues, ScalarValue, - TableReference, UnnestOptions, + assert_contains, Constraint, Constraints, DFSchema, DataFusionError, ParamValues, + ScalarValue, TableReference, UnnestOptions, }; use datafusion_common_runtime::SpawnedTask; use datafusion_datasource::file_format::format_as_file_type; @@ -79,10 +80,16 @@ use datafusion_expr::{ LogicalPlanBuilder, ScalarFunctionImplementation, SortExpr, WindowFrame, WindowFrameBound, WindowFrameUnits, WindowFunctionDefinition, }; +use datafusion_physical_expr::aggregate::AggregateExprBuilder; use datafusion_physical_expr::expressions::Column; use datafusion_physical_expr::Partitioning; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; -use datafusion_physical_plan::{displayable, ExecutionPlanProperties}; +use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; +use datafusion_physical_plan::aggregates::{ + AggregateExec, AggregateMode, PhysicalGroupBy, +}; +use datafusion_physical_plan::empty::EmptyExec; +use datafusion_physical_plan::{displayable, ExecutionPlan, ExecutionPlanProperties}; // Get string representation of the plan async fn physical_plan_to_string(df: &DataFrame) -> String { @@ -6322,3 +6329,105 @@ async fn test_copy_to_preserves_order() -> Result<()> { ); Ok(()) } + +#[tokio::test] +async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> { + let ctx = SessionContext::new(); + + // Simple schema with just the fields we need + let file_schema = Arc::new(Schema::new(vec![ + Field::new( + "timestamp", + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + true, + ), + Field::new("ticker", DataType::Utf8, true), + Field::new("value", DataType::Float64, true), + Field::new("date", DataType::Utf8, false), + ])); + + let df_schema = DFSchema::try_from(file_schema.clone())?; + + let timestamp = col("timestamp"); + let value = col("value"); + let ticker = col("ticker"); + let date = col("date"); + + let mock_exec = Arc::new(EmptyExec::new(file_schema.clone())); + + // Build first_value aggregate + let first_value = Arc::new( + AggregateExprBuilder::new( + datafusion_functions_aggregate::first_last::first_value_udaf(), + vec![ctx.create_physical_expr(value.clone(), &df_schema)?], + ) + .alias("first_value(value)") + .order_by(vec![PhysicalSortExpr::new( + ctx.create_physical_expr(timestamp.clone(), &df_schema)?, + SortOptions::new(false, false), + )]) + .schema(file_schema.clone()) + .build() + .expect("Failed to build first_value"), + ); + + // Build last_value aggregate + let last_value = Arc::new( + AggregateExprBuilder::new( + datafusion_functions_aggregate::first_last::last_value_udaf(), + vec![ctx.create_physical_expr(value.clone(), &df_schema)?], + ) + .alias("last_value(value)") + .order_by(vec![PhysicalSortExpr::new( + ctx.create_physical_expr(timestamp.clone(), &df_schema)?, + SortOptions::new(false, false), + )]) + .schema(file_schema.clone()) + .build() + .expect("Failed to build last_value"), + ); + + let partial_agg = AggregateExec::try_new( + AggregateMode::Partial, + PhysicalGroupBy::new_single(vec![ + ( + ctx.create_physical_expr(date.clone(), &df_schema)?, + "date".to_string(), + ), + ( + ctx.create_physical_expr(ticker.clone(), &df_schema)?, + "ticker".to_string(), + ), + ]), + vec![first_value, last_value], + vec![None, None], + mock_exec, + file_schema, + ) + .expect("Failed to build partial agg"); + + // Assert that the schema field names match the expected names + let expected_field_names = vec![ + "date", + "ticker", + "first_value(value)[first_value]", + "timestamp@0", + "is_set", + "last_value(value)[last_value]", + "timestamp@0", + "is_set", + ]; + + let binding = partial_agg.schema(); + let actual_field_names: Vec<_> = binding.fields().iter().map(|f| f.name()).collect(); + assert_eq!(actual_field_names, expected_field_names); + + // Ensure that DFSchema::try_from does not fail + let partial_agg_exec_schema = DFSchema::try_from(partial_agg.schema()); + assert!( + partial_agg_exec_schema.is_ok(), + "Expected get AggregateExec schema to succeed with duplicate state fields" + ); + + Ok(()) +} From 14da942f81153d7f10644fba16add6b298e5b536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Mon, 22 Sep 2025 18:19:44 +0100 Subject: [PATCH 140/177] feat: expose `udafs` and `udwfs` methods on `FunctionRegistry` (#17650) (#17725) * expose udafs and udwfs method on `FunctionRegistry` * fix doc test * add default implementations not to trigger backward incompatible change for others --- datafusion/core/src/execution/context/mod.rs | 8 ++++++ .../core/src/execution/session_state.rs | 8 ++++++ datafusion/execution/src/task.rs | 8 ++++++ datafusion/expr/src/registry.rs | 26 ++++++++++++++++++- datafusion/proto/src/bytes/mod.rs | 8 ++++++ datafusion/proto/src/bytes/registry.rs | 8 ++++++ datafusion/spark/src/lib.rs | 2 ++ 7 files changed, 67 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/execution/context/mod.rs b/datafusion/core/src/execution/context/mod.rs index de1d40dda3493..fb5e2e02234ca 100644 --- a/datafusion/core/src/execution/context/mod.rs +++ b/datafusion/core/src/execution/context/mod.rs @@ -1727,6 +1727,14 @@ impl FunctionRegistry for SessionContext { ) -> Result<()> { self.state.write().register_expr_planner(expr_planner) } + + fn udafs(&self) -> HashSet { + self.state.read().udafs() + } + + fn udwfs(&self) -> HashSet { + self.state.read().udwfs() + } } /// Create a new task context instance from SessionContext diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index a7b3bdeeace84..38212167f9316 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -1875,6 +1875,14 @@ impl FunctionRegistry for SessionState { self.expr_planners.push(expr_planner); Ok(()) } + + fn udafs(&self) -> HashSet { + self.aggregate_functions.keys().cloned().collect() + } + + fn udwfs(&self) -> HashSet { + self.window_functions.keys().cloned().collect() + } } impl OptimizerConfig for SessionState { diff --git a/datafusion/execution/src/task.rs b/datafusion/execution/src/task.rs index b11596c4a30f4..19f97f9e79ec2 100644 --- a/datafusion/execution/src/task.rs +++ b/datafusion/execution/src/task.rs @@ -201,6 +201,14 @@ impl FunctionRegistry for TaskContext { fn expr_planners(&self) -> Vec> { vec![] } + + fn udafs(&self) -> HashSet { + self.aggregate_functions.keys().cloned().collect() + } + + fn udwfs(&self) -> HashSet { + self.window_functions.keys().cloned().collect() + } } #[cfg(test)] diff --git a/datafusion/expr/src/registry.rs b/datafusion/expr/src/registry.rs index 4eb49710bcf85..8ea9e34dac12c 100644 --- a/datafusion/expr/src/registry.rs +++ b/datafusion/expr/src/registry.rs @@ -27,9 +27,25 @@ use std::sync::Arc; /// A registry knows how to build logical expressions out of user-defined function' names pub trait FunctionRegistry { - /// Set of all available udfs. + /// Returns names of all available scalar user defined functions. fn udfs(&self) -> HashSet; + /// Returns names of all available aggregate user defined functions. + fn udafs(&self) -> HashSet { + // This default implementation is provided temporarily + // to maintain backward compatibility for the 50.1 release. + // It will be reverted to a required method in future versions. + HashSet::default() + } + + /// Returns names of all available window user defined functions. + fn udwfs(&self) -> HashSet { + // This default implementation is provided temporarily + // to maintain backward compatibility for the 50.1 release. + // It will be reverted to a required method in future versions. + HashSet::default() + } + /// Returns a reference to the user defined scalar function (udf) named /// `name`. fn udf(&self, name: &str) -> Result>; @@ -200,4 +216,12 @@ impl FunctionRegistry for MemoryFunctionRegistry { fn expr_planners(&self) -> Vec> { vec![] } + + fn udafs(&self) -> HashSet { + self.udafs.keys().cloned().collect() + } + + fn udwfs(&self) -> HashSet { + self.udwfs.keys().cloned().collect() + } } diff --git a/datafusion/proto/src/bytes/mod.rs b/datafusion/proto/src/bytes/mod.rs index da01d89c0c3d1..16d65c419ae6c 100644 --- a/datafusion/proto/src/bytes/mod.rs +++ b/datafusion/proto/src/bytes/mod.rs @@ -170,6 +170,14 @@ impl Serializeable for Expr { fn expr_planners(&self) -> Vec> { vec![] } + + fn udafs(&self) -> std::collections::HashSet { + std::collections::HashSet::default() + } + + fn udwfs(&self) -> std::collections::HashSet { + std::collections::HashSet::default() + } } Expr::from_bytes_with_registry(&bytes, &PlaceHolderRegistry)?; diff --git a/datafusion/proto/src/bytes/registry.rs b/datafusion/proto/src/bytes/registry.rs index eae2425f8ac19..5d46d41f793ed 100644 --- a/datafusion/proto/src/bytes/registry.rs +++ b/datafusion/proto/src/bytes/registry.rs @@ -59,4 +59,12 @@ impl FunctionRegistry for NoRegistry { fn expr_planners(&self) -> Vec> { vec![] } + + fn udafs(&self) -> HashSet { + HashSet::new() + } + + fn udwfs(&self) -> HashSet { + HashSet::new() + } } diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs index 531883a6c4b20..bec7d90062eb8 100644 --- a/datafusion/spark/src/lib.rs +++ b/datafusion/spark/src/lib.rs @@ -53,6 +53,8 @@ //! # impl FunctionRegistry for SessionContext { //! # fn register_udf(&mut self, _udf: Arc) -> Result>> { Ok (None) } //! # fn udfs(&self) -> HashSet { unimplemented!() } +//! # fn udafs(&self) -> HashSet { unimplemented!() } +//! # fn udwfs(&self) -> HashSet { unimplemented!() } //! # fn udf(&self, _name: &str) -> Result> { unimplemented!() } //! # fn udaf(&self, name: &str) -> Result> {unimplemented!() } //! # fn udwf(&self, name: &str) -> Result> { unimplemented!() } From f7f6a2a7ef5338bd7685acfa520cf598a31cb288 Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Fri, 26 Sep 2025 18:08:49 +0800 Subject: [PATCH 141/177] change physical plan details loglevel from debug to trace for potential overflow --- datafusion/core/src/physical_planner.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 6618d9495d78f..be474cabef144 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -100,7 +100,7 @@ use async_trait::async_trait; use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper}; use futures::{StreamExt, TryStreamExt}; use itertools::{multiunzip, Itertools}; -use log::debug; +use log::{debug, trace}; use tokio::sync::Mutex; /// Physical query planner that converts a `LogicalPlan` to an @@ -2093,7 +2093,15 @@ impl DefaultPhysicalPlanner { "Optimized physical plan:\n{}\n", displayable(new_plan.as_ref()).indent(false) ); - debug!("Detailed optimized physical plan:\n{new_plan:?}"); + + // This is potentially very large, so only log at trace level, + // otherwise it can overflow the tokio runtime stack because without + // tree_maximum_render_width setting. + // + // For example: + // thread 'tokio-runtime-worker' has overflowed its stack + // fatal runtime error: stack overflow, aborting + trace!("Detailed optimized physical plan:\n{new_plan:?}"); Ok(new_plan) } From d198e90b3b4f2b477420f6c434a279e9e0a3af6d Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Fri, 26 Sep 2025 18:21:21 +0800 Subject: [PATCH 142/177] fix comments. --- datafusion/core/src/physical_planner.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index be474cabef144..dabbb1c6c84bb 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2095,8 +2095,7 @@ impl DefaultPhysicalPlanner { ); // This is potentially very large, so only log at trace level, - // otherwise it can overflow the tokio runtime stack because without - // tree_maximum_render_width setting. + // otherwise it can overflow the tokio runtime stack. // // For example: // thread 'tokio-runtime-worker' has overflowed its stack From a6cf5dec609c8c625a42b7067ff1a57c12ac61df Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Fri, 26 Sep 2025 22:43:40 +0800 Subject: [PATCH 143/177] debug to displayable --- datafusion/core/src/physical_planner.rs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index dabbb1c6c84bb..aaae319f27f22 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2094,13 +2094,10 @@ impl DefaultPhysicalPlanner { displayable(new_plan.as_ref()).indent(false) ); - // This is potentially very large, so only log at trace level, - // otherwise it can overflow the tokio runtime stack. - // - // For example: - // thread 'tokio-runtime-worker' has overflowed its stack - // fatal runtime error: stack overflow, aborting - trace!("Detailed optimized physical plan:\n{new_plan:?}"); + debug!( + "Detailed optimized physical plan:\n{}\n", + displayable(new_plan.as_ref()).indent(true) + ); Ok(new_plan) } From 172371bd8d15e8c058981894e90ddb7369e7e75d Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Fri, 26 Sep 2025 23:02:59 +0800 Subject: [PATCH 144/177] clippy --- datafusion/core/src/physical_planner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index aaae319f27f22..f6a19cc6928e1 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -100,7 +100,7 @@ use async_trait::async_trait; use datafusion_physical_plan::async_func::{AsyncFuncExec, AsyncMapper}; use futures::{StreamExt, TryStreamExt}; use itertools::{multiunzip, Itertools}; -use log::{debug, trace}; +use log::debug; use tokio::sync::Mutex; /// Physical query planner that converts a `LogicalPlan` to an From d7997804ad8a662af49672e923e7aee0aacaf844 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Tue, 14 Oct 2025 14:51:27 +0800 Subject: [PATCH 145/177] Impl `gather_filters_for_pushdown` for `CoalescePartitionsExec` (#18046) * Impl gather_filters_for_pushdown for CoalescePartitionsExec * add tests --- .../physical_optimizer/filter_pushdown/mod.rs | 134 +++++++++++++++++- .../filter_pushdown/util.rs | 2 +- .../physical-plan/src/coalesce_partitions.rs | 12 ++ 3 files changed, 146 insertions(+), 2 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs index eaf3be2b86edb..c3549060b6d5c 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs @@ -33,7 +33,11 @@ use datafusion::{ prelude::{ParquetReadOptions, SessionConfig, SessionContext}, scalar::ScalarValue, }; +use datafusion_catalog::memory::DataSourceExec; use datafusion_common::config::ConfigOptions; +use datafusion_datasource::{ + file_groups::FileGroup, file_scan_config::FileScanConfigBuilder, PartitionedFile, +}; use datafusion_execution::object_store::ObjectStoreUrl; use datafusion_expr::ScalarUDF; use datafusion_functions::math::random::RandomFunc; @@ -60,6 +64,8 @@ use futures::StreamExt; use object_store::{memory::InMemory, ObjectStore}; use util::{format_plan_for_test, OptimizationTest, TestNode, TestScanBuilder}; +use crate::physical_optimizer::filter_pushdown::util::TestSource; + mod util; #[test] @@ -834,6 +840,132 @@ async fn test_topk_dynamic_filter_pushdown_multi_column_sort() { assert!(stream.next().await.is_none()); } +#[tokio::test] +async fn test_topk_filter_passes_through_coalesce_partitions() { + // Create multiple batches for different partitions + let batches = vec![ + record_batch!( + ("a", Utf8, ["aa", "ab"]), + ("b", Utf8, ["bd", "bc"]), + ("c", Float64, [1.0, 2.0]) + ) + .unwrap(), + record_batch!( + ("a", Utf8, ["ac", "ad"]), + ("b", Utf8, ["bb", "ba"]), + ("c", Float64, [2.0, 1.0]) + ) + .unwrap(), + ]; + + // Create a source that supports all batches + let source = Arc::new(TestSource::new(true, batches)); + + let base_config = FileScanConfigBuilder::new( + ObjectStoreUrl::parse("test://").unwrap(), + Arc::clone(&schema()), + source, + ) + .with_file_groups(vec![ + // Partition 0 + FileGroup::new(vec![PartitionedFile::new("test1.parquet", 123)]), + // Partition 1 + FileGroup::new(vec![PartitionedFile::new("test2.parquet", 123)]), + ]) + .build(); + + let scan = DataSourceExec::from_data_source(base_config); + + // Add CoalescePartitionsExec to merge the two partitions + let coalesce = Arc::new(CoalescePartitionsExec::new(scan)) as Arc; + + // Add SortExec with TopK + let plan = Arc::new( + SortExec::new( + LexOrdering::new(vec![PhysicalSortExpr::new( + col("b", &schema()).unwrap(), + SortOptions::new(true, false), + )]) + .unwrap(), + coalesce, + ) + .with_fetch(Some(1)), + ) as Arc; + + // Test optimization - the filter SHOULD pass through CoalescePartitionsExec + // if it properly implements from_children (not all_unsupported) + insta::assert_snapshot!( + OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true), + @r" + OptimizationTest: + input: + - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] + - CoalescePartitionsExec + - DataSourceExec: file_groups={2 groups: [[test1.parquet], [test2.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true + output: + Ok: + - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] + - CoalescePartitionsExec + - DataSourceExec: file_groups={2 groups: [[test1.parquet], [test2.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] + " + ); +} + +#[tokio::test] +async fn test_topk_filter_passes_through_coalesce_batches() { + let batches = vec![ + record_batch!( + ("a", Utf8, ["aa", "ab"]), + ("b", Utf8, ["bd", "bc"]), + ("c", Float64, [1.0, 2.0]) + ) + .unwrap(), + record_batch!( + ("a", Utf8, ["ac", "ad"]), + ("b", Utf8, ["bb", "ba"]), + ("c", Float64, [2.0, 1.0]) + ) + .unwrap(), + ]; + + let scan = TestScanBuilder::new(schema()) + .with_support(true) + .with_batches(batches) + .build(); + + let coalesce_batches = + Arc::new(CoalesceBatchesExec::new(scan, 1024)) as Arc; + + // Add SortExec with TopK + let plan = Arc::new( + SortExec::new( + LexOrdering::new(vec![PhysicalSortExpr::new( + col("b", &schema()).unwrap(), + SortOptions::new(true, false), + )]) + .unwrap(), + coalesce_batches, + ) + .with_fetch(Some(1)), + ) as Arc; + + insta::assert_snapshot!( + OptimizationTest::new(Arc::clone(&plan), FilterPushdown::new_post_optimization(), true), + @r" + OptimizationTest: + input: + - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] + - CoalesceBatchesExec: target_batch_size=1024 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true + output: + Ok: + - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] + - CoalesceBatchesExec: target_batch_size=1024 + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] + " + ); +} + #[tokio::test] async fn test_hashjoin_dynamic_filter_pushdown() { use datafusion_common::JoinType; @@ -1478,7 +1610,7 @@ async fn test_topk_dynamic_filter_pushdown_integration() { ctx.sql( r" COPY ( - SELECT 1372708800 + value AS t + SELECT 1372708800 + value AS t FROM generate_series(0, 99999) ORDER BY t ) TO 'memory:///1.parquet' diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs index 7d0020b2e937f..2b009b3ed2184 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/util.rs @@ -111,7 +111,7 @@ pub struct TestSource { } impl TestSource { - fn new(support: bool, batches: Vec) -> Self { + pub fn new(support: bool, batches: Vec) -> Self { Self { support, metrics: ExecutionPlanMetricsSet::new(), diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index 685e751832eb4..21f0d61a81eb7 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -28,11 +28,14 @@ use super::{ Statistics, }; use crate::execution_plan::{CardinalityEffect, EvaluationType, SchedulingType}; +use crate::filter_pushdown::{FilterDescription, FilterPushdownPhase}; use crate::projection::{make_with_child, ProjectionExec}; use crate::{DisplayFormatType, ExecutionPlan, Partitioning}; +use datafusion_common::config::ConfigOptions; use datafusion_common::{internal_err, Result}; use datafusion_execution::TaskContext; +use datafusion_physical_expr::PhysicalExpr; /// Merge execution plan executes partitions in parallel and combines them into a single /// partition. No guarantees are made about the order of the resulting partition. @@ -270,6 +273,15 @@ impl ExecutionPlan for CoalescePartitionsExec { cache: self.cache.clone(), })) } + + fn gather_filters_for_pushdown( + &self, + _phase: FilterPushdownPhase, + parent_filters: Vec>, + _config: &ConfigOptions, + ) -> Result { + FilterDescription::from_children(parent_filters, &self.children()) + } } #[cfg(test)] From ab03bab1d06645a3e283f665e4d6e2658ab009c7 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Thu, 16 Oct 2025 14:28:58 +0800 Subject: [PATCH 146/177] Add independent configs for topk/join dynamic filter (#18090) * Add independent configs for topk/join dynamic filter * fix ci * update doc * fix typo --- datafusion/common/src/config.rs | 26 +- .../physical-plan/src/joins/hash_join/exec.rs | 2 +- datafusion/physical-plan/src/sorts/sort.rs | 2 +- .../dynamic_filter_pushdown_config.slt | 339 ++++++++++++++++++ .../test_files/information_schema.slt | 6 +- docs/source/user-guide/configs.md | 4 +- 6 files changed, 374 insertions(+), 5 deletions(-) create mode 100644 datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index 109ed8e4c4464..374e41aaf49ac 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -731,11 +731,21 @@ config_namespace! { /// past window functions, if possible pub enable_window_limits: bool, default = true - /// When set to true attempts to push down dynamic filters generated by operators into the file scan phase. + /// When set to true, the optimizer will attempt to push down TopK dynamic filters + /// into the file scan phase. + pub enable_topk_dynamic_filter_pushdown: bool, default = true + + /// When set to true, the optimizer will attempt to push down Join dynamic filters + /// into the file scan phase. + pub enable_join_dynamic_filter_pushdown: bool, default = true + + /// When set to true attempts to push down dynamic filters generated by operators (topk & join) into the file scan phase. /// For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer /// will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. /// This means that if we already have 10 timestamps in the year 2025 /// any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. + /// The config will suppress `enable_join_dynamic_filter_pushdown` & `enable_topk_dynamic_filter_pushdown` + /// So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. pub enable_dynamic_filter_pushdown: bool, default = true /// When set to true, the optimizer will insert filters before a join between @@ -1025,6 +1035,20 @@ impl ConfigOptions { }; if prefix == "datafusion" { + if key == "optimizer.enable_dynamic_filter_pushdown" { + let bool_value = value.parse::().map_err(|e| { + DataFusionError::Configuration(format!( + "Failed to parse '{value}' as bool: {e}", + )) + })?; + + { + self.optimizer.enable_dynamic_filter_pushdown = bool_value; + self.optimizer.enable_topk_dynamic_filter_pushdown = bool_value; + self.optimizer.enable_join_dynamic_filter_pushdown = bool_value; + } + return Ok(()); + } return ConfigField::set(self, key, value); } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 8b6767e8583d7..07dd84a3986ae 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -1137,7 +1137,7 @@ impl ExecutionPlan for HashJoinExec { // Add dynamic filters in Post phase if enabled if matches!(phase, FilterPushdownPhase::Post) - && config.optimizer.enable_dynamic_filter_pushdown + && config.optimizer.enable_join_dynamic_filter_pushdown { // Add actual dynamic filter to right side (probe side) let dynamic_filter = Self::create_dynamic_filter(&self.on); diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index e1a78e9369d92..c00c84c881247 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1376,7 +1376,7 @@ impl ExecutionPlan for SortExec { ChildFilterDescription::from_child(&parent_filters, self.input())?; if let Some(filter) = &self.filter { - if config.optimizer.enable_dynamic_filter_pushdown { + if config.optimizer.enable_topk_dynamic_filter_pushdown { child = child.with_self_filter(filter.read().expr()); } } diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt new file mode 100644 index 0000000000000..e5cd6d88b08f4 --- /dev/null +++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt @@ -0,0 +1,339 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Tests for dynamic filter pushdown configuration options +# - enable_topk_dynamic_filter_pushdown (for TopK dynamic filters) +# - enable_join_dynamic_filter_pushdown (for Join dynamic filters) +# - enable_dynamic_filter_pushdown (controls both) + +# Setup: Create parquet test files +statement ok +CREATE TABLE test_data(id INT, value INT, name VARCHAR) AS VALUES +(1, 100, 'a'), +(2, 200, 'b'), +(3, 300, 'c'), +(4, 400, 'd'), +(5, 500, 'e'), +(6, 600, 'f'), +(7, 700, 'g'), +(8, 800, 'h'), +(9, 900, 'i'), +(10, 1000, 'j'); + +statement ok +CREATE TABLE join_left(id INT, data VARCHAR) AS VALUES +(1, 'left1'), +(2, 'left2'), +(3, 'left3'), +(4, 'left4'), +(5, 'left5'); + +statement ok +CREATE TABLE join_right(id INT, info VARCHAR) AS VALUES +(1, 'right1'), +(3, 'right3'), +(5, 'right5'); + +# Copy data to parquet files +query I +COPY test_data TO 'test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet' STORED AS PARQUET; +---- +10 + +query I +COPY join_left TO 'test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet' STORED AS PARQUET; +---- +5 + +query I +COPY join_right TO 'test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet' STORED AS PARQUET; +---- +3 + +# Create external tables from parquet files +statement ok +CREATE EXTERNAL TABLE test_parquet(id INT, value INT, name VARCHAR) +STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet'; + +statement ok +CREATE EXTERNAL TABLE left_parquet(id INT, data VARCHAR) +STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet'; + +statement ok +CREATE EXTERNAL TABLE right_parquet(id INT, info VARCHAR) +STORED AS PARQUET +LOCATION 'test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet'; + +# Test 1: TopK dynamic filter pushdown with Parquet +query TT +EXPLAIN SELECT * FROM test_parquet ORDER BY value DESC LIMIT 3; +---- +logical_plan +01)Sort: test_parquet.value DESC NULLS FIRST, fetch=3 +02)--TableScan: test_parquet projection=[id, value, name] +physical_plan +01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ] + +# Disable TopK dynamic filter pushdown +statement ok +SET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = false; + +query TT +EXPLAIN SELECT * FROM test_parquet ORDER BY value DESC LIMIT 3; +---- +logical_plan +01)Sort: test_parquet.value DESC NULLS FIRST, fetch=3 +02)--TableScan: test_parquet projection=[id, value, name] +physical_plan +01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet + +# Re-enable for next tests +statement ok +SET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = true; + +# Test 2: Join dynamic filter pushdown with Parquet +query TT +EXPLAIN SELECT l.*, r.info +FROM left_parquet l +INNER JOIN right_parquet r ON l.id = r.id; +---- +logical_plan +01)Projection: l.id, l.data, r.info +02)--Inner Join: l.id = r.id +03)----SubqueryAlias: l +04)------TableScan: left_parquet projection=[id, data] +05)----SubqueryAlias: r +06)------TableScan: right_parquet projection=[id, info] +physical_plan +01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] + +# Disable Join dynamic filter pushdown +statement ok +SET datafusion.optimizer.enable_join_dynamic_filter_pushdown = false; + +# Without Join filter, HashJoin should NOT have filter=DynamicFilter +query TT +EXPLAIN SELECT l.*, r.info +FROM left_parquet l +INNER JOIN right_parquet r ON l.id = r.id; +---- +logical_plan +01)Projection: l.id, l.data, r.info +02)--Inner Join: l.id = r.id +03)----SubqueryAlias: l +04)------TableScan: left_parquet projection=[id, data] +05)----SubqueryAlias: r +06)------TableScan: right_parquet projection=[id, info] +physical_plan +01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet + +# Re-enable for next tests +statement ok +SET datafusion.optimizer.enable_join_dynamic_filter_pushdown = true; + +# Test 3: Test independent control + +# Disable TopK, keep Join enabled +statement ok +SET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = false; + +statement ok +SET datafusion.optimizer.enable_join_dynamic_filter_pushdown = true; + +# Join should still have dynamic filter +query TT +EXPLAIN SELECT l.*, r.info +FROM left_parquet l +INNER JOIN right_parquet r ON l.id = r.id; +---- +logical_plan +01)Projection: l.id, l.data, r.info +02)--Inner Join: l.id = r.id +03)----SubqueryAlias: l +04)------TableScan: left_parquet projection=[id, data] +05)----SubqueryAlias: r +06)------TableScan: right_parquet projection=[id, info] +physical_plan +01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] + +# Enable TopK, disable Join +statement ok +SET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = true; + +statement ok +SET datafusion.optimizer.enable_join_dynamic_filter_pushdown = false; + +# Join should NOT have dynamic filter +query TT +EXPLAIN SELECT l.*, r.info +FROM left_parquet l +INNER JOIN right_parquet r ON l.id = r.id; +---- +logical_plan +01)Projection: l.id, l.data, r.info +02)--Inner Join: l.id = r.id +03)----SubqueryAlias: l +04)------TableScan: left_parquet projection=[id, data] +05)----SubqueryAlias: r +06)------TableScan: right_parquet projection=[id, info] +physical_plan +01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet + +# Test 4: Backward compatibility + +# First, set both new configs to specific values +statement ok +SET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = true; + +statement ok +SET datafusion.optimizer.enable_join_dynamic_filter_pushdown = true; + +statement ok +set datafusion.catalog.information_schema = true + +# Setting the config should override both +statement ok +SET datafusion.optimizer.enable_dynamic_filter_pushdown = false; + +# Verify both configs are now false +query T +SELECT value FROM information_schema.df_settings +WHERE name = 'datafusion.optimizer.enable_topk_dynamic_filter_pushdown'; +---- +false + +query T +SELECT value FROM information_schema.df_settings +WHERE name = 'datafusion.optimizer.enable_join_dynamic_filter_pushdown'; +---- +false + +statement ok +set datafusion.catalog.information_schema = false + +# Join should NOT have dynamic filter +query TT +EXPLAIN SELECT l.*, r.info +FROM left_parquet l +INNER JOIN right_parquet r ON l.id = r.id; +---- +logical_plan +01)Projection: l.id, l.data, r.info +02)--Inner Join: l.id = r.id +03)----SubqueryAlias: l +04)------TableScan: left_parquet projection=[id, data] +05)----SubqueryAlias: r +06)------TableScan: right_parquet projection=[id, info] +physical_plan +01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet + +# Re-enable +statement ok +SET datafusion.optimizer.enable_dynamic_filter_pushdown = true; + +statement ok +set datafusion.catalog.information_schema = true + +# Verify both configs are now true +query T +SELECT value FROM information_schema.df_settings +WHERE name = 'datafusion.optimizer.enable_topk_dynamic_filter_pushdown'; +---- +true + +query T +SELECT value FROM information_schema.df_settings +WHERE name = 'datafusion.optimizer.enable_join_dynamic_filter_pushdown'; +---- +true + +statement ok +set datafusion.catalog.information_schema = false + +# Join should have dynamic filter again +query TT +EXPLAIN SELECT l.*, r.info +FROM left_parquet l +INNER JOIN right_parquet r ON l.id = r.id; +---- +logical_plan +01)Projection: l.id, l.data, r.info +02)--Inner Join: l.id = r.id +03)----SubqueryAlias: l +04)------TableScan: left_parquet projection=[id, data] +05)----SubqueryAlias: r +06)------TableScan: right_parquet projection=[id, info] +physical_plan +01)ProjectionExec: expr=[id@1 as id, data@2 as data, info@0 as info] +02)--CoalesceBatchesExec: target_batch_size=8192 +03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] +04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] + +# Cleanup + +statement ok +DROP TABLE test_data; + +statement ok +DROP TABLE join_left; + +statement ok +DROP TABLE join_right; + +statement ok +DROP TABLE test_parquet; + +statement ok +DROP TABLE left_parquet; + +statement ok +DROP TABLE right_parquet; + +# Reset configs to defaults +statement ok +SET datafusion.optimizer.enable_topk_dynamic_filter_pushdown = true; + +statement ok +SET datafusion.optimizer.enable_join_dynamic_filter_pushdown = true; + +statement ok +SET datafusion.optimizer.enable_dynamic_filter_pushdown = true; diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 23b8e16a31159..a9e9c74676b12 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -287,8 +287,10 @@ datafusion.optimizer.allow_symmetric_joins_without_pruning true datafusion.optimizer.default_filter_selectivity 20 datafusion.optimizer.enable_distinct_aggregation_soft_limit true datafusion.optimizer.enable_dynamic_filter_pushdown true +datafusion.optimizer.enable_join_dynamic_filter_pushdown true datafusion.optimizer.enable_round_robin_repartition true datafusion.optimizer.enable_topk_aggregation true +datafusion.optimizer.enable_topk_dynamic_filter_pushdown true datafusion.optimizer.enable_window_limits true datafusion.optimizer.expand_views_at_output false datafusion.optimizer.filter_null_join_keys false @@ -400,9 +402,11 @@ datafusion.format.types_info false Show types in visual representation batches datafusion.optimizer.allow_symmetric_joins_without_pruning true Should DataFusion allow symmetric hash joins for unbounded data sources even when its inputs do not have any ordering or filtering If the flag is not enabled, the SymmetricHashJoin operator will be unable to prune its internal buffers, resulting in certain join types - such as Full, Left, LeftAnti, LeftSemi, Right, RightAnti, and RightSemi - being produced only at the end of the execution. This is not typical in stream processing. Additionally, without proper design for long runner execution, all types of joins may encounter out-of-memory errors. datafusion.optimizer.default_filter_selectivity 20 The default filter selectivity used by Filter Statistics when an exact selectivity cannot be determined. Valid values are between 0 (no selectivity) and 100 (all rows are selected). datafusion.optimizer.enable_distinct_aggregation_soft_limit true When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. -datafusion.optimizer.enable_dynamic_filter_pushdown true When set to true attempts to push down dynamic filters generated by operators into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. +datafusion.optimizer.enable_dynamic_filter_pushdown true When set to true attempts to push down dynamic filters generated by operators (topk & join) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown` & `enable_topk_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. +datafusion.optimizer.enable_join_dynamic_filter_pushdown true When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. datafusion.optimizer.enable_round_robin_repartition true When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores datafusion.optimizer.enable_topk_aggregation true When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible +datafusion.optimizer.enable_topk_dynamic_filter_pushdown true When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. datafusion.optimizer.enable_window_limits true When set to true, the optimizer will attempt to push limit operations past window functions, if possible datafusion.optimizer.expand_views_at_output false When set to true, if the returned type is a view type then the output will be coerced to a non-view. Coerces `Utf8View` to `LargeUtf8`, and `BinaryView` to `LargeBinary`. datafusion.optimizer.filter_null_join_keys false When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index d9fb2b850fb5f..4c1dcb43766af 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -130,7 +130,9 @@ The following configuration settings are available: | datafusion.optimizer.enable_round_robin_repartition | true | When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores | | datafusion.optimizer.enable_topk_aggregation | true | When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible | | datafusion.optimizer.enable_window_limits | true | When set to true, the optimizer will attempt to push limit operations past window functions, if possible | -| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. | +| datafusion.optimizer.enable_topk_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_join_dynamic_filter_pushdown | true | When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. | +| datafusion.optimizer.enable_dynamic_filter_pushdown | true | When set to true attempts to push down dynamic filters generated by operators (topk & join) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown` & `enable_topk_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. | | datafusion.optimizer.filter_null_join_keys | false | When set to true, the optimizer will insert filters before a join between a nullable and non-nullable column to filter out nulls on the nullable side. This filter can add additional overhead when the file format does not fully support predicate push down. | | datafusion.optimizer.repartition_aggregations | true | Should DataFusion repartition data using the aggregate keys to execute aggregates in parallel using the provided `target_partitions` level | | datafusion.optimizer.repartition_file_min_size | 10485760 | Minimum total files size in bytes to perform file scan repartitioning. | From 9fdb7b3ea5ee3c4cfc1ed131d9a24053dfbc5724 Mon Sep 17 00:00:00 2001 From: Qi Zhu Date: Thu, 23 Oct 2025 23:57:14 +0800 Subject: [PATCH 147/177] CoalescePartitionsExec fetch is not consistent with one partition and more than one partition (#18245) ## Which issue does this PR close? - Closes [#18244](https://github.com/apache/datafusion/issues/18244) ## Rationale for this change In our internal project, the limit will not return right number when CoalescePartitionsExec follow up by our customer operator which is only one partition output. After my investigation i found: CoalescePartitionsExec fetch is not consistent with one partition and more than one partition. ## What changes are included in this PR? Make CoalescePartitionsExec fetch should be consistent when the partition number changes. ## Are these changes tested? Yes ## Are there any user-facing changes? No (cherry picked from commit be85bf41aed7c1eeba842af31997dfbad07be973) --- .../physical-plan/src/coalesce_partitions.rs | 120 +++++++++++++++++- 1 file changed, 118 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-plan/src/coalesce_partitions.rs b/datafusion/physical-plan/src/coalesce_partitions.rs index 21f0d61a81eb7..a0ca6898bd3c8 100644 --- a/datafusion/physical-plan/src/coalesce_partitions.rs +++ b/datafusion/physical-plan/src/coalesce_partitions.rs @@ -170,8 +170,18 @@ impl ExecutionPlan for CoalescePartitionsExec { "CoalescePartitionsExec requires at least one input partition" ), 1 => { - // bypass any threading / metrics if there is a single partition - self.input.execute(0, context) + // single-partition path: execute child directly, but ensure fetch is respected + // (wrap with ObservedStream only if fetch is present so we don't add overhead otherwise) + let child_stream = self.input.execute(0, context)?; + if self.fetch.is_some() { + let baseline_metrics = BaselineMetrics::new(&self.metrics, partition); + return Ok(Box::pin(ObservedStream::new( + child_stream, + baseline_metrics, + self.fetch, + ))); + } + Ok(child_stream) } _ => { let baseline_metrics = BaselineMetrics::new(&self.metrics, partition); @@ -361,4 +371,110 @@ mod tests { collect(coalesce_partitions_exec, task_ctx).await.unwrap(); } + + #[tokio::test] + async fn test_single_partition_with_fetch() -> Result<()> { + let task_ctx = Arc::new(TaskContext::default()); + + // Use existing scan_partitioned with 1 partition (returns 100 rows per partition) + let input = test::scan_partitioned(1); + + // Test with fetch=3 + let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(3)); + + let stream = coalesce.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!(row_count, 3, "Should only return 3 rows due to fetch=3"); + + Ok(()) + } + + #[tokio::test] + async fn test_multi_partition_with_fetch_one() -> Result<()> { + let task_ctx = Arc::new(TaskContext::default()); + + // Create 4 partitions, each with 100 rows + // This simulates the real-world scenario where each partition has data + let input = test::scan_partitioned(4); + + // Test with fetch=1 (the original bug: was returning multiple rows instead of 1) + let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(1)); + + let stream = coalesce.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!( + row_count, 1, + "Should only return 1 row due to fetch=1, not one per partition" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_single_partition_without_fetch() -> Result<()> { + let task_ctx = Arc::new(TaskContext::default()); + + // Use scan_partitioned with 1 partition + let input = test::scan_partitioned(1); + + // Test without fetch (should return all rows) + let coalesce = CoalescePartitionsExec::new(input); + + let stream = coalesce.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!( + row_count, 100, + "Should return all 100 rows when fetch is None" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_single_partition_fetch_larger_than_batch() -> Result<()> { + let task_ctx = Arc::new(TaskContext::default()); + + // Use scan_partitioned with 1 partition (returns 100 rows) + let input = test::scan_partitioned(1); + + // Test with fetch larger than available rows + let coalesce = CoalescePartitionsExec::new(input).with_fetch(Some(200)); + + let stream = coalesce.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!( + row_count, 100, + "Should return all available rows (100) when fetch (200) is larger" + ); + + Ok(()) + } + + #[tokio::test] + async fn test_multi_partition_fetch_exact_match() -> Result<()> { + let task_ctx = Arc::new(TaskContext::default()); + + // Create 4 partitions, each with 100 rows + let num_partitions = 4; + let csv = test::scan_partitioned(num_partitions); + + // Test with fetch=400 (exactly all rows) + let coalesce = CoalescePartitionsExec::new(csv).with_fetch(Some(400)); + + let stream = coalesce.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + let row_count: usize = batches.iter().map(|batch| batch.num_rows()).sum(); + assert_eq!(row_count, 400, "Should return exactly 400 rows"); + + Ok(()) + } } From 50e9973234f3384e90172c27582f01ece685fde4 Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Fri, 24 Oct 2025 13:28:37 +0800 Subject: [PATCH 148/177] fix DynamicFilterPhysicalExpr test --- .../core/tests/physical_optimizer/filter_pushdown/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs index c3549060b6d5c..32c4f030b0fc6 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs @@ -906,7 +906,7 @@ async fn test_topk_filter_passes_through_coalesce_partitions() { Ok: - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec - - DataSourceExec: file_groups={2 groups: [[test1.parquet], [test2.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] + - DataSourceExec: file_groups={2 groups: [[test1.parquet], [test2.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilterPhysicalExpr [ true ] " ); } @@ -961,7 +961,7 @@ async fn test_topk_filter_passes_through_coalesce_batches() { Ok: - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalesceBatchesExec: target_batch_size=1024 - - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] + - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilterPhysicalExpr [ true ] " ); } From f520827adf84ef40ae94f3f8985cec123cfeecfe Mon Sep 17 00:00:00 2001 From: Vyquos <75266055+Vyquos@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:43:22 +0000 Subject: [PATCH 149/177] fix: Remove parquet encryption feature from root deps (#17700) This fix relates to issue #16650 by completing #16649 . (cherry picked from commit 2ba2f1c4601027574bd793e4cd14665c915754b9) --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 53c35ed35f0d0..93dcfeb4da25e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -161,7 +161,6 @@ parquet = { version = "56.0.0", default-features = false, features = [ "arrow", "async", "object_store", - "encryption", ] } pbjson = { version = "0.7.0" } pbjson-types = "0.7" From 6420a014d758a76c4fc91df549841309734bdfee Mon Sep 17 00:00:00 2001 From: Qi Zhu <821684824@qq.com> Date: Tue, 4 Nov 2025 17:50:00 +0800 Subject: [PATCH 150/177] Continue to fix the datafusion will default to use encryption --- datafusion/sqllogictest/Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index 339904953385c..c242a5f0f4330 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -43,7 +43,7 @@ bigdecimal = { workspace = true } bytes = { workspace = true, optional = true } chrono = { workspace = true, optional = true } clap = { version = "4.5.47", features = ["derive", "env"] } -datafusion = { workspace = true, default-features = true, features = ["avro", "parquet_encryption"] } +datafusion = { workspace = true, default-features = true, features = ["avro"] } datafusion-spark = { workspace = true, default-features = true } datafusion-substrait = { workspace = true, default-features = true } futures = { workspace = true } @@ -78,6 +78,9 @@ postgres = [ "testcontainers-modules", "tokio-postgres", ] +parquet_encryption = [ + "datafusion/parquet_encryption" +] [dev-dependencies] env_logger = { workspace = true } From 375d2e676074c4fcffafce9f6511761ecb29bbc6 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Fri, 14 Nov 2025 15:11:28 +0800 Subject: [PATCH 151/177] Make CI green --- datafusion-testing | 2 +- datafusion/common/src/scalar/mod.rs | 2 +- datafusion/core/src/datasource/listing/table.rs | 2 +- .../core/tests/physical_optimizer/filter_pushdown/mod.rs | 2 +- datafusion/physical-expr/src/expressions/case.rs | 1 - datafusion/sqllogictest/Cargo.toml | 2 +- .../test_files/dynamic_filter_pushdown_config.slt | 8 ++++---- docs/source/user-guide/introduction.md | 2 +- typos.toml | 6 +++++- 9 files changed, 15 insertions(+), 12 deletions(-) diff --git a/datafusion-testing b/datafusion-testing index f72ac4075ada5..e9f9e22ccf091 160000 --- a/datafusion-testing +++ b/datafusion-testing @@ -1 +1 @@ -Subproject commit f72ac4075ada5ea9810551bc0c3e3161c61204a2 +Subproject commit e9f9e22ccf09145a7368f80fd6a871f11e2b4481 diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index 4d88f5a66732e..91058575723e5 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -2387,7 +2387,7 @@ impl ScalarValue { Arc::new(array) } // explicitly enumerate unsupported types so newly added - // types must be aknowledged, Time32 and Time64 types are + // types must be acknowledged, Time32 and Time64 types are // not supported if the TimeUnit is not valid (Time32 can // only be used with Second and Millisecond, Time64 only // with Microsecond and Nanosecond) diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 690ce31d0dc76..14b5bfa54eda2 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -1131,7 +1131,7 @@ impl ListingTable { } } -// Expressions can be used for parttion pruning if they can be evaluated using +// Expressions can be used for partition pruning if they can be evaluated using // only the partition columns and there are partition columns. fn can_be_evaluated_for_partition_pruning( partition_column_names: &[&str], diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs index 32c4f030b0fc6..ad6abec8cadca 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs @@ -1234,7 +1234,7 @@ async fn test_hashjoin_dynamic_filter_pushdown_partitioned() { Arc::new(CoalesceBatchesExec::new(hash_join, 8192)) as Arc; // Top-level CoalescePartitionsExec let cp = Arc::new(CoalescePartitionsExec::new(cb)) as Arc; - // Add a sort for determistic output + // Add a sort for deterministic output let plan = Arc::new(SortExec::new( LexOrdering::new(vec![PhysicalSortExpr::new( col("a", &probe_side_schema).unwrap(), diff --git a/datafusion/physical-expr/src/expressions/case.rs b/datafusion/physical-expr/src/expressions/case.rs index 65a2108266647..5409cfe8e7e45 100644 --- a/datafusion/physical-expr/src/expressions/case.rs +++ b/datafusion/physical-expr/src/expressions/case.rs @@ -1070,7 +1070,6 @@ mod tests { .into_iter() .collect(); - //let valid_array = vec![true, false, false, true, false, tru let null_buffer = Buffer::from([0b00101001u8]); let load4 = load4 .into_data() diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index c242a5f0f4330..2b92bb013c47f 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -79,7 +79,7 @@ postgres = [ "tokio-postgres", ] parquet_encryption = [ - "datafusion/parquet_encryption" + "datafusion/parquet_encryption", ] [dev-dependencies] diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt index e5cd6d88b08f4..00696fc4fb4f4 100644 --- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt +++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt @@ -89,7 +89,7 @@ logical_plan 02)--TableScan: test_parquet projection=[id, value, name] physical_plan 01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false] -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ] +02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] # Disable TopK dynamic filter pushdown statement ok @@ -127,7 +127,7 @@ physical_plan 02)--CoalesceBatchesExec: target_batch_size=8192 03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet -05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] # Disable Join dynamic filter pushdown statement ok @@ -184,7 +184,7 @@ physical_plan 02)--CoalesceBatchesExec: target_batch_size=8192 03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet -05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] # Enable TopK, disable Join statement ok @@ -306,7 +306,7 @@ physical_plan 02)--CoalesceBatchesExec: target_batch_size=8192 03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet -05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] +05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] # Cleanup diff --git a/docs/source/user-guide/introduction.md b/docs/source/user-guide/introduction.md index 040405f8f63e7..68164c1cbfedd 100644 --- a/docs/source/user-guide/introduction.md +++ b/docs/source/user-guide/introduction.md @@ -86,7 +86,7 @@ Here are some example systems built using DataFusion: By using DataFusion, projects are freed to focus on their specific features, and avoid reimplementing general (but still necessary) features such as an expression representation, standard optimizations, -parellelized streaming execution plans, file format support, etc. +parallelized streaming execution plans, file format support, etc. ## Known Users diff --git a/typos.toml b/typos.toml index 46f21febcf86b..09c5c55c452ab 100644 --- a/typos.toml +++ b/typos.toml @@ -34,6 +34,9 @@ alph = "alph" wih = "wih" Ded = "Ded" +# From SLT README +nteger = "nteger" + [files] extend-exclude = [ "*.slt", @@ -42,5 +45,6 @@ extend-exclude = [ "*.sql", "dev/changelog/**", "benchmarks/**", - "*.csv" + "*.csv", + "docs/source/contributor-guide/governance.md" ] From f37ac205f63c5ad9bd19e065d59fa555c825aca5 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sat, 18 Oct 2025 10:25:20 -0700 Subject: [PATCH 152/177] Make CI green --- .github/workflows/audit.yml | 16 +++++----- .github/workflows/rust.yml | 29 ++++++++++++++++++- Cargo.toml | 1 + datafusion/datasource-parquet/src/opener.rs | 6 ++++ datafusion/datasource-parquet/src/source.rs | 3 ++ .../test_files/encrypted_parquet.slt | 8 ++--- 6 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 2f12a8b5d2209..b525afa4acd7b 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -23,25 +23,27 @@ concurrency: on: push: + branches: + - main paths: - "**/Cargo.toml" - "**/Cargo.lock" - branches: - - main pull_request: paths: - "**/Cargo.toml" - "**/Cargo.lock" + merge_group: + jobs: security_audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install cargo-audit - run: cargo install cargo-audit + uses: taiki-e/install-action@f535147c22906d77695e11cb199e764aa610a4fc # v2.62.46 + with: + tool: cargo-audit - name: Run audit check - # Ignored until https://github.com/apache/datafusion/issues/15571 - # ignored py03 warning until arrow 55 upgrade - run: cargo audit --ignore RUSTSEC-2024-0370 --ignore RUSTSEC-2025-0020 --ignore RUSTSEC-2025-0047 + run: cargo audit --ignore RUSTSEC-2025-0111 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 05a6d70f0278a..6aaff95e74a5f 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -266,7 +266,21 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust + volumes: + - /usr/local:/host/usr/local steps: + - name: Remove unnecessary preinstalled software + run: | + echo "Disk space before cleanup:" + df -h + # remove tool cache: about 8.5GB (github has host /opt/hostedtoolcache mounted as /__t) + rm -rf /__t/* || true + # remove Haskell runtime: about 6.3GB (host /usr/local/.ghcup) + rm -rf /host/usr/local/.ghcup || true + # remove Android library: about 7.8GB (host /usr/local/lib/android) + rm -rf /host/usr/local/lib/android || true + echo "Disk space after cleanup:" + df -h - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: submodules: true @@ -347,6 +361,19 @@ jobs: with: save-if: ${{ github.ref_name == 'main' }} shared-key: "amd-ci-linux-test-example" + - name: Remove unnecessary preinstalled software + run: | + echo "Disk space before cleanup:" + df -h + apt-get clean + rm -rf /__t/CodeQL + rm -rf /__t/PyPy + rm -rf /__t/Java_Temurin-Hotspot_jdk + rm -rf /__t/Python + rm -rf /__t/go + rm -rf /__t/Ruby + echo "Disk space after cleanup:" + df -h - name: Run examples run: | # test datafusion-sql examples @@ -444,7 +471,7 @@ jobs: export RUST_MIN_STACK=20971520 export TPCH_DATA=`realpath datafusion/sqllogictest/test_files/tpch/data` cargo test plan_q --package datafusion-benchmarks --profile ci --features=ci -- --test-threads=1 - INCLUDE_TPCH=true cargo test --features backtrace --profile ci --package datafusion-sqllogictest --test sqllogictests + INCLUDE_TPCH=true cargo test --features backtrace,parquet_encryption --profile ci --package datafusion-sqllogictest --test sqllogictests - name: Verify Working Directory Clean run: git diff --exit-code diff --git a/Cargo.toml b/Cargo.toml index 93dcfeb4da25e..49affe557a328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,6 +198,7 @@ rpath = false strip = false # Retain debug info for flamegraphs [profile.ci] +debug = false inherits = "dev" incremental = false diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs index 93a3d4af54326..664b86e964eba 100644 --- a/datafusion/datasource-parquet/src/opener.rs +++ b/datafusion/datasource-parquet/src/opener.rs @@ -98,6 +98,7 @@ pub(super) struct ParquetOpener { /// Coerce INT96 timestamps to specific TimeUnit pub coerce_int96: Option, /// Optional parquet FileDecryptionProperties + #[cfg(feature = "parquet_encryption")] pub file_decryption_properties: Option>, /// Rewrite expressions in the context of the file schema pub(crate) expr_adapter_factory: Option>, @@ -151,9 +152,11 @@ impl FileOpener for ParquetOpener { let mut predicate_file_schema = Arc::clone(&self.logical_file_schema); let enable_page_index = self.enable_page_index; + #[cfg(feature = "parquet_encryption")] let encryption_context = self.get_encryption_context(); Ok(Box::pin(async move { + #[cfg(feature = "parquet_encryption")] let file_decryption_properties = encryption_context .get_file_decryption_properties(&file_location) .await?; @@ -502,6 +505,7 @@ where } #[derive(Default)] +#[cfg_attr(not(feature = "parquet_encryption"), allow(dead_code))] struct EncryptionContext { #[cfg(feature = "parquet_encryption")] file_decryption_properties: Option>, @@ -544,6 +548,7 @@ impl EncryptionContext { } #[cfg(not(feature = "parquet_encryption"))] +#[allow(dead_code)] impl EncryptionContext { async fn get_file_decryption_properties( &self, @@ -563,6 +568,7 @@ impl ParquetOpener { } #[cfg(not(feature = "parquet_encryption"))] + #[allow(dead_code)] fn get_encryption_context(&self) -> EncryptionContext { EncryptionContext::default() } diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index 007c239ef4928..644cea85ca0a9 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -52,6 +52,7 @@ use datafusion_physical_plan::metrics::Count; use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_physical_plan::DisplayFormatType; +#[cfg(feature = "parquet_encryption")] use datafusion_common::encryption::map_config_decryption_to_decryption; #[cfg(feature = "parquet_encryption")] use datafusion_execution::parquet_encryption::EncryptionFactory; @@ -541,6 +542,7 @@ impl FileSource for ParquetSource { Arc::new(DefaultParquetFileReaderFactory::new(object_store)) as _ }); + #[cfg(feature = "parquet_encryption")] let file_decryption_properties = self .table_parquet_options() .crypto @@ -576,6 +578,7 @@ impl FileSource for ParquetSource { enable_row_group_stats_pruning: self.table_parquet_options.global.pruning, schema_adapter_factory, coerce_int96, + #[cfg(feature = "parquet_encryption")] file_decryption_properties, expr_adapter_factory, #[cfg(feature = "parquet_encryption")] diff --git a/datafusion/sqllogictest/test_files/encrypted_parquet.slt b/datafusion/sqllogictest/test_files/encrypted_parquet.slt index d580b7d1ad2b8..326d7f42d3c83 100644 --- a/datafusion/sqllogictest/test_files/encrypted_parquet.slt +++ b/datafusion/sqllogictest/test_files/encrypted_parquet.slt @@ -29,11 +29,11 @@ STORED AS PARQUET LOCATION 'test_files/scratch/encrypted_parquet/' OPTIONS ( -- Encryption properties 'format.crypto.file_encryption.encrypt_footer' 'true', 'format.crypto.file_encryption.footer_key_as_hex' '30313233343536373839303132333435', -- b"0123456789012345" - 'format.crypto.file_encryption.column_key_as_hex::double_field' '31323334353637383930313233343530', -- b"1234567890123450" - 'format.crypto.file_encryption.column_key_as_hex::float_field' '31323334353637383930313233343531', -- b"1234567890123451" + 'format.crypto.file_encryption.column_key_as_hex::double_field' '31323334353637383930313233343530', -- b"1234567890123450" + 'format.crypto.file_encryption.column_key_as_hex::float_field' '31323334353637383930313233343531', -- b"1234567890123451" -- Decryption properties - 'format.crypto.file_decryption.footer_key_as_hex' '30313233343536373839303132333435', -- b"0123456789012345" - 'format.crypto.file_decryption.column_key_as_hex::double_field' '31323334353637383930313233343530', -- b"1234567890123450" + 'format.crypto.file_decryption.footer_key_as_hex' '30313233343536373839303132333435', -- b"0123456789012345" + 'format.crypto.file_decryption.column_key_as_hex::double_field' '31323334353637383930313233343530', -- b"1234567890123450" 'format.crypto.file_decryption.column_key_as_hex::float_field' '31323334353637383930313233343531', -- b"1234567890123451" ) From 19bbdff78f0cabeb73a3376a852eabf186382292 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Fri, 31 Oct 2025 15:06:33 +0800 Subject: [PATCH 153/177] Support row group limit pruning --- datafusion/core/tests/parquet/mod.rs | 52 ++- .../core/tests/parquet/row_group_pruning.rs | 336 +++++++++++++++- datafusion/datasource-parquet/src/metrics.rs | 14 + datafusion/datasource-parquet/src/opener.rs | 6 +- .../src/row_group_filter.rs | 96 ++++- datafusion/execution/src/memory_pool/pool.rs | 2 +- .../physical-expr/src/simplifier/mod.rs | 15 +- .../physical-expr/src/simplifier/not.rs | 377 ++++++++++++++++++ datafusion/pruning/src/pruning_predicate.rs | 1 - 9 files changed, 881 insertions(+), 18 deletions(-) create mode 100644 datafusion/physical-expr/src/simplifier/not.rs diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index c44d14abd381a..dad8472dedc42 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -150,6 +150,11 @@ impl TestOutput { self.metric_value("row_groups_matched_statistics") } + /// The number of row_groups fully matched by statistics + fn row_groups_fully_matched_statistics(&self) -> Option { + self.metric_value("row_groups_fully_matched_statistics") + } + /// The number of row_groups pruned by statistics fn row_groups_pruned_statistics(&self) -> Option { self.metric_value("row_groups_pruned_statistics") @@ -178,6 +183,11 @@ impl TestOutput { self.metric_value("page_index_rows_pruned") } + /// The number of row groups pruned by limit pruning + fn limit_pruned_row_groups(&self) -> Option { + self.metric_value("limit_pruned_row_groups") + } + fn description(&self) -> String { format!( "Input:\n{}\nQuery:\n{}\nOutput:\n{}\nMetrics:\n{}", @@ -191,20 +201,41 @@ impl TestOutput { /// and the appropriate scenario impl ContextWithParquet { async fn new(scenario: Scenario, unit: Unit) -> Self { - Self::with_config(scenario, unit, SessionConfig::new()).await + Self::with_config(scenario, unit, SessionConfig::new(), None, None).await + } + + /// Set custom schema and batches for the test + pub async fn with_custom_data( + scenario: Scenario, + unit: Unit, + schema: Arc, + batches: Vec, + ) -> Self { + Self::with_config( + scenario, + unit, + SessionConfig::new(), + Some(schema), + Some(batches), + ) + .await } async fn with_config( scenario: Scenario, unit: Unit, mut config: SessionConfig, + custom_schema: Option>, + custom_batches: Option>, ) -> Self { // Use a single partition for deterministic results no matter how many CPUs the host has config = config.with_target_partitions(1); let file = match unit { Unit::RowGroup(row_per_group) => { config = config.with_parquet_bloom_filter_pruning(true); - make_test_file_rg(scenario, row_per_group).await + config.options_mut().execution.parquet.pushdown_filters = true; + make_test_file_rg(scenario, row_per_group, custom_schema, custom_batches) + .await } Unit::Page(row_per_page) => { config = config.with_parquet_page_index_pruning(true); @@ -1030,7 +1061,12 @@ fn create_data_batch(scenario: Scenario) -> Vec { } /// Create a test parquet file with various data types -async fn make_test_file_rg(scenario: Scenario, row_per_group: usize) -> NamedTempFile { +async fn make_test_file_rg( + scenario: Scenario, + row_per_group: usize, + custom_schema: Option>, + custom_batches: Option>, +) -> NamedTempFile { let mut output_file = tempfile::Builder::new() .prefix("parquet_pruning") .suffix(".parquet") @@ -1043,8 +1079,14 @@ async fn make_test_file_rg(scenario: Scenario, row_per_group: usize) -> NamedTem .set_statistics_enabled(EnabledStatistics::Page) .build(); - let batches = create_data_batch(scenario); - let schema = batches[0].schema(); + let (batches, schema) = + if let (Some(schema), Some(batches)) = (custom_schema, custom_batches) { + (batches, schema) + } else { + let batches = create_data_batch(scenario); + let schema = batches[0].schema(); + (batches, schema) + }; let mut writer = ArrowWriter::try_new(&mut output_file, schema, Some(props)).unwrap(); diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs index 44409166d3ce3..42c4488d8444a 100644 --- a/datafusion/core/tests/parquet/row_group_pruning.rs +++ b/datafusion/core/tests/parquet/row_group_pruning.rs @@ -18,8 +18,12 @@ //! This file contains an end to end test of parquet pruning. It writes //! data into a parquet file and then verifies row groups are pruned as //! expected. +use std::sync::Arc; + +use arrow::array::{ArrayRef, Int32Array, RecordBatch}; +use arrow_schema::{DataType, Field, Schema}; use datafusion::prelude::SessionConfig; -use datafusion_common::ScalarValue; +use datafusion_common::{DataFusionError, ScalarValue}; use itertools::Itertools; use crate::parquet::Unit::RowGroup; @@ -30,10 +34,12 @@ struct RowGroupPruningTest { query: String, expected_errors: Option, expected_row_group_matched_by_statistics: Option, + expected_row_group_fully_matched_by_statistics: Option, expected_row_group_pruned_by_statistics: Option, expected_files_pruned_by_statistics: Option, expected_row_group_matched_by_bloom_filter: Option, expected_row_group_pruned_by_bloom_filter: Option, + expected_limit_pruned_row_groups: Option, expected_rows: usize, } impl RowGroupPruningTest { @@ -45,9 +51,11 @@ impl RowGroupPruningTest { expected_errors: None, expected_row_group_matched_by_statistics: None, expected_row_group_pruned_by_statistics: None, + expected_row_group_fully_matched_by_statistics: None, expected_files_pruned_by_statistics: None, expected_row_group_matched_by_bloom_filter: None, expected_row_group_pruned_by_bloom_filter: None, + expected_limit_pruned_row_groups: None, expected_rows: 0, } } @@ -76,6 +84,15 @@ impl RowGroupPruningTest { self } + // Set the expected fully matched row groups by statistics + fn with_fully_matched_by_stats( + mut self, + fully_matched_by_stats: Option, + ) -> Self { + self.expected_row_group_fully_matched_by_statistics = fully_matched_by_stats; + self + } + // Set the expected pruned row groups by statistics fn with_pruned_by_stats(mut self, pruned_by_stats: Option) -> Self { self.expected_row_group_pruned_by_statistics = pruned_by_stats; @@ -99,6 +116,11 @@ impl RowGroupPruningTest { self } + fn with_limit_pruned_row_groups(mut self, pruned_by_limit: Option) -> Self { + self.expected_limit_pruned_row_groups = pruned_by_limit; + self + } + /// Set the number of expected rows from the output of this test fn with_expected_rows(mut self, rows: usize) -> Self { self.expected_rows = rows; @@ -143,6 +165,65 @@ impl RowGroupPruningTest { self.expected_row_group_pruned_by_bloom_filter, "mismatched row_groups_pruned_bloom_filter", ); + + assert_eq!( + output.result_rows, + self.expected_rows, + "Expected {} rows, got {}: {}", + output.result_rows, + self.expected_rows, + output.description(), + ); + } + + // Execute the test with the current configuration + async fn test_row_group_prune_with_custom_data( + self, + schema: Arc, + batches: Vec, + max_row_per_group: usize, + ) { + let output = ContextWithParquet::with_custom_data( + self.scenario, + RowGroup(max_row_per_group), + schema, + batches, + ) + .await + .query(&self.query) + .await; + + println!("{}", output.description()); + assert_eq!( + output.predicate_evaluation_errors(), + self.expected_errors, + "mismatched predicate_evaluation error" + ); + assert_eq!( + output.row_groups_matched_statistics(), + self.expected_row_group_matched_by_statistics, + "mismatched row_groups_matched_statistics", + ); + assert_eq!( + output.row_groups_fully_matched_statistics(), + self.expected_row_group_fully_matched_by_statistics, + "mismatched row_groups_fully_matched_statistics", + ); + assert_eq!( + output.row_groups_pruned_statistics(), + self.expected_row_group_pruned_by_statistics, + "mismatched row_groups_pruned_statistics", + ); + assert_eq!( + output.files_ranges_pruned_statistics(), + self.expected_files_pruned_by_statistics, + "mismatched files_ranges_pruned_statistics", + ); + assert_eq!( + output.limit_pruned_row_groups(), + self.expected_limit_pruned_row_groups, + "mismatched limit_pruned_row_groups", + ); assert_eq!( output.result_rows, self.expected_rows, @@ -287,11 +368,16 @@ async fn prune_disabled() { let expected_rows = 10; let config = SessionConfig::new().with_parquet_pruning(false); - let output = - ContextWithParquet::with_config(Scenario::Timestamps, RowGroup(5), config) - .await - .query(query) - .await; + let output = ContextWithParquet::with_config( + Scenario::Timestamps, + RowGroup(5), + config, + None, + None, + ) + .await + .query(query) + .await; println!("{}", output.description()); // This should not prune any @@ -1634,3 +1720,241 @@ async fn test_bloom_filter_decimal_dict() { .test_row_group_prune() .await; } + +// Helper function to create a batch with a single Int32 column. +fn make_i32_batch( + name: &str, + values: Vec, +) -> datafusion_common::error::Result { + let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Int32, false)])); + let array: ArrayRef = Arc::new(Int32Array::from(values)); + RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from) +} + +// Helper function to create a batch with two Int32 columns +fn make_two_col_i32_batch( + name_a: &str, + name_b: &str, + values_a: Vec, + values_b: Vec, +) -> datafusion_common::error::Result { + let schema = Arc::new(Schema::new(vec![ + Field::new(name_a, DataType::Int32, false), + Field::new(name_b, DataType::Int32, false), + ])); + let array_a: ArrayRef = Arc::new(Int32Array::from(values_a)); + let array_b: ArrayRef = Arc::new(Int32Array::from(values_b)); + RecordBatch::try_new(schema, vec![array_a, array_b]).map_err(DataFusionError::from) +} + +#[tokio::test] +async fn test_limit_pruning_basic() -> datafusion_common::error::Result<()> { + // Scenario: Simple integer column, multiple row groups + // Query: SELECT c1 FROM t WHERE c1 = 0 LIMIT 2 + // We expect 2 rows in total. + + // Row Group 0: c1 = [0, -2] -> Partially matched, 1 row + // Row Group 1: c1 = [1, 2] -> Fully matched, 2 rows + // Row Group 2: c1 = [3, 4] -> Fully matched, 2 rows + // Row Group 3: c1 = [5, 6] -> Fully matched, 2 rows + // Row Group 4: c1 = [-1, -2] -> Not matched + + // If limit = 2, and RG1 is fully matched and has 2 rows, we should + // only scan RG1 and prune other row groups + // RG4 is pruned by statistics. RG2 and RG3 are pruned by limit. + // So 2 row groups are effectively pruned due to limit pruning. + + let schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); + let query = "SELECT c1 FROM t WHERE c1 >= 0 LIMIT 2"; + + let batches = vec![ + make_i32_batch("c1", vec![0, -2])?, + make_i32_batch("c1", vec![0, 0])?, + make_i32_batch("c1", vec![0, 0])?, + make_i32_batch("c1", vec![0, 0])?, + make_i32_batch("c1", vec![-1, -2])?, + ]; + + RowGroupPruningTest::new() + .with_scenario(Scenario::Int) // Assuming Scenario::Int can handle this data + .with_query(query) + .with_expected_errors(Some(0)) + .with_expected_rows(2) + .with_pruned_files(Some(0)) + .with_matched_by_stats(Some(4)) + .with_fully_matched_by_stats(Some(3)) + .with_pruned_by_stats(Some(1)) + .with_limit_pruned_row_groups(Some(3)) + .test_row_group_prune_with_custom_data(schema, batches, 2) + .await; + + Ok(()) +} + +#[tokio::test] +async fn test_limit_pruning_complex_filter() -> datafusion_common::error::Result<()> { + // Test Case 1: Complex filter with two columns (a = 1 AND b > 1 AND b < 4) + // Row Group 0: a=[1,1,1], b=[0,2,3] -> Partially matched, 2 rows match (b=2,3) + // Row Group 1: a=[1,1,1], b=[2,2,2] -> Fully matched, 3 rows + // Row Group 2: a=[1,1,1], b=[2,3,3] -> Fully matched, 3 rows + // Row Group 3: a=[1,1,1], b=[2,2,3] -> Fully matched, 3 rows + // Row Group 4: a=[2,2,2], b=[2,2,2] -> Not matched (a != 1) + // Row Group 5: a=[1,1,1], b=[5,6,7] -> Not matched (b >= 4) + + // With LIMIT 5, we need RG1 (3 rows) + RG2 (2 rows from 3) = 5 rows + // RG4 and RG5 should be pruned by statistics + // RG3 should be pruned by limit + // RG0 is partially matched, so it depends on the order + + let schema = Arc::new(Schema::new(vec![ + Field::new("a", DataType::Int32, false), + Field::new("b", DataType::Int32, false), + ])); + let query = "SELECT a, b FROM t WHERE a = 1 AND b > 1 AND b < 4 LIMIT 5"; + + let batches = vec![ + make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![0, 2, 3])?, + make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 2, 2])?, + make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 3, 3])?, + make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![2, 2, 3])?, + make_two_col_i32_batch("a", "b", vec![2, 2, 2], vec![2, 2, 2])?, + make_two_col_i32_batch("a", "b", vec![1, 1, 1], vec![5, 6, 7])?, + ]; + + RowGroupPruningTest::new() + .with_scenario(Scenario::Int) + .with_query(query) + .with_expected_errors(Some(0)) + .with_expected_rows(5) + .with_pruned_files(Some(0)) + .with_matched_by_stats(Some(4)) // RG0,1,2,3 are matched + .with_fully_matched_by_stats(Some(3)) + .with_pruned_by_stats(Some(2)) // RG4,5 are pruned + .with_limit_pruned_row_groups(Some(2)) // RG0, RG3 is pruned by limit + .test_row_group_prune_with_custom_data(schema, batches, 3) + .await; + + Ok(()) +} + +#[tokio::test] +async fn test_limit_pruning_multiple_fully_matched( +) -> datafusion_common::error::Result<()> { + // Test Case 2: Limit requires multiple fully matched row groups + // Row Group 0: a=[5,5,5,5] -> Fully matched, 4 rows + // Row Group 1: a=[5,5,5,5] -> Fully matched, 4 rows + // Row Group 2: a=[5,5,5,5] -> Fully matched, 4 rows + // Row Group 3: a=[5,5,5,5] -> Fully matched, 4 rows + // Row Group 4: a=[1,2,3,4] -> Not matched + + // With LIMIT 8, we need RG0 (4 rows) + RG1 (4 rows) 8 rows + // RG2,3 should be pruned by limit + // RG4 should be pruned by statistics + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let query = "SELECT a FROM t WHERE a = 5 LIMIT 8"; + + let batches = vec![ + make_i32_batch("a", vec![5, 5, 5, 5])?, + make_i32_batch("a", vec![5, 5, 5, 5])?, + make_i32_batch("a", vec![5, 5, 5, 5])?, + make_i32_batch("a", vec![5, 5, 5, 5])?, + make_i32_batch("a", vec![1, 2, 3, 4])?, + ]; + + RowGroupPruningTest::new() + .with_scenario(Scenario::Int) + .with_query(query) + .with_expected_errors(Some(0)) + .with_expected_rows(8) + .with_pruned_files(Some(0)) + .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched + .with_fully_matched_by_stats(Some(4)) + .with_pruned_by_stats(Some(1)) // RG4 pruned + .with_limit_pruned_row_groups(Some(2)) // RG2,3 pruned by limit + .test_row_group_prune_with_custom_data(schema, batches, 4) + .await; + + Ok(()) +} + +#[tokio::test] +async fn test_limit_pruning_no_fully_matched() -> datafusion_common::error::Result<()> { + // Test Case 3: No fully matched row groups - all are partially matched + // Row Group 0: a=[1,2,3] -> Partially matched, 1 row (a=2) + // Row Group 1: a=[2,3,4] -> Partially matched, 1 row (a=2) + // Row Group 2: a=[2,5,6] -> Partially matched, 1 row (a=2) + // Row Group 3: a=[2,7,8] -> Partially matched, 1 row (a=2) + // Row Group 4: a=[9,10,11] -> Not matched + + // With LIMIT 3, we need to scan RG0,1,2 to get 3 matching rows + // Cannot prune much by limit since all matching RGs are partial + // RG4 should be pruned by statistics + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let query = "SELECT a FROM t WHERE a = 2 LIMIT 3"; + + let batches = vec![ + make_i32_batch("a", vec![1, 2, 3])?, + make_i32_batch("a", vec![2, 3, 4])?, + make_i32_batch("a", vec![2, 5, 6])?, + make_i32_batch("a", vec![2, 7, 8])?, + make_i32_batch("a", vec![9, 10, 11])?, + ]; + + RowGroupPruningTest::new() + .with_scenario(Scenario::Int) + .with_query(query) + .with_expected_errors(Some(0)) + .with_expected_rows(3) + .with_pruned_files(Some(0)) + .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched + .with_fully_matched_by_stats(Some(0)) + .with_pruned_by_stats(Some(1)) // RG4 pruned + .with_limit_pruned_row_groups(Some(0)) // RG3 pruned by limit + .test_row_group_prune_with_custom_data(schema, batches, 3) + .await; + + Ok(()) +} + +#[tokio::test] +async fn test_limit_pruning_exceeds_fully_matched() -> datafusion_common::error::Result<()> +{ + // Test Case 4: Limit exceeds all fully matched rows, need partially matched + // Row Group 0: a=[10,11,12,12] -> Partially matched, 1 row (a=10) + // Row Group 1: a=[10,10,10,10] -> Fully matched, 4 rows + // Row Group 2: a=[10,10,10,10] -> Fully matched, 4 rows + // Row Group 3: a=[10,13,14,11] -> Partially matched, 1 row (a=10) + // Row Group 4: a=[20,21,22,22] -> Not matched + + // With LIMIT 10, we need RG1 (4) + RG2 (4) = 8 from fully matched + // Still need 2 more, so we need to scan partially matched RG0 and RG3 + // All matching row groups should be scanned, only RG4 pruned by statistics + + let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); + let query = "SELECT a FROM t WHERE a = 10 LIMIT 10"; + + let batches = vec![ + make_i32_batch("a", vec![10, 11, 12, 12])?, + make_i32_batch("a", vec![10, 10, 10, 10])?, + make_i32_batch("a", vec![10, 10, 10, 10])?, + make_i32_batch("a", vec![10, 13, 14, 11])?, + make_i32_batch("a", vec![20, 21, 22, 22])?, + ]; + + RowGroupPruningTest::new() + .with_scenario(Scenario::Int) + .with_query(query) + .with_expected_errors(Some(0)) + .with_expected_rows(10) // Total: 1 + 3 + 4 + 1 = 9 (less than limit) + .with_pruned_files(Some(0)) + .with_matched_by_stats(Some(4)) // RG0,1,2,3 matched + .with_fully_matched_by_stats(Some(2)) + .with_pruned_by_stats(Some(1)) // RG4 pruned + .with_limit_pruned_row_groups(Some(0)) // No limit pruning since we need all RGs + .test_row_group_prune_with_custom_data(schema, batches, 4) + .await; + + Ok(()) +} diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rs index 574fe2a040eab..8c1c97873a6a6 100644 --- a/datafusion/datasource-parquet/src/metrics.rs +++ b/datafusion/datasource-parquet/src/metrics.rs @@ -48,6 +48,10 @@ pub struct ParquetFileMetrics { pub row_groups_matched_bloom_filter: Count, /// Number of row groups pruned by bloom filters pub row_groups_pruned_bloom_filter: Count, + /// Number of row groups pruned due to limit pruning. + pub limit_pruned_row_groups: Count, + /// Number of row groups whose statistics were checked and fully matched + pub row_groups_fully_matched_statistics: Count, /// Number of row groups whose statistics were checked and matched (not pruned) pub row_groups_matched_statistics: Count, /// Number of row groups pruned by statistics @@ -93,6 +97,14 @@ impl ParquetFileMetrics { .with_new_label("filename", filename.to_string()) .counter("row_groups_pruned_bloom_filter", partition); + let limit_pruned_row_groups = MetricBuilder::new(metrics) + .with_new_label("filename", filename.to_string()) + .counter("limit_pruned_row_groups", partition); + + let row_groups_fully_matched_statistics = MetricBuilder::new(metrics) + .with_new_label("filename", filename.to_string()) + .counter("row_groups_fully_matched_statistics", partition); + let row_groups_matched_statistics = MetricBuilder::new(metrics) .with_new_label("filename", filename.to_string()) .counter("row_groups_matched_statistics", partition); @@ -145,8 +157,10 @@ impl ParquetFileMetrics { predicate_evaluation_errors, row_groups_matched_bloom_filter, row_groups_pruned_bloom_filter, + row_groups_fully_matched_statistics, row_groups_matched_statistics, row_groups_pruned_statistics, + limit_pruned_row_groups, bytes_scanned, pushdown_rows_pruned, pushdown_rows_matched, diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs index 664b86e964eba..fc7c5611af714 100644 --- a/datafusion/datasource-parquet/src/opener.rs +++ b/datafusion/datasource-parquet/src/opener.rs @@ -376,8 +376,12 @@ impl FileOpener for ParquetOpener { } } - let mut access_plan = row_groups.build(); + // Prune by limit + if let Some(limit) = limit { + row_groups.prune_by_limit(limit, rg_metadata, &file_metrics); + } + let mut access_plan = row_groups.build(); // page index pruning: if all data on individual pages can // be ruled using page metadata, rows from other columns // with that range can be skipped as well diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index 51d50d780f103..a13143e5d1643 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -24,6 +24,8 @@ use arrow::datatypes::Schema; use datafusion_common::pruning::PruningStatistics; use datafusion_common::{Column, Result, ScalarValue}; use datafusion_datasource::FileRange; +use datafusion_physical_expr::expressions::NotExpr; +use datafusion_physical_expr::PhysicalExprSimplifier; use datafusion_pruning::PruningPredicate; use parquet::arrow::arrow_reader::statistics::StatisticsConverter; use parquet::arrow::parquet_column; @@ -46,13 +48,19 @@ use parquet::{ pub struct RowGroupAccessPlanFilter { /// which row groups should be accessed access_plan: ParquetAccessPlan, + /// which row groups are fully contained within the pruning predicate + is_fully_matched: Vec, } impl RowGroupAccessPlanFilter { /// Create a new `RowGroupPlanBuilder` for pruning out the groups to scan /// based on metadata and statistics pub fn new(access_plan: ParquetAccessPlan) -> Self { - Self { access_plan } + let num_row_groups = access_plan.len(); + Self { + access_plan, + is_fully_matched: vec![false; num_row_groups], + } } /// Return true if there are no row groups @@ -65,6 +73,49 @@ impl RowGroupAccessPlanFilter { self.access_plan } + /// Returns the is_fully_matched vector + pub fn is_fully_matched(&self) -> &Vec { + &self.is_fully_matched + } + + /// Prunes the access plan based on the limit and fully contained row groups. + pub fn prune_by_limit( + &mut self, + limit: usize, + rg_metadata: &[RowGroupMetaData], + metrics: &ParquetFileMetrics, + ) { + let mut fully_matched_row_group_indexes: Vec = Vec::new(); + let mut fully_matched_rows_count: usize = 0; + + // Iterate through the currently accessible row groups + for &idx in self.access_plan.row_group_indexes().iter() { + if self.is_fully_matched[idx] { + let row_group_row_count = rg_metadata[idx].num_rows() as usize; + fully_matched_row_group_indexes.push(idx); + fully_matched_rows_count += row_group_row_count; + if fully_matched_rows_count >= limit { + break; + } + } + } + + if fully_matched_rows_count >= limit { + let original_num_accessible_row_groups = + self.access_plan.row_group_indexes().len(); + let new_num_accessible_row_groups = fully_matched_row_group_indexes.len(); + let pruned_count = original_num_accessible_row_groups + .saturating_sub(new_num_accessible_row_groups); + metrics.limit_pruned_row_groups.add(pruned_count); + + let mut new_access_plan = ParquetAccessPlan::new_none(rg_metadata.len()); + for &idx in &fully_matched_row_group_indexes { + new_access_plan.scan(idx); + } + self.access_plan = new_access_plan; + } + } + /// Prune remaining row groups to only those within the specified range. /// /// Updates this set to mark row groups that should not be scanned @@ -130,15 +181,56 @@ impl RowGroupAccessPlanFilter { // try to prune the row groups in a single call match predicate.prune(&pruning_stats) { Ok(values) => { - // values[i] is false means the predicate could not be true for row group i + let mut fully_contained_candidates_original_idx: Vec = Vec::new(); for (idx, &value) in row_group_indexes.iter().zip(values.iter()) { if !value { self.access_plan.skip(*idx); metrics.row_groups_pruned_statistics.add(1); } else { + fully_contained_candidates_original_idx.push(*idx); metrics.row_groups_matched_statistics.add(1); } } + + // Note: this part of code shouldn't be expensive with a limited number of row groups + // If we do find it's expensive, we can consider optimizing it further. + if !fully_contained_candidates_original_idx.is_empty() { + // Use NotExpr to create the inverted predicate + let inverted_expr = + Arc::new(NotExpr::new(Arc::clone(predicate.orig_expr()))); + // Simplify the NOT expression (e.g., NOT(c1 = 0) -> c1 != 0) + // before building the pruning predicate + let mut simplifier = PhysicalExprSimplifier::new(arrow_schema); + let inverted_expr = simplifier.simplify(inverted_expr).unwrap(); + if let Ok(inverted_predicate) = PruningPredicate::try_new( + inverted_expr, + Arc::clone(predicate.schema()), + ) { + let inverted_pruning_stats = RowGroupPruningStatistics { + parquet_schema, + row_group_metadatas: fully_contained_candidates_original_idx + .iter() + .map(|&i| &groups[i]) + .collect::>(), + arrow_schema, + }; + + if let Ok(inverted_values) = + inverted_predicate.prune(&inverted_pruning_stats) + { + for (i, &original_row_group_idx) in + fully_contained_candidates_original_idx.iter().enumerate() + { + // If the inverted predicate *also* prunes this row group (meaning inverted_values[i] is false), + // it implies that *all* rows in this group satisfy the original predicate. + if !inverted_values[i] { + self.is_fully_matched[original_row_group_idx] = true; + metrics.row_groups_fully_matched_statistics.add(1); + } + } + } + } + } } // stats filter array could not be built, so we can't prune Err(e) => { diff --git a/datafusion/execution/src/memory_pool/pool.rs b/datafusion/execution/src/memory_pool/pool.rs index da456b7071f77..8e5a82ef998a7 100644 --- a/datafusion/execution/src/memory_pool/pool.rs +++ b/datafusion/execution/src/memory_pool/pool.rs @@ -260,7 +260,7 @@ fn insufficient_capacity_err( additional: usize, available: usize, ) -> DataFusionError { - resources_datafusion_err!("Failed to allocate additional {} for {} with {} already allocated for this reservation - {} remain available for the total pool", + resources_datafusion_err!("Failed to allocate additional {} for {} with {} already allocated for this reservation - {} remain available for the total pool", human_readable_size(additional), reservation.registration.consumer.name, human_readable_size(reservation.size), human_readable_size(available)) } diff --git a/datafusion/physical-expr/src/simplifier/mod.rs b/datafusion/physical-expr/src/simplifier/mod.rs index 80d6ee0a7b914..d0c787867dd06 100644 --- a/datafusion/physical-expr/src/simplifier/mod.rs +++ b/datafusion/physical-expr/src/simplifier/mod.rs @@ -24,8 +24,9 @@ use datafusion_common::{ }; use std::sync::Arc; -use crate::PhysicalExpr; +use crate::{simplifier::not::simplify_not_expr_recursive, PhysicalExpr}; +pub mod not; pub mod unwrap_cast; /// Simplifies physical expressions by applying various optimizations @@ -56,6 +57,11 @@ impl<'a> TreeNodeRewriter for PhysicalExprSimplifier<'a> { type Node = Arc; fn f_up(&mut self, node: Self::Node) -> Result> { + // Apply NOT expression simplification first + let not_simplified = simplify_not_expr_recursive(node, self.schema)?; + let node = not_simplified.data; + let transformed = not_simplified.transformed; + // Apply unwrap cast optimization #[cfg(test)] let original_type = node.data_type(self.schema).unwrap(); @@ -66,7 +72,12 @@ impl<'a> TreeNodeRewriter for PhysicalExprSimplifier<'a> { original_type, "Simplified expression should have the same data type as the original" ); - Ok(unwrapped) + // Combine transformation results + let final_transformed = transformed || unwrapped.transformed; + Ok(Transformed::new_transformed( + unwrapped.data, + final_transformed, + )) } } diff --git a/datafusion/physical-expr/src/simplifier/not.rs b/datafusion/physical-expr/src/simplifier/not.rs new file mode 100644 index 0000000000000..d3e69bc74904e --- /dev/null +++ b/datafusion/physical-expr/src/simplifier/not.rs @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Simplify NOT expressions in physical expressions +//! +//! This module provides optimizations for NOT expressions such as: +//! - Double negation elimination: NOT(NOT(expr)) -> expr +//! - NOT with binary comparisons: NOT(a = b) -> a != b +//! - NOT with IN expressions: NOT(a IN (list)) -> a NOT IN (list) +//! - De Morgan's laws: NOT(A AND B) -> NOT A OR NOT B +//! - Constant folding: NOT(TRUE) -> FALSE, NOT(FALSE) -> TRUE + +use std::sync::Arc; + +use arrow::datatypes::Schema; +use datafusion_common::{tree_node::Transformed, Result, ScalarValue}; +use datafusion_expr::Operator; + +use crate::expressions::{lit, BinaryExpr, Literal, NotExpr}; +use crate::PhysicalExpr; + +/// Attempts to simplify NOT expressions +pub(crate) fn simplify_not_expr( + expr: Arc, + schema: &Schema, +) -> Result>> { + // Check if this is a NOT expression + let not_expr = match expr.as_any().downcast_ref::() { + Some(not_expr) => not_expr, + None => return Ok(Transformed::no(expr)), + }; + + let inner_expr = not_expr.arg(); + + // Handle NOT(NOT(expr)) -> expr (double negation elimination) + if let Some(inner_not) = inner_expr.as_any().downcast_ref::() { + // Recursively simplify the inner expression + let simplified = + simplify_not_expr_recursive(Arc::clone(inner_not.arg()), schema)?; + // We eliminated double negation, so always return transformed=true + return Ok(Transformed::yes(simplified.data)); + } + + // Handle NOT(literal) -> !literal + if let Some(literal) = inner_expr.as_any().downcast_ref::() { + if let ScalarValue::Boolean(Some(val)) = literal.value() { + return Ok(Transformed::yes(lit(ScalarValue::Boolean(Some(!val))))); + } + if let ScalarValue::Boolean(None) = literal.value() { + return Ok(Transformed::yes(lit(ScalarValue::Boolean(None)))); + } + } + + // Handle NOT(binary_expr) where we can flip the operator + if let Some(binary_expr) = inner_expr.as_any().downcast_ref::() { + if let Some(negated_op) = negate_operator(binary_expr.op()) { + // Recursively simplify the left and right expressions first + let left_simplified = + simplify_not_expr_recursive(Arc::clone(binary_expr.left()), schema)?; + let right_simplified = + simplify_not_expr_recursive(Arc::clone(binary_expr.right()), schema)?; + + let new_binary = Arc::new(BinaryExpr::new( + left_simplified.data, + negated_op, + right_simplified.data, + )); + // We flipped the operator, so always return transformed=true + return Ok(Transformed::yes(new_binary)); + } + + // Handle De Morgan's laws for AND/OR + match binary_expr.op() { + Operator::And => { + // NOT(A AND B) -> NOT A OR NOT B + let not_left = Arc::new(NotExpr::new(Arc::clone(binary_expr.left()))); + let not_right = Arc::new(NotExpr::new(Arc::clone(binary_expr.right()))); + + // Recursively simplify the NOT expressions + let simplified_left = simplify_not_expr_recursive(not_left, schema)?; + let simplified_right = simplify_not_expr_recursive(not_right, schema)?; + + let new_binary = Arc::new(BinaryExpr::new( + simplified_left.data, + Operator::Or, + simplified_right.data, + )); + return Ok(Transformed::yes(new_binary)); + } + Operator::Or => { + // NOT(A OR B) -> NOT A AND NOT B + let not_left = Arc::new(NotExpr::new(Arc::clone(binary_expr.left()))); + let not_right = Arc::new(NotExpr::new(Arc::clone(binary_expr.right()))); + + // Recursively simplify the NOT expressions + let simplified_left = simplify_not_expr_recursive(not_left, schema)?; + let simplified_right = simplify_not_expr_recursive(not_right, schema)?; + + let new_binary = Arc::new(BinaryExpr::new( + simplified_left.data, + Operator::And, + simplified_right.data, + )); + return Ok(Transformed::yes(new_binary)); + } + _ => {} + } + } + + // If no simplification possible, return the original expression + Ok(Transformed::no(expr)) +} + +/// Helper function that recursively simplifies expressions, including NOT expressions +pub fn simplify_not_expr_recursive( + expr: Arc, + schema: &Schema, +) -> Result>> { + // First, try to simplify any NOT expressions in this expression + let not_simplified = simplify_not_expr(Arc::clone(&expr), schema)?; + + // If the expression was transformed, we might have created new opportunities for simplification + if not_simplified.transformed { + // Recursively simplify the result + let further_simplified = + simplify_not_expr_recursive(Arc::clone(¬_simplified.data), schema)?; + if further_simplified.transformed { + return Ok(Transformed::yes(further_simplified.data)); + } else { + return Ok(not_simplified); + } + } + + // If this expression wasn't a NOT expression, try to simplify its children + // This handles cases where NOT expressions might be nested deeper in the tree + if let Some(binary_expr) = expr.as_any().downcast_ref::() { + let left_simplified = + simplify_not_expr_recursive(Arc::clone(binary_expr.left()), schema)?; + let right_simplified = + simplify_not_expr_recursive(Arc::clone(binary_expr.right()), schema)?; + + if left_simplified.transformed || right_simplified.transformed { + let new_binary = Arc::new(BinaryExpr::new( + left_simplified.data, + *binary_expr.op(), + right_simplified.data, + )); + return Ok(Transformed::yes(new_binary)); + } + } + + Ok(not_simplified) +} + +/// Returns the negated version of a comparison operator, if possible +fn negate_operator(op: &Operator) -> Option { + match op { + Operator::Eq => Some(Operator::NotEq), + Operator::NotEq => Some(Operator::Eq), + Operator::Lt => Some(Operator::GtEq), + Operator::LtEq => Some(Operator::Gt), + Operator::Gt => Some(Operator::LtEq), + Operator::GtEq => Some(Operator::Lt), + Operator::IsDistinctFrom => Some(Operator::IsNotDistinctFrom), + Operator::IsNotDistinctFrom => Some(Operator::IsDistinctFrom), + // For other operators, we can't directly negate them + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::expressions::{col, lit, BinaryExpr, NotExpr}; + use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_common::ScalarValue; + use datafusion_expr::Operator; + + fn test_schema() -> Schema { + Schema::new(vec![ + Field::new("a", DataType::Boolean, false), + Field::new("b", DataType::Int32, false), + ]) + } + + #[test] + fn test_double_negation_elimination() -> Result<()> { + let schema = test_schema(); + + // Create NOT(NOT(b > 5)) + let inner_expr: Arc = Arc::new(BinaryExpr::new( + col("b", &schema)?, + Operator::Gt, + lit(ScalarValue::Int32(Some(5))), + )); + let inner_not = Arc::new(NotExpr::new(Arc::clone(&inner_expr))); + let double_not = Arc::new(NotExpr::new(inner_not)); + + let result = simplify_not_expr_recursive(double_not, &schema)?; + + assert!(result.transformed); + // Should be simplified back to the original b > 5 + assert_eq!(result.data.to_string(), inner_expr.to_string()); + Ok(()) + } + + #[test] + fn test_not_literal() -> Result<()> { + let schema = test_schema(); + + // NOT(TRUE) -> FALSE + let not_true = Arc::new(NotExpr::new(lit(ScalarValue::Boolean(Some(true))))); + let result = simplify_not_expr(not_true, &schema)?; + assert!(result.transformed); + + if let Some(literal) = result.data.as_any().downcast_ref::() { + assert_eq!(literal.value(), &ScalarValue::Boolean(Some(false))); + } else { + panic!("Expected literal result"); + } + + // NOT(FALSE) -> TRUE + let not_false = Arc::new(NotExpr::new(lit(ScalarValue::Boolean(Some(false))))); + let result = simplify_not_expr_recursive(not_false, &schema)?; + assert!(result.transformed); + + if let Some(literal) = result.data.as_any().downcast_ref::() { + assert_eq!(literal.value(), &ScalarValue::Boolean(Some(true))); + } else { + panic!("Expected literal result"); + } + + Ok(()) + } + + #[test] + fn test_negate_comparison() -> Result<()> { + let schema = test_schema(); + + // NOT(b = 5) -> b != 5 + let eq_expr = Arc::new(BinaryExpr::new( + col("b", &schema)?, + Operator::Eq, + lit(ScalarValue::Int32(Some(5))), + )); + let not_eq = Arc::new(NotExpr::new(eq_expr)); + + let result = simplify_not_expr_recursive(not_eq, &schema)?; + assert!(result.transformed); + + if let Some(binary) = result.data.as_any().downcast_ref::() { + assert_eq!(binary.op(), &Operator::NotEq); + } else { + panic!("Expected binary expression result"); + } + + Ok(()) + } + + #[test] + fn test_demorgans_law_and() -> Result<()> { + let schema = test_schema(); + + // NOT(a AND b) -> NOT a OR NOT b + let and_expr = Arc::new(BinaryExpr::new( + col("a", &schema)?, + Operator::And, + col("b", &schema)?, + )); + let not_and = Arc::new(NotExpr::new(and_expr)); + + let result = simplify_not_expr_recursive(not_and, &schema)?; + assert!(result.transformed); + + if let Some(binary) = result.data.as_any().downcast_ref::() { + assert_eq!(binary.op(), &Operator::Or); + // Left and right should both be NOT expressions + assert!(binary.left().as_any().downcast_ref::().is_some()); + assert!(binary.right().as_any().downcast_ref::().is_some()); + } else { + panic!("Expected binary expression result"); + } + + Ok(()) + } + + #[test] + fn test_demorgans_law_or() -> Result<()> { + let schema = test_schema(); + + // NOT(a OR b) -> NOT a AND NOT b + let or_expr = Arc::new(BinaryExpr::new( + col("a", &schema)?, + Operator::Or, + col("b", &schema)?, + )); + let not_or = Arc::new(NotExpr::new(or_expr)); + + let result = simplify_not_expr_recursive(not_or, &schema)?; + assert!(result.transformed); + + if let Some(binary) = result.data.as_any().downcast_ref::() { + assert_eq!(binary.op(), &Operator::And); + // Left and right should both be NOT expressions + assert!(binary.left().as_any().downcast_ref::().is_some()); + assert!(binary.right().as_any().downcast_ref::().is_some()); + } else { + panic!("Expected binary expression result"); + } + + Ok(()) + } + + #[test] + fn test_demorgans_with_comparison_simplification() -> Result<()> { + let schema = test_schema(); + + // NOT(b = 1 AND b = 2) -> b != 1 OR b != 2 + // This tests the combination of De Morgan's law and operator negation + let eq1 = Arc::new(BinaryExpr::new( + col("b", &schema)?, + Operator::Eq, + lit(ScalarValue::Int32(Some(1))), + )); + let eq2 = Arc::new(BinaryExpr::new( + col("b", &schema)?, + Operator::Eq, + lit(ScalarValue::Int32(Some(2))), + )); + let and_expr = Arc::new(BinaryExpr::new(eq1, Operator::And, eq2)); + let not_and = Arc::new(NotExpr::new(and_expr)); + + let result = simplify_not_expr_recursive(not_and, &schema)?; + assert!(result.transformed, "Expression should be transformed"); + + // Verify the result is an OR expression + if let Some(or_binary) = result.data.as_any().downcast_ref::() { + assert_eq!(or_binary.op(), &Operator::Or, "Top level should be OR"); + + // Verify left side is b != 1 + if let Some(left_binary) = + or_binary.left().as_any().downcast_ref::() + { + assert_eq!(left_binary.op(), &Operator::NotEq, "Left should be NotEq"); + } else { + panic!("Expected left to be a binary expression with !="); + } + + // Verify right side is b != 2 + if let Some(right_binary) = + or_binary.right().as_any().downcast_ref::() + { + assert_eq!(right_binary.op(), &Operator::NotEq, "Right should be NotEq"); + } else { + panic!("Expected right to be a binary expression with !="); + } + } else { + panic!("Expected binary OR expression result"); + } + + Ok(()) + } +} diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 5e92dbe227fdd..63e1c571a4ae1 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -473,7 +473,6 @@ impl PruningPredicate { // Simplify the newly created predicate to get rid of redundant casts, comparisons, etc. let predicate_expr = PhysicalExprSimplifier::new(&predicate_schema).simplify(predicate_expr)?; - let literal_guarantees = LiteralGuarantee::analyze(&expr); Ok(Self { From ff301c8742de85be16a14b011aba66dee5b9fff9 Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Wed, 19 Nov 2025 14:20:01 +0800 Subject: [PATCH 154/177] Add restriction for enabling limit pruning (#21) --- .../core/src/datasource/listing/table.rs | 1 + datafusion/core/tests/parquet/mod.rs | 16 ++++++---- .../core/tests/parquet/row_group_pruning.rs | 30 ++++++++++--------- datafusion/datasource-parquet/src/opener.rs | 15 ++++++++-- datafusion/datasource-parquet/src/source.rs | 1 + datafusion/datasource/src/file_scan_config.rs | 14 +++++++++ 6 files changed, 55 insertions(+), 22 deletions(-) diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index 14b5bfa54eda2..82ab966dbd8b8 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -1253,6 +1253,7 @@ impl TableProvider for ListingTable { .with_output_ordering(output_ordering) .with_table_partition_cols(table_partition_cols) .with_expr_adapter(self.expr_adapter_factory.clone()) + .with_limit_pruning(limit.is_some()) .build(), ) .await diff --git a/datafusion/core/tests/parquet/mod.rs b/datafusion/core/tests/parquet/mod.rs index dad8472dedc42..6ce962f1b786c 100644 --- a/datafusion/core/tests/parquet/mod.rs +++ b/datafusion/core/tests/parquet/mod.rs @@ -150,11 +150,18 @@ impl TestOutput { self.metric_value("row_groups_matched_statistics") } + /* /// The number of row_groups fully matched by statistics fn row_groups_fully_matched_statistics(&self) -> Option { self.metric_value("row_groups_fully_matched_statistics") } + /// The number of row groups pruned by limit pruning + fn limit_pruned_row_groups(&self) -> Option { + self.metric_value("limit_pruned_row_groups") + } + */ + /// The number of row_groups pruned by statistics fn row_groups_pruned_statistics(&self) -> Option { self.metric_value("row_groups_pruned_statistics") @@ -183,11 +190,6 @@ impl TestOutput { self.metric_value("page_index_rows_pruned") } - /// The number of row groups pruned by limit pruning - fn limit_pruned_row_groups(&self) -> Option { - self.metric_value("limit_pruned_row_groups") - } - fn description(&self) -> String { format!( "Input:\n{}\nQuery:\n{}\nOutput:\n{}\nMetrics:\n{}", @@ -204,7 +206,8 @@ impl ContextWithParquet { Self::with_config(scenario, unit, SessionConfig::new(), None, None).await } - /// Set custom schema and batches for the test + // Set custom schema and batches for the test + /* pub async fn with_custom_data( scenario: Scenario, unit: Unit, @@ -220,6 +223,7 @@ impl ContextWithParquet { ) .await } + */ async fn with_config( scenario: Scenario, diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs index 42c4488d8444a..81cbdfb27121e 100644 --- a/datafusion/core/tests/parquet/row_group_pruning.rs +++ b/datafusion/core/tests/parquet/row_group_pruning.rs @@ -18,12 +18,8 @@ //! This file contains an end to end test of parquet pruning. It writes //! data into a parquet file and then verifies row groups are pruned as //! expected. -use std::sync::Arc; - -use arrow::array::{ArrayRef, Int32Array, RecordBatch}; -use arrow_schema::{DataType, Field, Schema}; use datafusion::prelude::SessionConfig; -use datafusion_common::{DataFusionError, ScalarValue}; +use datafusion_common::ScalarValue; use itertools::Itertools; use crate::parquet::Unit::RowGroup; @@ -34,12 +30,12 @@ struct RowGroupPruningTest { query: String, expected_errors: Option, expected_row_group_matched_by_statistics: Option, - expected_row_group_fully_matched_by_statistics: Option, + // expected_row_group_fully_matched_by_statistics: Option, expected_row_group_pruned_by_statistics: Option, expected_files_pruned_by_statistics: Option, expected_row_group_matched_by_bloom_filter: Option, expected_row_group_pruned_by_bloom_filter: Option, - expected_limit_pruned_row_groups: Option, + // expected_limit_pruned_row_groups: Option, expected_rows: usize, } impl RowGroupPruningTest { @@ -51,11 +47,11 @@ impl RowGroupPruningTest { expected_errors: None, expected_row_group_matched_by_statistics: None, expected_row_group_pruned_by_statistics: None, - expected_row_group_fully_matched_by_statistics: None, + // expected_row_group_fully_matched_by_statistics: None, expected_files_pruned_by_statistics: None, expected_row_group_matched_by_bloom_filter: None, expected_row_group_pruned_by_bloom_filter: None, - expected_limit_pruned_row_groups: None, + // expected_limit_pruned_row_groups: None, expected_rows: 0, } } @@ -85,6 +81,7 @@ impl RowGroupPruningTest { } // Set the expected fully matched row groups by statistics + /* fn with_fully_matched_by_stats( mut self, fully_matched_by_stats: Option, @@ -93,6 +90,12 @@ impl RowGroupPruningTest { self } + fn with_limit_pruned_row_groups(mut self, pruned_by_limit: Option) -> Self { + self.expected_limit_pruned_row_groups = pruned_by_limit; + self + } + */ + // Set the expected pruned row groups by statistics fn with_pruned_by_stats(mut self, pruned_by_stats: Option) -> Self { self.expected_row_group_pruned_by_statistics = pruned_by_stats; @@ -116,11 +119,6 @@ impl RowGroupPruningTest { self } - fn with_limit_pruned_row_groups(mut self, pruned_by_limit: Option) -> Self { - self.expected_limit_pruned_row_groups = pruned_by_limit; - self - } - /// Set the number of expected rows from the output of this test fn with_expected_rows(mut self, rows: usize) -> Self { self.expected_rows = rows; @@ -177,6 +175,7 @@ impl RowGroupPruningTest { } // Execute the test with the current configuration + /* async fn test_row_group_prune_with_custom_data( self, schema: Arc, @@ -233,6 +232,7 @@ impl RowGroupPruningTest { output.description(), ); } + */ } #[tokio::test] @@ -1721,6 +1721,7 @@ async fn test_bloom_filter_decimal_dict() { .await; } +/* // Helper function to create a batch with a single Int32 column. fn make_i32_batch( name: &str, @@ -1958,3 +1959,4 @@ async fn test_limit_pruning_exceeds_fully_matched() -> datafusion_common::error: Ok(()) } +*/ diff --git a/datafusion/datasource-parquet/src/opener.rs b/datafusion/datasource-parquet/src/opener.rs index fc7c5611af714..3b1f1ae35917b 100644 --- a/datafusion/datasource-parquet/src/opener.rs +++ b/datafusion/datasource-parquet/src/opener.rs @@ -97,6 +97,8 @@ pub(super) struct ParquetOpener { pub enable_row_group_stats_pruning: bool, /// Coerce INT96 timestamps to specific TimeUnit pub coerce_int96: Option, + /// Should limit pruning be applied + pub enable_limit_pruning: bool, /// Optional parquet FileDecryptionProperties #[cfg(feature = "parquet_encryption")] pub file_decryption_properties: Option>, @@ -144,6 +146,7 @@ impl FileOpener for ParquetOpener { let enable_bloom_filter = self.enable_bloom_filter; let enable_row_group_stats_pruning = self.enable_row_group_stats_pruning; let limit = self.limit; + let enable_limit_pruning = self.enable_limit_pruning; let predicate_creation_errors = MetricBuilder::new(&self.metrics) .global_counter("num_predicate_creation_errors"); @@ -377,8 +380,10 @@ impl FileOpener for ParquetOpener { } // Prune by limit - if let Some(limit) = limit { - row_groups.prune_by_limit(limit, rg_metadata, &file_metrics); + if enable_limit_pruning { + if let Some(limit) = limit { + row_groups.prune_by_limit(limit, rg_metadata, &file_metrics); + } } let mut access_plan = row_groups.build(); @@ -826,6 +831,7 @@ mod test { reorder_filters: false, enable_page_index: false, enable_bloom_filter: false, + enable_limit_pruning: false, schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), enable_row_group_stats_pruning: true, coerce_int96: None, @@ -914,6 +920,7 @@ mod test { reorder_filters: false, enable_page_index: false, enable_bloom_filter: false, + enable_limit_pruning: false, schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), enable_row_group_stats_pruning: true, coerce_int96: None, @@ -1018,6 +1025,7 @@ mod test { reorder_filters: false, enable_page_index: false, enable_bloom_filter: false, + enable_limit_pruning: false, schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), enable_row_group_stats_pruning: true, coerce_int96: None, @@ -1132,6 +1140,7 @@ mod test { reorder_filters: true, enable_page_index: false, enable_bloom_filter: false, + enable_limit_pruning: false, schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), enable_row_group_stats_pruning: false, // note that this is false! coerce_int96: None, @@ -1247,6 +1256,7 @@ mod test { reorder_filters: false, enable_page_index: false, enable_bloom_filter: false, + enable_limit_pruning: false, schema_adapter_factory: Arc::new(DefaultSchemaAdapterFactory), enable_row_group_stats_pruning: true, coerce_int96: None, @@ -1429,6 +1439,7 @@ mod test { reorder_filters: false, enable_page_index: false, enable_bloom_filter: false, + enable_limit_pruning: false, schema_adapter_factory: Arc::new(CustomSchemaAdapterFactory), enable_row_group_stats_pruning: false, coerce_int96: None, diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index 644cea85ca0a9..3bfc1463ae960 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -576,6 +576,7 @@ impl FileSource for ParquetSource { enable_page_index: self.enable_page_index(), enable_bloom_filter: self.bloom_filter_on_read(), enable_row_group_stats_pruning: self.table_parquet_options.global.pruning, + enable_limit_pruning: base_config.limit_pruning, schema_adapter_factory, coerce_int96, #[cfg(feature = "parquet_encryption")] diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs index 4e2235eae8fec..e5bbbdb028e7d 100644 --- a/datafusion/datasource/src/file_scan_config.rs +++ b/datafusion/datasource/src/file_scan_config.rs @@ -196,6 +196,8 @@ pub struct FileScanConfig { /// Expression adapter used to adapt filters and projections that are pushed down into the scan /// from the logical schema to the physical schema of the file. pub expr_adapter_factory: Option>, + /// If there is a limit pushed down at the logical plan level, we can enable limit_pruning + pub limit_pruning: bool, } /// A builder for [`FileScanConfig`]'s. @@ -275,6 +277,8 @@ pub struct FileScanConfigBuilder { new_lines_in_values: Option, batch_size: Option, expr_adapter_factory: Option>, + /// If there is a limit pushed down at the logical plan level, we can enable limit_pruning + limit_pruning: bool, } impl FileScanConfigBuilder { @@ -304,6 +308,7 @@ impl FileScanConfigBuilder { constraints: None, batch_size: None, expr_adapter_factory: None, + limit_pruning: false, } } @@ -426,6 +431,12 @@ impl FileScanConfigBuilder { self } + /// Enable or disable limit pruning. + pub fn with_limit_pruning(mut self, limit_pruning: bool) -> Self { + self.limit_pruning = limit_pruning; + self + } + /// Build the final [`FileScanConfig`] with all the configured settings. /// /// This method takes ownership of the builder and returns the constructed `FileScanConfig`. @@ -446,6 +457,7 @@ impl FileScanConfigBuilder { new_lines_in_values, batch_size, expr_adapter_factory: expr_adapter, + limit_pruning, } = self; let constraints = constraints.unwrap_or_default(); @@ -473,6 +485,7 @@ impl FileScanConfigBuilder { new_lines_in_values, batch_size, expr_adapter_factory: expr_adapter, + limit_pruning, } } } @@ -494,6 +507,7 @@ impl From for FileScanConfigBuilder { constraints: Some(config.constraints), batch_size: config.batch_size, expr_adapter_factory: config.expr_adapter_factory, + limit_pruning: config.limit_pruning, } } } From 7a11979960dde1994274b8f10299458707904568 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 11:47:43 +0530 Subject: [PATCH 155/177] fix: resolve conflict by picking upstream --- datafusion/common/src/config.rs | 6 ------ datafusion/sqllogictest/Cargo.toml | 4 ---- 2 files changed, 10 deletions(-) diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index a49cac40e875a..a77fd764eea06 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -2676,7 +2676,6 @@ config_namespace! { // The input regex for Nulls when loading CSVs. pub null_regex: Option, default = None pub comment: Option, default = None -<<<<<<< HEAD /// Whether to allow truncated rows when parsing, both within a single file and across files. /// /// When set to false (default), reading a single CSV file which has rows of different lengths will @@ -2686,11 +2685,6 @@ config_namespace! { /// rows with null values for the missing columns; if reading multiple CSV files with different number /// of columns, it creates a union schema containing all columns found across the files, and will /// pad any files missing columns with null values for their rows. -======= - // Whether to allow truncated rows when parsing. - // By default this is set to false and will error if the CSV rows have different lengths. - // When set to true then it will allow records with less than the expected number of columns ->>>>>>> origin/branch-51 pub truncated_rows: Option, default = None } } diff --git a/datafusion/sqllogictest/Cargo.toml b/datafusion/sqllogictest/Cargo.toml index eeae52df85049..177761e4af54e 100644 --- a/datafusion/sqllogictest/Cargo.toml +++ b/datafusion/sqllogictest/Cargo.toml @@ -45,11 +45,7 @@ async-trait = { workspace = true } bigdecimal = { workspace = true } bytes = { workspace = true, optional = true } chrono = { workspace = true, optional = true } -<<<<<<< HEAD clap = { version = "4.5.50", features = ["derive", "env"] } -======= -clap = { version = "4.5.47", features = ["derive", "env"] } ->>>>>>> origin/branch-51 datafusion = { workspace = true, default-features = true, features = ["avro"] } datafusion-spark = { workspace = true, default-features = true } datafusion-substrait = { workspace = true, default-features = true } From 942bab830f21a7d80b0cb9ccd309ae2ad17faf7f Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 11:48:25 +0530 Subject: [PATCH 156/177] fix: required methods in `FunctionRegistry` See: https://datafusion.apache.org/library-user-guide/upgrading.html#functionregistry-exposes-two-additional-methods --- datafusion/expr/src/registry.rs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/datafusion/expr/src/registry.rs b/datafusion/expr/src/registry.rs index 558b4fd1b14f8..9554dd68e1758 100644 --- a/datafusion/expr/src/registry.rs +++ b/datafusion/expr/src/registry.rs @@ -31,27 +31,10 @@ pub trait FunctionRegistry { fn udfs(&self) -> HashSet; /// Returns names of all available aggregate user defined functions. -<<<<<<< HEAD fn udafs(&self) -> HashSet; /// Returns names of all available window user defined functions. fn udwfs(&self) -> HashSet; -======= - fn udafs(&self) -> HashSet { - // This default implementation is provided temporarily - // to maintain backward compatibility for the 50.1 release. - // It will be reverted to a required method in future versions. - HashSet::default() - } - - /// Returns names of all available window user defined functions. - fn udwfs(&self) -> HashSet { - // This default implementation is provided temporarily - // to maintain backward compatibility for the 50.1 release. - // It will be reverted to a required method in future versions. - HashSet::default() - } ->>>>>>> origin/branch-51 /// Returns a reference to the user defined scalar function (udf) named /// `name`. From 08cdacc0f7a7a0e58431c401099c2f98d3049914 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 11:59:05 +0530 Subject: [PATCH 157/177] fix: uses `ScalarValue::try_cmp` See https://github.com/apache/datafusion/pull/16624 --- .../src/simplify_expressions/simplify_predicates.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs index b481f87fae641..241b370666d65 100644 --- a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs +++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs @@ -204,7 +204,6 @@ fn find_most_restrictive_predicate( if let Some(scalar) = scalar_value { if let Some(current_best) = best_value { -<<<<<<< HEAD let comparison = scalar.try_cmp(current_best)?; let is_better = if find_greater { comparison == std::cmp::Ordering::Greater @@ -215,20 +214,12 @@ fn find_most_restrictive_predicate( || (comparison == std::cmp::Ordering::Equal && op == &Operator::Lt) }; -======= - if let Some(comparison) = scalar.partial_cmp(current_best) { - let is_better = if find_greater { - comparison == std::cmp::Ordering::Greater - } else { - comparison == std::cmp::Ordering::Less - }; ->>>>>>> origin/branch-51 if is_better { best_value = Some(scalar); most_restrictive_idx = idx; } - } + } else { best_value = Some(scalar); most_restrictive_idx = idx; From d00b42a659cd1f90f06c626868d0ebf7253b8c17 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:00:29 +0530 Subject: [PATCH 158/177] fix: keep `with_node_id` --- datafusion/physical-plan/src/aggregates/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/datafusion/physical-plan/src/aggregates/mod.rs b/datafusion/physical-plan/src/aggregates/mod.rs index 08ae2d6799ef0..d0925a8aed2de 100644 --- a/datafusion/physical-plan/src/aggregates/mod.rs +++ b/datafusion/physical-plan/src/aggregates/mod.rs @@ -1031,7 +1031,6 @@ impl ExecutionPlan for AggregateExec { fn cardinality_effect(&self) -> CardinalityEffect { CardinalityEffect::LowerEqual } -<<<<<<< HEAD /// Push down parent filters when possible (see implementation comment for details), /// but do not introduce any new self filters. @@ -1113,7 +1112,8 @@ impl ExecutionPlan for AggregateExec { ); Ok(FilterDescription::new().with_child(child_desc)) -======= + } + fn with_node_id( self: Arc, node_id: usize, @@ -1136,7 +1136,6 @@ impl ExecutionPlan for AggregateExec { let new_props: PlanProperties = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) ->>>>>>> origin/branch-51 } } From 46565c4c9f5a9c9751363aa5b73b3ba30e712854 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:03:48 +0530 Subject: [PATCH 159/177] fix: remove duplicate definition --- datafusion/functions-aggregate/src/string_agg.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/datafusion/functions-aggregate/src/string_agg.rs b/datafusion/functions-aggregate/src/string_agg.rs index e8b705bf67e06..4a040df7b4a3b 100644 --- a/datafusion/functions-aggregate/src/string_agg.rs +++ b/datafusion/functions-aggregate/src/string_agg.rs @@ -222,10 +222,6 @@ impl AggregateUDFImpl for StringAgg { datafusion_expr::ReversedUDAF::Reversed(string_agg_udaf()) } - fn reverse_expr(&self) -> datafusion_expr::ReversedUDAF { - datafusion_expr::ReversedUDAF::Reversed(string_agg_udaf()) - } - fn documentation(&self) -> Option<&Documentation> { self.doc() } From c4415768ec90da54336f5d4667aaa229d8871c2a Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:06:21 +0530 Subject: [PATCH 160/177] fix: `AnalyzeExec::new` now takes 5 arguments instead of 4 https://docs.rs/datafusion/51.0.0/datafusion/physical_plan/analyze/struct.AnalyzeExec.html#method.new --- datafusion/physical-plan/src/analyze.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/physical-plan/src/analyze.rs b/datafusion/physical-plan/src/analyze.rs index 9a101104cbce4..571c51ffbd5ee 100644 --- a/datafusion/physical-plan/src/analyze.rs +++ b/datafusion/physical-plan/src/analyze.rs @@ -225,6 +225,7 @@ impl ExecutionPlan for AnalyzeExec { let mut new_plan = AnalyzeExec::new( self.verbose, self.show_statistics, + self.metric_types.clone(), Arc::clone(self.input()), Arc::clone(&self.schema), ); From 2298984ff250bf7644d9862721c0f6725e880723 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:16:16 +0530 Subject: [PATCH 161/177] fix: use `expr()` method --- datafusion/physical-plan/src/projection.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/projection.rs b/datafusion/physical-plan/src/projection.rs index 2850fefa53b0a..cfdaa4e9d9fd4 100644 --- a/datafusion/physical-plan/src/projection.rs +++ b/datafusion/physical-plan/src/projection.rs @@ -307,7 +307,7 @@ impl ExecutionPlan for ProjectionExec { node_id: usize, ) -> Result>> { let mut new_plan = - ProjectionExec::try_new(self.expr.clone(), Arc::clone(self.input()))?; + ProjectionExec::try_new(self.expr().to_vec(), Arc::clone(self.input()))?; let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) From e5237bfc1374c68bf82b31c6f353ccca53fba11e Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:39:05 +0530 Subject: [PATCH 162/177] fix: allow deprecated `UnionExec::new` https://docs.rs/datafusion/51.0.0/datafusion/physical_plan/union/struct.UnionExec.html#method.new --- datafusion/physical-plan/src/union.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 33aa205d00a40..1b939c003d65b 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -334,6 +334,7 @@ impl ExecutionPlan for UnionExec { true } + #[allow(deprecated)] fn with_node_id( self: Arc, node_id: usize, From 8e99a4215efecc111244ecf4be1ce9b86209843f Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:43:01 +0530 Subject: [PATCH 163/177] fix: `UnnestExec::new` returns a `Result` type --- datafusion/physical-plan/src/unnest.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index 977fe6fc78d13..f6ef85dc59795 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -278,7 +278,7 @@ impl ExecutionPlan for UnnestExec { self.struct_column_indices.clone(), Arc::clone(&self.schema), self.options.clone(), - ); + )?; let new_props = new_plan.cache.clone().with_node_id(node_id); new_plan.cache = new_props; Ok(Some(Arc::new(new_plan))) From a57ddc59d31136b9f953e4e96cbfddce542f4625 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 12:50:02 +0530 Subject: [PATCH 164/177] fix: keep upstream `FileScanConfig` changes * use `with_projection_indices()` - https://datafusion.apache.org/library-user-guide/upgrading.html#filescanconfig-projection-renamed-to-filescanconfig-projection-exprs * PR refactoring `reassign_predicate_columns`: https://github.com/apache/datafusion/pull/17703 --- datafusion/datasource/src/file_scan_config.rs | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/datafusion/datasource/src/file_scan_config.rs b/datafusion/datasource/src/file_scan_config.rs index db4b61c2bff80..02d9762a4a396 100644 --- a/datafusion/datasource/src/file_scan_config.rs +++ b/datafusion/datasource/src/file_scan_config.rs @@ -625,26 +625,8 @@ impl DataSource for FileScanConfig { if let Some(filter) = self.file_source.filter() { // We need to remap column indexes to match the projected schema since that's what the equivalence properties deal with. // Note that this will *ignore* any non-projected columns: these don't factor into ordering / equivalence. -<<<<<<< HEAD match Self::add_filter_equivalence_info(filter, &mut eq_properties, &schema) { Ok(()) => {} -======= - match reassign_predicate_columns(filter, &schema, true) { - Ok(filter) => { - match Self::add_filter_equivalence_info( - filter, - &mut eq_properties, - &schema, - ) { - Ok(()) => {} - Err(e) => { - warn!("Failed to add filter equivalence info: {e}"); - #[cfg(debug_assertions)] - panic!("Failed to add filter equivalence info: {e}"); - } - } - } ->>>>>>> origin/branch-51 Err(e) => { warn!("Failed to add filter equivalence info: {e}"); #[cfg(debug_assertions)] @@ -848,7 +830,6 @@ impl FileScanConfig { eq_properties: &mut EquivalenceProperties, schema: &Schema, ) -> Result<()> { -<<<<<<< HEAD // Gather valid equality pairs from the filter expression let equal_pairs = split_conjunction(&filter).into_iter().filter_map(|expr| { // Ignore any binary expressions that reference non-existent columns in the current schema @@ -865,25 +846,6 @@ impl FileScanConfig { for (lhs, rhs) in equal_pairs { eq_properties.add_equal_conditions(lhs, rhs)? -======= - macro_rules! ignore_dangling_col { - ($col:expr) => { - if let Some(col) = $col.as_any().downcast_ref::() { - if schema.index_of(col.name()).is_err() { - continue; - } - } - }; - } - - let (equal_pairs, _) = collect_columns_from_predicate(&filter); - for (lhs, rhs) in equal_pairs { - // Ignore any binary expressions that reference non-existent columns in the current schema - // (e.g. due to unnecessary projections being removed) - ignore_dangling_col!(lhs); - ignore_dangling_col!(rhs); - eq_properties.add_equal_conditions(Arc::clone(lhs), Arc::clone(rhs))? ->>>>>>> origin/branch-51 } Ok(()) @@ -1571,12 +1533,8 @@ mod tests { use datafusion_common::{assert_batches_eq, internal_err}; use datafusion_expr::{Operator, SortExpr}; use datafusion_physical_expr::create_physical_sort_expr; -<<<<<<< HEAD use datafusion_physical_expr::expressions::{BinaryExpr, Column, Literal}; use datafusion_physical_expr_common::sort_expr::PhysicalSortExpr; -======= - use datafusion_physical_expr::expressions::{BinaryExpr, Literal}; ->>>>>>> origin/branch-51 /// Returns the column names on the schema pub fn columns(schema: &Schema) -> Vec { @@ -2352,11 +2310,7 @@ mod tests { Arc::clone(&file_schema), Arc::clone(&file_source), ) -<<<<<<< HEAD .with_projection_indices(Some(vec![0, 1, 2])) -======= - .with_projection(Some(vec![0, 1, 2])) ->>>>>>> origin/branch-51 .build(); // Simulate projection being updated. Since the filter has already been pushed down, From f1322fe01e0a93f064bbda12cc2dc1f7a8f8e3f2 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 13:36:27 +0530 Subject: [PATCH 165/177] fix: parquet metrics + row_group_filter + reader + source --- datafusion/datasource-parquet/src/metrics.rs | 19 +++++-------------- datafusion/datasource-parquet/src/reader.rs | 4 ---- .../src/row_group_filter.rs | 4 ---- datafusion/datasource-parquet/src/source.rs | 5 ----- 4 files changed, 5 insertions(+), 27 deletions(-) diff --git a/datafusion/datasource-parquet/src/metrics.rs b/datafusion/datasource-parquet/src/metrics.rs index 7a88f06c2608a..c45d234f3b512 100644 --- a/datafusion/datasource-parquet/src/metrics.rs +++ b/datafusion/datasource-parquet/src/metrics.rs @@ -44,25 +44,18 @@ pub struct ParquetFileMetrics { pub files_ranges_pruned_statistics: PruningMetrics, /// Number of times the predicate could not be evaluated pub predicate_evaluation_errors: Count, -<<<<<<< HEAD /// Number of row groups whose bloom filters were checked, tracked with matched/pruned counts pub row_groups_pruned_bloom_filter: PruningMetrics, /// Number of row groups whose statistics were checked, tracked with matched/pruned counts pub row_groups_pruned_statistics: PruningMetrics, -======= /// Number of row groups whose bloom filters were checked and matched (not pruned) pub row_groups_matched_bloom_filter: Count, - /// Number of row groups pruned by bloom filters - pub row_groups_pruned_bloom_filter: Count, /// Number of row groups pruned due to limit pruning. pub limit_pruned_row_groups: Count, /// Number of row groups whose statistics were checked and fully matched pub row_groups_fully_matched_statistics: Count, /// Number of row groups whose statistics were checked and matched (not pruned) pub row_groups_matched_statistics: Count, - /// Number of row groups pruned by statistics - pub row_groups_pruned_statistics: Count, ->>>>>>> origin/branch-51 /// Total number of bytes scanned pub bytes_scanned: Count, /// Total rows filtered out by predicates pushed into parquet scan @@ -100,13 +93,14 @@ impl ParquetFileMetrics { // ----------------------- // 'summary' level metrics // ----------------------- + let row_groups_matched_bloom_filter = MetricBuilder::new(metrics) + .with_new_label("filename", filename.to_string()) + .counter("row_groups_matched_bloom_filter", partition); + let row_groups_pruned_bloom_filter = MetricBuilder::new(metrics) .with_new_label("filename", filename.to_string()) -<<<<<<< HEAD .with_type(MetricType::SUMMARY) .pruning_metrics("row_groups_pruned_bloom_filter", partition); -======= - .counter("row_groups_pruned_bloom_filter", partition); let limit_pruned_row_groups = MetricBuilder::new(metrics) .with_new_label("filename", filename.to_string()) @@ -119,7 +113,6 @@ impl ParquetFileMetrics { let row_groups_matched_statistics = MetricBuilder::new(metrics) .with_new_label("filename", filename.to_string()) .counter("row_groups_matched_statistics", partition); ->>>>>>> origin/branch-51 let row_groups_pruned_statistics = MetricBuilder::new(metrics) .with_new_label("filename", filename.to_string()) @@ -184,12 +177,10 @@ impl ParquetFileMetrics { Self { files_ranges_pruned_statistics, predicate_evaluation_errors, + row_groups_matched_bloom_filter, row_groups_pruned_bloom_filter, -<<<<<<< HEAD -======= row_groups_fully_matched_statistics, row_groups_matched_statistics, ->>>>>>> origin/branch-51 row_groups_pruned_statistics, limit_pruned_row_groups, bytes_scanned, diff --git a/datafusion/datasource-parquet/src/reader.rs b/datafusion/datasource-parquet/src/reader.rs index da97cd253aa7a..88a3cea5623bc 100644 --- a/datafusion/datasource-parquet/src/reader.rs +++ b/datafusion/datasource-parquet/src/reader.rs @@ -269,11 +269,7 @@ impl AsyncFileReader for CachedParquetFileReader { #[cfg(not(feature = "parquet_encryption"))] let file_decryption_properties = None; -<<<<<<< HEAD DFParquetMetadata::new(&self.store, &object_meta) -======= - DFParquetMetadata::new(&self.store, &file_meta.object_meta) ->>>>>>> origin/branch-51 .with_decryption_properties(file_decryption_properties) .with_file_metadata_cache(Some(Arc::clone(&metadata_cache))) .with_metadata_size_hint(self.metadata_size_hint) diff --git a/datafusion/datasource-parquet/src/row_group_filter.rs b/datafusion/datasource-parquet/src/row_group_filter.rs index 7c07d419d997f..f0d483ba35b10 100644 --- a/datafusion/datasource-parquet/src/row_group_filter.rs +++ b/datafusion/datasource-parquet/src/row_group_filter.rs @@ -192,12 +192,8 @@ impl RowGroupAccessPlanFilter { self.access_plan.skip(*idx); metrics.row_groups_pruned_statistics.add_pruned(1); } else { -<<<<<<< HEAD metrics.row_groups_pruned_statistics.add_matched(1); -======= fully_contained_candidates_original_idx.push(*idx); - metrics.row_groups_matched_statistics.add(1); ->>>>>>> origin/branch-51 } } diff --git a/datafusion/datasource-parquet/src/source.rs b/datafusion/datasource-parquet/src/source.rs index baa39d70a16f3..339d36b57cc35 100644 --- a/datafusion/datasource-parquet/src/source.rs +++ b/datafusion/datasource-parquet/src/source.rs @@ -53,11 +53,6 @@ use datafusion_physical_plan::metrics::Count; use datafusion_physical_plan::metrics::ExecutionPlanMetricsSet; use datafusion_physical_plan::DisplayFormatType; -<<<<<<< HEAD -======= -#[cfg(feature = "parquet_encryption")] -use datafusion_common::encryption::map_config_decryption_to_decryption; ->>>>>>> origin/branch-51 #[cfg(feature = "parquet_encryption")] use datafusion_execution::parquet_encryption::EncryptionFactory; use itertools::Itertools; From b3413745c8aca4788693ffa1d05614e22b0c868a Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 13:38:39 +0530 Subject: [PATCH 166/177] fix: proto + regen --- datafusion/proto-common/src/generated/pbjson.rs | 5 ----- datafusion/proto/src/lib.rs | 3 --- 2 files changed, 8 deletions(-) diff --git a/datafusion/proto-common/src/generated/pbjson.rs b/datafusion/proto-common/src/generated/pbjson.rs index 8061a9404d321..e63f345459b8f 100644 --- a/datafusion/proto-common/src/generated/pbjson.rs +++ b/datafusion/proto-common/src/generated/pbjson.rs @@ -2007,12 +2007,7 @@ impl<'de> serde::Deserialize<'de> for CsvOptions { if truncated_rows__.is_some() { return Err(serde::de::Error::duplicate_field("truncatedRows")); } -<<<<<<< HEAD truncated_rows__ = -======= - - truncated_rows__ = ->>>>>>> origin/branch-51 Some(map_.next_value::<::pbjson::private::BytesDeserialize<_>>()?.0) ; } diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rs index d84d58a65af6b..b16b12bc05162 100644 --- a/datafusion/proto/src/lib.rs +++ b/datafusion/proto/src/lib.rs @@ -97,7 +97,6 @@ //! assert_eq!(format!("{:?}", plan), format!("{:?}", logical_round_trip)); //! # Ok(()) //! # } -<<<<<<< HEAD //! ``` //! # Example: Serializing [`ExecutionPlan`]s //! @@ -121,8 +120,6 @@ //! # Ok(()) //! # } //! ``` -======= ->>>>>>> origin/branch-51 pub mod bytes; pub mod common; pub mod generated; From ab8723c446e24d7556041dfc48747c2a813f228a Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:14:03 +0530 Subject: [PATCH 167/177] fix: upstream reorg of `ListingTable` into `datafusion-catalog-listing` crate See https://datafusion.apache.org/library-user-guide/upgrading.html#reorganization-of-listingtable-into-datafusion-catalog-listing-crate --- .../core/src/datasource/listing/table.rs | 1198 ----------------- 1 file changed, 1198 deletions(-) diff --git a/datafusion/core/src/datasource/listing/table.rs b/datafusion/core/src/datasource/listing/table.rs index b0fd41a4deeae..3333b70676203 100644 --- a/datafusion/core/src/datasource/listing/table.rs +++ b/datafusion/core/src/datasource/listing/table.rs @@ -103,1204 +103,6 @@ impl ListingTableConfigExt for ListingTableConfig { async fn infer(self, state: &dyn Session) -> datafusion_common::Result { self.infer_options(state).await?.infer_schema(state).await } -<<<<<<< HEAD -======= - - /// Infer the partition columns from `table_paths`. - /// - /// # Errors - /// * if `self.options` is not set. See [`Self::with_listing_options`] - pub async fn infer_partitions_from_path(self, state: &dyn Session) -> Result { - match self.options { - Some(options) => { - let Some(url) = self.table_paths.first() else { - return config_err!("No table path found"); - }; - let partitions = options - .infer_partitions(state, url) - .await? - .into_iter() - .map(|col_name| { - ( - col_name, - DataType::Dictionary( - Box::new(DataType::UInt16), - Box::new(DataType::Utf8), - ), - ) - }) - .collect::>(); - let options = options.with_table_partition_cols(partitions); - Ok(Self { - table_paths: self.table_paths, - file_schema: self.file_schema, - options: Some(options), - schema_source: self.schema_source, - schema_adapter_factory: self.schema_adapter_factory, - expr_adapter_factory: self.expr_adapter_factory, - }) - } - None => config_err!("No `ListingOptions` set for inferring schema"), - } - } - - /// Set the [`SchemaAdapterFactory`] for the [`ListingTable`] - /// - /// The schema adapter factory is used to create schema adapters that can - /// handle schema evolution and type conversions when reading files with - /// different schemas than the table schema. - /// - /// If not provided, a default schema adapter factory will be used. - /// - /// # Example: Custom Schema Adapter for Type Coercion - /// ```rust - /// # use std::sync::Arc; - /// # use datafusion::datasource::listing::{ListingTableConfig, ListingOptions, ListingTableUrl}; - /// # use datafusion::datasource::schema_adapter::{SchemaAdapterFactory, SchemaAdapter}; - /// # use datafusion::datasource::file_format::parquet::ParquetFormat; - /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType}; - /// # - /// # #[derive(Debug)] - /// # struct MySchemaAdapterFactory; - /// # impl SchemaAdapterFactory for MySchemaAdapterFactory { - /// # fn create(&self, _projected_table_schema: SchemaRef, _file_schema: SchemaRef) -> Box { - /// # unimplemented!() - /// # } - /// # } - /// # let table_paths = ListingTableUrl::parse("file:///path/to/data").unwrap(); - /// # let listing_options = ListingOptions::new(Arc::new(ParquetFormat::default())); - /// # let table_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); - /// let config = ListingTableConfig::new(table_paths) - /// .with_listing_options(listing_options) - /// .with_schema(table_schema) - /// .with_schema_adapter_factory(Arc::new(MySchemaAdapterFactory)); - /// ``` - pub fn with_schema_adapter_factory( - self, - schema_adapter_factory: Arc, - ) -> Self { - Self { - schema_adapter_factory: Some(schema_adapter_factory), - ..self - } - } - - /// Get the [`SchemaAdapterFactory`] for this configuration - pub fn schema_adapter_factory(&self) -> Option<&Arc> { - self.schema_adapter_factory.as_ref() - } - - /// Set the [`PhysicalExprAdapterFactory`] for the [`ListingTable`] - /// - /// The expression adapter factory is used to create physical expression adapters that can - /// handle schema evolution and type conversions when evaluating expressions - /// with different schemas than the table schema. - /// - /// If not provided, a default physical expression adapter factory will be used unless a custom - /// `SchemaAdapterFactory` is set, in which case only the `SchemaAdapterFactory` will be used. - /// - /// See for details on this transition. - pub fn with_expr_adapter_factory( - self, - expr_adapter_factory: Arc, - ) -> Self { - Self { - expr_adapter_factory: Some(expr_adapter_factory), - ..self - } - } -} - -/// Options for creating a [`ListingTable`] -#[derive(Clone, Debug)] -pub struct ListingOptions { - /// A suffix on which files should be filtered (leave empty to - /// keep all files on the path) - pub file_extension: String, - /// The file format - pub format: Arc, - /// The expected partition column names in the folder structure. - /// See [Self::with_table_partition_cols] for details - pub table_partition_cols: Vec<(String, DataType)>, - /// Set true to try to guess statistics from the files. - /// This can add a lot of overhead as it will usually require files - /// to be opened and at least partially parsed. - pub collect_stat: bool, - /// Group files to avoid that the number of partitions exceeds - /// this limit - pub target_partitions: usize, - /// Optional pre-known sort order(s). Must be `SortExpr`s. - /// - /// DataFusion may take advantage of this ordering to omit sorts - /// or use more efficient algorithms. Currently sortedness must be - /// provided if it is known by some external mechanism, but may in - /// the future be automatically determined, for example using - /// parquet metadata. - /// - /// See - /// - /// NOTE: This attribute stores all equivalent orderings (the outer `Vec`) - /// where each ordering consists of an individual lexicographic - /// ordering (encapsulated by a `Vec`). If there aren't - /// multiple equivalent orderings, the outer `Vec` will have a - /// single element. - pub file_sort_order: Vec>, -} - -impl ListingOptions { - /// Creates an options instance with the given format - /// Default values: - /// - use default file extension filter - /// - no input partition to discover - /// - one target partition - /// - do not collect statistics - pub fn new(format: Arc) -> Self { - Self { - file_extension: format.get_ext(), - format, - table_partition_cols: vec![], - collect_stat: false, - target_partitions: 1, - file_sort_order: vec![], - } - } - - /// Set options from [`SessionConfig`] and returns self. - /// - /// Currently this sets `target_partitions` and `collect_stat` - /// but if more options are added in the future that need to be coordinated - /// they will be synchronized through this method. - pub fn with_session_config_options(mut self, config: &SessionConfig) -> Self { - self = self.with_target_partitions(config.target_partitions()); - self = self.with_collect_stat(config.collect_statistics()); - self - } - - /// Set file extension on [`ListingOptions`] and returns self. - /// - /// # Example - /// ``` - /// # use std::sync::Arc; - /// # use datafusion::prelude::SessionContext; - /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat}; - /// - /// let listing_options = ListingOptions::new(Arc::new( - /// ParquetFormat::default() - /// )) - /// .with_file_extension(".parquet"); - /// - /// assert_eq!(listing_options.file_extension, ".parquet"); - /// ``` - pub fn with_file_extension(mut self, file_extension: impl Into) -> Self { - self.file_extension = file_extension.into(); - self - } - - /// Optionally set file extension on [`ListingOptions`] and returns self. - /// - /// If `file_extension` is `None`, the file extension will not be changed - /// - /// # Example - /// ``` - /// # use std::sync::Arc; - /// # use datafusion::prelude::SessionContext; - /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat}; - /// let extension = Some(".parquet"); - /// let listing_options = ListingOptions::new(Arc::new( - /// ParquetFormat::default() - /// )) - /// .with_file_extension_opt(extension); - /// - /// assert_eq!(listing_options.file_extension, ".parquet"); - /// ``` - pub fn with_file_extension_opt(mut self, file_extension: Option) -> Self - where - S: Into, - { - if let Some(file_extension) = file_extension { - self.file_extension = file_extension.into(); - } - self - } - - /// Set `table partition columns` on [`ListingOptions`] and returns self. - /// - /// "partition columns," used to support [Hive Partitioning], are - /// columns added to the data that is read, based on the folder - /// structure where the data resides. - /// - /// For example, give the following files in your filesystem: - /// - /// ```text - /// /mnt/nyctaxi/year=2022/month=01/tripdata.parquet - /// /mnt/nyctaxi/year=2021/month=12/tripdata.parquet - /// /mnt/nyctaxi/year=2021/month=11/tripdata.parquet - /// ``` - /// - /// A [`ListingTable`] created at `/mnt/nyctaxi/` with partition - /// columns "year" and "month" will include new `year` and `month` - /// columns while reading the files. The `year` column would have - /// value `2022` and the `month` column would have value `01` for - /// the rows read from - /// `/mnt/nyctaxi/year=2022/month=01/tripdata.parquet` - /// - ///# Notes - /// - /// - If only one level (e.g. `year` in the example above) is - /// specified, the other levels are ignored but the files are - /// still read. - /// - /// - Files that don't follow this partitioning scheme will be - /// ignored. - /// - /// - Since the columns have the same value for all rows read from - /// each individual file (such as dates), they are typically - /// dictionary encoded for efficiency. You may use - /// [`wrap_partition_type_in_dict`] to request a - /// dictionary-encoded type. - /// - /// - The partition columns are solely extracted from the file path. Especially they are NOT part of the parquet files itself. - /// - /// # Example - /// - /// ``` - /// # use std::sync::Arc; - /// # use arrow::datatypes::DataType; - /// # use datafusion::prelude::col; - /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat}; - /// - /// // listing options for files with paths such as `/mnt/data/col_a=x/col_b=y/data.parquet` - /// // `col_a` and `col_b` will be included in the data read from those files - /// let listing_options = ListingOptions::new(Arc::new( - /// ParquetFormat::default() - /// )) - /// .with_table_partition_cols(vec![("col_a".to_string(), DataType::Utf8), - /// ("col_b".to_string(), DataType::Utf8)]); - /// - /// assert_eq!(listing_options.table_partition_cols, vec![("col_a".to_string(), DataType::Utf8), - /// ("col_b".to_string(), DataType::Utf8)]); - /// ``` - /// - /// [Hive Partitioning]: https://docs.cloudera.com/HDPDocuments/HDP2/HDP-2.1.3/bk_system-admin-guide/content/hive_partitioned_tables.html - /// [`wrap_partition_type_in_dict`]: crate::datasource::physical_plan::wrap_partition_type_in_dict - pub fn with_table_partition_cols( - mut self, - table_partition_cols: Vec<(String, DataType)>, - ) -> Self { - self.table_partition_cols = table_partition_cols; - self - } - - /// Set stat collection on [`ListingOptions`] and returns self. - /// - /// ``` - /// # use std::sync::Arc; - /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat}; - /// - /// let listing_options = ListingOptions::new(Arc::new( - /// ParquetFormat::default() - /// )) - /// .with_collect_stat(true); - /// - /// assert_eq!(listing_options.collect_stat, true); - /// ``` - pub fn with_collect_stat(mut self, collect_stat: bool) -> Self { - self.collect_stat = collect_stat; - self - } - - /// Set number of target partitions on [`ListingOptions`] and returns self. - /// - /// ``` - /// # use std::sync::Arc; - /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat}; - /// - /// let listing_options = ListingOptions::new(Arc::new( - /// ParquetFormat::default() - /// )) - /// .with_target_partitions(8); - /// - /// assert_eq!(listing_options.target_partitions, 8); - /// ``` - pub fn with_target_partitions(mut self, target_partitions: usize) -> Self { - self.target_partitions = target_partitions; - self - } - - /// Set file sort order on [`ListingOptions`] and returns self. - /// - /// ``` - /// # use std::sync::Arc; - /// # use datafusion::prelude::col; - /// # use datafusion::datasource::{listing::ListingOptions, file_format::parquet::ParquetFormat}; - /// - /// // Tell datafusion that the files are sorted by column "a" - /// let file_sort_order = vec![vec![ - /// col("a").sort(true, true) - /// ]]; - /// - /// let listing_options = ListingOptions::new(Arc::new( - /// ParquetFormat::default() - /// )) - /// .with_file_sort_order(file_sort_order.clone()); - /// - /// assert_eq!(listing_options.file_sort_order, file_sort_order); - /// ``` - pub fn with_file_sort_order(mut self, file_sort_order: Vec>) -> Self { - self.file_sort_order = file_sort_order; - self - } - - /// Infer the schema of the files at the given path on the provided object store. - /// - /// If the table_path contains one or more files (i.e. it is a directory / - /// prefix of files) their schema is merged by calling [`FileFormat::infer_schema`] - /// - /// Note: The inferred schema does not include any partitioning columns. - /// - /// This method is called as part of creating a [`ListingTable`]. - pub async fn infer_schema<'a>( - &'a self, - state: &dyn Session, - table_path: &'a ListingTableUrl, - ) -> Result { - let store = state.runtime_env().object_store(table_path)?; - - let files: Vec<_> = table_path - .list_all_files(state, store.as_ref(), &self.file_extension) - .await? - // Empty files cannot affect schema but may throw when trying to read for it - .try_filter(|object_meta| future::ready(object_meta.size > 0)) - .try_collect() - .await?; - - let schema = self.format.infer_schema(state, &store, &files).await?; - - Ok(schema) - } - - /// Infers the partition columns stored in `LOCATION` and compares - /// them with the columns provided in `PARTITIONED BY` to help prevent - /// accidental corrupts of partitioned tables. - /// - /// Allows specifying partial partitions. - pub async fn validate_partitions( - &self, - state: &dyn Session, - table_path: &ListingTableUrl, - ) -> Result<()> { - if self.table_partition_cols.is_empty() { - return Ok(()); - } - - if !table_path.is_collection() { - return plan_err!( - "Can't create a partitioned table backed by a single file, \ - perhaps the URL is missing a trailing slash?" - ); - } - - let inferred = self.infer_partitions(state, table_path).await?; - - // no partitioned files found on disk - if inferred.is_empty() { - return Ok(()); - } - - let table_partition_names = self - .table_partition_cols - .iter() - .map(|(col_name, _)| col_name.clone()) - .collect_vec(); - - if inferred.len() < table_partition_names.len() { - return plan_err!( - "Inferred partitions to be {:?}, but got {:?}", - inferred, - table_partition_names - ); - } - - // match prefix to allow creating tables with partial partitions - for (idx, col) in table_partition_names.iter().enumerate() { - if &inferred[idx] != col { - return plan_err!( - "Inferred partitions to be {:?}, but got {:?}", - inferred, - table_partition_names - ); - } - } - - Ok(()) - } - - /// Infer the partitioning at the given path on the provided object store. - /// For performance reasons, it doesn't read all the files on disk - /// and therefore may fail to detect invalid partitioning. - pub(crate) async fn infer_partitions( - &self, - state: &dyn Session, - table_path: &ListingTableUrl, - ) -> Result> { - let store = state.runtime_env().object_store(table_path)?; - - // only use 10 files for inference - // This can fail to detect inconsistent partition keys - // A DFS traversal approach of the store can help here - let files: Vec<_> = table_path - .list_all_files(state, store.as_ref(), &self.file_extension) - .await? - .take(10) - .try_collect() - .await?; - - let stripped_path_parts = files.iter().map(|file| { - table_path - .strip_prefix(&file.location) - .unwrap() - .collect_vec() - }); - - let partition_keys = stripped_path_parts - .map(|path_parts| { - path_parts - .into_iter() - .rev() - .skip(1) // get parents only; skip the file itself - .rev() - .map(|s| s.split('=').take(1).collect()) - .collect_vec() - }) - .collect_vec(); - - match partition_keys.into_iter().all_equal_value() { - Ok(v) => Ok(v), - Err(None) => Ok(vec![]), - Err(Some(diff)) => { - let mut sorted_diff = [diff.0, diff.1]; - sorted_diff.sort(); - plan_err!("Found mixed partition values on disk {:?}", sorted_diff) - } - } - } -} - -/// Built in [`TableProvider`] that reads data from one or more files as a single table. -/// -/// The files are read using an [`ObjectStore`] instance, for example from -/// local files or objects from AWS S3. -/// -/// # Features: -/// * Reading multiple files as a single table -/// * Hive style partitioning (e.g., directories named `date=2024-06-01`) -/// * Merges schemas from files with compatible but not identical schemas (see [`ListingTableConfig::file_schema`]) -/// * `limit`, `filter` and `projection` pushdown for formats that support it (e.g., -/// Parquet) -/// * Statistics collection and pruning based on file metadata -/// * Pre-existing sort order (see [`ListingOptions::file_sort_order`]) -/// * Metadata caching to speed up repeated queries (see [`FileMetadataCache`]) -/// * Statistics caching (see [`FileStatisticsCache`]) -/// -/// [`FileMetadataCache`]: datafusion_execution::cache::cache_manager::FileMetadataCache -/// -/// # Reading Directories and Hive Style Partitioning -/// -/// For example, given the `table1` directory (or object store prefix) -/// -/// ```text -/// table1 -/// ├── file1.parquet -/// └── file2.parquet -/// ``` -/// -/// A `ListingTable` would read the files `file1.parquet` and `file2.parquet` as -/// a single table, merging the schemas if the files have compatible but not -/// identical schemas. -/// -/// Given the `table2` directory (or object store prefix) -/// -/// ```text -/// table2 -/// ├── date=2024-06-01 -/// │ ├── file3.parquet -/// │ └── file4.parquet -/// └── date=2024-06-02 -/// └── file5.parquet -/// ``` -/// -/// A `ListingTable` would read the files `file3.parquet`, `file4.parquet`, and -/// `file5.parquet` as a single table, again merging schemas if necessary. -/// -/// Given the hive style partitioning structure (e.g,. directories named -/// `date=2024-06-01` and `date=2026-06-02`), `ListingTable` also adds a `date` -/// column when reading the table: -/// * The files in `table2/date=2024-06-01` will have the value `2024-06-01` -/// * The files in `table2/date=2024-06-02` will have the value `2024-06-02`. -/// -/// If the query has a predicate like `WHERE date = '2024-06-01'` -/// only the corresponding directory will be read. -/// -/// # See Also -/// -/// 1. [`ListingTableConfig`]: Configuration options -/// 1. [`DataSourceExec`]: `ExecutionPlan` used by `ListingTable` -/// -/// [`DataSourceExec`]: crate::datasource::source::DataSourceExec -/// -/// # Caching Metadata -/// -/// Some formats, such as Parquet, use the `FileMetadataCache` to cache file -/// metadata that is needed to execute but expensive to read, such as row -/// groups and statistics. The cache is scoped to the [`SessionContext`] and can -/// be configured via the [runtime config options]. -/// -/// [`SessionContext`]: crate::prelude::SessionContext -/// [runtime config options]: https://datafusion.apache.org/user-guide/configs.html#runtime-configuration-settings -/// -/// # Example: Read a directory of parquet files using a [`ListingTable`] -/// -/// ```no_run -/// # use datafusion::prelude::SessionContext; -/// # use datafusion::error::Result; -/// # use std::sync::Arc; -/// # use datafusion::datasource::{ -/// # listing::{ -/// # ListingOptions, ListingTable, ListingTableConfig, ListingTableUrl, -/// # }, -/// # file_format::parquet::ParquetFormat, -/// # }; -/// # #[tokio::main] -/// # async fn main() -> Result<()> { -/// let ctx = SessionContext::new(); -/// let session_state = ctx.state(); -/// let table_path = "/path/to/parquet"; -/// -/// // Parse the path -/// let table_path = ListingTableUrl::parse(table_path)?; -/// -/// // Create default parquet options -/// let file_format = ParquetFormat::new(); -/// let listing_options = ListingOptions::new(Arc::new(file_format)) -/// .with_file_extension(".parquet"); -/// -/// // Resolve the schema -/// let resolved_schema = listing_options -/// .infer_schema(&session_state, &table_path) -/// .await?; -/// -/// let config = ListingTableConfig::new(table_path) -/// .with_listing_options(listing_options) -/// .with_schema(resolved_schema); -/// -/// // Create a new TableProvider -/// let provider = Arc::new(ListingTable::try_new(config)?); -/// -/// // This provider can now be read as a dataframe: -/// let df = ctx.read_table(provider.clone()); -/// -/// // or registered as a named table: -/// ctx.register_table("my_table", provider); -/// -/// # Ok(()) -/// # } -/// ``` -#[derive(Debug, Clone)] -pub struct ListingTable { - table_paths: Vec, - /// `file_schema` contains only the columns physically stored in the data files themselves. - /// - Represents the actual fields found in files like Parquet, CSV, etc. - /// - Used when reading the raw data from files - file_schema: SchemaRef, - /// `table_schema` combines `file_schema` + partition columns - /// - Partition columns are derived from directory paths (not stored in files) - /// - These are columns like "year=2022/month=01" in paths like `/data/year=2022/month=01/file.parquet` - table_schema: SchemaRef, - /// Indicates how the schema was derived (inferred or explicitly specified) - schema_source: SchemaSource, - /// Options used to configure the listing table such as the file format - /// and partitioning information - options: ListingOptions, - /// The SQL definition for this table, if any - definition: Option, - /// Cache for collected file statistics - collected_statistics: FileStatisticsCache, - /// Constraints applied to this table - constraints: Constraints, - /// Column default expressions for columns that are not physically present in the data files - column_defaults: HashMap, - /// Optional [`SchemaAdapterFactory`] for creating schema adapters - schema_adapter_factory: Option>, - /// Optional [`PhysicalExprAdapterFactory`] for creating physical expression adapters - expr_adapter_factory: Option>, -} - -impl ListingTable { - /// Create new [`ListingTable`] - /// - /// See documentation and example on [`ListingTable`] and [`ListingTableConfig`] - pub fn try_new(config: ListingTableConfig) -> Result { - // Extract schema_source before moving other parts of the config - let schema_source = config.schema_source(); - - let file_schema = config - .file_schema - .ok_or_else(|| DataFusionError::Internal("No schema provided.".into()))?; - - let options = config.options.ok_or_else(|| { - DataFusionError::Internal("No ListingOptions provided".into()) - })?; - - // Add the partition columns to the file schema - let mut builder = SchemaBuilder::from(file_schema.as_ref().to_owned()); - for (part_col_name, part_col_type) in &options.table_partition_cols { - builder.push(Field::new(part_col_name, part_col_type.clone(), false)); - } - - let table_schema = Arc::new( - builder - .finish() - .with_metadata(file_schema.metadata().clone()), - ); - - let table = Self { - table_paths: config.table_paths, - file_schema, - table_schema, - schema_source, - options, - definition: None, - collected_statistics: Arc::new(DefaultFileStatisticsCache::default()), - constraints: Constraints::default(), - column_defaults: HashMap::new(), - schema_adapter_factory: config.schema_adapter_factory, - expr_adapter_factory: config.expr_adapter_factory, - }; - - Ok(table) - } - - /// Assign constraints - pub fn with_constraints(mut self, constraints: Constraints) -> Self { - self.constraints = constraints; - self - } - - /// Assign column defaults - pub fn with_column_defaults( - mut self, - column_defaults: HashMap, - ) -> Self { - self.column_defaults = column_defaults; - self - } - - /// Set the [`FileStatisticsCache`] used to cache parquet file statistics. - /// - /// Setting a statistics cache on the `SessionContext` can avoid refetching statistics - /// multiple times in the same session. - /// - /// If `None`, creates a new [`DefaultFileStatisticsCache`] scoped to this query. - pub fn with_cache(mut self, cache: Option) -> Self { - self.collected_statistics = - cache.unwrap_or_else(|| Arc::new(DefaultFileStatisticsCache::default())); - self - } - - /// Specify the SQL definition for this table, if any - pub fn with_definition(mut self, definition: Option) -> Self { - self.definition = definition; - self - } - - /// Get paths ref - pub fn table_paths(&self) -> &Vec { - &self.table_paths - } - - /// Get options ref - pub fn options(&self) -> &ListingOptions { - &self.options - } - - /// Get the schema source - pub fn schema_source(&self) -> SchemaSource { - self.schema_source - } - - /// Set the [`SchemaAdapterFactory`] for this [`ListingTable`] - /// - /// The schema adapter factory is used to create schema adapters that can - /// handle schema evolution and type conversions when reading files with - /// different schemas than the table schema. - /// - /// # Example: Adding Schema Evolution Support - /// ```rust - /// # use std::sync::Arc; - /// # use datafusion::datasource::listing::{ListingTable, ListingTableConfig, ListingOptions, ListingTableUrl}; - /// # use datafusion::datasource::schema_adapter::{DefaultSchemaAdapterFactory, SchemaAdapter}; - /// # use datafusion::datasource::file_format::parquet::ParquetFormat; - /// # use arrow::datatypes::{SchemaRef, Schema, Field, DataType}; - /// # let table_path = ListingTableUrl::parse("file:///path/to/data").unwrap(); - /// # let options = ListingOptions::new(Arc::new(ParquetFormat::default())); - /// # let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); - /// # let config = ListingTableConfig::new(table_path).with_listing_options(options).with_schema(schema); - /// # let table = ListingTable::try_new(config).unwrap(); - /// let table_with_evolution = table - /// .with_schema_adapter_factory(Arc::new(DefaultSchemaAdapterFactory)); - /// ``` - /// See [`ListingTableConfig::with_schema_adapter_factory`] for an example of custom SchemaAdapterFactory. - pub fn with_schema_adapter_factory( - self, - schema_adapter_factory: Arc, - ) -> Self { - Self { - schema_adapter_factory: Some(schema_adapter_factory), - ..self - } - } - - /// Get the [`SchemaAdapterFactory`] for this table - pub fn schema_adapter_factory(&self) -> Option<&Arc> { - self.schema_adapter_factory.as_ref() - } - - /// Creates a schema adapter for mapping between file and table schemas - /// - /// Uses the configured schema adapter factory if available, otherwise falls back - /// to the default implementation. - fn create_schema_adapter(&self) -> Box { - let table_schema = self.schema(); - match &self.schema_adapter_factory { - Some(factory) => { - factory.create_with_projected_schema(Arc::clone(&table_schema)) - } - None => DefaultSchemaAdapterFactory::from_schema(Arc::clone(&table_schema)), - } - } - - /// Creates a file source and applies schema adapter factory if available - fn create_file_source_with_schema_adapter(&self) -> Result> { - let mut source = self.options.format.file_source(); - // Apply schema adapter to source if available - // - // The source will use this SchemaAdapter to adapt data batches as they flow up the plan. - // Note: ListingTable also creates a SchemaAdapter in `scan()` but that is only used to adapt collected statistics. - if let Some(factory) = &self.schema_adapter_factory { - source = source.with_schema_adapter_factory(Arc::clone(factory))?; - } - Ok(source) - } - - /// If file_sort_order is specified, creates the appropriate physical expressions - fn try_create_output_ordering(&self) -> Result> { - create_ordering(&self.table_schema, &self.options.file_sort_order) - } -} - -// Expressions can be used for partition pruning if they can be evaluated using -// only the partition columns and there are partition columns. -fn can_be_evaluated_for_partition_pruning( - partition_column_names: &[&str], - expr: &Expr, -) -> bool { - !partition_column_names.is_empty() - && expr_applicable_for_cols(partition_column_names, expr) -} - -#[async_trait] -impl TableProvider for ListingTable { - fn as_any(&self) -> &dyn Any { - self - } - - fn schema(&self) -> SchemaRef { - Arc::clone(&self.table_schema) - } - - fn constraints(&self) -> Option<&Constraints> { - Some(&self.constraints) - } - - fn table_type(&self) -> TableType { - TableType::Base - } - - async fn scan( - &self, - state: &dyn Session, - projection: Option<&Vec>, - filters: &[Expr], - limit: Option, - ) -> Result> { - // extract types of partition columns - let table_partition_cols = self - .options - .table_partition_cols - .iter() - .map(|col| Ok(self.table_schema.field_with_name(&col.0)?.clone())) - .collect::>>()?; - - let table_partition_col_names = table_partition_cols - .iter() - .map(|field| field.name().as_str()) - .collect::>(); - // If the filters can be resolved using only partition cols, there is no need to - // pushdown it to TableScan, otherwise, `unhandled` pruning predicates will be generated - let (partition_filters, filters): (Vec<_>, Vec<_>) = - filters.iter().cloned().partition(|filter| { - can_be_evaluated_for_partition_pruning(&table_partition_col_names, filter) - }); - - // We should not limit the number of partitioned files to scan if there are filters and limit - // at the same time. This is because the limit should be applied after the filters are applied. - let statistic_file_limit = if filters.is_empty() { limit } else { None }; - - let (mut partitioned_file_lists, statistics) = self - .list_files_for_scan(state, &partition_filters, statistic_file_limit) - .await?; - - // if no files need to be read, return an `EmptyExec` - if partitioned_file_lists.is_empty() { - let projected_schema = project_schema(&self.schema(), projection)?; - return Ok(Arc::new(EmptyExec::new(projected_schema))); - } - - let output_ordering = self.try_create_output_ordering()?; - match state - .config_options() - .execution - .split_file_groups_by_statistics - .then(|| { - output_ordering.first().map(|output_ordering| { - FileScanConfig::split_groups_by_statistics_with_target_partitions( - &self.table_schema, - &partitioned_file_lists, - output_ordering, - self.options.target_partitions, - ) - }) - }) - .flatten() - { - Some(Err(e)) => log::debug!("failed to split file groups by statistics: {e}"), - Some(Ok(new_groups)) => { - if new_groups.len() <= self.options.target_partitions { - partitioned_file_lists = new_groups; - } else { - log::debug!("attempted to split file groups by statistics, but there were more file groups than target_partitions; falling back to unordered") - } - } - None => {} // no ordering required - }; - - let Some(object_store_url) = - self.table_paths.first().map(ListingTableUrl::object_store) - else { - return Ok(Arc::new(EmptyExec::new(Arc::new(Schema::empty())))); - }; - - let file_source = self.create_file_source_with_schema_adapter()?; - - // create the execution plan - self.options - .format - .create_physical_plan( - state, - FileScanConfigBuilder::new( - object_store_url, - Arc::clone(&self.file_schema), - file_source, - ) - .with_file_groups(partitioned_file_lists) - .with_constraints(self.constraints.clone()) - .with_statistics(statistics) - .with_projection(projection.cloned()) - .with_limit(limit) - .with_output_ordering(output_ordering) - .with_table_partition_cols(table_partition_cols) - .with_expr_adapter(self.expr_adapter_factory.clone()) - .with_limit_pruning(limit.is_some()) - .build(), - ) - .await - } - - fn supports_filters_pushdown( - &self, - filters: &[&Expr], - ) -> Result> { - let partition_column_names = self - .options - .table_partition_cols - .iter() - .map(|col| col.0.as_str()) - .collect::>(); - filters - .iter() - .map(|filter| { - if can_be_evaluated_for_partition_pruning(&partition_column_names, filter) - { - // if filter can be handled by partition pruning, it is exact - return Ok(TableProviderFilterPushDown::Exact); - } - - Ok(TableProviderFilterPushDown::Inexact) - }) - .collect() - } - - fn get_table_definition(&self) -> Option<&str> { - self.definition.as_deref() - } - - async fn insert_into( - &self, - state: &dyn Session, - input: Arc, - insert_op: InsertOp, - ) -> Result> { - // Check that the schema of the plan matches the schema of this table. - self.schema() - .logically_equivalent_names_and_types(&input.schema())?; - - let table_path = &self.table_paths()[0]; - if !table_path.is_collection() { - return plan_err!( - "Inserting into a ListingTable backed by a single file is not supported, URL is possibly missing a trailing `/`. \ - To append to an existing file use StreamTable, e.g. by using CREATE UNBOUNDED EXTERNAL TABLE" - ); - } - - // Get the object store for the table path. - let store = state.runtime_env().object_store(table_path)?; - - let file_list_stream = pruned_partition_list( - state, - store.as_ref(), - table_path, - &[], - &self.options.file_extension, - &self.options.table_partition_cols, - ) - .await?; - - let file_group = file_list_stream.try_collect::>().await?.into(); - let keep_partition_by_columns = - state.config_options().execution.keep_partition_by_columns; - - // Sink related option, apart from format - let config = FileSinkConfig { - original_url: String::default(), - object_store_url: self.table_paths()[0].object_store(), - table_paths: self.table_paths().clone(), - file_group, - output_schema: self.schema(), - table_partition_cols: self.options.table_partition_cols.clone(), - insert_op, - keep_partition_by_columns, - file_extension: self.options().format.get_ext(), - }; - - let orderings = self.try_create_output_ordering()?; - // It is sufficient to pass only one of the equivalent orderings: - let order_requirements = orderings.into_iter().next().map(Into::into); - - self.options() - .format - .create_writer_physical_plan(input, state, config, order_requirements) - .await - } - - fn get_column_default(&self, column: &str) -> Option<&Expr> { - self.column_defaults.get(column) - } -} - -impl ListingTable { - /// Get the list of files for a scan as well as the file level statistics. - /// The list is grouped to let the execution plan know how the files should - /// be distributed to different threads / executors. - async fn list_files_for_scan<'a>( - &'a self, - ctx: &'a dyn Session, - filters: &'a [Expr], - limit: Option, - ) -> Result<(Vec, Statistics)> { - let store = if let Some(url) = self.table_paths.first() { - ctx.runtime_env().object_store(url)? - } else { - return Ok((vec![], Statistics::new_unknown(&self.file_schema))); - }; - // list files (with partitions) - let file_list = future::try_join_all(self.table_paths.iter().map(|table_path| { - pruned_partition_list( - ctx, - store.as_ref(), - table_path, - filters, - &self.options.file_extension, - &self.options.table_partition_cols, - ) - })) - .await?; - let meta_fetch_concurrency = - ctx.config_options().execution.meta_fetch_concurrency; - let file_list = stream::iter(file_list).flatten_unordered(meta_fetch_concurrency); - // collect the statistics if required by the config - let files = file_list - .map(|part_file| async { - let part_file = part_file?; - let statistics = if self.options.collect_stat { - self.do_collect_statistics(ctx, &store, &part_file).await? - } else { - Arc::new(Statistics::new_unknown(&self.file_schema)) - }; - Ok(part_file.with_statistics(statistics)) - }) - .boxed() - .buffer_unordered(ctx.config_options().execution.meta_fetch_concurrency); - - let (file_group, inexact_stats) = - get_files_with_limit(files, limit, self.options.collect_stat).await?; - - let file_groups = file_group.split_files(self.options.target_partitions); - let (mut file_groups, mut stats) = compute_all_files_statistics( - file_groups, - self.schema(), - self.options.collect_stat, - inexact_stats, - )?; - - let schema_adapter = self.create_schema_adapter(); - let (schema_mapper, _) = schema_adapter.map_schema(self.file_schema.as_ref())?; - - stats.column_statistics = - schema_mapper.map_column_statistics(&stats.column_statistics)?; - file_groups.iter_mut().try_for_each(|file_group| { - if let Some(stat) = file_group.statistics_mut() { - stat.column_statistics = - schema_mapper.map_column_statistics(&stat.column_statistics)?; - } - Ok::<_, DataFusionError>(()) - })?; - Ok((file_groups, stats)) - } - - /// Collects statistics for a given partitioned file. - /// - /// This method first checks if the statistics for the given file are already cached. - /// If they are, it returns the cached statistics. - /// If they are not, it infers the statistics from the file and stores them in the cache. - async fn do_collect_statistics( - &self, - ctx: &dyn Session, - store: &Arc, - part_file: &PartitionedFile, - ) -> Result> { - match self - .collected_statistics - .get_with_extra(&part_file.object_meta.location, &part_file.object_meta) - { - Some(statistics) => Ok(statistics), - None => { - let statistics = self - .options - .format - .infer_stats( - ctx, - store, - Arc::clone(&self.file_schema), - &part_file.object_meta, - ) - .await?; - let statistics = Arc::new(statistics); - self.collected_statistics.put_with_extra( - &part_file.object_meta.location, - Arc::clone(&statistics), - &part_file.object_meta, - ); - Ok(statistics) - } - } - } -} - -/// Processes a stream of partitioned files and returns a `FileGroup` containing the files. -/// -/// This function collects files from the provided stream until either: -/// 1. The stream is exhausted -/// 2. The accumulated number of rows exceeds the provided `limit` (if specified) -/// -/// # Arguments -/// * `files` - A stream of `Result` items to process -/// * `limit` - An optional row count limit. If provided, the function will stop collecting files -/// once the accumulated number of rows exceeds this limit -/// * `collect_stats` - Whether to collect and accumulate statistics from the files -/// -/// # Returns -/// A `Result` containing a `FileGroup` with the collected files -/// and a boolean indicating whether the statistics are inexact. -/// -/// # Note -/// The function will continue processing files if statistics are not available or if the -/// limit is not provided. If `collect_stats` is false, statistics won't be accumulated -/// but files will still be collected. -async fn get_files_with_limit( - files: impl Stream>, - limit: Option, - collect_stats: bool, -) -> Result<(FileGroup, bool)> { - let mut file_group = FileGroup::default(); - // Fusing the stream allows us to call next safely even once it is finished. - let mut all_files = Box::pin(files.fuse()); - enum ProcessingState { - ReadingFiles, - ReachedLimit, - } - - let mut state = ProcessingState::ReadingFiles; - let mut num_rows = Precision::Absent; - - while let Some(file_result) = all_files.next().await { - // Early exit if we've already reached our limit - if matches!(state, ProcessingState::ReachedLimit) { - break; - } - - let file = file_result?; - - // Update file statistics regardless of state - if collect_stats { - if let Some(file_stats) = &file.statistics { - num_rows = if file_group.is_empty() { - // For the first file, just take its row count - file_stats.num_rows - } else { - // For subsequent files, accumulate the counts - num_rows.add(&file_stats.num_rows) - }; - } - } - - // Always add the file to our group - file_group.push(file); - - // Check if we've hit the limit (if one was specified) - if let Some(limit) = limit { - if let Precision::Exact(row_count) = num_rows { - if row_count > limit { - state = ProcessingState::ReachedLimit; - } - } - } - } - // If we still have files in the stream, it means that the limit kicked - // in, and the statistic could have been different had we processed the - // files in a different order. - let inexact_stats = all_files.next().await.is_some(); - Ok((file_group, inexact_stats)) ->>>>>>> origin/branch-51 } #[cfg(test)] From 9ae854f74ba8c1b228f72206e494ac572a35d89c Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:15:56 +0530 Subject: [PATCH 168/177] fix: keep comment --- datafusion/core/src/physical_planner.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index e40fe041a3305..c280b50a9f07a 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -2296,13 +2296,10 @@ impl DefaultPhysicalPlanner { displayable(new_plan.as_ref()).indent(false) ); -<<<<<<< HEAD // Don't print new_plan directly, as that may overflow the stack. // For example: // thread 'tokio-runtime-worker' has overflowed its stack // fatal runtime error: stack overflow, aborting -======= ->>>>>>> origin/branch-51 debug!( "Detailed optimized physical plan:\n{}\n", displayable(new_plan.as_ref()).indent(true) From 4167f9febee04ae7f03deba103b6248242789707 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:23:39 +0530 Subject: [PATCH 169/177] fix: formatting --- .../src/simplify_expressions/simplify_predicates.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs index 241b370666d65..e811ce7313102 100644 --- a/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs +++ b/datafusion/optimizer/src/simplify_expressions/simplify_predicates.rs @@ -215,11 +215,10 @@ fn find_most_restrictive_predicate( && op == &Operator::Lt) }; - if is_better { - best_value = Some(scalar); - most_restrictive_idx = idx; - } - + if is_better { + best_value = Some(scalar); + most_restrictive_idx = idx; + } } else { best_value = Some(scalar); most_restrictive_idx = idx; From 53f069656e1b0effc520d803d6fbc0cefeffb829 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:23:54 +0530 Subject: [PATCH 170/177] fix: test compilation errors --- datafusion/core/tests/dataframe/mod.rs | 15 --------------- .../proto/tests/cases/roundtrip_physical_plan.rs | 11 +---------- 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/datafusion/core/tests/dataframe/mod.rs b/datafusion/core/tests/dataframe/mod.rs index a121e45676dec..4d52345a2adc5 100644 --- a/datafusion/core/tests/dataframe/mod.rs +++ b/datafusion/core/tests/dataframe/mod.rs @@ -66,13 +66,8 @@ use datafusion::test_util::{ use datafusion_catalog::TableProvider; use datafusion_common::test_util::{batches_to_sort_string, batches_to_string}; use datafusion_common::{ -<<<<<<< HEAD assert_contains, internal_datafusion_err, Constraint, Constraints, DFSchema, DataFusionError, ScalarValue, SchemaError, TableReference, UnnestOptions, -======= - assert_contains, Constraint, Constraints, DFSchema, DataFusionError, ParamValues, - ScalarValue, TableReference, UnnestOptions, ->>>>>>> origin/branch-51 }; use datafusion_common_runtime::SpawnedTask; use datafusion_datasource::file_format::format_as_file_type; @@ -96,12 +91,9 @@ use datafusion_physical_plan::aggregates::{ }; use datafusion_physical_plan::empty::EmptyExec; use datafusion_physical_plan::{displayable, ExecutionPlan, ExecutionPlanProperties}; -<<<<<<< HEAD use datafusion::error::Result as DataFusionResult; use datafusion_functions_window::expr_fn::lag; -======= ->>>>>>> origin/branch-51 // Get string representation of the plan async fn physical_plan_to_string(df: &DataFrame) -> String { @@ -6755,17 +6747,10 @@ async fn test_duplicate_state_fields_for_dfschema_construct() -> Result<()> { "ticker", "first_value(value)[first_value]", "timestamp@0", -<<<<<<< HEAD "first_value(value)[first_value_is_set]", "last_value(value)[last_value]", "timestamp@0", "last_value(value)[last_value_is_set]", -======= - "is_set", - "last_value(value)[last_value]", - "timestamp@0", - "is_set", ->>>>>>> origin/branch-51 ]; let binding = partial_agg.schema(); diff --git a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs index 14b84e8efd45c..0fd6a3b834741 100644 --- a/datafusion/proto/tests/cases/roundtrip_physical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_physical_plan.rs @@ -32,13 +32,10 @@ use arrow::csv::WriterBuilder; use arrow::datatypes::{Fields, TimeUnit}; use datafusion::physical_expr::aggregate::AggregateExprBuilder; use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec; -<<<<<<< HEAD use datafusion::physical_plan::metrics::MetricType; -======= use datafusion::physical_plan::node_id::{ annotate_node_id_for_execution_plan, NodeIdAnnotator, }; ->>>>>>> origin/branch-51 use datafusion_expr::dml::InsertOp; use datafusion_functions_aggregate::approx_percentile_cont::approx_percentile_cont_udaf; use datafusion_functions_aggregate::array_agg::array_agg_udaf; @@ -150,14 +147,8 @@ fn roundtrip_test_and_return( let proto: protobuf::PhysicalPlanNode = protobuf::PhysicalPlanNode::try_from_physical_plan(exec_plan.clone(), codec) .expect("to proto"); -<<<<<<< HEAD - let result_exec_plan: Arc = proto - .try_into_physical_plan(&ctx.task_ctx(), codec) -======= - let runtime = ctx.runtime_env(); let mut result_exec_plan: Arc = proto - .try_into_physical_plan(ctx, runtime.deref(), codec) ->>>>>>> origin/branch-51 + .try_into_physical_plan(&ctx.task_ctx(), codec) .expect("from proto"); // Qi: workaround for NodeId not being serialized/deserialized, From 99ddba71fb3f3fe2c39ce627f5d60b37d0cdca22 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:28:23 +0530 Subject: [PATCH 171/177] fix: drop ignore of `RUSTSEC-2025-0111` See PR: https://github.com/apache/datafusion/pull/18305 --- .github/workflows/audit.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 5086d35ebb96a..55b2a6010a1e6 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -44,16 +44,8 @@ jobs: steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - name: Install cargo-audit -<<<<<<< HEAD uses: taiki-e/install-action@f535147c22906d77695e11cb199e764aa610a4fc # v2.62.46 with: tool: cargo-audit - name: Run audit check run: cargo audit -======= - uses: taiki-e/install-action@f535147c22906d77695e11cb199e764aa610a4fc # v2.62.46 - with: - tool: cargo-audit - - name: Run audit check - run: cargo audit --ignore RUSTSEC-2025-0111 ->>>>>>> origin/branch-51 From 2c240e520dc05141eaed6f25650bfad0227afe7c Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:38:51 +0530 Subject: [PATCH 172/177] fix: filter pushdown test --- .../core/tests/physical_optimizer/filter_pushdown/mod.rs | 8 -------- 1 file changed, 8 deletions(-) diff --git a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs index bb533e85ce4ef..de61149508904 100644 --- a/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs +++ b/datafusion/core/tests/physical_optimizer/filter_pushdown/mod.rs @@ -906,11 +906,7 @@ async fn test_topk_filter_passes_through_coalesce_partitions() { Ok: - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalescePartitionsExec -<<<<<<< HEAD - DataSourceExec: file_groups={2 groups: [[test1.parquet], [test2.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] -======= - - DataSourceExec: file_groups={2 groups: [[test1.parquet], [test2.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilterPhysicalExpr [ true ] ->>>>>>> origin/branch-51 " ); } @@ -965,11 +961,7 @@ async fn test_topk_filter_passes_through_coalesce_batches() { Ok: - SortExec: TopK(fetch=1), expr=[b@1 DESC NULLS LAST], preserve_partitioning=[false] - CoalesceBatchesExec: target_batch_size=1024 -<<<<<<< HEAD - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilter [ empty ] -======= - - DataSourceExec: file_groups={1 group: [[test.parquet]]}, projection=[a, b, c], file_type=test, pushdown_supported=true, predicate=DynamicFilterPhysicalExpr [ true ] ->>>>>>> origin/branch-51 " ); } From fe01720b95a0184eee7102baf9c3799cf2a5d82b Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 14:45:48 +0530 Subject: [PATCH 173/177] fix: sqllogictests --- datafusion/sqllogictest/test_files/aggregate.slt | 8 -------- datafusion/sqllogictest/test_files/describe.slt | 8 +------- .../dynamic_filter_pushdown_config.slt | 16 ---------------- .../test_files/information_schema.slt | 6 ------ datafusion/sqllogictest/test_files/map.slt | 6 ------ .../sqllogictest/test_files/projection.slt | 4 ---- 6 files changed, 1 insertion(+), 47 deletions(-) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index c2b72d9ac04e0..a5973afc0a93d 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -6601,11 +6601,7 @@ query TT explain select string_agg(k, ',' order by v) from t; ---- logical_plan -<<<<<<< HEAD 01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v ASC NULLS LAST]]] -======= -01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v ASC NULLS LAST]]] ->>>>>>> origin/branch-51 02)--TableScan: t projection=[k, v] physical_plan 01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v ASC NULLS LAST]] @@ -6621,11 +6617,7 @@ query TT explain select string_agg(k, ',' order by v desc) from t; ---- logical_plan -<<<<<<< HEAD 01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]]] -======= -01)Aggregate: groupBy=[[]], aggr=[[string_agg(t.k, Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]]] ->>>>>>> origin/branch-51 02)--TableScan: t projection=[k, v] physical_plan 01)AggregateExec: mode=Single, gby=[], aggr=[string_agg(t.k,Utf8(",")) ORDER BY [t.v DESC NULLS FIRST]] diff --git a/datafusion/sqllogictest/test_files/describe.slt b/datafusion/sqllogictest/test_files/describe.slt index 5643c0e5737ea..0fe9f03a797b0 100644 --- a/datafusion/sqllogictest/test_files/describe.slt +++ b/datafusion/sqllogictest/test_files/describe.slt @@ -81,15 +81,9 @@ int_col Int32 YES bigint_col Int64 YES float_col Float32 YES double_col Float64 YES -<<<<<<< HEAD -date_string_col Utf8View YES -string_col Utf8View YES -timestamp_col Timestamp(ns) YES -======= date_string_col Utf8 YES string_col Utf8 YES -timestamp_col Timestamp(Nanosecond, None) YES ->>>>>>> origin/branch-51 +timestamp_col Timestamp(ns) YES year Int32 YES month Int32 YES diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt index 60ae111016cf1..e5cd6d88b08f4 100644 --- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt +++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt @@ -89,11 +89,7 @@ logical_plan 02)--TableScan: test_parquet projection=[id, value, name] physical_plan 01)SortExec: TopK(fetch=3), expr=[value@1 DESC], preserve_partitioning=[false] -<<<<<<< HEAD 02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilter [ empty ] -======= -02)--DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value, name], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] ->>>>>>> origin/branch-51 # Disable TopK dynamic filter pushdown statement ok @@ -131,11 +127,7 @@ physical_plan 02)--CoalesceBatchesExec: target_batch_size=8192 03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet -<<<<<<< HEAD 05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] -======= -05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] ->>>>>>> origin/branch-51 # Disable Join dynamic filter pushdown statement ok @@ -192,11 +184,7 @@ physical_plan 02)--CoalesceBatchesExec: target_batch_size=8192 03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet -<<<<<<< HEAD 05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] -======= -05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] ->>>>>>> origin/branch-51 # Enable TopK, disable Join statement ok @@ -318,11 +306,7 @@ physical_plan 02)--CoalesceBatchesExec: target_batch_size=8192 03)----HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(id@0, id@0)], projection=[info@1, id@2, data@3] 04)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_right.parquet]]}, projection=[id, info], file_type=parquet -<<<<<<< HEAD 05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilter [ empty ] -======= -05)------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/join_left.parquet]]}, projection=[id, data], file_type=parquet, predicate=DynamicFilterPhysicalExpr [ true ] ->>>>>>> origin/branch-51 # Cleanup diff --git a/datafusion/sqllogictest/test_files/information_schema.slt b/datafusion/sqllogictest/test_files/information_schema.slt index 63f12d21d04ae..dcf336c9be86e 100644 --- a/datafusion/sqllogictest/test_files/information_schema.slt +++ b/datafusion/sqllogictest/test_files/information_schema.slt @@ -292,10 +292,7 @@ datafusion.optimizer.default_filter_selectivity 20 datafusion.optimizer.enable_distinct_aggregation_soft_limit true datafusion.optimizer.enable_dynamic_filter_pushdown true datafusion.optimizer.enable_join_dynamic_filter_pushdown true -<<<<<<< HEAD datafusion.optimizer.enable_piecewise_merge_join false -======= ->>>>>>> origin/branch-51 datafusion.optimizer.enable_round_robin_repartition true datafusion.optimizer.enable_topk_aggregation true datafusion.optimizer.enable_topk_dynamic_filter_pushdown true @@ -416,10 +413,7 @@ datafusion.optimizer.default_filter_selectivity 20 The default filter selectivit datafusion.optimizer.enable_distinct_aggregation_soft_limit true When set to true, the optimizer will push a limit operation into grouped aggregations which have no aggregate expressions, as a soft limit, emitting groups once the limit is reached, before all rows in the group are read. datafusion.optimizer.enable_dynamic_filter_pushdown true When set to true attempts to push down dynamic filters generated by operators (topk & join) into the file scan phase. For example, for a query such as `SELECT * FROM t ORDER BY timestamp DESC LIMIT 10`, the optimizer will attempt to push down the current top 10 timestamps that the TopK operator references into the file scans. This means that if we already have 10 timestamps in the year 2025 any files that only have timestamps in the year 2024 can be skipped / pruned at various stages in the scan. The config will suppress `enable_join_dynamic_filter_pushdown` & `enable_topk_dynamic_filter_pushdown` So if you disable `enable_topk_dynamic_filter_pushdown`, then enable `enable_dynamic_filter_pushdown`, the `enable_topk_dynamic_filter_pushdown` will be overridden. datafusion.optimizer.enable_join_dynamic_filter_pushdown true When set to true, the optimizer will attempt to push down Join dynamic filters into the file scan phase. -<<<<<<< HEAD datafusion.optimizer.enable_piecewise_merge_join false When set to true, piecewise merge join is enabled. PiecewiseMergeJoin is currently experimental. Physical planner will opt for PiecewiseMergeJoin when there is only one range filter. -======= ->>>>>>> origin/branch-51 datafusion.optimizer.enable_round_robin_repartition true When set to true, the physical plan optimizer will try to add round robin repartitioning to increase parallelism to leverage more CPU cores datafusion.optimizer.enable_topk_aggregation true When set to true, the optimizer will attempt to perform limit operations during aggregations, if possible datafusion.optimizer.enable_topk_dynamic_filter_pushdown true When set to true, the optimizer will attempt to push down TopK dynamic filters into the file scan phase. diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt index a2d4a6071a59a..9b61c3a9acf78 100644 --- a/datafusion/sqllogictest/test_files/map.slt +++ b/datafusion/sqllogictest/test_files/map.slt @@ -43,15 +43,9 @@ LOCATION '../core/tests/data/parquet_map.parquet'; query TTT describe data; ---- -<<<<<<< HEAD ints Map("entries": Struct("key": Utf8, "value": Int64), unsorted) NO strings Map("entries": Struct("key": Utf8, "value": Utf8), unsorted) NO -timestamp Utf8View NO -======= -ints Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Int64, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO -strings Map(Field { name: "entries", data_type: Struct([Field { name: "key", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, Field { name: "value", data_type: Utf8, nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }]), nullable: false, dict_id: 0, dict_is_ordered: false, metadata: {} }, false) NO timestamp Utf8 NO ->>>>>>> origin/branch-51 query ??T SELECT * FROM data ORDER by ints['bytes'] DESC LIMIT 10; diff --git a/datafusion/sqllogictest/test_files/projection.slt b/datafusion/sqllogictest/test_files/projection.slt index b1e3535c7f6c8..9f840e7bdc2f0 100644 --- a/datafusion/sqllogictest/test_files/projection.slt +++ b/datafusion/sqllogictest/test_files/projection.slt @@ -253,11 +253,7 @@ physical_plan statement ok drop table t; -<<<<<<< HEAD # Regression test for -======= -# Regression test for ->>>>>>> origin/branch-51 # https://github.com/apache/datafusion/issues/17513 query I From 218cf37d51913345874aaae341c68cac2809e7da Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 15:44:07 +0530 Subject: [PATCH 174/177] fix: annotate `node_id` in roundtripped physical plan --- datafusion/proto/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion/proto/src/lib.rs b/datafusion/proto/src/lib.rs index b16b12bc05162..27917739dee99 100644 --- a/datafusion/proto/src/lib.rs +++ b/datafusion/proto/src/lib.rs @@ -104,6 +104,8 @@ //! # use datafusion::prelude::*; //! # use datafusion_common::Result; //! # use datafusion_proto::bytes::{physical_plan_from_bytes,physical_plan_to_bytes}; +//! # use datafusion_physical_plan::node_id::annotate_node_id_for_execution_plan; +//! # use datafusion_physical_plan::node_id::NodeIdAnnotator; //! # #[tokio::main] //! # async fn main() -> Result<()>{ //! // Create a plan that scans table 't' @@ -116,6 +118,11 @@ //! //! // Decode bytes from somewhere (over network, etc.) back to ExecutionPlan //! let physical_round_trip = physical_plan_from_bytes(&bytes, &ctx.task_ctx())?; +//! +//! // Workaround for `node_id` not being serializable: +//! let mut annotator = NodeIdAnnotator::new(); +//! let physical_round_trip = annotate_node_id_for_execution_plan(&physical_round_trip, &mut annotator)?; +//! //! assert_eq!(format!("{:?}", physical_plan), format!("{:?}", physical_round_trip)); //! # Ok(()) //! # } From f1c06945a8bc8a283f3518f252de1d545527b097 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Sun, 23 Nov 2025 19:17:24 +0530 Subject: [PATCH 175/177] fix: remove duplicate merge_group --- .github/workflows/audit.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 55b2a6010a1e6..73e5c5f810eb8 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -36,8 +36,6 @@ on: merge_group: - merge_group: - jobs: security_audit: runs-on: ubuntu-latest From 319cb70a4a6b63a6179f19935af8cf0730f0b320 Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Mon, 24 Nov 2025 12:36:03 +0530 Subject: [PATCH 176/177] refactor: remove unused values from physical plan module --- datafusion/physical-plan/src/values.rs | 343 ------------------------- 1 file changed, 343 deletions(-) delete mode 100644 datafusion/physical-plan/src/values.rs diff --git a/datafusion/physical-plan/src/values.rs b/datafusion/physical-plan/src/values.rs deleted file mode 100644 index e5dc3b1162338..0000000000000 --- a/datafusion/physical-plan/src/values.rs +++ /dev/null @@ -1,343 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//! Values execution plan - -use std::any::Any; -use std::sync::Arc; - -use crate::execution_plan::{Boundedness, EmissionType}; -use crate::memory::MemoryStream; -use crate::{common, DisplayAs, PlanProperties, SendableRecordBatchStream, Statistics}; -use crate::{ - ColumnarValue, DisplayFormatType, ExecutionPlan, Partitioning, PhysicalExpr, -}; -use arrow::datatypes::{Schema, SchemaRef}; -use arrow::record_batch::{RecordBatch, RecordBatchOptions}; -use datafusion_common::{internal_err, plan_err, Result, ScalarValue}; -use datafusion_execution::TaskContext; -use datafusion_physical_expr::EquivalenceProperties; - -/// Execution plan for values list based relation (produces constant rows) -#[deprecated( - since = "45.0.0", - note = "Use `MemorySourceConfig::try_new_as_values` instead" -)] -#[derive(Debug, Clone)] -pub struct ValuesExec { - /// The schema - schema: SchemaRef, - /// The data - data: Vec, - /// Cache holding plan properties like equivalences, output partitioning etc. - cache: PlanProperties, -} - -#[allow(deprecated)] -impl ValuesExec { - /// Create a new values exec from data as expr - #[deprecated(since = "45.0.0", note = "Use `MemoryExec::try_new` instead")] - pub fn try_new( - schema: SchemaRef, - data: Vec>>, - ) -> Result { - if data.is_empty() { - return plan_err!("Values list cannot be empty"); - } - let n_row = data.len(); - let n_col = schema.fields().len(); - // We have this single row batch as a placeholder to satisfy evaluation argument - // and generate a single output row - let batch = RecordBatch::try_new_with_options( - Arc::new(Schema::empty()), - vec![], - &RecordBatchOptions::new().with_row_count(Some(1)), - )?; - - let arr = (0..n_col) - .map(|j| { - (0..n_row) - .map(|i| { - let r = data[i][j].evaluate(&batch); - - match r { - Ok(ColumnarValue::Scalar(scalar)) => Ok(scalar), - Ok(ColumnarValue::Array(a)) if a.len() == 1 => { - ScalarValue::try_from_array(&a, 0) - } - Ok(ColumnarValue::Array(a)) => { - plan_err!( - "Cannot have array values {a:?} in a values list" - ) - } - Err(err) => Err(err), - } - }) - .collect::>>() - .and_then(ScalarValue::iter_to_array) - }) - .collect::>>()?; - let batch = RecordBatch::try_new_with_options( - Arc::clone(&schema), - arr, - &RecordBatchOptions::new().with_row_count(Some(n_row)), - )?; - let data: Vec = vec![batch]; - Self::try_new_from_batches(schema, data) - } - - /// Create a new plan using the provided schema and batches. - /// - /// Errors if any of the batches don't match the provided schema, or if no - /// batches are provided. - #[deprecated( - since = "45.0.0", - note = "Use `MemoryExec::try_new_from_batches` instead" - )] - pub fn try_new_from_batches( - schema: SchemaRef, - batches: Vec, - ) -> Result { - if batches.is_empty() { - return plan_err!("Values list cannot be empty"); - } - - for batch in &batches { - let batch_schema = batch.schema(); - if batch_schema != schema { - return plan_err!( - "Batch has invalid schema. Expected: {schema}, got: {batch_schema}" - ); - } - } - - let cache = Self::compute_properties(Arc::clone(&schema)); - #[allow(deprecated)] - Ok(ValuesExec { - schema, - data: batches, - cache, - }) - } - - /// Provides the data - pub fn data(&self) -> Vec { - #[allow(deprecated)] - self.data.clone() - } - - /// This function creates the cache object that stores the plan properties such as schema, equivalence properties, ordering, partitioning, etc. - fn compute_properties(schema: SchemaRef) -> PlanProperties { - PlanProperties::new( - EquivalenceProperties::new(schema), - Partitioning::UnknownPartitioning(1), - EmissionType::Incremental, - Boundedness::Bounded, - ) - } -} - -#[allow(deprecated)] -impl DisplayAs for ValuesExec { - fn fmt_as( - &self, - t: DisplayFormatType, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - match t { - DisplayFormatType::Default | DisplayFormatType::Verbose => { - write!(f, "ValuesExec") - } - DisplayFormatType::TreeRender => { - // TODO: collect info - write!(f, "") - } - } - } -} - -#[allow(deprecated)] -impl ExecutionPlan for ValuesExec { - fn name(&self) -> &'static str { - "ValuesExec" - } - - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - - fn properties(&self) -> &PlanProperties { - #[allow(deprecated)] - &self.cache - } - - fn children(&self) -> Vec<&Arc> { - vec![] - } - - fn with_new_children( - self: Arc, - _: Vec>, - ) -> Result> { - #[allow(deprecated)] - ValuesExec::try_new_from_batches(Arc::clone(&self.schema), self.data.clone()) - .map(|e| Arc::new(e) as _) - } - - fn execute( - &self, - partition: usize, - _context: Arc, - ) -> Result { - // ValuesExec has a single output partition - if 0 != partition { - return internal_err!( - "ValuesExec invalid partition {partition} (expected 0)" - ); - } - - Ok(Box::pin(MemoryStream::try_new( - self.data(), - #[allow(deprecated)] - Arc::clone(&self.schema), - None, - )?)) - } - - fn statistics(&self) -> Result { - let batch = self.data(); - Ok(common::compute_record_batch_statistics( - &[batch], - #[allow(deprecated)] - &self.schema, - None, - )) - } - - fn with_node_id( - self: Arc, - node_id: usize, - ) -> Result>> { - let mut new_plan = ValuesExec::try_new_from_batches( - Arc::clone(&self.schema), - self.data.clone(), - )?; - let new_props = new_plan.cache.clone().with_node_id(node_id); - new_plan.cache = new_props; - Ok(Some(Arc::new(new_plan))) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::expressions::lit; - use crate::test::{self, make_partition}; - - use arrow::datatypes::{DataType, Field}; - use datafusion_common::stats::{ColumnStatistics, Precision}; - - #[tokio::test] - async fn values_empty_case() -> Result<()> { - let schema = test::aggr_test_schema(); - #[allow(deprecated)] - let empty = ValuesExec::try_new(schema, vec![]); - assert!(empty.is_err()); - Ok(()) - } - - #[test] - fn new_exec_with_batches() { - let batch = make_partition(7); - let schema = batch.schema(); - let batches = vec![batch.clone(), batch]; - #[allow(deprecated)] - let _exec = ValuesExec::try_new_from_batches(schema, batches).unwrap(); - } - - #[test] - fn new_exec_with_batches_empty() { - let batch = make_partition(7); - let schema = batch.schema(); - #[allow(deprecated)] - let _ = ValuesExec::try_new_from_batches(schema, Vec::new()).unwrap_err(); - } - - #[test] - fn new_exec_with_batches_invalid_schema() { - let batch = make_partition(7); - let batches = vec![batch.clone(), batch]; - - let invalid_schema = Arc::new(Schema::new(vec![ - Field::new("col0", DataType::UInt32, false), - Field::new("col1", DataType::Utf8, false), - ])); - #[allow(deprecated)] - let _ = ValuesExec::try_new_from_batches(invalid_schema, batches).unwrap_err(); - } - - // Test issue: https://github.com/apache/datafusion/issues/8763 - #[test] - fn new_exec_with_non_nullable_schema() { - let schema = Arc::new(Schema::new(vec![Field::new( - "col0", - DataType::UInt32, - false, - )])); - #[allow(deprecated)] - let _ = ValuesExec::try_new(Arc::clone(&schema), vec![vec![lit(1u32)]]).unwrap(); - // Test that a null value is rejected - #[allow(deprecated)] - let _ = ValuesExec::try_new(schema, vec![vec![lit(ScalarValue::UInt32(None))]]) - .unwrap_err(); - } - - #[test] - fn values_stats_with_nulls_only() -> Result<()> { - let data = vec![ - vec![lit(ScalarValue::Null)], - vec![lit(ScalarValue::Null)], - vec![lit(ScalarValue::Null)], - ]; - let rows = data.len(); - #[allow(deprecated)] - let values = ValuesExec::try_new( - Arc::new(Schema::new(vec![Field::new("col0", DataType::Null, true)])), - data, - )?; - - #[allow(deprecated)] - let stats = values.statistics()?; - assert_eq!( - stats, - Statistics { - num_rows: Precision::Exact(rows), - total_byte_size: Precision::Exact(8), // not important - column_statistics: vec![ColumnStatistics { - null_count: Precision::Exact(rows), // there are only nulls - distinct_count: Precision::Absent, - max_value: Precision::Absent, - min_value: Precision::Absent, - sum_value: Precision::Absent, - },], - } - ); - - Ok(()) - } -} From df445a2a9ac36d00683f199cbee2560e6fefbfcb Mon Sep 17 00:00:00 2001 From: Jacob Sherin Date: Mon, 24 Nov 2025 14:54:26 +0530 Subject: [PATCH 177/177] feat: make `DefaultSchemaAdapter` public --- datafusion/datasource/src/schema_adapter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/datasource/src/schema_adapter.rs b/datafusion/datasource/src/schema_adapter.rs index 4c7b37113d58d..3bc1bddd2a8dc 100644 --- a/datafusion/datasource/src/schema_adapter.rs +++ b/datafusion/datasource/src/schema_adapter.rs @@ -243,7 +243,7 @@ impl SchemaAdapterFactory for DefaultSchemaAdapterFactory { /// This SchemaAdapter requires both the table schema and the projected table /// schema. See [`SchemaMapping`] for more details #[derive(Clone, Debug)] -pub(crate) struct DefaultSchemaAdapter { +pub struct DefaultSchemaAdapter { /// The schema for the table, projected to include only the fields being output (projected) by the /// associated ParquetSource projected_table_schema: SchemaRef,