From bb4f7ad97621411f66a478a3df0c204fe6717bb0 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 09:42:21 -0500 Subject: [PATCH 01/12] Bump DataFusion version to 44 --- Cargo.lock | 377 ++++++++++++++++++++++++++++------------------------- 1 file changed, 199 insertions(+), 178 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 352771cd..ff19bc5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,9 +111,9 @@ dependencies = [ [[package]] name = "allocator-api2" -version = "0.2.20" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45862d1c77f2228b9e10bc609d5bc203d86ebc9b87ad8d5d5167a6c9abf739d9" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android-tzdata" @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.93" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "apache-avro" @@ -444,18 +444,18 @@ checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "async-trait" -version = "0.1.83" +version = "0.1.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" +checksum = "3f934833b4b7233644e5848f235df3f57ed8c80f1528a26c3dfa13d2147fa056" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -611,9 +611,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.2" +version = "1.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" dependencies = [ "jobserver", "libc", @@ -634,9 +634,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -708,9 +708,9 @@ dependencies = [ [[package]] name = "const_panic" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "013b6c2c3a14d678f38cd23994b02da3a1a1b6a5d1eedddfe63a5a5f11b13a81" +checksum = "2459fc9262a1aa204eb4b5764ad4f189caec88aea9634389c0a25f8be7f6265e" [[package]] name = "constant_time_eq" @@ -778,18 +778,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" @@ -1365,7 +1365,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -1404,9 +1404,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fixedbitset" @@ -1416,9 +1416,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" [[package]] name = "flatbuffers" -version = "24.3.25" +version = "24.12.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" dependencies = [ "bitflags 1.3.2", "rustc_version", @@ -1505,7 +1505,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -1578,9 +1578,9 @@ checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" [[package]] name = "glob" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "h2" @@ -1654,9 +1654,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "http" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" dependencies = [ "bytes", "fnv", @@ -1700,9 +1700,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -1720,9 +1720,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1893,7 +1893,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -1919,9 +1919,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" dependencies = [ "equivalent", "hashbrown 0.15.2", @@ -1983,9 +1983,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a865e038f7f6ed956f788f0d7d60c541fff74c7bd74272c5d4cf15c63743e705" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" dependencies = [ "once_cell", "wasm-bindgen", @@ -1999,9 +1999,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0431c65b318a590c1de6b8fd6e72798c92291d27762d94c9e6c37ed7a73d8458" +checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -2012,9 +2012,9 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb17a4bdb9b418051aa59d41d65b1c9be5affab314a872e5ad7f06231fb3b4e0" +checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" dependencies = [ "lexical-parse-integer", "lexical-util", @@ -2023,9 +2023,9 @@ dependencies = [ [[package]] name = "lexical-parse-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5df98f4a4ab53bf8b175b363a34c7af608fe31f93cc1fb1bf07130622ca4ef61" +checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" dependencies = [ "lexical-util", "static_assertions", @@ -2033,18 +2033,18 @@ dependencies = [ [[package]] name = "lexical-util" -version = "1.0.3" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85314db53332e5c192b6bca611fb10c114a80d1b831ddac0af1e9be1b9232ca0" +checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" dependencies = [ "static_assertions", ] [[package]] name = "lexical-write-float" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7c3ad4e37db81c1cbe7cf34610340adc09c322871972f74877a712abc6c809" +checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" dependencies = [ "lexical-util", "lexical-write-integer", @@ -2053,9 +2053,9 @@ dependencies = [ [[package]] name = "lexical-write-integer" -version = "1.0.2" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb89e9f6958b83258afa3deed90b5de9ef68eef090ad5086c791cd2345610162" +checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" dependencies = [ "lexical-util", "static_assertions", @@ -2063,9 +2063,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.167" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libflate" @@ -2119,9 +2119,9 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.14" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" [[package]] name = "litemap" @@ -2207,9 +2207,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -2317,24 +2317,25 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6eb4c22c6154a1e759d7099f9ffad7cc5ef8245f9efbab4a41b92623079c82f3" +checksum = "3cfccb68961a56facde1163f9319e0d15743352344e7808a11795fb99698dcaf" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", "futures", + "httparse", "humantime", "hyper", "itertools", @@ -2505,18 +2506,18 @@ dependencies = [ [[package]] name = "phf" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_shared", ] [[package]] name = "phf_codegen" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8d39688d359e6b34654d328e262234662d16cc0f60ec8dcbe5e718709342a5a" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", "phf_shared", @@ -2524,9 +2525,9 @@ dependencies = [ [[package]] name = "phf_generator" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared", "rand", @@ -2534,18 +2535,18 @@ dependencies = [ [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.15" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" [[package]] name = "pin-utils" @@ -2576,12 +2577,12 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64d1ec885c64d0457d564db4ec299b2dae3f9c02808b8ad9c3a089c591b18033" +checksum = "483f8c21f64f3ea09fe0f30f5d48c3e8eefe5dac9129f0075f76593b4c1da705" dependencies = [ "proc-macro2", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -2595,9 +2596,9 @@ dependencies = [ [[package]] name = "prost" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" dependencies = [ "bytes", "prost-derive", @@ -2605,11 +2606,10 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1318b19085f08681016926435853bbf7858f9c082d0999b80550ff5d9abe15" +checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "bytes", "heck 0.5.0", "itertools", "log", @@ -2620,28 +2620,28 @@ dependencies = [ "prost", "prost-types", "regex", - "syn 2.0.90", + "syn 2.0.95", "tempfile", ] [[package]] name = "prost-derive" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" dependencies = [ "anyhow", "itertools", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "prost-types" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4759aa0d3a6232fb8dbdb97b61de2c20047c68aca932c7ed76da9d788508d670" +checksum = "cc2f1e56baa61e93533aebc21af4d2134b70f66275e0fcdf3cbe43d77ff7e8fc" dependencies = [ "prost", ] @@ -2715,7 +2715,7 @@ dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -2728,7 +2728,7 @@ dependencies = [ "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -2739,9 +2739,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.36.2" +version = "0.37.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" dependencies = [ "memchr", "serde", @@ -2760,7 +2760,7 @@ dependencies = [ "rustc-hash", "rustls", "socket2", - "thiserror 2.0.3", + "thiserror 2.0.10", "tokio", "tracing", ] @@ -2779,7 +2779,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.3", + "thiserror 2.0.10", "tinyvec", "tracing", "web-time", @@ -2787,9 +2787,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "1c40286217b4ba3a71d644d752e6a0b71f13f1b6a2c5311acfcbe0c2418ed904" dependencies = [ "cfg_aliases", "libc", @@ -2801,9 +2801,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -2840,9 +2840,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -2903,9 +2903,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.9" +version = "0.12.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f" +checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da" dependencies = [ "base64 0.22.1", "bytes", @@ -2937,6 +2937,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", + "tower", "tower-service", "url", "wasm-bindgen", @@ -2990,22 +2991,22 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "once_cell", "ring", @@ -3038,9 +3039,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" dependencies = [ "web-time", ] @@ -3058,9 +3059,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -3107,7 +3108,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3118,9 +3119,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "3.0.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316" dependencies = [ "bitflags 2.6.0", "core-foundation", @@ -3131,9 +3132,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" dependencies = [ "core-foundation-sys", "libc", @@ -3141,9 +3142,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" dependencies = [ "serde", ] @@ -3156,22 +3157,22 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3182,14 +3183,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -3206,7 +3207,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3253,9 +3254,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "siphasher" -version = "0.3.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "slab" @@ -3290,7 +3291,7 @@ dependencies = [ "heck 0.5.0", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3333,7 +3334,7 @@ checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3373,7 +3374,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3386,7 +3387,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3410,7 +3411,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", - "syn 2.0.90", + "syn 2.0.95", "typify", "walkdir", ] @@ -3434,9 +3435,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -3460,7 +3461,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3471,12 +3472,13 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "tempfile" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" dependencies = [ "cfg-if", "fastrand", + "getrandom", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3493,11 +3495,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3" dependencies = [ - "thiserror-impl 2.0.3", + "thiserror-impl 2.0.10", ] [[package]] @@ -3508,18 +3510,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3554,9 +3556,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" +checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" dependencies = [ "tinyvec_macros", ] @@ -3569,9 +3571,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.43.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e" dependencies = [ "backtrace", "bytes", @@ -3585,31 +3587,30 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls", - "rustls-pki-types", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61e7c3654c13bcd040d4a03abee2c75b1d14a37b423cf5a813ceae1cc903ec6a" +checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", @@ -3618,6 +3619,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -3643,7 +3665,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3709,7 +3731,7 @@ checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -3743,7 +3765,7 @@ dependencies = [ "semver", "serde", "serde_json", - "syn 2.0.90", + "syn 2.0.95", "thiserror 1.0.69", "unicode-ident", ] @@ -3761,7 +3783,7 @@ dependencies = [ "serde", "serde_json", "serde_tokenstream", - "syn 2.0.90", + "syn 2.0.95", "typify-impl", ] @@ -3867,9 +3889,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d15e63b4482863c109d70a7b8706c1e364eb6ea449b201a76c5b89cedcec2d5c" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" dependencies = [ "cfg-if", "once_cell", @@ -3878,24 +3900,23 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d36ef12e3aaca16ddd3f67922bc63e48e953f126de60bd33ccc0101ef9998cd" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" dependencies = [ "bumpalo", "log", - "once_cell", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.47" +version = "0.4.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dfaf8f50e5f293737ee323940c7d8b08a66a95a419223d9f41610ca08b0833d" +checksum = "38176d9b44ea84e9184eff0bc34cc167ed044f816accfe5922e54d84cf48eca2" dependencies = [ "cfg-if", "js-sys", @@ -3906,9 +3927,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "705440e08b42d3e4b36de7d66c944be628d579796b8090bfa3471478a2260051" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3916,22 +3937,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c9ae5a76e46f4deecd0f0255cc223cfa18dc9b261213b8aa0c7b36f61b3f1d" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.97" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ee99da9c5ba11bd675621338ef6fa52296b76b83305e9b6e5c77d4c286d6d49" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" [[package]] name = "wasm-streams" @@ -3948,9 +3969,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.74" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a98bc3c33f0fe7e59ad7cd041b89034fa82a7c2d4365ca538dda6cdaf513863c" +checksum = "04dd7223427d52553d3702c004d3b2fe07c148165faa56313cb00211e31c12bc" dependencies = [ "js-sys", "wasm-bindgen", @@ -4159,7 +4180,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "synstructure", ] @@ -4181,7 +4202,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] @@ -4201,7 +4222,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", "synstructure", ] @@ -4230,7 +4251,7 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.95", ] [[package]] From 96a647ee9308d5e9252f0e3fefc29770bce6d402 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 09:45:22 -0500 Subject: [PATCH 02/12] Trait definition for plan properties now returns LexOrdering --- src/dataset_exec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 2759aa67..3c46dfd0 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -34,7 +34,7 @@ use datafusion::error::{DataFusionError as InnerDataFusionError, Result as DFRes use datafusion::execution::context::TaskContext; use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::Expr; -use datafusion::physical_expr::{EquivalenceProperties, PhysicalSortExpr}; +use datafusion::physical_expr::{EquivalenceProperties, LexOrdering, PhysicalSortExpr}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, @@ -251,7 +251,7 @@ impl ExecutionPlanProperties for DatasetExec { self.plan_properties.output_partitioning() } - fn output_ordering(&self) -> Option<&[PhysicalSortExpr]> { + fn output_ordering(&self) -> Option<&LexOrdering> { None } From ceb034857d5f6f94eea685235bf34e9b68988ced Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 09:48:15 -0500 Subject: [PATCH 03/12] find_df_window_func was removed upstream --- src/functions.rs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/functions.rs b/src/functions.rs index ccc1981b..ae032d70 100644 --- a/src/functions.rs +++ b/src/functions.rs @@ -36,10 +36,7 @@ use datafusion::functions_aggregate; use datafusion::functions_window; use datafusion::logical_expr::expr::Alias; use datafusion::logical_expr::sqlparser::ast::NullTreatment as DFNullTreatment; -use datafusion::logical_expr::{ - expr::{find_df_window_func, WindowFunction}, - lit, Expr, WindowFunctionDefinition, -}; +use datafusion::logical_expr::{expr::WindowFunction, lit, Expr, WindowFunctionDefinition}; fn add_builder_fns_to_aggregate( agg_fn: Expr, @@ -232,12 +229,6 @@ fn when(when: PyExpr, then: PyExpr) -> PyResult { /// /// NOTE: we search the built-ins first because the `UDAF` versions currently do not have the same behavior. fn find_window_fn(name: &str, ctx: Option) -> PyResult { - // search built in window functions (soon to be deprecated) - let df_window_func = find_df_window_func(name); - if let Some(df_window_func) = df_window_func { - return Ok(df_window_func); - } - if let Some(ctx) = ctx { // search UDAFs let udaf = ctx From 8e1ce9d5c2ba90cf549b9f9cb6fd8e8cbf4845a7 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 09:49:45 -0500 Subject: [PATCH 04/12] Prepare and Execute variants were removed from LogicalPlan --- src/sql/logical.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sql/logical.rs b/src/sql/logical.rs index 40f0a6a6..a541889c 100644 --- a/src/sql/logical.rs +++ b/src/sql/logical.rs @@ -85,12 +85,10 @@ impl PyLogicalPlan { | LogicalPlan::Union(_) | LogicalPlan::Statement(_) | LogicalPlan::Values(_) - | LogicalPlan::Prepare(_) | LogicalPlan::Dml(_) | LogicalPlan::Ddl(_) | LogicalPlan::Copy(_) | LogicalPlan::DescribeTable(_) - | LogicalPlan::Execute(_) | LogicalPlan::RecursiveQuery(_) => Err(py_unsupported_variant_err(format!( "Conversion of variant not implemented: {:?}", self.plan From 384983cdbbaa019dc9bb8a4976b8ecd28e9cf263 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 09:51:42 -0500 Subject: [PATCH 05/12] Substrait functions now take SessionState instead of SessionContext --- src/substrait.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/substrait.rs b/src/substrait.rs index f89b6b09..16e8c950 100644 --- a/src/substrait.rs +++ b/src/substrait.rs @@ -114,7 +114,8 @@ impl PySubstraitProducer { /// Convert DataFusion LogicalPlan to Substrait Plan #[staticmethod] pub fn to_substrait_plan(plan: PyLogicalPlan, ctx: &PySessionContext) -> PyResult { - match producer::to_substrait_plan(&plan.plan, &ctx.ctx) { + let session_state = ctx.ctx.state(); + match producer::to_substrait_plan(&plan.plan, &session_state) { Ok(plan) => Ok(PyPlan { plan: *plan }), Err(e) => Err(py_datafusion_err(e)), } @@ -134,7 +135,8 @@ impl PySubstraitConsumer { plan: PyPlan, py: Python, ) -> PyResult { - let result = consumer::from_substrait_plan(&ctx.ctx, &plan.plan); + let session_state = ctx.ctx.state(); + let result = consumer::from_substrait_plan(&session_state, &plan.plan); let logical_plan = wait_for_future(py, result).map_err(DataFusionError::from)?; Ok(PyLogicalPlan::new(logical_plan)) } From d2d4c3ef12d4251e2eda3f44634d3b517aaca305 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 10:00:43 -0500 Subject: [PATCH 06/12] Remove unused import --- src/dataset_exec.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index 3c46dfd0..d73a7af9 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -34,7 +34,7 @@ use datafusion::error::{DataFusionError as InnerDataFusionError, Result as DFRes use datafusion::execution::context::TaskContext; use datafusion::logical_expr::utils::conjunction; use datafusion::logical_expr::Expr; -use datafusion::physical_expr::{EquivalenceProperties, LexOrdering, PhysicalSortExpr}; +use datafusion::physical_expr::{EquivalenceProperties, LexOrdering}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, From ce7c3bc86fdb9c64cf690f294699c4e45c0015ed Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 10:02:02 -0500 Subject: [PATCH 07/12] RuntimeConfig is now deprecated --- src/context.rs | 64 +++++++++++++++++++++++++------------------------- src/lib.rs | 2 +- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/context.rs b/src/context.rs index 88c90e0f..bab7fd42 100644 --- a/src/context.rs +++ b/src/context.rs @@ -62,7 +62,7 @@ use datafusion::execution::context::{ use datafusion::execution::disk_manager::DiskManagerConfig; use datafusion::execution::memory_pool::{FairSpillPool, GreedyMemoryPool, UnboundedMemoryPool}; use datafusion::execution::options::ReadOptions; -use datafusion::execution::runtime_env::{RuntimeConfig, RuntimeEnv}; +use datafusion::execution::runtime_env::RuntimeEnvBuilder; use datafusion::physical_plan::SendableRecordBatchStream; use datafusion::prelude::{ AvroReadOptions, CsvReadOptions, DataFrame, NdJsonReadOptions, ParquetReadOptions, @@ -165,62 +165,62 @@ impl PySessionConfig { } /// Runtime options for a SessionContext -#[pyclass(name = "RuntimeConfig", module = "datafusion", subclass)] +#[pyclass(name = "RuntimeEnvBuilder", module = "datafusion", subclass)] #[derive(Clone)] -pub struct PyRuntimeConfig { - pub config: RuntimeConfig, +pub struct PyRuntimeEnvBuilder { + pub builder: RuntimeEnvBuilder, } #[pymethods] -impl PyRuntimeConfig { +impl PyRuntimeEnvBuilder { #[new] fn new() -> Self { Self { - config: RuntimeConfig::default(), + builder: RuntimeEnvBuilder::default(), } } fn with_disk_manager_disabled(&self) -> Self { - let config = self.config.clone(); - let config = config.with_disk_manager(DiskManagerConfig::Disabled); - Self { config } + let mut builder = self.builder.clone(); + builder = builder.with_disk_manager(DiskManagerConfig::Disabled); + Self { builder } } fn with_disk_manager_os(&self) -> Self { - let config = self.config.clone(); - let config = config.with_disk_manager(DiskManagerConfig::NewOs); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_disk_manager(DiskManagerConfig::NewOs); + Self { builder } } fn with_disk_manager_specified(&self, paths: Vec) -> Self { - let config = self.config.clone(); + let builder = self.builder.clone(); let paths = paths.iter().map(|s| s.into()).collect(); - let config = config.with_disk_manager(DiskManagerConfig::NewSpecified(paths)); - Self { config } + let builder = builder.with_disk_manager(DiskManagerConfig::NewSpecified(paths)); + Self { builder } } fn with_unbounded_memory_pool(&self) -> Self { - let config = self.config.clone(); - let config = config.with_memory_pool(Arc::new(UnboundedMemoryPool::default())); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_memory_pool(Arc::new(UnboundedMemoryPool::default())); + Self { builder } } fn with_fair_spill_pool(&self, size: usize) -> Self { - let config = self.config.clone(); - let config = config.with_memory_pool(Arc::new(FairSpillPool::new(size))); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_memory_pool(Arc::new(FairSpillPool::new(size))); + Self { builder } } fn with_greedy_memory_pool(&self, size: usize) -> Self { - let config = self.config.clone(); - let config = config.with_memory_pool(Arc::new(GreedyMemoryPool::new(size))); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_memory_pool(Arc::new(GreedyMemoryPool::new(size))); + Self { builder } } fn with_temp_file_path(&self, path: &str) -> Self { - let config = self.config.clone(); - let config = config.with_temp_file_path(path); - Self { config } + let builder = self.builder.clone(); + let builder = builder.with_temp_file_path(path); + Self { builder } } } @@ -276,19 +276,19 @@ impl PySessionContext { #[new] pub fn new( config: Option, - runtime: Option, + runtime: Option, ) -> PyResult { let config = if let Some(c) = config { c.config } else { SessionConfig::default().with_information_schema(true) }; - let runtime_config = if let Some(c) = runtime { - c.config + let runtime_env_builder = if let Some(c) = runtime { + c.builder } else { - RuntimeConfig::default() + RuntimeEnvBuilder::default() }; - let runtime = Arc::new(RuntimeEnv::try_new(runtime_config)?); + let runtime = Arc::new(runtime_env_builder.build()?); let session_state = SessionStateBuilder::new() .with_config(config) .with_runtime_env(runtime) diff --git a/src/lib.rs b/src/lib.rs index 0b57e099..1111d5d0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -78,7 +78,7 @@ fn _internal(py: Python, m: Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; - m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; From 789c3e603efdcc3c2a46bdb3bf9cd7e12f8815b7 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 20:17:09 -0500 Subject: [PATCH 08/12] Switch from RuntimeConfig to RuntimeEnvBuilder --- README.md | 2 +- benchmarks/db-benchmark/groupby-datafusion.py | 6 +- benchmarks/tpch/tpch.py | 2 +- docs/source/user-guide/configuration.rst | 8 +-- examples/create-context.py | 4 +- python/datafusion/__init__.py | 4 +- python/datafusion/context.py | 55 +++++++++++-------- python/tests/test_context.py | 10 ++-- 8 files changed, 52 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 83b307e7..ca612c1a 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ It is possible to configure runtime (memory and disk settings) and configuration ```python runtime = ( - RuntimeConfig() + RuntimeEnvBuilder() .with_disk_manager_os() .with_fair_spill_pool(10000000) ) diff --git a/benchmarks/db-benchmark/groupby-datafusion.py b/benchmarks/db-benchmark/groupby-datafusion.py index 3a4399f7..960c8ba9 100644 --- a/benchmarks/db-benchmark/groupby-datafusion.py +++ b/benchmarks/db-benchmark/groupby-datafusion.py @@ -22,7 +22,7 @@ from datafusion import ( col, functions as f, - RuntimeConfig, + RuntimeEnvBuilder, SessionConfig, SessionContext, ) @@ -85,7 +85,9 @@ def execute(df): # create a session context with explicit runtime and config settings runtime = ( - RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(64 * 1024 * 1024 * 1024) + RuntimeEnvBuilder() + .with_disk_manager_os() + .with_fair_spill_pool(64 * 1024 * 1024 * 1024) ) config = ( SessionConfig() diff --git a/benchmarks/tpch/tpch.py b/benchmarks/tpch/tpch.py index 7f104a4c..daa831b5 100644 --- a/benchmarks/tpch/tpch.py +++ b/benchmarks/tpch/tpch.py @@ -28,7 +28,7 @@ def bench(data_path, query_path): # create context # runtime = ( - # RuntimeConfig() + # RuntimeEnvBuilder() # .with_disk_manager_os() # .with_fair_spill_pool(10000000) # ) diff --git a/docs/source/user-guide/configuration.rst b/docs/source/user-guide/configuration.rst index 7d330019..db200a46 100644 --- a/docs/source/user-guide/configuration.rst +++ b/docs/source/user-guide/configuration.rst @@ -19,18 +19,18 @@ Configuration ============= Let's look at how we can configure DataFusion. When creating a :py:class:`~datafusion.context.SessionContext`, you can pass in -a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.context.RuntimeConfig` object. These two cover a wide range of options. +a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.context.RuntimeEnvBuilder` object. These two cover a wide range of options. .. code-block:: python - from datafusion import RuntimeConfig, SessionConfig, SessionContext + from datafusion import RuntimeEnvBuilder, SessionConfig, SessionContext # create a session context with default settings ctx = SessionContext() print(ctx) # create a session context with explicit runtime and config settings - runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) + runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) @@ -48,4 +48,4 @@ a :py:class:`~datafusion.context.SessionConfig` and :py:class:`~datafusion.conte You can read more about available :py:class:`~datafusion.context.SessionConfig` options in the `rust DataFusion Configuration guide `_, -and about :code:`RuntimeConfig` options in the rust `online API documentation `_. +and about :code:`RuntimeEnvBuilder` options in the rust `online API documentation `_. diff --git a/examples/create-context.py b/examples/create-context.py index 11525d8b..760c8513 100644 --- a/examples/create-context.py +++ b/examples/create-context.py @@ -15,14 +15,14 @@ # specific language governing permissions and limitations # under the License. -from datafusion import RuntimeConfig, SessionConfig, SessionContext +from datafusion import RuntimeEnvBuilder, SessionConfig, SessionContext # create a session context with default settings ctx = SessionContext() print(ctx) # create a session context with explicit runtime and config settings -runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) +runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) diff --git a/python/datafusion/__init__.py b/python/datafusion/__init__.py index 7367b0d3..2d8db42c 100644 --- a/python/datafusion/__init__.py +++ b/python/datafusion/__init__.py @@ -29,7 +29,7 @@ from .context import ( SessionContext, SessionConfig, - RuntimeConfig, + RuntimeEnvBuilder, SQLOptions, ) @@ -66,7 +66,7 @@ "SessionContext", "SessionConfig", "SQLOptions", - "RuntimeConfig", + "RuntimeEnvBuilder", "Expr", "ScalarUDF", "WindowFrame", diff --git a/python/datafusion/context.py b/python/datafusion/context.py index 6d7f574c..3c284c9f 100644 --- a/python/datafusion/context.py +++ b/python/datafusion/context.py @@ -20,7 +20,7 @@ from __future__ import annotations from ._internal import SessionConfig as SessionConfigInternal -from ._internal import RuntimeConfig as RuntimeConfigInternal +from ._internal import RuntimeEnvBuilder as RuntimeEnvBuilderInternal from ._internal import SQLOptions as SQLOptionsInternal from ._internal import SessionContext as SessionContextInternal @@ -265,39 +265,41 @@ def set(self, key: str, value: str) -> SessionConfig: return self -class RuntimeConfig: +class RuntimeEnvBuilder: """Runtime configuration options.""" def __init__(self) -> None: - """Create a new :py:class:`RuntimeConfig` with default values.""" - self.config_internal = RuntimeConfigInternal() + """Create a new :py:class:`RuntimeEnvBuilder` with default values.""" + self.config_internal = RuntimeEnvBuilderInternal() - def with_disk_manager_disabled(self) -> RuntimeConfig: + def with_disk_manager_disabled(self) -> RuntimeEnvBuilder: """Disable the disk manager, attempts to create temporary files will error. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ self.config_internal = self.config_internal.with_disk_manager_disabled() return self - def with_disk_manager_os(self) -> RuntimeConfig: + def with_disk_manager_os(self) -> RuntimeEnvBuilder: """Use the operating system's temporary directory for disk manager. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ self.config_internal = self.config_internal.with_disk_manager_os() return self - def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConfig: + def with_disk_manager_specified( + self, *paths: str | pathlib.Path + ) -> RuntimeEnvBuilder: """Use the specified paths for the disk manager's temporary files. Args: paths: Paths to use for the disk manager's temporary files. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ paths_list = [str(p) for p in paths] self.config_internal = self.config_internal.with_disk_manager_specified( @@ -305,16 +307,16 @@ def with_disk_manager_specified(self, *paths: str | pathlib.Path) -> RuntimeConf ) return self - def with_unbounded_memory_pool(self) -> RuntimeConfig: + def with_unbounded_memory_pool(self) -> RuntimeEnvBuilder: """Use an unbounded memory pool. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. """ self.config_internal = self.config_internal.with_unbounded_memory_pool() return self - def with_fair_spill_pool(self, size: int) -> RuntimeConfig: + def with_fair_spill_pool(self, size: int) -> RuntimeEnvBuilder: """Use a fair spill pool with the specified size. This pool works best when you know beforehand the query has multiple spillable @@ -335,16 +337,16 @@ def with_fair_spill_pool(self, size: int) -> RuntimeConfig: size: Size of the memory pool in bytes. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. Examples usage:: - config = RuntimeConfig().with_fair_spill_pool(1024) + config = RuntimeEnvBuilder().with_fair_spill_pool(1024) """ self.config_internal = self.config_internal.with_fair_spill_pool(size) return self - def with_greedy_memory_pool(self, size: int) -> RuntimeConfig: + def with_greedy_memory_pool(self, size: int) -> RuntimeEnvBuilder: """Use a greedy memory pool with the specified size. This pool works well for queries that do not need to spill or have a single @@ -355,32 +357,39 @@ def with_greedy_memory_pool(self, size: int) -> RuntimeConfig: size: Size of the memory pool in bytes. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. Example usage:: - config = RuntimeConfig().with_greedy_memory_pool(1024) + config = RuntimeEnvBuilder().with_greedy_memory_pool(1024) """ self.config_internal = self.config_internal.with_greedy_memory_pool(size) return self - def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeConfig: + def with_temp_file_path(self, path: str | pathlib.Path) -> RuntimeEnvBuilder: """Use the specified path to create any needed temporary files. Args: path: Path to use for temporary files. Returns: - A new :py:class:`RuntimeConfig` object with the updated setting. + A new :py:class:`RuntimeEnvBuilder` object with the updated setting. Example usage:: - config = RuntimeConfig().with_temp_file_path("/tmp") + config = RuntimeEnvBuilder().with_temp_file_path("/tmp") """ self.config_internal = self.config_internal.with_temp_file_path(str(path)) return self +@deprecated("Use `RuntimeEnvBuilder` instead.") +class RuntimeConfig(RuntimeEnvBuilder): + """See `RuntimeEnvBuilder`.""" + + pass + + class SQLOptions: """Options to be used when performing SQL queries.""" @@ -454,7 +463,9 @@ class SessionContext: """ def __init__( - self, config: SessionConfig | None = None, runtime: RuntimeConfig | None = None + self, + config: SessionConfig | None = None, + runtime: RuntimeEnvBuilder | None = None, ) -> None: """Main interface for executing queries with DataFusion. diff --git a/python/tests/test_context.py b/python/tests/test_context.py index ab86faa9..10e8ad0e 100644 --- a/python/tests/test_context.py +++ b/python/tests/test_context.py @@ -25,7 +25,7 @@ from datafusion import ( DataFrame, - RuntimeConfig, + RuntimeEnvBuilder, SessionConfig, SessionContext, SQLOptions, @@ -43,7 +43,7 @@ def test_create_context_session_config_only(): def test_create_context_runtime_config_only(): - SessionContext(runtime=RuntimeConfig()) + SessionContext(runtime=RuntimeEnvBuilder()) @pytest.mark.parametrize("path_to_str", (True, False)) @@ -54,7 +54,7 @@ def test_runtime_configs(tmp_path, path_to_str): path1 = str(path1) if path_to_str else path1 path2 = str(path2) if path_to_str else path2 - runtime = RuntimeConfig().with_disk_manager_specified(path1, path2) + runtime = RuntimeEnvBuilder().with_disk_manager_specified(path1, path2) config = SessionConfig().with_default_catalog_and_schema("foo", "bar") ctx = SessionContext(config, runtime) assert ctx is not None @@ -67,7 +67,7 @@ def test_runtime_configs(tmp_path, path_to_str): def test_temporary_files(tmp_path, path_to_str): path = str(tmp_path) if path_to_str else tmp_path - runtime = RuntimeConfig().with_temp_file_path(path) + runtime = RuntimeEnvBuilder().with_temp_file_path(path) config = SessionConfig().with_default_catalog_and_schema("foo", "bar") ctx = SessionContext(config, runtime) assert ctx is not None @@ -77,7 +77,7 @@ def test_temporary_files(tmp_path, path_to_str): def test_create_context_with_all_valid_args(): - runtime = RuntimeConfig().with_disk_manager_os().with_fair_spill_pool(10000000) + runtime = RuntimeEnvBuilder().with_disk_manager_os().with_fair_spill_pool(10000000) config = ( SessionConfig() .with_create_default_catalog_and_schema(True) From ab8e3a2a06dde5c32a0ae2cf58023dc84cb20bc6 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Fri, 13 Dec 2024 20:21:28 -0500 Subject: [PATCH 09/12] Update return types on unit tests --- python/tests/test_functions.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 5dce188e..0d98c4b6 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -103,8 +103,11 @@ def test_lit_arith(df): result = df.collect() assert len(result) == 1 result = result[0] + assert result.column(0) == pa.array([5, 6, 7]) - assert result.column(1) == pa.array(["Hello!", "World!", "!!"]) + assert result.column(1) == pa.array( + ["Hello!", "World!", "!!"], type=pa.string_view() + ) def test_math_functions(): @@ -661,7 +664,7 @@ def test_array_function_obj_tests(stmt, py_expr): ), ( f.concat(column("a").cast(pa.string()), literal("?")), - pa.array(["Hello?", "World?", "!?"]), + pa.array(["Hello?", "World?", "!?"], type=pa.string_view()), ), (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), @@ -871,8 +874,8 @@ def test_temporal_functions(df): result = df.collect() assert len(result) == 1 result = result[0] - assert result.column(0) == pa.array([12, 6, 7], type=pa.float64()) - assert result.column(1) == pa.array([2022, 2027, 2020], type=pa.float64()) + assert result.column(0) == pa.array([12, 6, 7], type=pa.int32()) + assert result.column(1) == pa.array([2022, 2027, 2020], type=pa.int32()) assert result.column(2) == pa.array( [datetime(2022, 12, 1), datetime(2027, 6, 1), datetime(2020, 7, 1)], type=pa.timestamp("us"), @@ -904,7 +907,7 @@ def test_temporal_functions(df): assert result.column(9) == pa.array( [datetime(2023, 9, 7, 5, 6, 14, 523952)] * 3, type=pa.timestamp("us") ) - assert result.column(10) == pa.array([31, 26, 2], type=pa.float64()) + assert result.column(10) == pa.array([31, 26, 2], type=pa.int32()) def test_arrow_cast(df): From 1e88816ff7dfdc08650f46078eaebb5e7b9028dd Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 9 Jan 2025 07:11:49 -0500 Subject: [PATCH 10/12] DF 44 changes the execution plan properties to have boundedness and emission type --- Cargo.lock | 408 +++++++++++++++++++++++--------------------- Cargo.toml | 9 +- src/dataset_exec.rs | 16 +- src/udwf.rs | 6 - 4 files changed, 225 insertions(+), 214 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff19bc5c..105cc30c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,14 +138,14 @@ checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "apache-avro" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb7c683b2f8f40970b70e39ff8be514c95b96fcb9c4af87e1ed2cb2e10801a0" +checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" dependencies = [ - "bzip2", + "bigdecimal", + "bzip2 0.4.4", "crc32fast", "digest", - "lazy_static", "libflate", "log", "num-bigint", @@ -153,15 +153,16 @@ dependencies = [ "rand", "regex-lite", "serde", + "serde_bytes", "serde_json", "snap", - "strum 0.25.0", - "strum_macros 0.25.3", + "strum", + "strum_macros", "thiserror 1.0.69", "typed-builder", "uuid", "xz2", - "zstd 0.12.4", + "zstd", ] [[package]] @@ -415,16 +416,15 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df895a515f70646414f4b45c0b79082783b80552b373a68283012928df56f522" dependencies = [ - "bzip2", + "bzip2 0.4.4", "flate2", "futures-core", - "futures-io", "memchr", "pin-project-lite", "tokio", "xz2", - "zstd 0.13.2", - "zstd-safe 7.2.1", + "zstd", + "zstd-safe", ] [[package]] @@ -506,6 +506,20 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -598,6 +612,16 @@ dependencies = [ "libc", ] +[[package]] +name = "bzip2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bafdbf26611df8c14810e268ddceda071c297570a5fb360ceddf617fe417ef58" +dependencies = [ + "bzip2-sys", + "libc", +] + [[package]] name = "bzip2-sys" version = "0.1.11+1.0.8" @@ -681,8 +705,8 @@ version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24f165e7b643266ea80cb858aed492ad9280e3e05ce24d4a99d7d7b889b6a4d9" dependencies = [ - "strum 0.26.3", - "strum_macros 0.26.4", + "strum", + "strum_macros", "unicode-width", ] @@ -850,11 +874,10 @@ dependencies = [ [[package]] name = "datafusion" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbba0799cf6913b456ed07a94f0f3b6e12c62a5d88b10809e2284a0f2b915c05" +checksum = "014fc8c384ecacedaabb3bc8359c2a6c6e9d8f7bea65be3434eccacfc37f52d9" dependencies = [ - "ahash", "apache-avro", "arrow", "arrow-array", @@ -863,7 +886,7 @@ dependencies = [ "async-compression", "async-trait", "bytes", - "bzip2", + "bzip2 0.5.0", "chrono", "dashmap", "datafusion-catalog", @@ -874,6 +897,7 @@ dependencies = [ "datafusion-functions", "datafusion-functions-aggregate", "datafusion-functions-nested", + "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", "datafusion-physical-expr", @@ -884,19 +908,14 @@ dependencies = [ "flate2", "futures", "glob", - "half", - "hashbrown 0.14.5", - "indexmap", "itertools", "log", "num-traits", - "num_cpus", "object_store", "parking_lot", "parquet", - "paste", - "pin-project-lite", "rand", + "regex", "sqlparser", "tempfile", "tokio", @@ -904,14 +923,14 @@ dependencies = [ "url", "uuid", "xz2", - "zstd 0.13.2", + "zstd", ] [[package]] name = "datafusion-catalog" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7493c5c2d40eec435b13d92e5703554f4efc7059451fcb8d3a79580ff0e45560" +checksum = "ee60d33e210ef96070377ae667ece7caa0e959c8387496773d4a1a72f1a5012e" dependencies = [ "arrow-schema", "async-trait", @@ -924,9 +943,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24953049ebbd6f8964f91f60aa3514e121b5e81e068e33b60e77815ab369b25c" +checksum = "0b42b7d720fe21ed9cca2ebb635f3f13a12cfab786b41e0fba184fb2e620525b" dependencies = [ "ahash", "apache-avro", @@ -934,44 +953,48 @@ dependencies = [ "arrow-array", "arrow-buffer", "arrow-schema", - "chrono", "half", "hashbrown 0.14.5", "indexmap", - "instant", "libc", - "num_cpus", + "log", "object_store", "parquet", "paste", "pyo3", + "recursive", "sqlparser", "tokio", + "web-time", ] [[package]] name = "datafusion-common-runtime" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f06df4ef76872e11c924d3c814fd2a8dd09905ed2e2195f71c857d78abd19685" +checksum = "72fbf14d4079f7ce5306393084fe5057dddfdc2113577e0049310afa12e94281" dependencies = [ "log", "tokio", ] +[[package]] +name = "datafusion-doc" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c278dbd64860ed0bb5240fc1f4cb6aeea437153910aea69bcf7d5a8d6d0454f3" + [[package]] name = "datafusion-execution" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bbdcb628d690f3ce5fea7de81642b514486d58ff9779a51f180a69a4eadb361" +checksum = "e22cb02af47e756468b3cbfee7a83e3d4f2278d452deb4b033ba933c75169486" dependencies = [ "arrow", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", "futures", - "hashbrown 0.14.5", "log", "object_store", "parking_lot", @@ -982,45 +1005,41 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8036495980e3131f706b7d33ab00b4492d73dc714e3cb74d11b50f9602a73246" +checksum = "62298eadb1d15b525df1315e61a71519ffc563d41d5c3b2a30fda2d70f77b93c" dependencies = [ - "ahash", "arrow", - "arrow-array", - "arrow-buffer", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", "indexmap", "paste", + "recursive", "serde_json", "sqlparser", - "strum 0.26.3", - "strum_macros 0.26.4", ] [[package]] name = "datafusion-expr-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da0f3cb4669f9523b403d6b5a0ec85023e0ab3bf0183afd1517475b3e64fdd2" +checksum = "dda7f73c5fc349251cd3dcb05773c5bf55d2505a698ef9d38dfc712161ea2f55" dependencies = [ "arrow", "datafusion-common", "itertools", - "paste", ] [[package]] name = "datafusion-ffi" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e923c459b53a26d92a8806d1f6a37fdf48bde51507a39eaed6f42a60f2bfd160" +checksum = "114e944790756b84c2cc5971eae24f5430980149345601939ac222885d4db5f7" dependencies = [ "abi_stable", "arrow", @@ -1028,7 +1047,6 @@ dependencies = [ "async-trait", "datafusion", "datafusion-proto", - "doc-comment", "futures", "log", "prost", @@ -1036,9 +1054,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52c4012648b34853e40a2c6bcaa8772f837831019b68aca384fb38436dba162" +checksum = "fd197f3b2975424d3a4898ea46651be855a46721a56727515dbd5c9e2fb597da" dependencies = [ "arrow", "arrow-buffer", @@ -1047,8 +1065,11 @@ dependencies = [ "blake3", "chrono", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hashbrown 0.14.5", "hex", "itertools", @@ -1063,44 +1084,44 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5b8bb624597ba28ed7446df4a9bd7c7a7bde7c578b6b527da3f47371d5f6741" +checksum = "aabbe48fba18f9981b134124381bee9e46f93518b8ad2f9721ee296cef5affb9" dependencies = [ "ahash", "arrow", "arrow-schema", "datafusion-common", + "datafusion-doc", "datafusion-execution", "datafusion-expr", "datafusion-functions-aggregate-common", + "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", "half", - "indexmap", "log", "paste", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb06208fc470bc8cf1ce2d9a1159d42db591f2c7264a8c1776b53ad8f675143" +checksum = "d7a3fefed9c8c11268d446d924baca8cabf52fe32f73fdaa20854bac6473590c" dependencies = [ "ahash", "arrow", "datafusion-common", "datafusion-expr-common", "datafusion-physical-expr-common", - "rand", ] [[package]] name = "datafusion-functions-nested" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fca25bbb87323716d05e54114666e942172ccca23c5a507e9c7851db6e965317" +checksum = "6360f27464fab857bec698af39b2ae331dc07c8bf008fb4de387a19cdc6815a5" dependencies = [ "arrow", "arrow-array", @@ -1116,18 +1137,35 @@ dependencies = [ "itertools", "log", "paste", - "rand", +] + +[[package]] +name = "datafusion-functions-table" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c35c070eb705c12795dab399c3809f4dfbc290678c624d3989490ca9b8449c1" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", ] [[package]] name = "datafusion-functions-window" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ae23356c634e54c59f7c51acb7a5b9f6240ffb2cf997049a1a24a8a88598dbe" +checksum = "52229bca26b590b140900752226c829f15fc1a99840e1ca3ce1a9534690b82a8" dependencies = [ "datafusion-common", + "datafusion-doc", "datafusion-expr", "datafusion-functions-window-common", + "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", "log", @@ -1136,48 +1174,54 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4b3d6ff7794acea026de36007077a06b18b89e4f9c3fea7f2215f9f7dd9059b" +checksum = "367befc303b64a668a10ae6988a064a9289e1999e71a7f8e526b6e14d6bdd9d6" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", ] +[[package]] +name = "datafusion-macros" +version = "44.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5de3c8f386ea991696553afe241a326ecbc3c98a12c562867e4be754d3a060c" +dependencies = [ + "quote", + "syn 2.0.95", +] + [[package]] name = "datafusion-optimizer" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec6241eb80c595fa0e1a8a6b69686b5cf3bd5fdacb8319582a0943b0bd788aa" +checksum = "53b520413906f755910422b016fb73884ae6e9e1b376de4f9584b6c0e031da75" dependencies = [ "arrow", - "async-trait", "chrono", "datafusion-common", "datafusion-expr", "datafusion-physical-expr", - "hashbrown 0.14.5", "indexmap", "itertools", "log", - "paste", + "recursive", + "regex", "regex-syntax", ] [[package]] name = "datafusion-physical-expr" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3370357b8fc75ec38577700644e5d1b0bc78f38babab99c0b8bd26bafb3e4335" +checksum = "acd6ddc378f6ad19af95ccd6790dec8f8e1264bc4c70e99ddc1830c1a1c78ccd" dependencies = [ "ahash", "arrow", "arrow-array", "arrow-buffer", - "arrow-ord", "arrow-schema", - "arrow-string", - "chrono", "datafusion-common", "datafusion-expr", "datafusion-expr-common", @@ -1194,39 +1238,40 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7734d94bf2fa6f6e570935b0ddddd8421179ce200065be97874e13d46a47b" +checksum = "06e6c05458eccd74b4c77ed6a1fe63d52434240711de7f6960034794dad1caf5" dependencies = [ "ahash", "arrow", "datafusion-common", "datafusion-expr-common", "hashbrown 0.14.5", - "rand", + "itertools", ] [[package]] name = "datafusion-physical-optimizer" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eee8c479522df21d7b395640dff88c5ed05361852dce6544d7c98e9dbcebffe" +checksum = "9dc3a82190f49c37d377f31317e07ab5d7588b837adadba8ac367baad5dc2351" dependencies = [ "arrow", - "arrow-schema", "datafusion-common", "datafusion-execution", "datafusion-expr-common", "datafusion-physical-expr", "datafusion-physical-plan", "itertools", + "log", + "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17e1fc2e2c239d14e8556f2622b19a726bf6bc6962cc00c71fc52626274bee24" +checksum = "6a6608bc9844b4ddb5ed4e687d173e6c88700b1d0482f43894617d18a1fe75da" dependencies = [ "ahash", "arrow", @@ -1240,7 +1285,6 @@ dependencies = [ "datafusion-common-runtime", "datafusion-execution", "datafusion-expr", - "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", @@ -1250,18 +1294,16 @@ dependencies = [ "indexmap", "itertools", "log", - "once_cell", "parking_lot", "pin-project-lite", - "rand", "tokio", ] [[package]] name = "datafusion-proto" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f730f7fc5a20134d4e5ecdf7bbf392002ac58163d58423ea28a702dc077b06e1" +checksum = "8e23b0998195e495bfa7b37cdceb317129a6c40522219f6872d2e0c9ae9f4fcb" dependencies = [ "arrow", "chrono", @@ -1275,14 +1317,12 @@ dependencies = [ [[package]] name = "datafusion-proto-common" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12c225fe49e4f943e35446b263613ada7a9e9f8d647544e6b07037b9803567df" +checksum = "cfc59992a29eed2d2c1dd779deac99083b217774ebcf90ee121840607a4d866f" dependencies = [ "arrow", - "chrono", "datafusion-common", - "object_store", "prost", ] @@ -1294,7 +1334,6 @@ dependencies = [ "async-trait", "datafusion", "datafusion-ffi", - "datafusion-functions-window-common", "datafusion-proto", "datafusion-substrait", "futures", @@ -1312,30 +1351,32 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e3a4ed41dbee20a5d947a59ca035c225d67dc9cbe869c10f66dcdf25e7ce51" +checksum = "6a884061c79b33d0c8e84a6f4f4be8bdc12c0f53f5af28ddf5d6d95ac0b15fdc" dependencies = [ "arrow", "arrow-array", "arrow-schema", + "bigdecimal", "datafusion-common", "datafusion-expr", "indexmap", "log", + "recursive", "regex", "sqlparser", - "strum 0.26.3", ] [[package]] name = "datafusion-substrait" -version = "43.0.0" +version = "44.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9c768d2b4c4485c43afbaeeb86dd1f2ac3fb34a9e6e8c8b06180d2a223d5ba" +checksum = "d2ec36dd38512b1ecc7a3bb92e72046b944611b2f0d709445c1e51b0143bffd4" dependencies = [ "arrow-buffer", "async-recursion", + "async-trait", "chrono", "datafusion", "itertools", @@ -1368,12 +1409,6 @@ dependencies = [ "syn 2.0.95", ] -[[package]] -name = "doc-comment" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" - [[package]] name = "dyn-clone" version = "1.0.17" @@ -1628,24 +1663,12 @@ version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" -[[package]] -name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" -[[package]] -name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - [[package]] name = "hex" version = "0.4.3" @@ -1933,18 +1956,6 @@ version = "2.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5" -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "integer-encoding" version = "3.0.4" @@ -2253,6 +2264,7 @@ checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" dependencies = [ "num-integer", "num-traits", + "serde", ] [[package]] @@ -2305,16 +2317,6 @@ dependencies = [ "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" version = "0.36.7" @@ -2432,7 +2434,7 @@ dependencies = [ "thrift", "tokio", "twox-hash", - "zstd 0.13.2", + "zstd", "zstd-sys", ] @@ -2467,7 +2469,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6eea3058763d6e656105d1403cb04e0a41b7bbac6362d413e7c33be0c32279c9" dependencies = [ - "heck 0.5.0", + "heck", "itertools", "prost", "prost-types", @@ -2610,7 +2612,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0f3e5beed80eb580c68e2c600937ac2c4eedabdfd5ef1e5b7ea4f3fba84497b" dependencies = [ - "heck 0.5.0", + "heck", "itertools", "log", "multimap", @@ -2655,6 +2657,15 @@ dependencies = [ "cmake", ] +[[package]] +name = "psm" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200b9ff220857e53e184257720a14553b2f4aa02577d2ed9842d45d4b9654810" +dependencies = [ + "cc", +] + [[package]] name = "pyo3" version = "0.22.6" @@ -2724,7 +2735,7 @@ version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "36c011a03ba1e50152b4b394b479826cad97e7a21eb52df179cd91ac411cbfbe" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "pyo3-build-config", "quote", @@ -2838,6 +2849,26 @@ dependencies = [ "getrandom", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.95", +] + [[package]] name = "redox_syscall" version = "0.5.8" @@ -3164,6 +3195,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + [[package]] name = "serde_derive" version = "1.0.217" @@ -3288,7 +3328,7 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "syn 2.0.95", @@ -3318,9 +3358,9 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.51.0" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fe11944a61da0da3f592e19a45ebe5ab92dc14a779907ff1f08fbb797bfefc7" +checksum = "05a528114c392209b3264855ad491fcce534b94a38771b0a0b97a79379275ce8" dependencies = [ "log", "sqlparser_derive", @@ -3328,9 +3368,9 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01b2e185515564f15375f593fb966b5718bc624ba77fe49fa4616ad619690554" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" dependencies = [ "proc-macro2", "quote", @@ -3344,38 +3384,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] -name = "static_assertions" -version = "1.1.0" +name = "stacker" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "799c883d55abdb5e98af1a7b3f23b9b6de8ecada0ecac058672d7635eb48ca7b" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "windows-sys 0.59.0", +] [[package]] -name = "strum" -version = "0.25.0" +name = "static_assertions" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strum" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" -dependencies = [ - "strum_macros 0.26.4", -] - -[[package]] -name = "strum_macros" -version = "0.25.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" -dependencies = [ - "heck 0.4.1", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.95", -] [[package]] name = "strum_macros" @@ -3383,7 +3414,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.5.0", + "heck", "proc-macro2", "quote", "rustversion", @@ -3392,11 +3423,11 @@ dependencies = [ [[package]] name = "substrait" -version = "0.45.5" +version = "0.50.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a127ae9d8e443cea5c2122eb2ffe5fe489e802a1e746a09c5a5cb59d074c0aeb" +checksum = "b1772d041c37cc7e6477733c76b2acf4ee36bd52b2ae4d9ea0ec9c87d003db32" dependencies = [ - "heck 0.5.0", + "heck", "pbjson", "pbjson-build", "pbjson-types", @@ -3716,18 +3747,18 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" [[package]] name = "typed-builder" -version = "0.16.2" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34085c17941e36627a879208083e25d357243812c30e7d7387c3b954f30ade16" +checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" dependencies = [ "typed-builder-macro", ] [[package]] name = "typed-builder-macro" -version = "0.16.2" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f03ca4cb38206e2bef0700092660bb74d696f808514dae47fa1467cbfe26e96e" +checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" dependencies = [ "proc-macro2", "quote", @@ -3756,7 +3787,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d59ab345b6c0d8ae9500b9ff334a4c7c0d316c1c628dc55726b95887eb8dbd11" dependencies = [ - "heck 0.5.0", + "heck", "log", "proc-macro2", "quote", @@ -4254,32 +4285,13 @@ dependencies = [ "syn 2.0.95", ] -[[package]] -name = "zstd" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a27595e173641171fc74a1232b7b1c7a7cb6e18222c11e9dfb9888fa424c53c" -dependencies = [ - "zstd-safe 6.0.6", -] - [[package]] name = "zstd" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ - "zstd-safe 7.2.1", -] - -[[package]] -name = "zstd-safe" -version = "6.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee98ffd0b48ee95e6c5168188e44a54550b1564d9d530ee21d5f0eaed1069581" -dependencies = [ - "libc", - "zstd-sys", + "zstd-safe", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d2884468..48219414 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,11 +38,10 @@ tokio = { version = "1.41", features = ["macros", "rt", "rt-multi-thread", "sync pyo3 = { version = "0.22", features = ["extension-module", "abi3", "abi3-py38"] } pyo3-async-runtimes = { version = "0.22", features = ["tokio-runtime"]} arrow = { version = "53", features = ["pyarrow"] } -datafusion = { version = "43.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } -datafusion-substrait = { version = "43.0.0", optional = true } -datafusion-proto = { version = "43.0.0" } -datafusion-ffi = { version = "43.0.0" } -datafusion-functions-window-common = { version = "43.0.0" } +datafusion = { version = "44.0.0", features = ["pyarrow", "avro", "unicode_expressions"] } +datafusion-substrait = { version = "44.0.0", optional = true } +datafusion-proto = { version = "44.0.0" } +datafusion-ffi = { version = "44.0.0" } prost = "0.13" # keep in line with `datafusion-substrait` uuid = { version = "1.11", features = ["v4"] } mimalloc = { version = "0.1", optional = true, default-features = false, features = ["local_dynamic_tls"] } diff --git a/src/dataset_exec.rs b/src/dataset_exec.rs index d73a7af9..9d255942 100644 --- a/src/dataset_exec.rs +++ b/src/dataset_exec.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; /// Implements a Datafusion physical ExecutionPlan that delegates to a PyArrow Dataset /// This actually performs the projection, filtering and scanning of a Dataset use pyo3::prelude::*; @@ -37,8 +38,8 @@ use datafusion::logical_expr::Expr; use datafusion::physical_expr::{EquivalenceProperties, LexOrdering}; use datafusion::physical_plan::stream::RecordBatchStreamAdapter; use datafusion::physical_plan::{ - DisplayAs, DisplayFormatType, ExecutionMode, ExecutionPlan, ExecutionPlanProperties, - Partitioning, SendableRecordBatchStream, Statistics, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, Partitioning, + SendableRecordBatchStream, Statistics, }; use crate::errors::DataFusionError; @@ -136,7 +137,8 @@ impl DatasetExec { let plan_properties = datafusion::physical_plan::PlanProperties::new( EquivalenceProperties::new(schema.clone()), Partitioning::UnknownPartitioning(fragments.len()), - ExecutionMode::Bounded, + EmissionType::Final, + Boundedness::Bounded, ); Ok(DatasetExec { @@ -255,8 +257,12 @@ impl ExecutionPlanProperties for DatasetExec { None } - fn execution_mode(&self) -> datafusion::physical_plan::ExecutionMode { - self.plan_properties.execution_mode + fn boundedness(&self) -> Boundedness { + self.plan_properties.boundedness + } + + fn pipeline_behavior(&self) -> EmissionType { + self.plan_properties.emission_type } fn equivalence_properties(&self) -> &datafusion::physical_expr::EquivalenceProperties { diff --git a/src/udwf.rs b/src/udwf.rs index 3f5ad0b1..689eb79e 100644 --- a/src/udwf.rs +++ b/src/udwf.rs @@ -22,9 +22,7 @@ use std::sync::Arc; use arrow::array::{make_array, Array, ArrayData, ArrayRef}; use datafusion::logical_expr::function::{PartitionEvaluatorArgs, WindowUDFFieldArgs}; use datafusion::logical_expr::window_state::WindowAggState; -use datafusion::physical_plan::PhysicalExpr; use datafusion::scalar::ScalarValue; -use datafusion_functions_window_common::expr::ExpressionArgs; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; @@ -319,8 +317,4 @@ impl WindowUDFImpl for MultiColumnWindowUDF { let _ = _partition_evaluator_args; (self.partition_evaluator_factory)() } - - fn expressions(&self, expr_args: ExpressionArgs) -> Vec> { - expr_args.input_exprs().into() - } } From a6457ed787bdd9867e2fcd5b415a9d1f1555c322 Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 9 Jan 2025 07:13:09 -0500 Subject: [PATCH 11/12] Initcap now returns stringview --- python/tests/test_functions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/tests/test_functions.py b/python/tests/test_functions.py index 0d98c4b6..01c6c9ce 100644 --- a/python/tests/test_functions.py +++ b/python/tests/test_functions.py @@ -666,7 +666,10 @@ def test_array_function_obj_tests(stmt, py_expr): f.concat(column("a").cast(pa.string()), literal("?")), pa.array(["Hello?", "World?", "!?"], type=pa.string_view()), ), - (f.initcap(column("c")), pa.array(["Hello ", " World ", " !"])), + ( + f.initcap(column("c")), + pa.array(["Hello ", " World ", " !"], type=pa.string_view()), + ), (f.left(column("a"), literal(3)), pa.array(["Hel", "Wor", "!"])), (f.length(column("c")), pa.array([6, 7, 2], type=pa.int32())), (f.lower(column("a")), pa.array(["hello", "world", "!"])), From 951e1156cd31e55e3c1733e0542f09117228cbbd Mon Sep 17 00:00:00 2001 From: Tim Saucer Date: Thu, 9 Jan 2025 08:24:40 -0500 Subject: [PATCH 12/12] Bump datafusion version in example --- examples/ffi-table-provider/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/ffi-table-provider/Cargo.toml b/examples/ffi-table-provider/Cargo.toml index 4e54eaf0..4e6f91f3 100644 --- a/examples/ffi-table-provider/Cargo.toml +++ b/examples/ffi-table-provider/Cargo.toml @@ -21,8 +21,8 @@ version = "0.1.0" edition = "2021" [dependencies] -datafusion = { version = "43.0.0" } -datafusion-ffi = { version = "43.0.0" } +datafusion = { version = "44.0.0" } +datafusion-ffi = { version = "44.0.0" } pyo3 = { version = "0.22.6", features = ["extension-module", "abi3", "abi3-py38"] } arrow = { version = "53.2.0" } arrow-array = { version = "53.2.0" }