This is an automated email from the ASF dual-hosted git repository.
milenkovicm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-ballista.git
The following commit(s) were added to refs/heads/main by this push:
new 7829e0205 chore(deps): update to datafusion v.53 (#1486)
7829e0205 is described below
commit 7829e02054db72bf7984eeb9b1647bc001b80f01
Author: Marko Milenković <[email protected]>
AuthorDate: Tue Mar 24 18:53:59 2026 +0000
chore(deps): update to datafusion v.53 (#1486)
---
Cargo.lock | 613 +++++++++++----------
Cargo.toml | 22 +-
ballista-cli/Cargo.toml | 4 +-
ballista/client/Cargo.toml | 12 +-
ballista/client/tests/context_checks.rs | 38 ++
ballista/client/tests/context_unsupported.rs | 47 --
ballista/core/Cargo.toml | 4 +-
ballista/core/src/diagram.rs | 2 +
.../core/src/execution_plans/distributed_query.rs | 27 +-
.../core/src/execution_plans/shuffle_reader.rs | 8 +-
.../core/src/execution_plans/shuffle_writer.rs | 8 +-
.../src/execution_plans/sort_shuffle/writer.rs | 8 +-
.../core/src/execution_plans/unresolved_shuffle.rs | 16 +-
ballista/core/src/serde/mod.rs | 14 +-
ballista/executor/Cargo.toml | 4 +-
ballista/executor/src/collect.rs | 8 +-
ballista/executor/src/executor.rs | 14 +-
ballista/scheduler/Cargo.toml | 4 +-
ballista/scheduler/src/cluster/mod.rs | 1 +
.../src/physical_optimizer/join_selection.rs | 69 +--
ballista/scheduler/src/state/aqe/execution_plan.rs | 10 +-
.../state/aqe/optimizer_rule/propagate_empty.rs | 2 +
.../scheduler/src/state/aqe/test/alter_stages.rs | 18 +-
.../scheduler/src/state/execution_graph_dot.rs | 3 +-
benchmarks/Cargo.toml | 6 +-
examples/Cargo.toml | 10 +-
26 files changed, 488 insertions(+), 484 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index d59076b38..df85b80ce 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -93,9 +93,9 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.13"
+version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anstyle-parse"
@@ -148,7 +148,7 @@ dependencies = [
"miniz_oxide",
"num-bigint",
"quad-rand",
- "rand",
+ "rand 0.9.2",
"regex-lite",
"serde",
"serde_bytes",
@@ -184,9 +184,9 @@ checksum =
"7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "arrow"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8"
+checksum = "d441fdda254b65f3e9025910eb2c2066b6295d9c8ed409522b8d2ace1ff8574c"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -205,9 +205,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b"
+checksum = "ced5406f8b720cc0bc3aa9cf5758f93e8593cda5490677aa194e4b4b383f9a59"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -219,9 +219,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef"
+checksum = "772bd34cacdda8baec9418d80d23d0fb4d50ef0735685bd45158b83dfeb6e62d"
dependencies = [
"ahash",
"arrow-buffer",
@@ -238,9 +238,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2"
+checksum = "898f4cf1e9598fdb77f356fdf2134feedfd0ee8d5a4e0a5f573e7d0aec16baa4"
dependencies = [
"bytes",
"half",
@@ -250,9 +250,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5"
+checksum = "b0127816c96533d20fc938729f48c52d3e48f99717e7a0b5ade77d742510736d"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -272,9 +272,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a"
+checksum = "ca025bd0f38eeecb57c2153c0123b960494138e6a957bbda10da2b25415209fe"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -287,9 +287,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304"
+checksum = "42d10beeab2b1c3bb0b53a00f7c944a178b622173a5c7bcabc3cb45d90238df4"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -300,9 +300,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "58c5b083668e6230eae3eab2fc4b5fb989974c845d0aa538dde61a4327c78675"
+checksum = "302b2e036335f3f04d65dad3f74ff1f2aae6dc671d6aa04dc6b61193761e16fb"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -328,9 +328,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1"
+checksum = "609a441080e338147a84e8e6904b6da482cefb957c5cdc0f3398872f69a315d0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -344,9 +344,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86"
+checksum = "6ead0914e4861a531be48fe05858265cf854a4880b9ed12618b1d08cba9bebc8"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -368,9 +368,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b"
+checksum = "763a7ba279b20b52dad300e68cfc37c17efa65e68623169076855b3a9e941ca5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -381,9 +381,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0"
+checksum = "e14fe367802f16d7668163ff647830258e6e0aeea9a4d79aaedf273af3bdcd3e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -394,9 +394,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68"
+checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743"
dependencies = [
"serde_core",
"serde_json",
@@ -404,9 +404,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b"
+checksum = "78694888660a9e8ac949853db393af2a8b8fc82c19ce333132dfa2e72cc1a7fe"
dependencies = [
"ahash",
"arrow-array",
@@ -418,9 +418,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8"
+checksum = "61e04a01f8bb73ce54437514c5fd3ee2aa3e8abe4c777ee5cc55853b1652f79e"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -435,9 +435,9 @@ dependencies = [
[[package]]
name = "astral-tokio-tar"
-version = "0.5.6"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ec179a06c1769b1e42e1e2cbe74c7dcdb3d6383c838454d063eaac5bbb7ebbe5"
+checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9"
dependencies = [
"filetime",
"futures-core",
@@ -528,9 +528,9 @@ checksum =
"c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "aws-config"
-version = "1.8.13"
+version = "1.8.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c456581cb3c77fafcc8c67204a70680d40b61112d6da78c77bd31d945b65f1b5"
+checksum = "11493b0bad143270fb8ad284a096dd529ba91924c5409adeac856cc1bf047dbc"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -548,7 +548,7 @@ dependencies = [
"fastrand",
"hex",
"http 1.4.0",
- "ring",
+ "sha1",
"time",
"tokio",
"tracing",
@@ -558,9 +558,9 @@ dependencies = [
[[package]]
name = "aws-credential-types"
-version = "1.2.11"
+version = "1.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3cd362783681b15d136480ad555a099e82ecd8e2d10a841e14dfd0078d67fee3"
+checksum = "8f20799b373a1be121fe3005fba0c2090af9411573878f224df44b42727fcaf7"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -570,9 +570,9 @@ dependencies = [
[[package]]
name = "aws-lc-rs"
-version = "1.16.1"
+version = "1.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "94bffc006df10ac2a68c83692d734a465f8ee6c5b384d8545a636f81d858f4bf"
+checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc"
dependencies = [
"aws-lc-sys",
"zeroize",
@@ -580,9 +580,9 @@ dependencies = [
[[package]]
name = "aws-lc-sys"
-version = "0.38.0"
+version = "0.39.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4321e568ed89bb5a7d291a7f37997c2c0df89809d7b6d12062c81ddb54aa782e"
+checksum = "1fa7e52a4c5c547c741610a2c6f123f3881e409b714cd27e6798ef020c514f0a"
dependencies = [
"cc",
"cmake",
@@ -592,9 +592,9 @@ dependencies = [
[[package]]
name = "aws-runtime"
-version = "1.6.0"
+version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c635c2dc792cb4a11ce1a4f392a925340d1bdf499289b5ec1ec6810954eb43f5"
+checksum = "5fc0651c57e384202e47153c1260b84a9936e19803d747615edf199dc3b98d17"
dependencies = [
"aws-credential-types",
"aws-sigv4",
@@ -605,6 +605,7 @@ dependencies = [
"aws-smithy-types",
"aws-types",
"bytes",
+ "bytes-utils",
"fastrand",
"http 1.4.0",
"http-body 1.0.1",
@@ -616,9 +617,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sso"
-version = "1.93.0"
+version = "1.97.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9dcb38bb33fc0a11f1ffc3e3e85669e0a11a37690b86f77e75306d8f369146a0"
+checksum = "9aadc669e184501caaa6beafb28c6267fc1baef0810fb58f9b205485ca3f2567"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -640,9 +641,9 @@ dependencies = [
[[package]]
name = "aws-sdk-ssooidc"
-version = "1.95.0"
+version = "1.99.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2ada8ffbea7bd1be1f53df1dadb0f8fdb04badb13185b3321b929d1ee3caad09"
+checksum = "1342a7db8f358d3de0aed2007a0b54e875458e39848d54cc1d46700b2bfcb0a8"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -664,9 +665,9 @@ dependencies = [
[[package]]
name = "aws-sdk-sts"
-version = "1.97.0"
+version = "1.101.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e6443ccadc777095d5ed13e21f5c364878c9f5bad4e35187a6cdbd863b0afcad"
+checksum = "ab41ad64e4051ecabeea802d6a17845a91e83287e1dd249e6963ea1ba78c428a"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -689,9 +690,9 @@ dependencies = [
[[package]]
name = "aws-sigv4"
-version = "1.3.8"
+version = "1.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "efa49f3c607b92daae0c078d48a4571f599f966dce3caee5f1ea55c4d9073f99"
+checksum = "b0b660013a6683ab23797778e21f1f854744fdf05f68204b4cca4c8c04b5d1f4"
dependencies = [
"aws-credential-types",
"aws-smithy-http",
@@ -711,9 +712,9 @@ dependencies = [
[[package]]
name = "aws-smithy-async"
-version = "1.2.11"
+version = "1.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "52eec3db979d18cb807fc1070961cc51d87d069abe9ab57917769687368a8c6c"
+checksum = "2ffcaf626bdda484571968400c326a244598634dc75fd451325a54ad1a59acfc"
dependencies = [
"futures-util",
"pin-project-lite",
@@ -722,9 +723,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http"
-version = "0.63.3"
+version = "0.63.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "630e67f2a31094ffa51b210ae030855cb8f3b7ee1329bdd8d085aaf61e8b97fc"
+checksum = "ba1ab2dc1c2c3749ead27180d333c42f11be8b0e934058fb4b2258ee8dbe5231"
dependencies = [
"aws-smithy-runtime-api",
"aws-smithy-types",
@@ -743,9 +744,9 @@ dependencies = [
[[package]]
name = "aws-smithy-http-client"
-version = "1.1.9"
+version = "1.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "12fb0abf49ff0cab20fd31ac1215ed7ce0ea92286ba09e2854b42ba5cabe7525"
+checksum = "6a2f165a7feee6f263028b899d0a181987f4fa7179a6411a32a439fba7c5f769"
dependencies = [
"aws-smithy-async",
"aws-smithy-runtime-api",
@@ -767,27 +768,27 @@ dependencies = [
[[package]]
name = "aws-smithy-json"
-version = "0.62.3"
+version = "0.62.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3cb96aa208d62ee94104645f7b2ecaf77bf27edf161590b6224bfbac2832f979"
+checksum = "9648b0bb82a2eedd844052c6ad2a1a822d1f8e3adee5fbf668366717e428856a"
dependencies = [
"aws-smithy-types",
]
[[package]]
name = "aws-smithy-observability"
-version = "0.2.4"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c0a46543fbc94621080b3cf553eb4cbbdc41dd9780a30c4756400f0139440a1d"
+checksum = "a06c2315d173edbf1920da8ba3a7189695827002e4c0fc961973ab1c54abca9c"
dependencies = [
"aws-smithy-runtime-api",
]
[[package]]
name = "aws-smithy-query"
-version = "0.60.13"
+version = "0.60.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "0cebbddb6f3a5bd81553643e9c7daf3cc3dc5b0b5f398ac668630e8a84e6fff0"
+checksum = "1a56d79744fb3edb5d722ef79d86081e121d3b9422cb209eb03aea6aa4f21ebd"
dependencies = [
"aws-smithy-types",
"urlencoding",
@@ -795,9 +796,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime"
-version = "1.10.0"
+version = "1.10.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f3df87c14f0127a0d77eb261c3bc45d5b4833e2a1f63583ebfb728e4852134ee"
+checksum = "028999056d2d2fd58a697232f9eec4a643cf73a71cf327690a7edad1d2af2110"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
@@ -820,9 +821,9 @@ dependencies = [
[[package]]
name = "aws-smithy-runtime-api"
-version = "1.11.3"
+version = "1.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "49952c52f7eebb72ce2a754d3866cc0f87b97d2a46146b79f80f3a93fb2b3716"
+checksum = "876ab3c9c29791ba4ba02b780a3049e21ec63dabda09268b175272c3733a79e6"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
@@ -837,9 +838,9 @@ dependencies = [
[[package]]
name = "aws-smithy-types"
-version = "1.4.3"
+version = "1.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3b3a26048eeab0ddeba4b4f9d51654c79af8c3b32357dc5f336cee85ab331c33"
+checksum = "9d73dbfbaa8e4bc57b9045137680b958d274823509a360abfd8e1d514d40c95c"
dependencies = [
"base64-simd",
"bytes",
@@ -860,18 +861,18 @@ dependencies = [
[[package]]
name = "aws-smithy-xml"
-version = "0.60.13"
+version = "0.60.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "11b2f670422ff42bf7065031e72b45bc52a3508bd089f743ea90731ca2b6ea57"
+checksum = "0ce02add1aa3677d022f8adf81dcbe3046a95f17a1b1e8979c145cd21d3d22b3"
dependencies = [
"xmlparser",
]
[[package]]
name = "aws-types"
-version = "1.3.11"
+version = "1.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1d980627d2dd7bfc32a3c025685a033eeab8d365cc840c631ef59d1b8f428164"
+checksum = "47c8323699dd9b3c8d5b3c13051ae9cdef58fd179957c882f8374dd8725962d9"
dependencies = [
"aws-credential-types",
"aws-smithy-async",
@@ -935,7 +936,7 @@ dependencies = [
[[package]]
name = "ballista"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"async-trait",
"ballista-core",
@@ -955,7 +956,7 @@ dependencies = [
[[package]]
name = "ballista-benchmarks"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"ballista",
"ballista-core",
@@ -965,7 +966,7 @@ dependencies = [
"env_logger",
"futures",
"mimalloc",
- "rand",
+ "rand 0.9.2",
"serde",
"serde_json",
"structopt",
@@ -975,7 +976,7 @@ dependencies = [
[[package]]
name = "ballista-cli"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"ballista",
"clap 4.6.0",
@@ -990,7 +991,7 @@ dependencies = [
[[package]]
name = "ballista-core"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"arrow-flight",
"async-trait",
@@ -1010,7 +1011,7 @@ dependencies = [
"parking_lot",
"prost",
"prost-types",
- "rand",
+ "rand 0.9.2",
"rustc_version",
"serde",
"tempfile",
@@ -1026,7 +1027,7 @@ dependencies = [
[[package]]
name = "ballista-examples"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"arrow-flight",
"ballista",
@@ -1052,7 +1053,7 @@ dependencies = [
[[package]]
name = "ballista-executor"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"arrow",
"arrow-flight",
@@ -1080,7 +1081,7 @@ dependencies = [
[[package]]
name = "ballista-scheduler"
-version = "52.0.0"
+version = "53.0.0"
dependencies = [
"arrow-flight",
"async-trait",
@@ -1102,7 +1103,7 @@ dependencies = [
"prometheus",
"prost",
"prost-types",
- "rand",
+ "rand 0.9.2",
"rstest",
"serde",
"tokio",
@@ -1184,7 +1185,7 @@ dependencies = [
"cc",
"cfg-if",
"constant_time_eq",
- "cpufeatures",
+ "cpufeatures 0.2.17",
]
[[package]]
@@ -1198,9 +1199,9 @@ dependencies = [
[[package]]
name = "bollard"
-version = "0.20.1"
+version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "227aa051deec8d16bd9c34605e7aaf153f240e35483dd42f6f78903847934738"
+checksum = "ee04c4c84f1f811b017f2fbb7dd8815c976e7ca98593de9c1e2afad0f636bff4"
dependencies = [
"async-stream",
"base64 0.22.1",
@@ -1222,7 +1223,7 @@ dependencies = [
"log",
"num",
"pin-project-lite",
- "rand",
+ "rand 0.9.2",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
@@ -1272,9 +1273,9 @@ dependencies = [
[[package]]
name = "bon"
-version = "3.9.0"
+version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83"
+checksum = "f47dbe92550676ee653353c310dfb9cf6ba17ee70396e1f7cf0a2020ad49b2fe"
dependencies = [
"bon-macros",
"rustversion",
@@ -1282,11 +1283,11 @@ dependencies = [
[[package]]
name = "bon-macros"
-version = "3.9.0"
+version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0"
+checksum = "519bd3116aeeb42d5372c29d982d16d0170d3d4a5ed85fc7dd91642ffff3c67c"
dependencies = [
- "darling 0.23.0",
+ "darling",
"ident_case",
"prettyplease",
"proc-macro2",
@@ -1361,9 +1362,9 @@ checksum =
"37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.2.56"
+version = "1.2.57"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
dependencies = [
"find-msvc-tools",
"jobserver",
@@ -1383,6 +1384,17 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
+[[package]]
+name = "chacha20"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601"
+dependencies = [
+ "cfg-if",
+ "cpufeatures 0.3.0",
+ "rand_core 0.10.0",
+]
+
[[package]]
name = "chrono"
version = "0.4.44"
@@ -1479,9 +1491,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "1.0.0"
+version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]]
name = "clipboard-win"
@@ -1503,9 +1515,9 @@ dependencies = [
[[package]]
name = "colorchoice"
-version = "1.0.4"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]]
name = "comfy-table"
@@ -1601,6 +1613,15 @@ dependencies = [
"libc",
]
+[[package]]
+name = "cpufeatures"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+dependencies = [
+ "libc",
+]
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -1732,38 +1753,14 @@ version = "0.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1"
-[[package]]
-name = "darling"
-version = "0.21.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0"
-dependencies = [
- "darling_core 0.21.3",
- "darling_macro 0.21.3",
-]
-
[[package]]
name = "darling"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d"
dependencies = [
- "darling_core 0.23.0",
- "darling_macro 0.23.0",
-]
-
-[[package]]
-name = "darling_core"
-version = "0.21.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4"
-dependencies = [
- "fnv",
- "ident_case",
- "proc-macro2",
- "quote",
- "strsim",
- "syn 2.0.117",
+ "darling_core",
+ "darling_macro",
]
[[package]]
@@ -1779,24 +1776,13 @@ dependencies = [
"syn 2.0.117",
]
-[[package]]
-name = "darling_macro"
-version = "0.21.3"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81"
-dependencies = [
- "darling_core 0.21.3",
- "quote",
- "syn 2.0.117",
-]
-
[[package]]
name = "darling_macro"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d"
dependencies = [
- "darling_core 0.23.0",
+ "darling_core",
"quote",
"syn 2.0.117",
]
@@ -1817,9 +1803,9 @@ dependencies = [
[[package]]
name = "datafusion"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ea28305c211e3541c9cfcf06a23d0d8c7c824b4502ed1fdf0a6ff4ad24ee531c"
+checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16"
dependencies = [
"arrow",
"arrow-schema",
@@ -1861,7 +1847,7 @@ dependencies = [
"object_store",
"parking_lot",
"parquet",
- "rand",
+ "rand 0.9.2",
"regex",
"sqlparser",
"tempfile",
@@ -1873,9 +1859,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "78ab99b6df5f60a6ddbc515e4c05caee1192d395cf3cb67ce5d1c17e3c9b9b74"
+checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137"
dependencies = [
"arrow",
"async-trait",
@@ -1898,9 +1884,9 @@ dependencies = [
[[package]]
name = "datafusion-catalog-listing"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "77ae3d14912c0d779ada98d30dc60f3244f3c26c2446b87394629ea5c076a31c"
+checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6"
dependencies = [
"arrow",
"async-trait",
@@ -1921,9 +1907,9 @@ dependencies = [
[[package]]
name = "datafusion-cli"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d6cc57c2a8889e722be7913bb3c053c554f23abafa2e99005ad6fe84c765f7ce"
+checksum = "8735220c84a731c3917dce75ec837a8376eddf5462b0c5dbaf5a2e354c9b6e05"
dependencies = [
"arrow",
"async-trait",
@@ -1949,9 +1935,9 @@ dependencies = [
[[package]]
name = "datafusion-common"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd"
+checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293"
dependencies = [
"ahash",
"apache-avro",
@@ -1962,6 +1948,7 @@ dependencies = [
"hashbrown 0.16.1",
"hex",
"indexmap 2.13.0",
+ "itertools 0.14.0",
"libc",
"log",
"object_store",
@@ -1975,9 +1962,9 @@ dependencies = [
[[package]]
name = "datafusion-common-runtime"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "42639baa0049d5fffd7e283504b9b5e7b9b2e7a2dea476eed60ab0d40d999b85"
+checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd"
dependencies = [
"futures",
"log",
@@ -1986,9 +1973,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "25951b617bb22a9619e1520450590cb2004bfcad10bcb396b961f4a1a10dcec5"
+checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79"
dependencies = [
"arrow",
"async-compression",
@@ -2012,7 +1999,7 @@ dependencies = [
"liblzma",
"log",
"object_store",
- "rand",
+ "rand 0.9.2",
"tokio",
"tokio-util",
"url",
@@ -2021,9 +2008,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-arrow"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dc0b28226960ba99c50d78ac6f736ebe09eb5cb3bb9bb58194266278000ca41f"
+checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15"
dependencies = [
"arrow",
"arrow-ipc",
@@ -2045,9 +2032,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-avro"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "18de2e914c2c9ed4b31a4920940b181b0957bc164eec4fc04c294533219bf0a7"
+checksum = "49dda81c79b6ba57b1853a9158abc66eb85a3aa1cede0c517dabec6d8a4ed3aa"
dependencies = [
"apache-avro",
"arrow",
@@ -2065,9 +2052,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-csv"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f538b57b052a678b1ce860181c65d3ace5a8486312dc50b41c01dd585a773a51"
+checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c"
dependencies = [
"arrow",
"async-trait",
@@ -2088,9 +2075,9 @@ dependencies = [
[[package]]
name = "datafusion-datasource-json"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "89fbc1d32b1b03c9734e27c0c5f041232b68621c8455f22769838634750a196c"
+checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8"
dependencies = [
"arrow",
"async-trait",
@@ -2105,14 +2092,16 @@ dependencies = [
"datafusion-session",
"futures",
"object_store",
+ "serde_json",
"tokio",
+ "tokio-stream",
]
[[package]]
name = "datafusion-datasource-parquet"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "203271d31fe5613a5943181db70ec98162121d1de94a9a300d5e5f19f9500a32"
+checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946"
dependencies = [
"arrow",
"async-trait",
@@ -2140,37 +2129,39 @@ dependencies = [
[[package]]
name = "datafusion-doc"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5b6450dc702b3d39e8ced54c3356abb453bd2f3cea86d90d555a4b92f7a38462"
+checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a"
[[package]]
name = "datafusion-execution"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e66a02fa601de49da5181dbdcf904a18b16a184db2b31f5e5534552ea2d5e660"
+checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df"
dependencies = [
"arrow",
+ "arrow-buffer",
"async-trait",
"chrono",
"dashmap",
"datafusion-common",
"datafusion-expr",
+ "datafusion-physical-expr-common",
"futures",
"log",
"object_store",
"parking_lot",
"parquet",
- "rand",
+ "rand 0.9.2",
"tempfile",
"url",
]
[[package]]
name = "datafusion-expr"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "cdf59a9b308a1a07dc2eb2f85e6366bc0226dc390b40f3aa0a72d79f1cfe2465"
+checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf"
dependencies = [
"arrow",
"async-trait",
@@ -2191,9 +2182,9 @@ dependencies = [
[[package]]
name = "datafusion-expr-common"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bd99eac4c6538c708638db43e7a3bd88e0e57955ddb722d420fb9a6d38dfc28f"
+checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9"
dependencies = [
"arrow",
"datafusion-common",
@@ -2204,9 +2195,9 @@ dependencies = [
[[package]]
name = "datafusion-functions"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "11aa2c492ac046397b36d57c62a72982aad306495bbcbcdbcabd424d4a2fe245"
+checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e"
dependencies = [
"arrow",
"arrow-buffer",
@@ -2225,8 +2216,9 @@ dependencies = [
"itertools 0.14.0",
"log",
"md-5",
+ "memchr",
"num-traits",
- "rand",
+ "rand 0.9.2",
"regex",
"sha2",
"unicode-segmentation",
@@ -2235,9 +2227,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-aggregate"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787"
+checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6"
dependencies = [
"ahash",
"arrow",
@@ -2251,14 +2243,15 @@ dependencies = [
"datafusion-physical-expr-common",
"half",
"log",
+ "num-traits",
"paste",
]
[[package]]
name = "datafusion-functions-aggregate-common"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae"
+checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311"
dependencies = [
"ahash",
"arrow",
@@ -2269,9 +2262,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-nested"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "29ebaa5d7024ef45973e0a7db1e9aeaa647936496f4d4061c0448f23d77d6320"
+checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790"
dependencies = [
"arrow",
"arrow-ord",
@@ -2285,16 +2278,18 @@ dependencies = [
"datafusion-functions-aggregate-common",
"datafusion-macros",
"datafusion-physical-expr-common",
+ "hashbrown 0.16.1",
"itertools 0.14.0",
+ "itoa",
"log",
"paste",
]
[[package]]
name = "datafusion-functions-table"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "60eab6f39df9ee49a2c7fa38eddc01fa0086ee31b29c7d19f38e72f479609752"
+checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6"
dependencies = [
"arrow",
"async-trait",
@@ -2308,9 +2303,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "e00b2c15e342a90e65a846199c9e49293dd09fe1bcd63d8be2544604892f7eb8"
+checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1"
dependencies = [
"arrow",
"datafusion-common",
@@ -2326,9 +2321,9 @@ dependencies = [
[[package]]
name = "datafusion-functions-window-common"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "493e2e1d1f4753dfc139a5213f1b5d0b97eea46a82d9bda3c7908aa96981b74b"
+checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9"
dependencies = [
"datafusion-common",
"datafusion-physical-expr-common",
@@ -2336,9 +2331,9 @@ dependencies = [
[[package]]
name = "datafusion-macros"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "ba01c55ade8278a791b429f7bf5cb1de64de587a342d084b18245edfae7096e2"
+checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578"
dependencies = [
"datafusion-doc",
"quote",
@@ -2347,9 +2342,9 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a80c6dfbba6a2163a9507f6353ac78c69d8deb26232c9e419160e58ff7c3e047"
+checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46"
dependencies = [
"arrow",
"chrono",
@@ -2367,9 +2362,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6"
+checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36"
dependencies = [
"ahash",
"arrow",
@@ -2391,9 +2386,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-adapter"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3f5e00e524ac33500be6c5eeac940bd3f6b984ba9b7df0cd5f6c34a8a2cc4d6b"
+checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64"
dependencies = [
"arrow",
"datafusion-common",
@@ -2406,9 +2401,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr-common"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce"
+checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c"
dependencies = [
"ahash",
"arrow",
@@ -2423,9 +2418,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-optimizer"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f3588753ab2b47b0e43cd823fe5e7944df6734dabd6dafb72e2cc1c2a22f1944"
+checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941"
dependencies = [
"arrow",
"datafusion-common",
@@ -2442,9 +2437,9 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61"
+checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d"
dependencies = [
"ahash",
"arrow",
@@ -2466,6 +2461,7 @@ dependencies = [
"indexmap 2.13.0",
"itertools 0.14.0",
"log",
+ "num-traits",
"parking_lot",
"pin-project-lite",
"tokio",
@@ -2473,9 +2469,9 @@ dependencies = [
[[package]]
name = "datafusion-proto"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "51eb50848366adf7ff2734e767442916bb45b7e739d5edbf33e9e4c47c5fa797"
+checksum = "677ee4448a010ed5faeff8d73ff78972c2ace59eff3cd7bd15833a1dafa00492"
dependencies = [
"arrow",
"chrono",
@@ -2496,13 +2492,14 @@ dependencies = [
"datafusion-proto-common",
"object_store",
"prost",
+ "rand 0.9.2",
]
[[package]]
name = "datafusion-proto-common"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b16bf2f2a9f99c48b61537870736f0aa9500c4128407c9999261ff3e8a170523"
+checksum = "965eca01edc8259edbbd95883a00b6d81e329fd44a019cfac3a03b026a83eade"
dependencies = [
"arrow",
"datafusion-common",
@@ -2511,9 +2508,9 @@ dependencies = [
[[package]]
name = "datafusion-pruning"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6434e2ee8a39d04b95fed688ff34dc251af6e4a0c2e1714716b6e3846690d589"
+checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6"
dependencies = [
"arrow",
"datafusion-common",
@@ -2528,9 +2525,9 @@ dependencies = [
[[package]]
name = "datafusion-session"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c91efb8302b4877d499c37e9a71886b90236ab27d9cc42fd51112febf341abd6"
+checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba"
dependencies = [
"async-trait",
"datafusion-common",
@@ -2542,38 +2539,43 @@ dependencies = [
[[package]]
name = "datafusion-spark"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "25f2e5519037772210eee5bb87a95dc953e1bd94bc2f9c9d6bb14b0c7fb9ab0a"
+checksum = "923a8b871962a9d860f036f743a20af50ff04729f1da2468ed220dab4f61c97d"
dependencies = [
"arrow",
"bigdecimal",
"chrono",
"crc32fast",
+ "datafusion",
"datafusion-catalog",
"datafusion-common",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
+ "datafusion-functions-aggregate",
"datafusion-functions-nested",
"log",
"percent-encoding",
- "rand",
+ "rand 0.9.2",
+ "serde_json",
"sha1",
+ "sha2",
"url",
]
[[package]]
name = "datafusion-sql"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "3f01eef7bcf4d00e87305b55f1b75792384e130fe0258bac02cd48378ae5ff87"
+checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81"
dependencies = [
"arrow",
"bigdecimal",
"chrono",
"datafusion-common",
"datafusion-expr",
+ "datafusion-functions-nested",
"indexmap 2.13.0",
"log",
"recursive",
@@ -2583,9 +2585,9 @@ dependencies = [
[[package]]
name = "datafusion-substrait"
-version = "52.3.0"
+version = "53.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fe00df31ca03a167d3e40054120930fe5fb689e66bc625b602fac7153b222aea"
+checksum = "d5e5656a7e63d51dd3e5af3dbd347ea83bbe993a77c66b854b74961570d16490"
dependencies = [
"async-recursion",
"async-trait",
@@ -2640,7 +2642,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2727,9 +2729,9 @@ checksum =
"c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d"
[[package]]
name = "env_filter"
-version = "1.0.0"
+version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f"
+checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef"
dependencies = [
"log",
"regex",
@@ -2761,7 +2763,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -2804,7 +2806,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "bb330bbd4cb7a5b9f559427f06f98a4f853a137c8298f3bd3f8ca57663e21986"
dependencies = [
"portable-atomic",
- "rand",
+ "rand 0.9.2",
"web-time",
]
@@ -3025,6 +3027,7 @@ dependencies = [
"cfg-if",
"libc",
"r-efi 6.0.0",
+ "rand_core 0.10.0",
"wasip2",
"wasip3",
]
@@ -3047,7 +3050,7 @@ dependencies = [
"into-attr-derive",
"pest",
"pest_derive",
- "rand",
+ "rand 0.9.2",
"tempfile",
]
@@ -3541,9 +3544,9 @@ checksum =
"d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "iri-string"
-version = "0.7.10"
+version = "0.7.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a"
+checksum = "d8e7418f59cc01c88316161279a7f665217ae316b388e58a0d10e29f54f1e5eb"
dependencies = [
"memchr",
"serde",
@@ -3575,9 +3578,9 @@ dependencies = [
[[package]]
name = "itoa"
-version = "1.0.17"
+version = "1.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
[[package]]
name = "jiff"
@@ -3742,9 +3745,9 @@ dependencies = [
[[package]]
name = "libredox"
-version = "0.1.14"
+version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a"
+checksum = "7ddbf48fd451246b1f8c2610bd3b4ac0cc6e149d89832867093ab69a17194f08"
dependencies = [
"bitflags 2.11.0",
"libc",
@@ -3793,9 +3796,9 @@ checksum =
"112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "lz4_flex"
-version = "0.12.1"
+version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746"
+checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a"
dependencies = [
"twox-hash",
]
@@ -3900,7 +3903,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3995,16 +3998,18 @@ dependencies = [
[[package]]
name = "object_store"
-version = "0.12.5"
+version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00"
+checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"chrono",
"form_urlencoded",
- "futures",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
"http 1.4.0",
"http-body-util",
"humantime",
@@ -4014,10 +4019,10 @@ dependencies = [
"parking_lot",
"percent-encoding",
"quick-xml",
- "rand",
+ "rand 0.10.0",
"reqwest",
"ring",
- "rustls-pemfile",
+ "rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
@@ -4110,14 +4115,13 @@ dependencies = [
[[package]]
name = "parquet"
-version = "57.3.0"
+version = "58.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb"
+checksum = "7d3f9f2205199603564127932b89695f52b62322f541d0fc7179d57c2e1c9877"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
- "arrow-cast",
"arrow-data",
"arrow-ipc",
"arrow-schema",
@@ -4373,9 +4377,9 @@ checksum =
"c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
[[package]]
name = "portable-atomic-util"
-version = "0.2.5"
+version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5"
+checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3"
dependencies = [
"portable-atomic",
]
@@ -4580,9 +4584,9 @@ dependencies = [
[[package]]
name = "pulldown-cmark"
-version = "0.13.1"
+version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "83c41efbf8f90ac44de7f3a868f0867851d261b56291732d0cbf7cceaaeb55a6"
+checksum = "7c3a14896dfa883796f1cb410461aef38810ea05f2b2c33c5aded3649095fdad"
dependencies = [
"bitflags 2.11.0",
"memchr",
@@ -4606,9 +4610,9 @@ checksum =
"5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40"
[[package]]
name = "quick-xml"
-version = "0.38.4"
+version = "0.39.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c"
+checksum = "958f21e8e7ceb5a1aa7fa87fab28e7c75976e0bfe7e23ff069e0a260f894067d"
dependencies = [
"memchr",
"serde",
@@ -4643,7 +4647,7 @@ dependencies = [
"bytes",
"getrandom 0.3.4",
"lru-slab",
- "rand",
+ "rand 0.9.2",
"ring",
"rustc-hash",
"rustls",
@@ -4666,7 +4670,7 @@ dependencies = [
"once_cell",
"socket2",
"tracing",
- "windows-sys 0.52.0",
+ "windows-sys 0.60.2",
]
[[package]]
@@ -4707,7 +4711,18 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
dependencies = [
"rand_chacha",
- "rand_core",
+ "rand_core 0.9.5",
+]
+
+[[package]]
+name = "rand"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8"
+dependencies = [
+ "chacha20",
+ "getrandom 0.4.2",
+ "rand_core 0.10.0",
]
[[package]]
@@ -4717,7 +4732,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.9.5",
]
[[package]]
@@ -4729,6 +4744,12 @@ dependencies = [
"getrandom 0.3.4",
]
+[[package]]
+name = "rand_core"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index";
+checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba"
+
[[package]]
name = "rayon"
version = "1.11.0"
@@ -4992,7 +5013,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys 0.12.1",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -5023,15 +5044,6 @@ dependencies = [
"security-framework",
]
-[[package]]
-name = "rustls-pemfile"
-version = "2.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50"
-dependencies = [
- "rustls-pki-types",
-]
-
[[package]]
name = "rustls-pki-types"
version = "1.14.0"
@@ -5311,9 +5323,9 @@ dependencies = [
[[package]]
name = "serde_with"
-version = "3.17.0"
+version = "3.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9"
+checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f"
dependencies = [
"base64 0.22.1",
"chrono",
@@ -5330,11 +5342,11 @@ dependencies = [
[[package]]
name = "serde_with_macros"
-version = "3.17.0"
+version = "3.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0"
+checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65"
dependencies = [
- "darling 0.21.3",
+ "darling",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -5360,7 +5372,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba"
dependencies = [
"cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
"digest",
]
@@ -5371,7 +5383,7 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
dependencies = [
"cfg-if",
- "cpufeatures",
+ "cpufeatures 0.2.17",
"digest",
]
@@ -5454,9 +5466,9 @@ dependencies = [
[[package]]
name = "sqlparser"
-version = "0.59.0"
+version = "0.61.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f"
+checksum = "dbf5ea8d4d7c808e1af1cbabebca9a2abe603bcefc22294c5b95018d53200cb7"
dependencies = [
"log",
"recursive",
@@ -5465,9 +5477,9 @@ dependencies = [
[[package]]
name = "sqlparser_derive"
-version = "0.3.0"
+version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c"
+checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289"
dependencies = [
"proc-macro2",
"quote",
@@ -5647,14 +5659,14 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix 1.1.4",
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
name = "testcontainers"
-version = "0.27.1"
+version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "c1c0624faaa317c56d6d19136580be889677259caf5c897941c6f446b4655068"
+checksum = "0bd36b06a2a6c0c3c81a83be1ab05fe86460d054d4d51bf513bc56b3e15bdc22"
dependencies = [
"astral-tokio-tar",
"async-trait",
@@ -5821,9 +5833,9 @@ dependencies = [
[[package]]
name = "tinyvec"
-version = "1.10.0"
+version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
+checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3"
dependencies = [
"tinyvec_macros",
]
@@ -5881,6 +5893,7 @@ dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
+ "tokio-util",
]
[[package]]
@@ -5898,18 +5911,18 @@ dependencies = [
[[package]]
name = "toml_datetime"
-version = "1.0.0+spec-1.1.0"
+version = "1.1.0+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e"
+checksum = "97251a7c317e03ad83774a8752a7e81fb6067740609f75ea2b585b569a59198f"
dependencies = [
"serde_core",
]
[[package]]
name = "toml_edit"
-version = "0.25.4+spec-1.1.0"
+version = "0.25.8+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2"
+checksum = "16bff38f1d86c47f9ff0647e6838d7bb362522bdf44006c7068c2b1e606f1f3c"
dependencies = [
"indexmap 2.13.0",
"toml_datetime",
@@ -5919,9 +5932,9 @@ dependencies = [
[[package]]
name = "toml_parser"
-version = "1.0.9+spec-1.1.0"
+version = "1.1.0+spec-1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
+checksum = "2334f11ee363607eb04df9b8fc8a13ca1715a72ba8662a26ac285c98aabb4011"
dependencies = [
"winnow",
]
@@ -6203,9 +6216,9 @@ checksum =
"e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
[[package]]
name = "unicode-segmentation"
-version = "1.12.0"
+version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+checksum = "a559e63b5d8004e12f9bce88af5c6d939c58de839b7532cfe9653846cedd2a9e"
[[package]]
name = "unicode-width"
@@ -6239,9 +6252,9 @@ checksum =
"8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
-version = "3.2.0"
+version = "3.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "fdc97a28575b85cfedf2a7e7d3cc64b3e11bd8ac766666318003abbacc7a21fc"
+checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0"
dependencies = [
"base64 0.22.1",
"log",
@@ -6249,14 +6262,14 @@ dependencies = [
"rustls",
"rustls-pki-types",
"ureq-proto",
- "utf-8",
+ "utf8-zero",
]
[[package]]
name = "ureq-proto"
-version = "0.5.3"
+version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "d81f9efa9df032be5934a46a068815a10a042b494b6a58cb0a1a97bb5467ed6f"
+checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c"
dependencies = [
"base64 0.22.1",
"http 1.4.0",
@@ -6284,10 +6297,10 @@ source =
"registry+https://github.com/rust-lang/crates.io-index";
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
-name = "utf-8"
-version = "0.7.6"
+name = "utf8-zero"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
[[package]]
name = "utf8_iter"
@@ -6522,7 +6535,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index";
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.59.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -6757,9 +6770,9 @@ checksum =
"d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
[[package]]
name = "winnow"
-version = "0.7.15"
+version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
+checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8"
dependencies = [
"memchr",
]
@@ -6899,18 +6912,18 @@ dependencies = [
[[package]]
name = "zerocopy"
-version = "0.8.42"
+version = "0.8.47"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
+checksum = "efbb2a062be311f2ba113ce66f697a4dc589f85e78a4aea276200804cea0ed87"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
-version = "0.8.42"
+version = "0.8.47"
source = "registry+https://github.com/rust-lang/crates.io-index";
-checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
+checksum = "0e8bc7269b54418e7aeeef514aa68f8690b8c0489a06b0136e5f57c4c5ccab89"
dependencies = [
"proc-macro2",
"quote",
diff --git a/Cargo.toml b/Cargo.toml
index dee6984b4..4b2deab45 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -26,17 +26,19 @@ edition = "2024"
rust-version = "1.88.0"
[workspace.dependencies]
-arrow = { version = "57", features = ["ipc_compression"] }
-arrow-flight = { version = "57", features = ["flight-sql-experimental"] }
-clap = { version = "4.6", features = ["derive", "cargo"] }
-datafusion = "52"
-datafusion-cli = "52"
-datafusion-proto = "52"
-datafusion-proto-common = "52"
-datafusion-spark = "52"
-datafusion-substrait = "52"
+arrow = { version = "58", features = ["ipc_compression"] }
+arrow-flight = { version = "58", features = ["flight-sql-experimental"] }
+clap = { version = "4.5", features = ["derive", "cargo"] }
+
+datafusion = "53"
+datafusion-cli = "53"
+datafusion-proto = "53"
+datafusion-proto-common = "53"
+datafusion-spark = "53"
+datafusion-substrait = "53"
+
insta = "1.46"
-object_store = "0.12"
+object_store = "0.13"
prost = "0.14"
prost-types = "0.14"
rstest = { version = "0.26" }
diff --git a/ballista-cli/Cargo.toml b/ballista-cli/Cargo.toml
index 31e6ed5e3..c241a842d 100644
--- a/ballista-cli/Cargo.toml
+++ b/ballista-cli/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-cli"
description = "Command Line Client for Ballista distributed query engine."
-version = "52.0.0"
+version = "53.0.0"
authors = ["Apache DataFusion <[email protected]>"]
edition = { workspace = true }
rust-version = { workspace = true }
@@ -29,7 +29,7 @@ repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
[dependencies]
-ballista = { path = "../ballista/client", version = "52.0.0", features =
["standalone"] }
+ballista = { path = "../ballista/client", version = "53.0.0", features =
["standalone"] }
clap = { workspace = true, features = ["derive", "cargo"] }
datafusion = { workspace = true }
datafusion-cli = { workspace = true }
diff --git a/ballista/client/Cargo.toml b/ballista/client/Cargo.toml
index 22245505b..bdb1022d5 100644
--- a/ballista/client/Cargo.toml
+++ b/ballista/client/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "52.0.0"
+version = "53.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -29,9 +29,9 @@ rust-version = { workspace = true }
[dependencies]
async-trait = { workspace = true }
-ballista-core = { path = "../core", version = "52.0.0" }
-ballista-executor = { path = "../executor", version = "52.0.0", optional =
true }
-ballista-scheduler = { path = "../scheduler", version = "52.0.0", optional =
true }
+ballista-core = { path = "../core", version = "53.0.0" }
+ballista-executor = { path = "../executor", version = "53.0.0", optional =
true }
+ballista-scheduler = { path = "../scheduler", version = "53.0.0", optional =
true }
datafusion = { workspace = true }
log = { workspace = true }
@@ -39,8 +39,8 @@ tokio = { workspace = true }
url = { workspace = true }
[dev-dependencies]
-ballista-executor = { path = "../executor", version = "52.0.0" }
-ballista-scheduler = { path = "../scheduler", version = "52.0.0" }
+ballista-executor = { path = "../executor", version = "53.0.0" }
+ballista-scheduler = { path = "../scheduler", version = "53.0.0" }
ctor = { workspace = true }
datafusion-proto = { workspace = true }
env_logger = { workspace = true }
diff --git a/ballista/client/tests/context_checks.rs
b/ballista/client/tests/context_checks.rs
index 0faf3b9b7..2afa4b9f8 100644
--- a/ballista/client/tests/context_checks.rs
+++ b/ballista/client/tests/context_checks.rs
@@ -1105,4 +1105,42 @@ mod supported {
Ok(())
}
+
+ #[rstest]
+ #[case::standalone(standalone_context())]
+ #[case::remote(remote_context())]
+ #[tokio::test]
+
+ async fn should_execute_sql_collect_from_arrow_file(
+ #[future(awt)]
+ #[case]
+ ctx: SessionContext,
+ test_data: String,
+ ) -> datafusion::error::Result<()> {
+ ctx.register_arrow(
+ "test",
+ &format!("{test_data}/alltypes_plain.arrow"),
+ Default::default(),
+ )
+ .await?;
+
+ let result = ctx
+ .sql("select string_col, timestamp_col from test where id > 4")
+ .await?
+ .collect()
+ .await?;
+ let expected = [
+ "+------------+---------------------+",
+ "| string_col | timestamp_col |",
+ "+------------+---------------------+",
+ "| 31 | 2009-03-01T00:01:00 |",
+ "| 30 | 2009-04-01T00:00:00 |",
+ "| 31 | 2009-04-01T00:01:00 |",
+ "+------------+---------------------+",
+ ];
+
+ assert_batches_eq!(expected, &result);
+
+ Ok(())
+ }
}
diff --git a/ballista/client/tests/context_unsupported.rs
b/ballista/client/tests/context_unsupported.rs
index 3c6b25a98..6e038034d 100644
--- a/ballista/client/tests/context_unsupported.rs
+++ b/ballista/client/tests/context_unsupported.rs
@@ -132,51 +132,4 @@ mod unsupported {
Ok(())
}
-
- #[rstest]
- #[case::standalone(standalone_context())]
- #[case::remote(remote_context())]
- #[tokio::test]
-
- async fn should_execute_sql_collect_from_arrow_file(
- #[future(awt)]
- #[case]
- ctx: SessionContext,
- test_data: String,
- ) -> datafusion::error::Result<()> {
- ctx.register_arrow(
- "test",
- &format!("{test_data}/alltypes_plain.arrow"),
- Default::default(),
- )
- .await?;
-
- let result = ctx
- .sql("select string_col, timestamp_col from test where id > 4")
- .await?
- .collect()
- .await;
-
- // at the moment its not supported
- assert!(result.is_err());
-
- // let result = ctx
- // .sql("select string_col, timestamp_col from test where id > 4")
- // .await?
- // .collect()
- // .await?;
- // let expected = [
- // "+------------+---------------------+",
- // "| string_col | timestamp_col |",
- // "+------------+---------------------+",
- // "| 31 | 2009-03-01T00:01:00 |",
- // "| 30 | 2009-04-01T00:00:00 |",
- // "| 31 | 2009-04-01T00:01:00 |",
- // "+------------+---------------------+",
- // ];
-
- // assert_batches_eq!(expected, &result);
-
- Ok(())
- }
}
diff --git a/ballista/core/Cargo.toml b/ballista/core/Cargo.toml
index 21353adff..91e0f4e50 100644
--- a/ballista/core/Cargo.toml
+++ b/ballista/core/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-core"
description = "Ballista Distributed Compute"
license = "Apache-2.0"
-version = "52.0.0"
+version = "53.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -53,7 +53,7 @@ clap = { workspace = true, optional = true }
datafusion = { workspace = true }
datafusion-proto = { workspace = true }
datafusion-proto-common = { workspace = true }
-datafusion-spark = { workspace = true, optional = true }
+datafusion-spark = { workspace = true, optional = true, features =
["datafusion"] }
futures = { workspace = true }
itertools = "0.14"
log = { workspace = true }
diff --git a/ballista/core/src/diagram.rs b/ballista/core/src/diagram.rs
index e1ec42681..5498c1161 100644
--- a/ballista/core/src/diagram.rs
+++ b/ballista/core/src/diagram.rs
@@ -23,6 +23,7 @@ use crate::execution_plans::{
use datafusion::datasource::source::DataSourceExec;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::physical_plan::aggregates::AggregateExec;
+#[allow(deprecated)]
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::filter::FilterExec;
@@ -75,6 +76,7 @@ pub fn produce_diagram(filename: &str, stages: &[Arc<dyn
ShuffleWriter>]) -> Res
Ok(())
}
+#[allow(deprecated)]
fn build_exec_plan_diagram(
w: &mut BufWriter<&File>,
plan: &dyn ExecutionPlan,
diff --git a/ballista/core/src/execution_plans/distributed_query.rs
b/ballista/core/src/execution_plans/distributed_query.rs
index 533ee3541..df8f765d0 100644
--- a/ballista/core/src/execution_plans/distributed_query.rs
+++ b/ballista/core/src/execution_plans/distributed_query.rs
@@ -39,7 +39,7 @@ use datafusion::physical_plan::metrics::{
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
- SendableRecordBatchStream, Statistics,
+ SendableRecordBatchStream,
};
use datafusion::prelude::SessionConfig;
use datafusion_proto::logical_plan::{
@@ -73,7 +73,7 @@ pub struct DistributedQueryExec<T: 'static + AsLogicalPlan> {
/// Session id
session_id: String,
/// Plan properties
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
/// Execution metrics, currently exposes:
/// - output_rows: Total number of rows returned
/// - transferred_bytes: Total bytes transferred from executors
@@ -92,8 +92,9 @@ impl<T: 'static + AsLogicalPlan> DistributedQueryExec<T> {
plan: LogicalPlan,
session_id: String,
) -> Self {
- let properties =
- Self::compute_properties(plan.schema().as_arrow().clone().into());
+ let properties = Arc::new(Self::compute_properties(
+ plan.schema().as_arrow().clone().into(),
+ ));
Self {
scheduler_url,
config,
@@ -114,8 +115,9 @@ impl<T: 'static + AsLogicalPlan> DistributedQueryExec<T> {
extension_codec: Arc<dyn LogicalExtensionCodec>,
session_id: String,
) -> Self {
- let properties =
- Self::compute_properties(plan.schema().as_arrow().clone().into());
+ let properties = Arc::new(Self::compute_properties(
+ plan.schema().as_arrow().clone().into(),
+ ));
Self {
scheduler_url,
config,
@@ -172,7 +174,7 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for
DistributedQueryExec<T> {
self.plan.schema().as_arrow().clone().into()
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
@@ -191,9 +193,9 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for
DistributedQueryExec<T> {
extension_codec: self.extension_codec.clone(),
plan_repr: self.plan_repr,
session_id: self.session_id.clone(),
- properties: Self::compute_properties(
+ properties: Arc::new(Self::compute_properties(
self.plan.schema().as_arrow().clone().into(),
- ),
+ )),
metrics: ExecutionPlanMetricsSet::new(),
}))
}
@@ -305,13 +307,6 @@ impl<T: 'static + AsLogicalPlan> ExecutionPlan for
DistributedQueryExec<T> {
}
}
- fn statistics(&self) -> Result<Statistics> {
- // This execution plan sends the logical plan to the scheduler without
- // performing the node by node conversion to a full physical plan.
- // This implies that we cannot infer the statistics at this stage.
- Ok(Statistics::new_unknown(&self.schema()))
- }
-
fn metrics(&self) -> Option<MetricsSet> {
Some(self.metrics.clone_inner())
}
diff --git a/ballista/core/src/execution_plans/shuffle_reader.rs
b/ballista/core/src/execution_plans/shuffle_reader.rs
index bc81b2150..628a22a15 100644
--- a/ballista/core/src/execution_plans/shuffle_reader.rs
+++ b/ballista/core/src/execution_plans/shuffle_reader.rs
@@ -69,7 +69,7 @@ pub struct ShuffleReaderExec {
pub partition: Vec<Vec<PartitionLocation>>,
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
}
impl ShuffleReaderExec {
@@ -80,12 +80,12 @@ impl ShuffleReaderExec {
schema: SchemaRef,
partitioning: Partitioning,
) -> Result<Self> {
- let properties = PlanProperties::new(
+ let properties = Arc::new(PlanProperties::new(
datafusion::physical_expr::EquivalenceProperties::new(schema.clone()),
partitioning,
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Ok(Self {
stage_id,
schema,
@@ -130,7 +130,7 @@ impl ExecutionPlan for ShuffleReaderExec {
self.schema.clone()
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
diff --git a/ballista/core/src/execution_plans/shuffle_writer.rs
b/ballista/core/src/execution_plans/shuffle_writer.rs
index 08ec387c5..8c5149ec1 100644
--- a/ballista/core/src/execution_plans/shuffle_writer.rs
+++ b/ballista/core/src/execution_plans/shuffle_writer.rs
@@ -85,7 +85,7 @@ pub struct ShuffleWriterExec {
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
/// Plan properties
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
}
impl std::fmt::Display for ShuffleWriterExec {
@@ -154,12 +154,12 @@ impl ShuffleWriterExec {
let partitioning = shuffle_output_partitioning
.clone()
.unwrap_or_else(||
plan.properties().output_partitioning().clone());
- let properties = PlanProperties::new(
+ let properties = Arc::new(PlanProperties::new(
datafusion::physical_expr::EquivalenceProperties::new(plan.schema()),
partitioning,
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Ok(Self {
job_id,
stage_id,
@@ -399,7 +399,7 @@ impl ExecutionPlan for ShuffleWriterExec {
self.plan.schema()
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
diff --git a/ballista/core/src/execution_plans/sort_shuffle/writer.rs
b/ballista/core/src/execution_plans/sort_shuffle/writer.rs
index 91fb34742..a24f5d729 100644
--- a/ballista/core/src/execution_plans/sort_shuffle/writer.rs
+++ b/ballista/core/src/execution_plans/sort_shuffle/writer.rs
@@ -83,7 +83,7 @@ pub struct SortShuffleWriterExec {
/// Execution metrics
metrics: ExecutionPlanMetricsSet,
/// Plan properties
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
}
#[derive(Debug, Clone)]
@@ -139,12 +139,12 @@ impl SortShuffleWriterExec {
}
}
- let properties = PlanProperties::new(
+ let properties = Arc::new(PlanProperties::new(
datafusion::physical_expr::EquivalenceProperties::new(plan.schema()),
shuffle_output_partitioning.clone(),
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Ok(Self {
job_id,
@@ -493,7 +493,7 @@ impl ExecutionPlan for SortShuffleWriterExec {
self.plan.schema()
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
diff --git a/ballista/core/src/execution_plans/unresolved_shuffle.rs
b/ballista/core/src/execution_plans/unresolved_shuffle.rs
index 2d7c05c37..a6a8b6623 100644
--- a/ballista/core/src/execution_plans/unresolved_shuffle.rs
+++ b/ballista/core/src/execution_plans/unresolved_shuffle.rs
@@ -23,7 +23,7 @@ use datafusion::error::{DataFusionError, Result};
use datafusion::execution::context::TaskContext;
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
- SendableRecordBatchStream, Statistics,
+ SendableRecordBatchStream,
};
/// UnresolvedShuffleExec represents a dependency on the results of a
ShuffleWriterExec node which hasn't computed yet.
@@ -41,18 +41,18 @@ pub struct UnresolvedShuffleExec {
/// The partition count this node will have once it is replaced with a
ShuffleReaderExec.
pub output_partition_count: usize,
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
}
impl UnresolvedShuffleExec {
/// Create a new UnresolvedShuffleExec
pub fn new(stage_id: usize, schema: SchemaRef, partitioning: Partitioning)
-> Self {
- let properties = PlanProperties::new(
+ let properties = Arc::new(PlanProperties::new(
datafusion::physical_expr::EquivalenceProperties::new(schema.clone()),
partitioning,
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Self {
stage_id,
schema,
@@ -100,7 +100,7 @@ impl ExecutionPlan for UnresolvedShuffleExec {
self.schema.clone()
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
@@ -131,10 +131,4 @@ impl ExecutionPlan for UnresolvedShuffleExec {
"Ballista UnresolvedShuffleExec does not support
execution".to_owned(),
))
}
-
- fn statistics(&self) -> Result<Statistics> {
- // The full statistics are computed in the `ShuffleReaderExec` node
- // that replaces this one once the previous stage is completed.
- Ok(Statistics::new_unknown(&self.schema()))
- }
}
diff --git a/ballista/core/src/serde/mod.rs b/ballista/core/src/serde/mod.rs
index 1a4433e9e..552a55b55 100644
--- a/ballista/core/src/serde/mod.rs
+++ b/ballista/core/src/serde/mod.rs
@@ -31,10 +31,12 @@ use datafusion_proto::logical_plan::file_formats::{
ArrowLogicalExtensionCodec, AvroLogicalExtensionCodec,
CsvLogicalExtensionCodec,
JsonLogicalExtensionCodec, ParquetLogicalExtensionCodec,
};
-use datafusion_proto::physical_plan::DefaultPhysicalExtensionCodec;
use
datafusion_proto::physical_plan::from_proto::parse_protobuf_hash_partitioning;
use datafusion_proto::physical_plan::from_proto::parse_protobuf_partitioning;
use datafusion_proto::physical_plan::to_proto::serialize_partitioning;
+use datafusion_proto::physical_plan::{
+ DefaultPhysicalExtensionCodec, DefaultPhysicalProtoConverter,
+};
use datafusion_proto::protobuf::proto_error;
use datafusion_proto::protobuf::{LogicalPlanNode, PhysicalPlanNode};
use datafusion_proto::{
@@ -336,7 +338,7 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
"Could not deserialize BallistaPhysicalPlanNode because
it's physical_plan_type is none".to_string()
)
})?;
-
+ let converter = DefaultPhysicalProtoConverter {};
match ballista_plan {
PhysicalPlanType::ShuffleWriter(shuffle_writer) => {
let input = inputs[0].clone();
@@ -346,6 +348,7 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
ctx,
input.schema().as_ref(),
self.default_codec.as_ref(),
+ &converter,
)?;
Ok(Arc::new(ShuffleWriterExec::try_new(
@@ -364,6 +367,7 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
ctx,
input.schema().as_ref(),
self.default_codec.as_ref(),
+ &converter,
)?;
let partitioning = shuffle_output_partitioning.ok_or_else(|| {
@@ -420,6 +424,7 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
ctx,
schema.as_ref(),
self.default_codec.as_ref(),
+ &converter,
)?;
let partitioning = partitioning
.ok_or_else(|| proto_error("missing required partitioning
field"))?;
@@ -439,6 +444,7 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
ctx,
schema.as_ref(),
self.default_codec.as_ref(),
+ &converter,
)?;
let partitioning = partitioning
.ok_or_else(|| proto_error("missing required partitioning
field"))?;
@@ -558,9 +564,11 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
.collect::<Result<Vec<_>, _>>()?,
});
}
+ let converter = DefaultPhysicalProtoConverter {};
let partitioning = serialize_partitioning(
&exec.properties().partitioning,
self.default_codec.as_ref(),
+ &converter,
)?;
let proto = protobuf::BallistaPhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::ShuffleReader(
@@ -580,9 +588,11 @@ impl PhysicalExtensionCodec for
BallistaPhysicalExtensionCodec {
Ok(())
} else if let Some(exec) =
node.as_any().downcast_ref::<UnresolvedShuffleExec>() {
+ let converter = DefaultPhysicalProtoConverter {};
let partitioning = serialize_partitioning(
&exec.properties().partitioning,
self.default_codec.as_ref(),
+ &converter,
)?;
let proto = protobuf::BallistaPhysicalPlanNode {
physical_plan_type: Some(PhysicalPlanType::UnresolvedShuffle(
diff --git a/ballista/executor/Cargo.toml b/ballista/executor/Cargo.toml
index 32816188d..a3e6c286f 100644
--- a/ballista/executor/Cargo.toml
+++ b/ballista/executor/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-executor"
description = "Ballista Distributed Compute - Executor"
license = "Apache-2.0"
-version = "52.0.0"
+version = "53.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -42,7 +42,7 @@ spark-compat = ["ballista-core/spark-compat"]
arrow = { workspace = true }
arrow-flight = { workspace = true }
async-trait = { workspace = true }
-ballista-core = { path = "../core", version = "52.0.0" }
+ballista-core = { path = "../core", version = "53.0.0" }
clap = { workspace = true, optional = true }
dashmap = { workspace = true }
datafusion = { workspace = true }
diff --git a/ballista/executor/src/collect.rs b/ballista/executor/src/collect.rs
index cb2dace01..40fae6cc5 100644
--- a/ballista/executor/src/collect.rs
+++ b/ballista/executor/src/collect.rs
@@ -43,18 +43,18 @@ pub struct CollectExec {
/// The underlying execution plan whose partitions will be merged.
plan: Arc<dyn ExecutionPlan>,
/// Properties describing this plan's partitioning and ordering.
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
}
impl CollectExec {
/// Creates a new CollectExec that merges all partitions of the given plan.
pub fn new(plan: Arc<dyn ExecutionPlan>) -> Self {
- let properties = PlanProperties::new(
+ let properties = Arc::new(PlanProperties::new(
datafusion::physical_expr::EquivalenceProperties::new(plan.schema()),
Partitioning::UnknownPartitioning(1),
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Self { plan, properties }
}
}
@@ -87,7 +87,7 @@ impl ExecutionPlan for CollectExec {
self.plan.schema()
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
diff --git a/ballista/executor/src/executor.rs
b/ballista/executor/src/executor.rs
index c0bcdfe95..03a2a0d2a 100644
--- a/ballista/executor/src/executor.rs
+++ b/ballista/executor/src/executor.rs
@@ -241,7 +241,7 @@ mod test {
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning,
PlanProperties,
- RecordBatchStream, SendableRecordBatchStream, Statistics,
+ RecordBatchStream, SendableRecordBatchStream,
};
use datafusion::prelude::SessionContext;
use futures::Stream;
@@ -275,20 +275,20 @@ mod test {
/// An ExecutionPlan which will never terminate
#[derive(Debug)]
pub struct NeverendingOperator {
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
}
impl NeverendingOperator {
fn new() -> Self {
NeverendingOperator {
- properties: PlanProperties::new(
+ properties: Arc::new(PlanProperties::new(
datafusion::physical_expr::EquivalenceProperties::new(Arc::new(
Schema::empty(),
)),
Partitioning::UnknownPartitioning(1),
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- ),
+ )),
}
}
}
@@ -322,7 +322,7 @@ mod test {
Arc::new(Schema::empty())
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
@@ -344,10 +344,6 @@ mod test {
) -> datafusion::common::Result<SendableRecordBatchStream> {
Ok(Box::pin(NeverendingRecordBatchStream))
}
-
- fn statistics(&self) -> Result<Statistics> {
- Ok(Statistics::new_unknown(&self.schema()))
- }
}
#[tokio::test]
diff --git a/ballista/scheduler/Cargo.toml b/ballista/scheduler/Cargo.toml
index 9bf90999e..263bc21da 100644
--- a/ballista/scheduler/Cargo.toml
+++ b/ballista/scheduler/Cargo.toml
@@ -19,7 +19,7 @@
name = "ballista-scheduler"
description = "Ballista Distributed Compute - Scheduler"
license = "Apache-2.0"
-version = "52.0.0"
+version = "53.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
readme = "README.md"
@@ -49,7 +49,7 @@ substrait = ["dep:datafusion-substrait"]
arrow-flight = { workspace = true }
async-trait = { workspace = true }
axum = "0.8.4"
-ballista-core = { path = "../core", version = "52.0.0" }
+ballista-core = { path = "../core", version = "53.0.0" }
clap = { workspace = true, optional = true }
dashmap = { workspace = true }
datafusion = { workspace = true }
diff --git a/ballista/scheduler/src/cluster/mod.rs
b/ballista/scheduler/src/cluster/mod.rs
index 5647ff89a..552faa265 100644
--- a/ballista/scheduler/src/cluster/mod.rs
+++ b/ballista/scheduler/src/cluster/mod.rs
@@ -1107,6 +1107,7 @@ mod test {
extensions: None,
statistics: None,
metadata_size_hint: None,
+ ordering: None,
}]);
}
vec![scan_files]
diff --git a/ballista/scheduler/src/physical_optimizer/join_selection.rs
b/ballista/scheduler/src/physical_optimizer/join_selection.rs
index 7873ac3e2..24a9b5c59 100644
--- a/ballista/scheduler/src/physical_optimizer/join_selection.rs
+++ b/ballista/scheduler/src/physical_optimizer/join_selection.rs
@@ -239,35 +239,30 @@ pub(crate) fn try_collect_left(
match (left_can_collect, right_can_collect) {
(true, true) => {
+ // Don't swap null-aware anti joins as they have specific side
requirements
if hash_join.join_type().supports_swap()
+ && !hash_join.null_aware
&& should_swap_join_order(&**left, &**right)?
{
Ok(Some(hash_join.swap_inputs(PartitionMode::CollectLeft)?))
} else {
- Ok(Some(Arc::new(HashJoinExec::try_new(
- Arc::clone(left),
- Arc::clone(right),
- hash_join.on().to_vec(),
- hash_join.filter().cloned(),
- hash_join.join_type(),
- hash_join.projection.clone(),
- PartitionMode::CollectLeft,
- hash_join.null_equality(),
- )?)))
+ Ok(Some(Arc::new(
+ hash_join
+ .builder()
+ .with_partition_mode(PartitionMode::CollectLeft)
+ .build()?,
+ )))
}
}
- (true, false) => Ok(Some(Arc::new(HashJoinExec::try_new(
- Arc::clone(left),
- Arc::clone(right),
- hash_join.on().to_vec(),
- hash_join.filter().cloned(),
- hash_join.join_type(),
- hash_join.projection.clone(),
- PartitionMode::CollectLeft,
- hash_join.null_equality(),
- )?))),
+ (true, false) => Ok(Some(Arc::new(
+ hash_join
+ .builder()
+ .with_partition_mode(PartitionMode::CollectLeft)
+ .build()?,
+ ))),
(false, true) => {
- if hash_join.join_type().supports_swap() {
+ // Don't swap null-aware anti joins as they have specific side
requirements
+ if hash_join.join_type().supports_swap() && !hash_join.null_aware {
hash_join.swap_inputs(PartitionMode::CollectLeft).map(Some)
} else {
Ok(None)
@@ -276,7 +271,6 @@ pub(crate) fn try_collect_left(
(false, false) => Ok(None),
}
}
-
/// Creates a partitioned hash join execution plan, swapping inputs if
beneficial.
///
/// Checks if the join order should be swapped based on the join type and
input statistics.
@@ -287,20 +281,29 @@ pub(crate) fn partitioned_hash_join(
) -> Result<Arc<dyn ExecutionPlan>> {
let left = hash_join.left();
let right = hash_join.right();
- if hash_join.join_type().supports_swap() &&
should_swap_join_order(&**left, &**right)?
+ // Don't swap null-aware anti joins as they have specific side requirements
+ if hash_join.join_type().supports_swap()
+ && !hash_join.null_aware
+ && should_swap_join_order(&**left, &**right)?
{
hash_join.swap_inputs(PartitionMode::Partitioned)
} else {
- Ok(Arc::new(HashJoinExec::try_new(
- Arc::clone(left),
- Arc::clone(right),
- hash_join.on().to_vec(),
- hash_join.filter().cloned(),
- hash_join.join_type(),
- hash_join.projection.clone(),
- PartitionMode::Partitioned,
- hash_join.null_equality(),
- )?))
+ // Null-aware anti joins must use CollectLeft mode because they track
probe-side state
+ // (probe_side_non_empty, probe_side_has_null) per-partition, but need
global knowledge
+ // for correct null handling. With partitioning, a partition might not
see probe rows
+ // even if the probe side is globally non-empty, leading to incorrect
NULL row handling.
+ let partition_mode = if hash_join.null_aware {
+ PartitionMode::CollectLeft
+ } else {
+ PartitionMode::Partitioned
+ };
+
+ Ok(Arc::new(
+ hash_join
+ .builder()
+ .with_partition_mode(partition_mode)
+ .build()?,
+ ))
}
}
diff --git a/ballista/scheduler/src/state/aqe/execution_plan.rs
b/ballista/scheduler/src/state/aqe/execution_plan.rs
index 38e04eecc..94fa3ee57 100644
--- a/ballista/scheduler/src/state/aqe/execution_plan.rs
+++ b/ballista/scheduler/src/state/aqe/execution_plan.rs
@@ -66,7 +66,7 @@ use std::sync::{Arc, atomic::AtomicI64};
#[derive(Debug)]
pub(crate) struct ExchangeExec {
input: Arc<dyn ExecutionPlan>,
- properties: PlanProperties,
+ properties: Arc<PlanProperties>,
pub(crate) partitioning: Option<Partitioning>,
pub(crate) plan_id: usize,
stage_id: Arc<AtomicI64>,
@@ -116,12 +116,12 @@ impl ExchangeExec {
None => input.output_partitioning().clone(),
};
let eq_properties = input.properties().eq_properties.clone();
- let properties = PlanProperties::new(
+ let properties = Arc::new(PlanProperties::new(
eq_properties,
plan_partitioning,
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Self {
input,
@@ -249,7 +249,7 @@ impl ExecutionPlan for ExchangeExec {
self
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.properties
}
@@ -443,7 +443,7 @@ impl ExecutionPlan for AdaptiveDatafusionExec {
self
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
self.input.properties()
}
diff --git a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
index 8e6de9c38..9fd581c6e 100644
--- a/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
+++ b/ballista/scheduler/src/state/aqe/optimizer_rule/propagate_empty.rs
@@ -22,6 +22,7 @@ use datafusion::config::ConfigOptions;
use datafusion::physical_optimizer::PhysicalOptimizerRule;
use datafusion::physical_plan::ExecutionPlan;
use datafusion::physical_plan::aggregates::AggregateExec;
+#[allow(deprecated)]
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::empty::EmptyExec;
use datafusion::physical_plan::filter::FilterExec;
@@ -51,6 +52,7 @@ macro_rules! empty_exec {
pub struct PropagateEmptyExecRule {}
impl PropagateEmptyExecRule {
+ #[allow(deprecated)]
fn transform(
plan: Arc<dyn ExecutionPlan>,
) -> datafusion::error::Result<Transformed<Arc<dyn ExecutionPlan>>> {
diff --git a/ballista/scheduler/src/state/aqe/test/alter_stages.rs
b/ballista/scheduler/src/state/aqe/test/alter_stages.rs
index 7188c45f9..10311004a 100644
--- a/ballista/scheduler/src/state/aqe/test/alter_stages.rs
+++ b/ballista/scheduler/src/state/aqe/test/alter_stages.rs
@@ -173,8 +173,7 @@ async fn should_insert_new_stage() ->
datafusion::error::Result<()> {
CrossJoinExec
CoalescePartitionsExec
ExchangeExec: partitioning=Hash([big_col@0], 2), plan_id=0,
stage_id=pending, stage_resolved=false
- CooperativeExec
- StatisticsExec: col_count=1, row_count=Inexact(262144)
+ StatisticsExec: col_count=1, row_count=Inexact(262144)
CooperativeExec
MockPartitionedScan: num_partitions=2,
statistics=[Rows=Inexact(1024), Bytes=Inexact(8192), [(Col[0]:)]]
");
@@ -184,8 +183,7 @@ async fn should_insert_new_stage() ->
datafusion::error::Result<()> {
assert_plan!(stages[0].plan.as_ref(), @ r"
ShuffleWriterExec: partitioning: Hash([big_col@0], 2)
- CooperativeExec
- StatisticsExec: col_count=1, row_count=Inexact(262144)
+ StatisticsExec: col_count=1, row_count=Inexact(262144)
");
planner.finalise_stage_internal(0, big_statistics_exchange())?;
@@ -424,7 +422,7 @@ fn get_thresholds() -> (usize, usize) {
struct MockPartitionedScan {
num_partitions: usize,
statistics: Statistics,
- plan_properties: PlanProperties,
+ plan_properties: Arc<PlanProperties>,
}
impl MockPartitionedScan {
@@ -433,12 +431,12 @@ impl MockPartitionedScan {
num_partitions: usize,
statistics: Statistics,
) -> Self {
- let plan_properties = PlanProperties::new(
+ let plan_properties = Arc::new(PlanProperties::new(
EquivalenceProperties::new(schema.clone()),
Partitioning::UnknownPartitioning(num_partitions),
datafusion::physical_plan::execution_plan::EmissionType::Incremental,
datafusion::physical_plan::execution_plan::Boundedness::Bounded,
- );
+ ));
Self {
num_partitions,
statistics,
@@ -473,7 +471,7 @@ impl ExecutionPlan for MockPartitionedScan {
self
}
- fn properties(&self) -> &PlanProperties {
+ fn properties(&self) -> &Arc<PlanProperties> {
&self.plan_properties
}
@@ -496,10 +494,6 @@ impl ExecutionPlan for MockPartitionedScan {
unimplemented!("should not be called")
}
- fn statistics(&self) -> datafusion::common::Result<Statistics> {
- Ok(self.statistics.clone())
- }
-
fn partition_statistics(
&self,
_partition: Option<usize>,
diff --git a/ballista/scheduler/src/state/execution_graph_dot.rs
b/ballista/scheduler/src/state/execution_graph_dot.rs
index 28f29711a..e49fca6a1 100644
--- a/ballista/scheduler/src/state/execution_graph_dot.rs
+++ b/ballista/scheduler/src/state/execution_graph_dot.rs
@@ -26,6 +26,7 @@ use datafusion::datasource::memory::MemorySourceConfig;
use datafusion::datasource::physical_plan::FileScanConfig;
use datafusion::datasource::source::DataSourceExec;
use datafusion::physical_plan::aggregates::AggregateExec;
+#[allow(deprecated)]
use datafusion::physical_plan::coalesce_batches::CoalesceBatchesExec;
use datafusion::physical_plan::coalesce_partitions::CoalescePartitionsExec;
use datafusion::physical_plan::filter::FilterExec;
@@ -227,7 +228,7 @@ fn sanitize(str: &str, max_len: Option<usize>) -> String {
}
sanitized
}
-
+#[allow(deprecated)]
fn get_operator_name(plan: &dyn ExecutionPlan) -> String {
if let Some(exec) = plan.as_any().downcast_ref::<FilterExec>() {
format!("Filter: {}", exec.predicate())
diff --git a/benchmarks/Cargo.toml b/benchmarks/Cargo.toml
index bf7f5f329..eeed7a421 100644
--- a/benchmarks/Cargo.toml
+++ b/benchmarks/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-benchmarks"
description = "Ballista Benchmarks"
-version = "52.0.0"
+version = "53.0.0"
edition = "2024"
authors = ["Apache DataFusion <[email protected]>"]
homepage = "https://datafusion.apache.org/ballista/";
@@ -31,8 +31,8 @@ ci = []
default = ["mimalloc"]
[dependencies]
-ballista = { path = "../ballista/client", version = "52.0.0" }
-ballista-core = { path = "../ballista/core", version = "52.0.0" }
+ballista = { path = "../ballista/client", version = "53.0.0" }
+ballista-core = { path = "../ballista/core", version = "53.0.0" }
datafusion = { workspace = true }
datafusion-proto = { workspace = true }
env_logger = { workspace = true }
diff --git a/examples/Cargo.toml b/examples/Cargo.toml
index 127c3ed0a..18fac834f 100644
--- a/examples/Cargo.toml
+++ b/examples/Cargo.toml
@@ -18,7 +18,7 @@
[package]
name = "ballista-examples"
description = "Ballista usage examples"
-version = "52.0.0"
+version = "53.0.0"
homepage = "https://datafusion.apache.org/ballista/";
repository = "https://github.com/apache/datafusion-ballista";
authors = ["Apache DataFusion <[email protected]>"]
@@ -52,10 +52,10 @@ rustls = { version = "0.23", features = ["ring"], optional
= true }
[dev-dependencies]
arrow-flight = { workspace = true }
-ballista = { path = "../ballista/client", version = "52.0.0" }
-ballista-core = { path = "../ballista/core", version = "52.0.0",
default-features = false }
-ballista-executor = { path = "../ballista/executor", version = "52.0.0",
default-features = false }
-ballista-scheduler = { path = "../ballista/scheduler", version = "52.0.0",
default-features = false }
+ballista = { path = "../ballista/client", version = "53.0.0" }
+ballista-core = { path = "../ballista/core", version = "53.0.0",
default-features = false }
+ballista-executor = { path = "../ballista/executor", version = "53.0.0",
default-features = false }
+ballista-scheduler = { path = "../ballista/scheduler", version = "53.0.0",
default-features = false }
ctor = { workspace = true }
datafusion = { workspace = true }
datafusion-proto = { workspace = true }
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
