tschwarzinger commented on code in PR #26387:
URL: https://github.com/apache/datafusion/pull/26387#discussion_r4441043551


##########
datafusion/core/tests/datasource/object_store_access.rs:
##########
@@ -1019,6 +1019,181 @@ async fn 
query_single_parquet_file_multi_row_groups_multiple_predicates() {
     );
 }
 
+#[tokio::test]
+async fn query_single_parquet_file_pushdown_filters_one_shot_io() {
+    let test = Test::new().with_single_file_parquet().await;
+    test.set("datafusion.execution.parquet.pushdown_filters", "true")
+        .await;
+
+    // With filter pushdown enabled and `progressive_io` disabled (the
+    // default), all data needed for each row group (the columns needed by
+    // the filter and by the projection) is fetched with a single request per
+    // row group, the same I/O pattern as when pushdown_filters is disabled
+    // (compare 
`query_single_parquet_file_multi_row_groups_multiple_predicates`):
+    //
+    // * Row group 1 (`a` in 0..100): `a` (4-534) and `b` (534-1064) are
+    //   fetched together, even though `a` is only needed to evaluate the
+    //   filter.
+    // * Row group 2 (`a` in 100..200): the statistics prove every row
+    //   matches, so the filter is skipped and only `b` (1594-2124) is read.
+    assert_snapshot!(
+        test.query("select min(b) from parquet_table WHERE a > 50").await,
+        @r"
+    ------- Query Output (1 rows) -------
+    +----------------------+
+    | min(parquet_table.b) |
+    +----------------------+
+    | 1051                 |
+    +----------------------+
+    ------- Object Store Request Summary -------
+    RequestCountingObjectStore()
+    Total Requests: 3
+    - GET  (opts) path=parquet_table.parquet head=true
+    - GET  (ranges) path=parquet_table.parquet ranges=4-534,534-1064
+    - GET  (ranges) path=parquet_table.parquet ranges=1594-2124
+    "
+    );
+}
+
+#[tokio::test]
+async fn query_single_parquet_file_pushdown_filters_progressive_io() {
+    let test = Test::new().with_single_file_parquet().await;
+    test.set("datafusion.execution.parquet.pushdown_filters", "true")
+        .await;
+    test.set("datafusion.execution.parquet.progressive_io", "true")
+        .await;
+
+    // With `progressive_io` enabled (and an offset index present in the
+    // file), a row group that needs the filter is fetched progressively:
+    // first the column needed to evaluate the filter (`a`, 4-534), then the
+    // remaining projected column (`b`, 534-1064) for the rows that passed,
+    // resulting in multiple requests for row group 1. Row group 2 is fully
+    // matched by its statistics, so the filter is skipped and only `b` is
+    // read, as in the one-shot case.
+    assert_snapshot!(
+        test.query("select min(b) from parquet_table WHERE a > 50").await,
+        @r"
+    ------- Query Output (1 rows) -------
+    +----------------------+
+    | min(parquet_table.b) |
+    +----------------------+
+    | 1051                 |
+    +----------------------+
+    ------- Object Store Request Summary -------
+    RequestCountingObjectStore()
+    Total Requests: 4
+    - GET  (opts) path=parquet_table.parquet head=true
+    - GET  (ranges) path=parquet_table.parquet ranges=4-534
+    - GET  (ranges) path=parquet_table.parquet ranges=534-951,951-1064
+    - GET  (ranges) path=parquet_table.parquet ranges=1594-2124
+    "
+    );
+}
+
+#[tokio::test]
+async fn query_single_parquet_file_multi_page_no_pushdown() {
+    let test = Test::new().with_single_file_parquet_multi_page().await;
+
+    // Baseline for the `*_page_index` tests below: without filter pushdown
+    // the page index still prunes the first two pages of `a` in row group 1
+    // (values 0..40 cannot match `a > 50`) and the matching rows of `b`, so
+    // only the dictionary page and the selected data pages of each column
+    // chunk are fetched, with a single request per row group.
+    assert_snapshot!(
+        test.query("select min(b) from parquet_table WHERE a > 50").await,
+        @r"
+    ------- Query Output (1 rows) -------
+    +----------------------+
+    | min(parquet_table.b) |
+    +----------------------+
+    | 1051                 |
+    +----------------------+
+    ------- Object Store Request Summary -------
+    RequestCountingObjectStore()
+    Total Requests: 3

Review Comment:
   I do love these kind of tests 🚀 



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to