From fb1c79c481c03f99631a5c4560223db205226cd5 Mon Sep 17 00:00:00 2001 From: Sami Imseih Date: Mon, 2 Jun 2025 13:15:47 -0500 Subject: [PATCH v2 1/1] Normalize variable fetch sizes in a FETCH command Prior to this patch, every FETCH call would generate a unique queryId. This led to significant bloat in pg_stat_statements, as repeatedly calling a specific cursor would result in a new queryId each time. For example, FETCH 1 c1; and FETCH 2 c1; would produce different queryIds. This patch improves the situation by normalizing the fetch size, so semantically similar statements generate the same queryId. As a result, statements like the below, which differ syntactically but have the same effect, will now share a single queryId: FETCH FROM c1; FETCH NEXT c1; FETCH 1 c1; Discussion: https://www.postgresql.org/message-id/flat/CAA5RZ0tA6LbHCg2qSS%2BKuM850BZC_%2BZgHV7Ug6BXw22TNyF%2BMA%40mail.gmail.com --- .../pg_stat_statements/expected/cursors.out | 4 +- .../expected/level_tracking.out | 4 +- .../pg_stat_statements/expected/utility.out | 163 +++++++++++++++++- contrib/pg_stat_statements/sql/utility.sql | 45 +++++ src/backend/parser/gram.y | 16 ++ src/include/nodes/parsenodes.h | 4 +- 6 files changed, 227 insertions(+), 9 deletions(-) diff --git a/contrib/pg_stat_statements/expected/cursors.out b/contrib/pg_stat_statements/expected/cursors.out index 0fc4b2c098d0..6781f99ea03e 100644 --- a/contrib/pg_stat_statements/expected/cursors.out +++ b/contrib/pg_stat_statements/expected/cursors.out @@ -57,8 +57,8 @@ SELECT calls, rows, query FROM pg_stat_statements ORDER BY query COLLATE "C"; 1 | 0 | COMMIT 1 | 0 | DECLARE cursor_stats_1 CURSOR WITH HOLD FOR SELECT $1 1 | 0 | DECLARE cursor_stats_2 CURSOR WITH HOLD FOR SELECT $1 - 1 | 1 | FETCH 1 IN cursor_stats_1 - 1 | 1 | FETCH 1 IN cursor_stats_2 + 1 | 1 | FETCH $1 IN cursor_stats_1 + 1 | 1 | FETCH $1 IN cursor_stats_2 1 | 1 | SELECT pg_stat_statements_reset() IS NOT NULL AS t (9 rows) diff --git a/contrib/pg_stat_statements/expected/level_tracking.out b/contrib/pg_stat_statements/expected/level_tracking.out index 75e785e1719e..ce4bd3a823d8 100644 --- a/contrib/pg_stat_statements/expected/level_tracking.out +++ b/contrib/pg_stat_statements/expected/level_tracking.out @@ -1142,7 +1142,7 @@ SELECT toplevel, calls, query FROM pg_stat_statements t | 1 | CLOSE foocur t | 1 | COMMIT t | 1 | DECLARE FOOCUR CURSOR FOR SELECT * from stats_track_tab - t | 1 | FETCH FORWARD 1 FROM foocur + t | 1 | FETCH FORWARD $1 FROM foocur f | 1 | SELECT * from stats_track_tab t | 1 | SELECT pg_stat_statements_reset() IS NOT NULL AS t (7 rows) @@ -1172,7 +1172,7 @@ SELECT toplevel, calls, query FROM pg_stat_statements t | 1 | CLOSE foocur t | 1 | COMMIT t | 1 | DECLARE FOOCUR CURSOR FOR SELECT * FROM stats_track_tab - t | 1 | FETCH FORWARD 1 FROM foocur + t | 1 | FETCH FORWARD $1 FROM foocur t | 1 | SELECT pg_stat_statements_reset() IS NOT NULL AS t (6 rows) diff --git a/contrib/pg_stat_statements/expected/utility.out b/contrib/pg_stat_statements/expected/utility.out index aa4f0f7e6280..ae6fad96ebbe 100644 --- a/contrib/pg_stat_statements/expected/utility.out +++ b/contrib/pg_stat_statements/expected/utility.out @@ -702,13 +702,11 @@ SELECT calls, rows, query FROM pg_stat_statements ORDER BY query COLLATE "C"; 1 | 13 | CREATE MATERIALIZED VIEW pgss_matv AS SELECT * FROM pgss_ctas 1 | 10 | CREATE TABLE pgss_ctas AS SELECT a, $1 b FROM generate_series($2, $3) a 1 | 0 | DECLARE pgss_cursor CURSOR FOR SELECT * FROM pgss_matv - 1 | 5 | FETCH FORWARD 5 pgss_cursor - 1 | 7 | FETCH FORWARD ALL pgss_cursor - 1 | 1 | FETCH NEXT pgss_cursor + 3 | 13 | FETCH NEXT pgss_cursor 1 | 13 | REFRESH MATERIALIZED VIEW pgss_matv 1 | 10 | SELECT generate_series($1, $2) c INTO pgss_select_into 1 | 1 | SELECT pg_stat_statements_reset() IS NOT NULL AS t -(12 rows) +(10 rows) DROP MATERIALIZED VIEW pgss_matv; DROP TABLE pgss_ctas; @@ -719,7 +717,164 @@ SELECT pg_stat_statements_reset() IS NOT NULL AS t; t (1 row) +-- Normalization of FETCH statements +-- this group will have the same queryId +BEGIN; +DECLARE pgss_cursor CURSOR FOR SELECT FROM generate_series(1, 50); +FETCH pgss_cursor; +-- +(1 row) + +FETCH FROM pgss_cursor; +-- +(1 row) + +FETCH NEXT pgss_cursor; +-- +(1 row) + +FETCH 1 pgss_cursor; +-- +(1 row) + +FETCH 2 pgss_cursor; +-- +(2 rows) + +FETCH 3 pgss_cursor; +-- +(3 rows) + +FETCH ALL pgss_cursor; +-- +(41 rows) + +FETCH FORWARD pgss_cursor; +-- +(0 rows) + +FETCH FORWARD 1 pgss_cursor; +-- +(0 rows) + +FETCH FORWARD 2 pgss_cursor; +-- +(0 rows) + +FETCH FORWARD 3 pgss_cursor; +-- +(0 rows) + +FETCH FORWARD ALL pgss_cursor; +-- +(0 rows) + +-- this next group will have the same queryId +FETCH PRIOR pgss_cursor; +-- +(1 row) + +FETCH BACKWARD 1 pgss_cursor; +-- +(1 row) + +FETCH BACKWARD 2 pgss_cursor; +-- +(2 rows) + +FETCH BACKWARD 3 pgss_cursor; +-- +(3 rows) + +FETCH BACKWARD ALL pgss_cursor; +-- +(43 rows) + +-- this next group will have the same queryId +FETCH FIRST pgss_cursor; +-- +(1 row) + +FETCH LAST pgss_cursor; +-- +(1 row) + +FETCH ABSOLUTE 1 pgss_cursor; +-- +(1 row) + +FETCH ABSOLUTE 2 pgss_cursor; +-- +(1 row) + +FETCH ABSOLUTE 3 pgss_cursor; +-- +(1 row) + +-- this next group will have the same queryId +FETCH RELATIVE 1 pgss_cursor; +-- +(1 row) + +FETCH RELATIVE 2 pgss_cursor; +-- +(1 row) + +FETCH RELATIVE 3 pgss_cursor; +-- +(1 row) + +COMMIT; +SELECT calls, query FROM pg_stat_statements ORDER BY query COLLATE "C"; + calls | query +-------+-------------------------------------------------------------------- + 1 | BEGIN + 1 | COMMIT + 1 | DECLARE pgss_cursor CURSOR FOR SELECT FROM generate_series($1, $2) + 5 | FETCH FIRST pgss_cursor + 5 | FETCH PRIOR pgss_cursor + 3 | FETCH RELATIVE $1 pgss_cursor + 12 | FETCH pgss_cursor + 1 | SELECT pg_stat_statements_reset() IS NOT NULL AS t +(8 rows) + +-- check that these statements normalize the fetch size +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + +BEGIN; +DECLARE pgss_cursor CURSOR FOR SELECT FROM generate_series(1, 50); +FETCH 1 pgss_cursor; +-- +(1 row) + +FETCH 2 pgss_cursor; +-- +(2 rows) + +FETCH 3 pgss_cursor; +-- +(3 rows) + +SELECT calls, query FROM pg_stat_statements ORDER BY query COLLATE "C"; + calls | query +-------+-------------------------------------------------------------------- + 1 | BEGIN + 1 | DECLARE pgss_cursor CURSOR FOR SELECT FROM generate_series($1, $2) + 3 | FETCH $1 pgss_cursor + 1 | SELECT pg_stat_statements_reset() IS NOT NULL AS t +(4 rows) + -- Special cases. Keep these ones at the end to avoid conflicts. +SELECT pg_stat_statements_reset() IS NOT NULL AS t; + t +--- + t +(1 row) + SET SCHEMA 'foo'; SET SCHEMA 'public'; RESET ALL; diff --git a/contrib/pg_stat_statements/sql/utility.sql b/contrib/pg_stat_statements/sql/utility.sql index dd97203c2102..db14f389eee8 100644 --- a/contrib/pg_stat_statements/sql/utility.sql +++ b/contrib/pg_stat_statements/sql/utility.sql @@ -365,7 +365,52 @@ DROP TABLE pgss_select_into; SELECT pg_stat_statements_reset() IS NOT NULL AS t; +-- Normalization of FETCH statements +-- this group will have the same queryId +BEGIN; +DECLARE pgss_cursor CURSOR FOR SELECT FROM generate_series(1, 50); +FETCH pgss_cursor; +FETCH FROM pgss_cursor; +FETCH NEXT pgss_cursor; +FETCH 1 pgss_cursor; +FETCH 2 pgss_cursor; +FETCH 3 pgss_cursor; +FETCH ALL pgss_cursor; +FETCH FORWARD pgss_cursor; +FETCH FORWARD 1 pgss_cursor; +FETCH FORWARD 2 pgss_cursor; +FETCH FORWARD 3 pgss_cursor; +FETCH FORWARD ALL pgss_cursor; +-- this next group will have the same queryId +FETCH PRIOR pgss_cursor; +FETCH BACKWARD 1 pgss_cursor; +FETCH BACKWARD 2 pgss_cursor; +FETCH BACKWARD 3 pgss_cursor; +FETCH BACKWARD ALL pgss_cursor; +-- this next group will have the same queryId +FETCH FIRST pgss_cursor; +FETCH LAST pgss_cursor; +FETCH ABSOLUTE 1 pgss_cursor; +FETCH ABSOLUTE 2 pgss_cursor; +FETCH ABSOLUTE 3 pgss_cursor; +-- this next group will have the same queryId +FETCH RELATIVE 1 pgss_cursor; +FETCH RELATIVE 2 pgss_cursor; +FETCH RELATIVE 3 pgss_cursor; +COMMIT; +SELECT calls, query FROM pg_stat_statements ORDER BY query COLLATE "C"; +-- check that these statements normalize the fetch size +SELECT pg_stat_statements_reset() IS NOT NULL AS t; +BEGIN; +DECLARE pgss_cursor CURSOR FOR SELECT FROM generate_series(1, 50); +FETCH 1 pgss_cursor; +FETCH 2 pgss_cursor; +FETCH 3 pgss_cursor; +SELECT calls, query FROM pg_stat_statements ORDER BY query COLLATE "C"; + + -- Special cases. Keep these ones at the end to avoid conflicts. +SELECT pg_stat_statements_reset() IS NOT NULL AS t; SET SCHEMA 'foo'; SET SCHEMA 'public'; RESET ALL; diff --git a/src/backend/parser/gram.y b/src/backend/parser/gram.y index 0b5652071d11..843dcc87a48b 100644 --- a/src/backend/parser/gram.y +++ b/src/backend/parser/gram.y @@ -7479,6 +7479,7 @@ fetch_args: cursor_name n->portalname = $1; n->direction = FETCH_FORWARD; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | from_in cursor_name @@ -7488,6 +7489,7 @@ fetch_args: cursor_name n->portalname = $2; n->direction = FETCH_FORWARD; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | NEXT opt_from_in cursor_name @@ -7497,6 +7499,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_FORWARD; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | PRIOR opt_from_in cursor_name @@ -7506,6 +7509,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_BACKWARD; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | FIRST_P opt_from_in cursor_name @@ -7515,6 +7519,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_ABSOLUTE; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | LAST_P opt_from_in cursor_name @@ -7524,6 +7529,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_ABSOLUTE; n->howMany = -1; + n->location = -1; $$ = (Node *) n; } | ABSOLUTE_P SignedIconst opt_from_in cursor_name @@ -7533,6 +7539,7 @@ fetch_args: cursor_name n->portalname = $4; n->direction = FETCH_ABSOLUTE; n->howMany = $2; + n->location = @2; $$ = (Node *) n; } | RELATIVE_P SignedIconst opt_from_in cursor_name @@ -7542,6 +7549,7 @@ fetch_args: cursor_name n->portalname = $4; n->direction = FETCH_RELATIVE; n->howMany = $2; + n->location = @2; $$ = (Node *) n; } | SignedIconst opt_from_in cursor_name @@ -7551,6 +7559,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_FORWARD; n->howMany = $1; + n->location = @1; $$ = (Node *) n; } | ALL opt_from_in cursor_name @@ -7560,6 +7569,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_FORWARD; n->howMany = FETCH_ALL; + n->location = -1; $$ = (Node *) n; } | FORWARD opt_from_in cursor_name @@ -7569,6 +7579,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_FORWARD; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | FORWARD SignedIconst opt_from_in cursor_name @@ -7578,6 +7589,7 @@ fetch_args: cursor_name n->portalname = $4; n->direction = FETCH_FORWARD; n->howMany = $2; + n->location = @2; $$ = (Node *) n; } | FORWARD ALL opt_from_in cursor_name @@ -7587,6 +7599,7 @@ fetch_args: cursor_name n->portalname = $4; n->direction = FETCH_FORWARD; n->howMany = FETCH_ALL; + n->location = -1; $$ = (Node *) n; } | BACKWARD opt_from_in cursor_name @@ -7596,6 +7609,7 @@ fetch_args: cursor_name n->portalname = $3; n->direction = FETCH_BACKWARD; n->howMany = 1; + n->location = -1; $$ = (Node *) n; } | BACKWARD SignedIconst opt_from_in cursor_name @@ -7605,6 +7619,7 @@ fetch_args: cursor_name n->portalname = $4; n->direction = FETCH_BACKWARD; n->howMany = $2; + n->location = @2; $$ = (Node *) n; } | BACKWARD ALL opt_from_in cursor_name @@ -7614,6 +7629,7 @@ fetch_args: cursor_name n->portalname = $4; n->direction = FETCH_BACKWARD; n->howMany = FETCH_ALL; + n->location = -1; $$ = (Node *) n; } ; diff --git a/src/include/nodes/parsenodes.h b/src/include/nodes/parsenodes.h index dd00ab420b8a..02bfce1ff38e 100644 --- a/src/include/nodes/parsenodes.h +++ b/src/include/nodes/parsenodes.h @@ -3428,9 +3428,11 @@ typedef struct FetchStmt { NodeTag type; FetchDirection direction; /* see above */ - long howMany; /* number of rows, or position argument */ + /* number of rows, or position argument */ + long howMany pg_node_attr(query_jumble_ignore); char *portalname; /* name of portal (cursor) */ bool ismove; /* true if MOVE */ + ParseLoc location pg_node_attr(query_jumble_location); } FetchStmt; /* ---------------------- -- 2.39.5 (Apple Git-154)