Hi, There is another thread [1] to add both pg_buffercache_evict_[relation | all] and pg_buffercache_mark_dirty[_all] functions to the pg_buffercache. I decided to create another thread as pg_buffercache_evict_[relation | all] functions are committed but pg_buffercache_mark_dirty[_all] functions still need review.
pg_buffercache_mark_dirty(): This function takes a buffer id as an argument and tries to mark this buffer as dirty. Returns true on success. pg_buffercache_mark_dirty_all(): This is very similar to the pg_buffercache_mark_dirty() function. The difference is pg_buffercache_mark_dirty_all() does not take an argument. Instead it just loops over the shared buffers and tries to mark all of them as dirty. It returns the number of buffers marked as dirty. Since that patch is targeted for the PG 19, pg_buffercache is bumped to v1.7. Latest version is attached and people who already reviewed the patches are CCed. [1] postgr.es/m/CAN55FZ0h_YoSqqutxV6DES1RW8ig6wcA8CR9rJk358YRMxZFmw%40mail.gmail.com -- Regards, Nazir Bilal Yavuz Microsoft
From 06f12f6174c0b6e1c5beeb4c1a8f4b33b89cc158 Mon Sep 17 00:00:00 2001 From: Nazir Bilal Yavuz <byavu...@gmail.com> Date: Fri, 4 Apr 2025 13:39:49 +0300 Subject: [PATCH v7] Add pg_buffercache_mark_dirty[_all]() functions for testing This commit introduces two new functions for marking shared buffers as dirty: pg_buffercache_mark_dirty(): Marks a specific shared buffer as dirty. pg_buffercache_mark_dirty_all(): Marks all shared buffers as dirty in a single operation. The pg_buffercache_mark_dirty_all() function provides an efficient way to dirty the entire buffer pool (e.g., ~550ms vs. ~70ms for 16GB of shared buffers), complementing pg_buffercache_mark_dirty() for more granular control. These functions are intended for developer testing and debugging scenarios, enabling users to simulate various buffer pool states and test write-back behavior. Both functions are superuser-only. Author: Nazir Bilal Yavuz <byavu...@gmail.com> Reviewed-by: Andres Freund <and...@anarazel.de> Reviewed-by: Aidar Imamov <a.ima...@postgrespro.ru> Reviewed-by: Joseph Koshakow <kosh...@gmail.com> Discussion: https://postgr.es/m/CAN55FZ0h_YoSqqutxV6DES1RW8ig6wcA8CR9rJk358YRMxZFmw%40mail.gmail.com --- src/include/storage/bufmgr.h | 2 + src/backend/storage/buffer/bufmgr.c | 75 +++++++++++++++++++ doc/src/sgml/pgbuffercache.sgml | 54 ++++++++++++- contrib/pg_buffercache/Makefile | 2 +- .../expected/pg_buffercache.out | 30 +++++++- contrib/pg_buffercache/meson.build | 1 + .../pg_buffercache--1.6--1.7.sql | 14 ++++ contrib/pg_buffercache/pg_buffercache.control | 2 +- contrib/pg_buffercache/pg_buffercache_pages.c | 33 ++++++++ contrib/pg_buffercache/sql/pg_buffercache.sql | 10 ++- 10 files changed, 216 insertions(+), 7 deletions(-) create mode 100644 contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql diff --git a/src/include/storage/bufmgr.h b/src/include/storage/bufmgr.h index 33a8b8c06fb..ec7fec6368a 100644 --- a/src/include/storage/bufmgr.h +++ b/src/include/storage/bufmgr.h @@ -312,6 +312,8 @@ extern void EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted, int32 *buffers_flushed, int32 *buffers_skipped); +extern bool MarkUnpinnedBufferDirty(Buffer buf); +extern void MarkAllUnpinnedBuffersDirty(int32 *buffers_dirtied); /* in buf_init.c */ extern void BufferManagerShmemInit(void); diff --git a/src/backend/storage/buffer/bufmgr.c b/src/backend/storage/buffer/bufmgr.c index db8f2b1754e..e2bdb86525b 100644 --- a/src/backend/storage/buffer/bufmgr.c +++ b/src/backend/storage/buffer/bufmgr.c @@ -6699,6 +6699,81 @@ EvictRelUnpinnedBuffers(Relation rel, int32 *buffers_evicted, } } +/* + * Try to mark the provided shared buffer as dirty. + * + * This function is intended for testing/development use only! + * + * Same as EvictUnpinnedBuffer() but with MarkBufferDirty() call inside. + * + * Returns true if the buffer was already dirty or it has successfully been + * marked as dirty. + */ +bool +MarkUnpinnedBufferDirty(Buffer buf) +{ + BufferDesc *desc; + uint32 buf_state; + + Assert(!BufferIsLocal(buf)); + + /* Make sure we can pin the buffer. */ + ResourceOwnerEnlarge(CurrentResourceOwner); + ReservePrivateRefCountEntry(); + + desc = GetBufferDescriptor(buf - 1); + + /* Lock the header and check if it's valid. */ + buf_state = LockBufHdr(desc); + if ((buf_state & BM_VALID) == 0) + { + UnlockBufHdr(desc, buf_state); + return false; + } + + /* Check that it's not pinned already. */ + if (BUF_STATE_GET_REFCOUNT(buf_state) > 0) + { + UnlockBufHdr(desc, buf_state); + return false; + } + + PinBuffer_Locked(desc); /* releases spinlock */ + + /* If it was not already dirty, mark it as dirty. */ + if (!(buf_state & BM_DIRTY)) + { + LWLockAcquire(BufferDescriptorGetContentLock(desc), LW_EXCLUSIVE); + MarkBufferDirty(buf); + LWLockRelease(BufferDescriptorGetContentLock(desc)); + } + + UnpinBuffer(desc); + + return true; +} + +/* + * Try to mark all the shared buffers as dirty. + * + * This function is intended for testing/development use only! See + * MarkUnpinnedBufferDirty(). + * + * The buffers_dirtied parameter is mandatory and indicate the total count of + * buffers were dirtied. + */ +void +MarkAllUnpinnedBuffersDirty(int32 *buffers_dirtied) +{ + *buffers_dirtied = 0; + + for (int buf = 1; buf <= NBuffers; buf++) + { + if (MarkUnpinnedBufferDirty(buf)) + (*buffers_dirtied)++; + } +} + /* * Generic implementation of the AIO handle staging callback for readv/writev * on local/shared buffers. diff --git a/doc/src/sgml/pgbuffercache.sgml b/doc/src/sgml/pgbuffercache.sgml index 537d6014942..bfccabd9c5e 100644 --- a/doc/src/sgml/pgbuffercache.sgml +++ b/doc/src/sgml/pgbuffercache.sgml @@ -35,6 +35,14 @@ <primary>pg_buffercache_evict_all</primary> </indexterm> + <indexterm> + <primary>pg_buffercache_mark_dirty</primary> + </indexterm> + + <indexterm> + <primary>pg_buffercache_mark_dirty_all</primary> + </indexterm> + <para> This module provides the <function>pg_buffercache_pages()</function> function (wrapped in the <structname>pg_buffercache</structname> view), @@ -42,9 +50,11 @@ <structname>pg_buffercache_numa</structname> view), the <function>pg_buffercache_summary()</function> function, the <function>pg_buffercache_usage_counts()</function> function, the - <function>pg_buffercache_evict()</function>, the - <function>pg_buffercache_evict_relation()</function> function and the - <function>pg_buffercache_evict_all()</function> function. + <function>pg_buffercache_evict()</function> function, the + <function>pg_buffercache_evict_relation()</function> function, the + <function>pg_buffercache_evict_all()</function> function, the + <function>pg_buffercache_mark_dirty()</function> function and the + <function>pg_buffercache_mark_dirty_all()</function> function. </para> <para> @@ -99,6 +109,18 @@ function is restricted to superusers only. </para> + <para> + The <function>pg_buffercache_mark_dirty()</function> function allows a block + to be marked as dirty from the buffer pool given a buffer identifier. Use of + this function is restricted to superusers only. + </para> + + <para> + The <function>pg_buffercache_mark_dirty_all()</function> function tries to + mark all buffers dirty in the buffer pool. Use of this function is + restricted to superusers only. + </para> + <sect2 id="pgbuffercache-pg-buffercache"> <title>The <structname>pg_buffercache</structname> View</title> @@ -522,6 +544,32 @@ </para> </sect2> + <sect2 id="pgbuffercache-pg-buffercache-mark-dirty"> + <title>The <structname>pg_buffercache_mark_dirty</structname> Function</title> + <para> + The <function>pg_buffercache_mark_dirty()</function> function takes a + buffer identifier, as shown in the <structfield>bufferid</structfield> + column of the <structname>pg_buffercache</structname> view. It returns + true on success, and false if the buffer wasn't valid or if it couldn't be + marked as dirty because it was pinned. The result is immediately out of + date upon return, as the buffer might become valid again at any time due to + concurrent activity. The function is intended for developer testing only. + </para> + </sect2> + + <sect2 id="pgbuffercache-pg-buffercache-mark-dirty-all"> + <title>The <structname>pg_buffercache_mark_dirty_all</structname> Function</title> + <para> + The <function>pg_buffercache_mark_dirty_all()</function> function is very + similar to the <function>pg_buffercache_mark_dirty()</function> function. + The difference is, the <function>pg_buffercache_mark_dirty_all()</function> + function does not take an argument; instead it tries to mark all buffers + dirty in the buffer pool. The result is immediately out of date upon + return, as the buffer might become valid again at any time due to + concurrent activity. The function is intended for developer testing only. + </para> + </sect2> + <sect2 id="pgbuffercache-sample-output"> <title>Sample Output</title> diff --git a/contrib/pg_buffercache/Makefile b/contrib/pg_buffercache/Makefile index 5f748543e2e..0e618f66aec 100644 --- a/contrib/pg_buffercache/Makefile +++ b/contrib/pg_buffercache/Makefile @@ -9,7 +9,7 @@ EXTENSION = pg_buffercache DATA = pg_buffercache--1.2.sql pg_buffercache--1.2--1.3.sql \ pg_buffercache--1.1--1.2.sql pg_buffercache--1.0--1.1.sql \ pg_buffercache--1.3--1.4.sql pg_buffercache--1.4--1.5.sql \ - pg_buffercache--1.5--1.6.sql + pg_buffercache--1.5--1.6.sql pg_buffercache--1.6--1.7.sql PGFILEDESC = "pg_buffercache - monitoring of shared buffer cache in real-time" REGRESS = pg_buffercache pg_buffercache_numa diff --git a/contrib/pg_buffercache/expected/pg_buffercache.out b/contrib/pg_buffercache/expected/pg_buffercache.out index 9a9216dc7b1..aa2c0328386 100644 --- a/contrib/pg_buffercache/expected/pg_buffercache.out +++ b/contrib/pg_buffercache/expected/pg_buffercache.out @@ -57,7 +57,7 @@ SELECT count(*) > 0 FROM pg_buffercache_usage_counts(); RESET role; ------ ----- Test pg_buffercache_evict* functions +---- Test pg_buffercache_evict* and pg_buffercache_mark_dirty* functions ------ CREATE ROLE regress_buffercache_normal; SET ROLE regress_buffercache_normal; @@ -68,6 +68,10 @@ SELECT * FROM pg_buffercache_evict_relation(1); ERROR: must be superuser to use pg_buffercache_evict_relation() SELECT * FROM pg_buffercache_evict_all(); ERROR: must be superuser to use pg_buffercache_evict_all() +SELECT * FROM pg_buffercache_mark_dirty(1); +ERROR: must be superuser to use pg_buffercache_mark_dirty() +SELECT * FROM pg_buffercache_mark_dirty_all(); +ERROR: must be superuser to use pg_buffercache_mark_dirty_all() RESET ROLE; -- These should return nothing, because these are STRICT functions SELECT * FROM pg_buffercache_evict(NULL); @@ -82,6 +86,12 @@ SELECT * FROM pg_buffercache_evict_relation(NULL); | | (1 row) +SELECT * FROM pg_buffercache_mark_dirty(NULL); + pg_buffercache_mark_dirty +--------------------------- + +(1 row) + -- These should fail because they are not called by valid range of buffers -- Number of the shared buffers are limited by max integer SELECT 2147483647 max_buffers \gset @@ -91,6 +101,12 @@ SELECT * FROM pg_buffercache_evict(0); ERROR: bad buffer ID: 0 SELECT * FROM pg_buffercache_evict(:max_buffers); ERROR: bad buffer ID: 2147483647 +SELECT * FROM pg_buffercache_mark_dirty(-1); +ERROR: bad buffer ID: -1 +SELECT * FROM pg_buffercache_mark_dirty(0); +ERROR: bad buffer ID: 0 +SELECT * FROM pg_buffercache_mark_dirty(:max_buffers); +ERROR: bad buffer ID: 2147483647 -- This should fail because pg_buffercache_evict_relation() doesn't accept -- local relations CREATE TEMP TABLE temp_pg_buffercache(); @@ -118,4 +134,16 @@ SELECT buffers_evicted IS NOT NULL FROM pg_buffercache_evict_relation('shared_pg (1 row) DROP TABLE shared_pg_buffercache; +SELECT pg_buffercache_mark_dirty(1) IS NOT NULL; + ?column? +---------- + t +(1 row) + +SELECT pg_buffercache_mark_dirty_all() IS NOT NULL; + ?column? +---------- + t +(1 row) + DROP ROLE regress_buffercache_normal; diff --git a/contrib/pg_buffercache/meson.build b/contrib/pg_buffercache/meson.build index 7cd039a1df9..7c31141881f 100644 --- a/contrib/pg_buffercache/meson.build +++ b/contrib/pg_buffercache/meson.build @@ -24,6 +24,7 @@ install_data( 'pg_buffercache--1.3--1.4.sql', 'pg_buffercache--1.4--1.5.sql', 'pg_buffercache--1.5--1.6.sql', + 'pg_buffercache--1.6--1.7.sql', 'pg_buffercache.control', kwargs: contrib_data_args, ) diff --git a/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql b/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql new file mode 100644 index 00000000000..db55c38e9b9 --- /dev/null +++ b/contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql @@ -0,0 +1,14 @@ +/* contrib/pg_buffercache/pg_buffercache--1.6--1.7.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "ALTER EXTENSION pg_buffercache UPDATE TO '1.7'" to load this file. \quit + +CREATE FUNCTION pg_buffercache_mark_dirty(IN int) +RETURNS bool +AS 'MODULE_PATHNAME', 'pg_buffercache_mark_dirty' +LANGUAGE C PARALLEL SAFE VOLATILE STRICT; + +CREATE FUNCTION pg_buffercache_mark_dirty_all() +RETURNS INT4 +AS 'MODULE_PATHNAME', 'pg_buffercache_mark_dirty_all' +LANGUAGE C PARALLEL SAFE VOLATILE; diff --git a/contrib/pg_buffercache/pg_buffercache.control b/contrib/pg_buffercache/pg_buffercache.control index b030ba3a6fa..11499550945 100644 --- a/contrib/pg_buffercache/pg_buffercache.control +++ b/contrib/pg_buffercache/pg_buffercache.control @@ -1,5 +1,5 @@ # pg_buffercache extension comment = 'examine the shared buffer cache' -default_version = '1.6' +default_version = '1.7' module_pathname = '$libdir/pg_buffercache' relocatable = true diff --git a/contrib/pg_buffercache/pg_buffercache_pages.c b/contrib/pg_buffercache/pg_buffercache_pages.c index e1701bd56ef..6136ab3a0ae 100644 --- a/contrib/pg_buffercache/pg_buffercache_pages.c +++ b/contrib/pg_buffercache/pg_buffercache_pages.c @@ -100,6 +100,8 @@ PG_FUNCTION_INFO_V1(pg_buffercache_usage_counts); PG_FUNCTION_INFO_V1(pg_buffercache_evict); PG_FUNCTION_INFO_V1(pg_buffercache_evict_relation); PG_FUNCTION_INFO_V1(pg_buffercache_evict_all); +PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty); +PG_FUNCTION_INFO_V1(pg_buffercache_mark_dirty_all); /* Only need to touch memory once per backend process lifetime */ @@ -772,3 +774,34 @@ pg_buffercache_evict_all(PG_FUNCTION_ARGS) PG_RETURN_DATUM(result); } + +/* + * Try to mark a shared buffer as dirty. + */ +Datum +pg_buffercache_mark_dirty(PG_FUNCTION_ARGS) +{ + Buffer buf = PG_GETARG_INT32(0); + + pg_buffercache_superuser_check("pg_buffercache_mark_dirty"); + + if (buf < 1 || buf > NBuffers) + elog(ERROR, "bad buffer ID: %d", buf); + + PG_RETURN_BOOL(MarkUnpinnedBufferDirty(buf)); +} + +/* + * Try to mark all the shared buffers as dirty. + */ +Datum +pg_buffercache_mark_dirty_all(PG_FUNCTION_ARGS) +{ + int32 buffers_dirtied = 0; + + pg_buffercache_superuser_check("pg_buffercache_mark_dirty_all"); + + MarkAllUnpinnedBuffersDirty(&buffers_dirtied); + + PG_RETURN_INT32(buffers_dirtied); +} diff --git a/contrib/pg_buffercache/sql/pg_buffercache.sql b/contrib/pg_buffercache/sql/pg_buffercache.sql index 47cca1907c7..9eac5214825 100644 --- a/contrib/pg_buffercache/sql/pg_buffercache.sql +++ b/contrib/pg_buffercache/sql/pg_buffercache.sql @@ -30,7 +30,7 @@ RESET role; ------ ----- Test pg_buffercache_evict* functions +---- Test pg_buffercache_evict* and pg_buffercache_mark_dirty* functions ------ CREATE ROLE regress_buffercache_normal; @@ -40,12 +40,15 @@ SET ROLE regress_buffercache_normal; SELECT * FROM pg_buffercache_evict(1); SELECT * FROM pg_buffercache_evict_relation(1); SELECT * FROM pg_buffercache_evict_all(); +SELECT * FROM pg_buffercache_mark_dirty(1); +SELECT * FROM pg_buffercache_mark_dirty_all(); RESET ROLE; -- These should return nothing, because these are STRICT functions SELECT * FROM pg_buffercache_evict(NULL); SELECT * FROM pg_buffercache_evict_relation(NULL); +SELECT * FROM pg_buffercache_mark_dirty(NULL); -- These should fail because they are not called by valid range of buffers -- Number of the shared buffers are limited by max integer @@ -53,6 +56,9 @@ SELECT 2147483647 max_buffers \gset SELECT * FROM pg_buffercache_evict(-1); SELECT * FROM pg_buffercache_evict(0); SELECT * FROM pg_buffercache_evict(:max_buffers); +SELECT * FROM pg_buffercache_mark_dirty(-1); +SELECT * FROM pg_buffercache_mark_dirty(0); +SELECT * FROM pg_buffercache_mark_dirty(:max_buffers); -- This should fail because pg_buffercache_evict_relation() doesn't accept -- local relations @@ -66,5 +72,7 @@ SELECT buffers_evicted IS NOT NULL FROM pg_buffercache_evict_all(); CREATE TABLE shared_pg_buffercache(); SELECT buffers_evicted IS NOT NULL FROM pg_buffercache_evict_relation('shared_pg_buffercache'); DROP TABLE shared_pg_buffercache; +SELECT pg_buffercache_mark_dirty(1) IS NOT NULL; +SELECT pg_buffercache_mark_dirty_all() IS NOT NULL; DROP ROLE regress_buffercache_normal; -- 2.49.0
