Hi all
While working on extensions I've often wanted to enable cache clobbering
for a targeted piece of code, without paying the price of running it for
all backends during postgres startup and any initial setup tasks that are
required.
So here's a patch that, when CLOBBER_CACHE_ALWAYS or
CLOBBER_CACHE_RECURSIVE are defined, adds a GUC named clobber_cache_depth .
It defaults to 1 for CLOBBER_CACHE_ALWAYS or 3 for CLOBBER_CACHE_RECURSIVE
to match the existing compiled-in behaviour. But with this change it's now
possible to run Pg with clobber_cache_depth=0 then set it to 1 only for
targeted tests.
clobber_cache_depth is treated as an unknown GUC if Pg was not built with
CLOBBER_CACHE_ALWAYS or CLOBBER_CACHE_RECURSIVE defined.
-----
On a side note, to make things like this easier to use, I personally patch
all pg_regress tests to include the following at the start of each sql
input file:
\set ECHO none
-- Put per-test settings or overrides here
\set ECHO queries
then patch the expected files accordingly. That way it's easy for me to
make per-test adjustments while still running the whole suite. It's not
always practical to run just one targeted test with TESTS=foo.
--
Craig Ringer http://www.2ndQuadrant.com/
2ndQuadrant - PostgreSQL Solutions for the Enterprise
From 372defde443d178fb4a9c8cf4092dea7debf72ea Mon Sep 17 00:00:00 2001
From: Craig Ringer <craig.rin...@2ndquadrant.com>
Date: Tue, 22 Sep 2020 09:51:00 +0800
Subject: [PATCH v1] Add runtime control over CLOBBER_CACHE_ALWAYS
When CLOBBER_CACHE_ALWAYS or CLOBBER_CACHE_RECURSIVE are defined,
a new GUC "clobber_cache_depth" becomes available. This allows runtime
control over the behaviour of cache clobber builds in order to allow
more targeted testing of new or changed features using aggressive
cache clobbering.
clobber_cache_depth defaults to 1 if CLOBBER_CACHE_ALWAYS is defined,
and to 3 if CLOBBER_CACHE_RECURSIVE is defined. That makes it match
the previous hardcoded behaviour of these macros, ensuring buildfarm
members won't get upset.
While we're at it, expose the macros in pg_config_manual.h
---
src/backend/utils/cache/inval.c | 51 +++++++++++++++++++--------------
src/backend/utils/misc/guc.c | 15 ++++++++++
src/include/pg_config_manual.h | 28 ++++++++++++++++--
src/include/utils/inval.h | 8 ++++++
4 files changed, 78 insertions(+), 24 deletions(-)
diff --git a/src/backend/utils/cache/inval.c b/src/backend/utils/cache/inval.c
index 628d6f5d0c..620c9558ac 100644
--- a/src/backend/utils/cache/inval.c
+++ b/src/backend/utils/cache/inval.c
@@ -109,6 +109,7 @@
#include "storage/sinval.h"
#include "storage/smgr.h"
#include "utils/catcache.h"
+#include "utils/guc.h"
#include "utils/inval.h"
#include "utils/memdebug.h"
#include "utils/memutils.h"
@@ -179,6 +180,9 @@ static SharedInvalidationMessage *SharedInvalidMessagesArray;
static int numSharedInvalidMessagesArray;
static int maxSharedInvalidMessagesArray;
+#if CACHE_CLOBBER_ALWAYS
+int clobber_cache_depth = 0;
+#endif
/*
* Dynamically-registered callback functions. Current implementation
@@ -689,35 +693,38 @@ AcceptInvalidationMessages(void)
/*
* Test code to force cache flushes anytime a flush could happen.
*
- * If used with CLOBBER_FREED_MEMORY, CLOBBER_CACHE_ALWAYS provides a
+ * CLOBBER_CACHE_ALWAYS (clobber_cache_depth = 1) provides a
* fairly thorough test that the system contains no cache-flush hazards.
* However, it also makes the system unbelievably slow --- the regression
- * tests take about 100 times longer than normal.
- *
- * If you're a glutton for punishment, try CLOBBER_CACHE_RECURSIVELY. This
- * slows things by at least a factor of 10000, so I wouldn't suggest
- * trying to run the entire regression tests that way. It's useful to try
- * a few simple tests, to make sure that cache reload isn't subject to
- * internal cache-flush hazards, but after you've done a few thousand
- * recursive reloads it's unlikely you'll learn more.
+ * tests take about 100 times longer than normal. To mitigate the
+ * slowdown it can be turned on and off per-backend or globally using
+ * the clobber_cache_depth GUC to allow targeted testing of specific
+ * code paths.
+ *
+ * If you're a glutton for punishment, try CLOBBER_CACHE_RECURSIVELY
+ * (set clobber_cache_depth to > 1). This slows things by at least a
+ * factor of 10000, so I wouldn't suggest trying to run the entire
+ * regression tests that way. It's useful to try a few simple tests,
+ * to make sure that cache reload isn't subject to internal cache-flush
+ * hazards, but after you've done a few thousand recursive reloads it's
+ * unlikely you'll learn more.
+ *
+ * You can use postgresql.conf or any other convenient means to disable
+ * clobbering by default then re-enable for targeted sections of tests,
+ * e.g you can edit a specific pg_regress test to SET
+ * clobber_cache_depth=1, then run the suite with:
+ *
+ * PGOPTIONS="-c clobber_cache_depth=0" make -C src/test/regress check
+ *
+ * Remember to use ALTER SYSTEM SET and reload the config if you want
+ * the change to apply to bgworkers, autovacuum, etc too.
*/
#if defined(CLOBBER_CACHE_ALWAYS)
- {
- static bool in_recursion = false;
-
- if (!in_recursion)
- {
- in_recursion = true;
- InvalidateSystemCaches();
- in_recursion = false;
- }
- }
-#elif defined(CLOBBER_CACHE_RECURSIVELY)
+ if (clobber_cache_depth > 0)
{
static int recursion_depth = 0;
- /* Maximum depth is arbitrary depending on your threshold of pain */
- if (recursion_depth < 3)
+ if (recursion_depth < clobber_cache_depth)
{
recursion_depth++;
InvalidateSystemCaches();
diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c
index 596bcb7b84..ae7d46cff0 100644
--- a/src/backend/utils/misc/guc.c
+++ b/src/backend/utils/misc/guc.c
@@ -99,6 +99,7 @@
#include "utils/rls.h"
#include "utils/snapmgr.h"
#include "utils/tzparser.h"
+#include "utils/inval.h"
#include "utils/varlena.h"
#include "utils/xml.h"
@@ -3399,7 +3400,21 @@ static struct config_int ConfigureNamesInt[] =
check_huge_page_size, NULL, NULL
},
+#if defined(CLOBBER_CACHE_ALWAYS) || defined(CLOBBER_CACHE_RECURSIVELY)
+ /* Only available in special debug builds, see pg_config_manual.h */
+ {
+ {"clobber_cache_depth", PGC_SUSET, DEVELOPER_OPTIONS,
+ gettext_noop("Runtime control over CLOBBER_CACHE_ALWAYS and CLOBBER_CACHE_RECURSIVELY",
+ NULL,
+ GUC_NOT_IN_SAMPLE
+ },
+ &clobber_cache_depth,
+ DEFAULT_CLOBBER_CACHE_DEPTH, 0, 5,
+ NULL, NULL, NULL
+ },
+
/* End-of-list marker */
+#endif
{
{NULL, 0, 0, NULL, NULL}, NULL, 0, 0, 0, NULL, NULL, NULL
}
diff --git a/src/include/pg_config_manual.h b/src/include/pg_config_manual.h
index 705dc69c06..615e0833f1 100644
--- a/src/include/pg_config_manual.h
+++ b/src/include/pg_config_manual.h
@@ -6,8 +6,15 @@
* for developers. If you edit any of these, be sure to do a *full*
* rebuild (and an initdb if noted).
*
+ * You can define these by editing pg_config_manual.h but it's usually
+ * sufficient to pass CFLAGS to configure, e.g.
+ *
+ * ./configure --enable-debug --enable-cassert CFLAGS="-DUSE_VALGRIND"
+ *
+ * The flags are persisted in Makefile.global so they will be correctly
+ * applied to extensions, including those built by PGXS.
+ *
* Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
- * Portions Copyright (c) 1994, Regents of the University of California
*
* src/include/pg_config_manual.h
*------------------------------------------------------------------------
@@ -280,7 +287,8 @@
* percentage points even when not run under Valgrind.
*
* You should normally use MEMORY_CONTEXT_CHECKING with USE_VALGRIND;
- * instrumentation of repalloc() is inferior without it.
+ * instrumentation of repalloc() is inferior without it. So it's defined
+ * automatically for USE_VALGRIND builds.
*/
/* #define USE_VALGRIND */
@@ -309,6 +317,22 @@
*/
/* #define RANDOMIZE_ALLOCATED_MEMORY */
+/*
+ * For cache invalidation debugging, clobber caches whenever an invalidation is
+ * possible. For CLOBBER_CACHE_RECURSIVELY, do so even when already processing
+ * invalidations. The GUC clobber_cache_depth is only available when these
+ * options are defined. See inval.c for details.
+ *
+ * It's strongly recommended that you use --enable-cassert and the
+ * implied CLOBBER_FREED_MEMORY if you enable either of these.
+ */
+/* #define CLOBBER_CACHE_ALWAYS */
+/* #define CLOBBER_CACHE_RECURSIVELY */
+
+#if defined(CLOBBER_CACHE_RECURSIVELY) && !defined(CLOBBER_CACHE_ALWAYS)
+#define CLOBBER_CACHE_ALWAYS
+#endif
+
/*
* Define this to force all parse and plan trees to be passed through
* copyObject(), to facilitate catching errors and omissions in
diff --git a/src/include/utils/inval.h b/src/include/utils/inval.h
index 463888c389..ddfae7a9d6 100644
--- a/src/include/utils/inval.h
+++ b/src/include/utils/inval.h
@@ -18,6 +18,14 @@
#include "storage/relfilenode.h"
#include "utils/relcache.h"
+#ifdef CLOBBER_CACHE_ALWAYS
+extern PGDLLIMPORT int clobber_cache_depth;
+#ifdef CLOBBER_CACHE_RECURSIVELY
+#define DEFAULT_CLOBBER_CACHE_DEPTH 3
+#else
+#define DEFAULT_CLOBBER_CACHE_DEPTH 1
+#endif
+#endif
typedef void (*SyscacheCallbackFunction) (Datum arg, int cacheid, uint32 hashvalue);
typedef void (*RelcacheCallbackFunction) (Datum arg, Oid relid);
--
2.26.2
