On Wed, Dec 28, 2022 at 03:13:23PM -0500, Tom Lane wrote: > I'm forced to the conclusion that we have to expose some VACUUM > options if we want this to work well. Attached is a draft patch > that invents SKIP_DATABASE_STATS and ONLY_DATABASE_STATS options > (name bikeshedding welcome) and teaches vacuumdb to use them.
This is the conclusion I arrived at, too. In fact, I was just about to post a similar patch set. I'm attaching it here anyway, but I'm fine with proceeding with your version. I think the main difference between your patch and mine is that I've exposed vac_update_datfrozenxid() via a function instead of a VACUUM option. IMHO that feels a little more natural, but I can't say I feel too strongly about it. -- Nathan Bossart Amazon Web Services: https://aws.amazon.com
>From a46934c1a6cec7a5efe8a25d49507a7a2f59c928 Mon Sep 17 00:00:00 2001 From: Nathan Bossart <nathandboss...@gmail.com> Date: Tue, 27 Dec 2022 15:21:26 -0800 Subject: [PATCH v1 1/3] add UPDATE_DATFROZENXID option to VACUUM --- doc/src/sgml/ref/vacuum.sgml | 22 ++++++++++++++++++++++ src/backend/commands/vacuum.c | 9 ++++++--- src/backend/postmaster/autovacuum.c | 2 +- src/include/commands/vacuum.h | 1 + src/test/regress/expected/vacuum.out | 2 ++ src/test/regress/sql/vacuum.sql | 3 +++ 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/doc/src/sgml/ref/vacuum.sgml b/doc/src/sgml/ref/vacuum.sgml index e14ead8826..1219614507 100644 --- a/doc/src/sgml/ref/vacuum.sgml +++ b/doc/src/sgml/ref/vacuum.sgml @@ -36,6 +36,7 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ <replaceable class="paramet PROCESS_TOAST [ <replaceable class="parameter">boolean</replaceable> ] TRUNCATE [ <replaceable class="parameter">boolean</replaceable> ] PARALLEL <replaceable class="parameter">integer</replaceable> + UPDATE_DATFROZENXID [ <replaceable class="parameter">boolean</replaceable> ] <phrase>and <replaceable class="parameter">table_and_columns</replaceable> is:</phrase> @@ -295,6 +296,27 @@ VACUUM [ FULL ] [ FREEZE ] [ VERBOSE ] [ ANALYZE ] [ <replaceable class="paramet </listitem> </varlistentry> + <varlistentry> + <term><literal>UPDATE_DATFROZENXID</literal></term> + <listitem> + <para> + Specifies that <command>VACUUM</command> should attempt to update the + current database's + <structname>pg_database</structname>.<structfield>datfrozenxid</structfield> + and + <structname>pg_database</structname>.<structfield>datminmxid</structfield> + values to the minimum of the per-table + <structname>pg_class</structname>.<structfield>relfrozenxid</structfield> + and + <structname>pg_class</structname>.<structfield>relminmxid</structfield> + values. This is usually the desired behavior and is the default. + Setting this option to false may be useful to avoid the extra work of + scanning all of the database's tables when executing several independent + <command>VACUUM</command> commands. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><replaceable class="parameter">boolean</replaceable></term> <listitem> diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index ba965b8c7b..51537aa5ba 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -114,6 +114,7 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) bool full = false; bool disable_page_skipping = false; bool process_toast = true; + bool update_datfrozenxid = vacstmt->is_vacuumcmd; ListCell *lc; /* index_cleanup and truncate values unspecified for now */ @@ -200,6 +201,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) params.nworkers = nworkers; } } + else if (strcmp(opt->defname, "update_datfrozenxid") == 0) + update_datfrozenxid = defGetBoolean(opt); else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), @@ -216,7 +219,8 @@ ExecVacuum(ParseState *pstate, VacuumStmt *vacstmt, bool isTopLevel) (freeze ? VACOPT_FREEZE : 0) | (full ? VACOPT_FULL : 0) | (disable_page_skipping ? VACOPT_DISABLE_PAGE_SKIPPING : 0) | - (process_toast ? VACOPT_PROCESS_TOAST : 0); + (process_toast ? VACOPT_PROCESS_TOAST : 0) | + (update_datfrozenxid ? VACOPT_UPDATE_DATFROZENXID : 0); /* sanity checks on options */ Assert(params.options & (VACOPT_VACUUM | VACOPT_ANALYZE)); @@ -528,11 +532,10 @@ vacuum(List *relations, VacuumParams *params, StartTransactionCommand(); } - if ((params->options & VACOPT_VACUUM) && !IsAutoVacuumWorkerProcess()) + if (params->options & VACOPT_UPDATE_DATFROZENXID) { /* * Update pg_database.datfrozenxid, and truncate pg_xact if possible. - * (autovacuum.c does this for itself.) */ vac_update_datfrozenxid(); } diff --git a/src/backend/postmaster/autovacuum.c b/src/backend/postmaster/autovacuum.c index 0746d80224..edc219c8f5 100644 --- a/src/backend/postmaster/autovacuum.c +++ b/src/backend/postmaster/autovacuum.c @@ -2854,7 +2854,7 @@ table_recheck_autovac(Oid relid, HTAB *table_toast_map, tab->at_relid = relid; tab->at_sharedrel = classForm->relisshared; - /* Note that this skips toast relations */ + /* Note that this skips toast relations and updating datfrozenxid */ tab->at_params.options = (dovacuum ? VACOPT_VACUUM : 0) | (doanalyze ? VACOPT_ANALYZE : 0) | (!wraparound ? VACOPT_SKIP_LOCKED : 0); diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 2f274f2bec..700489040e 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -188,6 +188,7 @@ typedef struct VacAttrStats #define VACOPT_SKIP_LOCKED 0x20 /* skip if cannot get lock */ #define VACOPT_PROCESS_TOAST 0x40 /* process the TOAST table, if any */ #define VACOPT_DISABLE_PAGE_SKIPPING 0x80 /* don't skip any pages */ +#define VACOPT_UPDATE_DATFROZENXID 0x100 /* update datfrozenxid afterwards */ /* * Values used by index_cleanup and truncate params. diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index 0035d158b7..19c405417d 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -282,6 +282,8 @@ ALTER TABLE vactst ALTER COLUMN t SET STORAGE EXTERNAL; VACUUM (PROCESS_TOAST FALSE) vactst; VACUUM (PROCESS_TOAST FALSE, FULL) vactst; ERROR: PROCESS_TOAST required with VACUUM FULL +-- UPDATE_DATFROZENXID option +VACUUM (UPDATE_DATFROZENXID FALSE) vactst; DROP TABLE vaccluster; DROP TABLE vactst; DROP TABLE vacparted; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index 9faa8a34a6..8ab3a93459 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -237,6 +237,9 @@ ALTER TABLE vactst ALTER COLUMN t SET STORAGE EXTERNAL; VACUUM (PROCESS_TOAST FALSE) vactst; VACUUM (PROCESS_TOAST FALSE, FULL) vactst; +-- UPDATE_DATFROZENXID option +VACUUM (UPDATE_DATFROZENXID FALSE) vactst; + DROP TABLE vaccluster; DROP TABLE vactst; DROP TABLE vacparted; -- 2.25.1
>From 5219ecc2f08d4fa76bc14b2c498f126ad96f0ae2 Mon Sep 17 00:00:00 2001 From: Nathan Bossart <nathandboss...@gmail.com> Date: Tue, 27 Dec 2022 16:20:01 -0800 Subject: [PATCH v1 2/3] introduce pg_update_datfrozenxid --- doc/src/sgml/func.sgml | 44 ++++++++++++++++++++++++++++ src/backend/commands/vacuum.c | 12 ++++++++ src/include/catalog/pg_proc.dat | 4 +++ src/include/commands/vacuum.h | 1 + src/test/regress/expected/vacuum.out | 6 ++++ src/test/regress/sql/vacuum.sql | 1 + 6 files changed, 68 insertions(+) diff --git a/doc/src/sgml/func.sgml b/doc/src/sgml/func.sgml index 3bf8d021c3..a97c7794f2 100644 --- a/doc/src/sgml/func.sgml +++ b/doc/src/sgml/func.sgml @@ -27285,6 +27285,50 @@ postgres=# SELECT '0/0'::pg_lsn + pd.segment_number * ps.setting::int + :offset </tgroup> </table> + <para> + <xref linkend="functions-admin-db"/> lists functions used to manage the + current database. + </para> + + <table id="functions-admin-db"> + <title>Database Management Functions</title> + <tgroup cols="1"> + <thead> + <row> + <entry role="func_table_entry"><para role="func_signature"> + Function + </para> + <para> + Description + </para></entry> + </row> + </thead> + + <tbody> + <row> + <entry role="func_table_entry"><para role="func_signature"> + <indexterm> + <primary>pg_update_datfrozenxid</primary> + </indexterm> + <function>pg_update_datfrozenxid</function> () + <returnvalue>void</returnvalue> + </para> + <para> + Updates the current database's + <structname>pg_database</structname>.<structfield>datfrozenxid</structfield> + and + <structname>pg_database</structname>.<structfield>relminmxid</structfield> + values to the minimum of the per-table + <structname>pg_class</structname>.<structfield>relfrozenxid</structfield> + and + <structname>pg_class</structname>.<structfield>relminmxid</structfield> + values. + </para></entry> + </row> + </tbody> + </tgroup> + </table> + <para> <xref linkend="functions-admin-collation"/> lists functions used to manage collations. diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 51537aa5ba..9b5049da2d 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -39,6 +39,7 @@ #include "catalog/pg_database.h" #include "catalog/pg_inherits.h" #include "catalog/pg_namespace.h" +#include "catalog/pg_proc.h" #include "commands/cluster.h" #include "commands/defrem.h" #include "commands/vacuum.h" @@ -1415,6 +1416,17 @@ vac_update_relstats(Relation relation, } +/* + * Allow calling vac_update_datfrozenxid() via SQL. + */ +Datum +pg_update_datfrozenxid(PG_FUNCTION_ARGS) +{ + vac_update_datfrozenxid(); + PG_RETURN_VOID(); +} + + /* * vac_update_datfrozenxid() -- update pg_database.datfrozenxid for our DB * diff --git a/src/include/catalog/pg_proc.dat b/src/include/catalog/pg_proc.dat index 7056c95371..23efafba49 100644 --- a/src/include/catalog/pg_proc.dat +++ b/src/include/catalog/pg_proc.dat @@ -11817,6 +11817,10 @@ proargmodes => '{i,o,o,o}', proargnames => '{slot_name,name,size,modification}', prosrc => 'pg_ls_replslotdir' }, +{ oid => '2173', + descr => 'update datfrozenxid and datminmxid for current database', + proname => 'pg_update_datfrozenxid', provolatile => 'v', prorettype => 'void', + proargtypes => '', prosrc => 'pg_update_datfrozenxid' }, # hash partitioning constraint function { oid => '5028', descr => 'hash partition CHECK constraint', diff --git a/src/include/commands/vacuum.h b/src/include/commands/vacuum.h index 700489040e..d594054186 100644 --- a/src/include/commands/vacuum.h +++ b/src/include/commands/vacuum.h @@ -329,6 +329,7 @@ extern void vac_update_relstats(Relation relation, extern bool vacuum_get_cutoffs(Relation rel, const VacuumParams *params, struct VacuumCutoffs *cutoffs); extern bool vacuum_xid_failsafe_check(const struct VacuumCutoffs *cutoffs); +extern Datum pg_update_datfrozenxid(PG_FUNCTION_ARGS); extern void vac_update_datfrozenxid(void); extern void vacuum_delay_point(void); extern bool vacuum_is_permitted_for_relation(Oid relid, Form_pg_class reltuple, diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out index 19c405417d..b559d2fe84 100644 --- a/src/test/regress/expected/vacuum.out +++ b/src/test/regress/expected/vacuum.out @@ -284,6 +284,12 @@ VACUUM (PROCESS_TOAST FALSE, FULL) vactst; ERROR: PROCESS_TOAST required with VACUUM FULL -- UPDATE_DATFROZENXID option VACUUM (UPDATE_DATFROZENXID FALSE) vactst; +SELECT pg_update_datfrozenxid(); + pg_update_datfrozenxid +------------------------ + +(1 row) + DROP TABLE vaccluster; DROP TABLE vactst; DROP TABLE vacparted; diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql index 8ab3a93459..4a19f2fa30 100644 --- a/src/test/regress/sql/vacuum.sql +++ b/src/test/regress/sql/vacuum.sql @@ -239,6 +239,7 @@ VACUUM (PROCESS_TOAST FALSE, FULL) vactst; -- UPDATE_DATFROZENXID option VACUUM (UPDATE_DATFROZENXID FALSE) vactst; +SELECT pg_update_datfrozenxid(); DROP TABLE vaccluster; DROP TABLE vactst; -- 2.25.1
>From 8b2ef45d668722a1728f4f3f2900654f8def1a10 Mon Sep 17 00:00:00 2001 From: Nathan Bossart <nathandboss...@gmail.com> Date: Tue, 27 Dec 2022 16:59:31 -0800 Subject: [PATCH v1 3/3] update datfrozenxid/datminmxid once per database in vacuumdb --- src/bin/scripts/t/100_vacuumdb.pl | 26 ++++++++-------- src/bin/scripts/vacuumdb.c | 52 +++++++++++++++++++++++++++++++ src/fe_utils/parallel_slot.c | 4 +++ 3 files changed, 69 insertions(+), 13 deletions(-) diff --git a/src/bin/scripts/t/100_vacuumdb.pl b/src/bin/scripts/t/100_vacuumdb.pl index e5343774fe..39eaae4e8e 100644 --- a/src/bin/scripts/t/100_vacuumdb.pl +++ b/src/bin/scripts/t/100_vacuumdb.pl @@ -22,15 +22,15 @@ $node->issues_sql_like( 'SQL VACUUM run'); $node->issues_sql_like( [ 'vacuumdb', '-f', 'postgres' ], - qr/statement: VACUUM \(FULL\).*;/, + qr/statement: VACUUM \(FULL, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb -f'); $node->issues_sql_like( [ 'vacuumdb', '-F', 'postgres' ], - qr/statement: VACUUM \(FREEZE\).*;/, + qr/statement: VACUUM \(FREEZE, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb -F'); $node->issues_sql_like( [ 'vacuumdb', '-zj2', 'postgres' ], - qr/statement: VACUUM \(ANALYZE\).*;/, + qr/statement: VACUUM \(ANALYZE, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb -zj2'); $node->issues_sql_like( [ 'vacuumdb', '-Z', 'postgres' ], @@ -38,11 +38,11 @@ $node->issues_sql_like( 'vacuumdb -Z'); $node->issues_sql_like( [ 'vacuumdb', '--disable-page-skipping', 'postgres' ], - qr/statement: VACUUM \(DISABLE_PAGE_SKIPPING\).*;/, + qr/statement: VACUUM \(DISABLE_PAGE_SKIPPING, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb --disable-page-skipping'); $node->issues_sql_like( [ 'vacuumdb', '--skip-locked', 'postgres' ], - qr/statement: VACUUM \(SKIP_LOCKED\).*;/, + qr/statement: VACUUM \(SKIP_LOCKED, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb --skip-locked'); $node->issues_sql_like( [ 'vacuumdb', '--skip-locked', '--analyze-only', 'postgres' ], @@ -53,32 +53,32 @@ $node->command_fails( '--analyze-only and --disable-page-skipping specified together'); $node->issues_sql_like( [ 'vacuumdb', '--no-index-cleanup', 'postgres' ], - qr/statement: VACUUM \(INDEX_CLEANUP FALSE\).*;/, + qr/statement: VACUUM \(INDEX_CLEANUP FALSE, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb --no-index-cleanup'); $node->command_fails( [ 'vacuumdb', '--analyze-only', '--no-index-cleanup', 'postgres' ], '--analyze-only and --no-index-cleanup specified together'); $node->issues_sql_like( [ 'vacuumdb', '--no-truncate', 'postgres' ], - qr/statement: VACUUM \(TRUNCATE FALSE\).*;/, + qr/statement: VACUUM \(TRUNCATE FALSE, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb --no-truncate'); $node->command_fails( [ 'vacuumdb', '--analyze-only', '--no-truncate', 'postgres' ], '--analyze-only and --no-truncate specified together'); $node->issues_sql_like( [ 'vacuumdb', '--no-process-toast', 'postgres' ], - qr/statement: VACUUM \(PROCESS_TOAST FALSE\).*;/, + qr/statement: VACUUM \(PROCESS_TOAST FALSE, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb --no-process-toast'); $node->command_fails( [ 'vacuumdb', '--analyze-only', '--no-process-toast', 'postgres' ], '--analyze-only and --no-process-toast specified together'); $node->issues_sql_like( [ 'vacuumdb', '-P', 2, 'postgres' ], - qr/statement: VACUUM \(PARALLEL 2\).*;/, + qr/statement: VACUUM \(PARALLEL 2, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb -P 2'); $node->issues_sql_like( [ 'vacuumdb', '-P', 0, 'postgres' ], - qr/statement: VACUUM \(PARALLEL 0\).*;/, + qr/statement: VACUUM \(PARALLEL 0, UPDATE_DATFROZENXID FALSE\).*;/, 'vacuumdb -P 0'); $node->command_ok([qw(vacuumdb -Z --table=pg_am dbname=template1)], 'vacuumdb with connection string'); @@ -119,7 +119,7 @@ $node->command_fails([ 'vacuumdb', '-P', -1, 'postgres' ], 'negative parallel degree'); $node->issues_sql_like( [ 'vacuumdb', '--analyze', '--table', 'vactable(a, b)', 'postgres' ], - qr/statement: VACUUM \(ANALYZE\) public.vactable\(a, b\);/, + qr/statement: VACUUM \(ANALYZE, UPDATE_DATFROZENXID FALSE\) public.vactable\(a, b\);/, 'vacuumdb --analyze with complete column list'); $node->issues_sql_like( [ 'vacuumdb', '--analyze-only', '--table', 'vactable(b)', 'postgres' ], @@ -150,11 +150,11 @@ $node->issues_sql_like( 'vacuumdb --table --min-xid-age'); $node->issues_sql_like( [ 'vacuumdb', '--schema', '"Foo"', 'postgres' ], - qr/VACUUM "Foo".bar/, + qr/VACUUM \(UPDATE_DATFROZENXID FALSE\) "Foo".bar/, 'vacuumdb --schema'); $node->issues_sql_like( [ 'vacuumdb', '--exclude-schema', '"Foo"', 'postgres' ], - qr/(?:(?!VACUUM "Foo".bar).)*/, + qr/(?:(?!VACUUM \(UPDATE_DATFROZENXID FALSE\) "Foo".bar).)*/, 'vacuumdb --exclude-schema'); $node->command_fails_like( [ 'vacuumdb', '-N', 'pg_catalog', '-t', 'pg_class', 'postgres', ], diff --git a/src/bin/scripts/vacuumdb.c b/src/bin/scripts/vacuumdb.c index 272e37d290..3ba1a37049 100644 --- a/src/bin/scripts/vacuumdb.c +++ b/src/bin/scripts/vacuumdb.c @@ -80,6 +80,8 @@ static void run_vacuum_command(PGconn *conn, const char *sql, bool echo, static void help(const char *progname); +static bool UpdateDatfrozenxidHandler(PGresult *res, PGconn *conn, void *context); + void check_objfilter(void); /* For analyze-in-stages mode */ @@ -791,6 +793,29 @@ vacuum_one_database(ConnParams *cparams, if (!ParallelSlotsWaitCompletion(sa)) failed = true; + else if (!vacopts->analyze_only && PQserverVersion(conn) >= 160000) + { + /* v16 and later updates datfrozenxid/datminmxid at the end */ + const char *cmd = "SELECT pg_catalog.pg_update_datfrozenxid()"; + ParallelSlot *free_slot = ParallelSlotsGetIdle(sa, NULL); + + if (!free_slot) + { + failed = true; + goto finish; + } + + ParallelSlotSetHandler(free_slot, UpdateDatfrozenxidHandler, NULL); + + if (echo) + printf("%s\n", cmd); + + if (PQsendQuery(free_slot->connection, cmd) != 1) + pg_log_error("\"pg_update_datfrozenxid()\" failed: %s", + PQerrorMessage(free_slot->connection)); + + failed = !ParallelSlotsWaitCompletion(sa); + } finish: ParallelSlotsTerminate(sa); @@ -802,6 +827,27 @@ finish: exit(1); } +/* + * UpdateDatfrozenxidHandler + * + * ParallelSlotResultHandler for result of pg_update_datfrozenxid(). Requires + * that the result status is PGRES_TUPLES_OK. Otherwise, logs an error and + * terminates further processing. + * + * res: PGresult from the query executed on the slot's connection + * conn: connection belonging to the slot + * context: unused + */ +static bool +UpdateDatfrozenxidHandler(PGresult *res, PGconn *conn, void *context) +{ + if (PQresultStatus(res) != PGRES_TUPLES_OK) + pg_log_error("\"pg_update_datfrozenxid()\" failed: %s", + PQerrorMessage(conn)); + + return (PQresultStatus(res) == PGRES_TUPLES_OK); +} + /* * Vacuum/analyze all connectable databases. * @@ -992,6 +1038,12 @@ prepare_vacuum_command(PQExpBuffer sql, int serverVersion, vacopts->parallel_workers); sep = comma; } + if (serverVersion >= 160000) + { + /* v16 and later calls pg_update_datfrozenxid() at the end */ + appendPQExpBuffer(sql, "%sUPDATE_DATFROZENXID FALSE", sep); + sep = comma; + } if (sep != paren) appendPQExpBufferChar(sql, ')'); } diff --git a/src/fe_utils/parallel_slot.c b/src/fe_utils/parallel_slot.c index 767256757f..aed9e40608 100644 --- a/src/fe_utils/parallel_slot.c +++ b/src/fe_utils/parallel_slot.c @@ -475,6 +475,10 @@ ParallelSlotsWaitCompletion(ParallelSlotArray *sa) continue; if (!consumeQueryResult(&sa->slots[i])) return false; + + /* mark slot as idle so it can be reused */ + sa->slots[i].inUse = false; + ParallelSlotClearHandler(&sa->slots[i]); } return true; -- 2.25.1