diff --git a/src/backend/catalog/system_views.sql b/src/backend/catalog/system_views.sql index ccc030f..9b8024c 100644 --- a/src/backend/catalog/system_views.sql +++ b/src/backend/catalog/system_views.sql @@ -631,6 +631,19 @@ CREATE VIEW pg_stat_activity AS WHERE S.datid = D.oid AND S.usesysid = U.oid; +CREATE VIEW pg_stat_vacuum_progress AS + SELECT + S.pid, + S.total_pages, + S.scanned_pages, + S.total_heap_pages, + S.scanned_heap_pages, + S.total_index_pages, + S.scanned_index_pages, + S.percent_complete, + S.table_name + FROM pg_stat_get_vacuum_progress() AS S; + CREATE VIEW pg_stat_replication AS SELECT S.pid, diff --git a/src/backend/commands/vacuum.c b/src/backend/commands/vacuum.c index 6d55148..b9ed102 100644 --- a/src/backend/commands/vacuum.c +++ b/src/backend/commands/vacuum.c @@ -284,6 +284,7 @@ vacuum(int options, RangeVar *relation, Oid relid, VacuumParams *params, VacuumPageMiss = 0; VacuumPageDirty = 0; + pgstat_report_activityflag(ACTIVITY_IS_VACUUM); /* * Loop to process each selected relation. */ @@ -355,6 +356,7 @@ vacuum(int options, RangeVar *relation, Oid relid, VacuumParams *params, vac_update_datfrozenxid(); } + pgstat_reset_activityflag; /* * Clean up working storage --- note we must do this after * StartTransactionCommand, else we might be trying to delete the active diff --git a/src/backend/commands/vacuumlazy.c b/src/backend/commands/vacuumlazy.c index a01cfb4..e2cb2b3 100644 --- a/src/backend/commands/vacuumlazy.c +++ b/src/backend/commands/vacuumlazy.c @@ -439,9 +439,17 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, Relation *Irel, int nindexes, bool scan_all) { BlockNumber nblocks, - blkno; + blkno, + total_pages, + scanned_total_pages = 0, + total_heap_pages, + rel_index_pages = 0, + total_index_pages = 0, + scanned_index_pages = 0; HeapTupleData tuple; char *relname; + char *schemaname; + int schemaname_len; BlockNumber empty_pages, vacuumed_pages; double num_tuples, @@ -456,14 +464,19 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, bool skipping_all_visible_blocks; xl_heap_freeze_tuple *frozen; StringInfoData buf; + uint32 progress_param[N_PROGRESS_PARAM]; + double progress_param_float[N_PROGRESS_PARAM]; + char progress_message[PROGRESS_MESSAGE_LENGTH][N_PROGRESS_PARAM]; pg_rusage_init(&ru0); relname = RelationGetRelationName(onerel); + schemaname = get_namespace_name(RelationGetNamespace(onerel)); ereport(elevel, (errmsg("vacuuming \"%s.%s\"", get_namespace_name(RelationGetNamespace(onerel)), relname))); + schemaname_len = strlen(schemaname); empty_pages = vacuumed_pages = 0; num_tuples = tups_vacuumed = nkeep = nunused = 0; @@ -471,7 +484,14 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, indstats = (IndexBulkDeleteResult **) palloc0(nindexes * sizeof(IndexBulkDeleteResult *)); - nblocks = RelationGetNumberOfBlocks(onerel); + total_heap_pages = nblocks = RelationGetNumberOfBlocks(onerel); + + for (i = 0; i < nindexes; i++) + rel_index_pages += RelationGetNumberOfBlocks(Irel[i]); + + total_index_pages = rel_index_pages; + total_pages = total_heap_pages + rel_index_pages; + vacrelstats->rel_pages = nblocks; vacrelstats->scanned_pages = 0; vacrelstats->nonempty_pages = 0; @@ -520,7 +540,14 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, vacuum_delay_point(); } if (next_not_all_visible_block >= SKIP_PAGES_THRESHOLD) + { skipping_all_visible_blocks = true; + if(!scan_all) + { + total_heap_pages = total_heap_pages - next_not_all_visible_block; + total_pages = total_pages - next_not_all_visible_block; + } + } else skipping_all_visible_blocks = false; @@ -559,7 +586,14 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, * following blocks. */ if (next_not_all_visible_block - blkno > SKIP_PAGES_THRESHOLD) + { skipping_all_visible_blocks = true; + if(!scan_all) + { + total_heap_pages = total_heap_pages - (next_not_all_visible_block - blkno); + total_pages = total_pages - (next_not_all_visible_block - blkno); + } + } else skipping_all_visible_blocks = false; all_visible_according_to_vm = false; @@ -596,11 +630,42 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, /* Log cleanup info before we touch indexes */ vacuum_log_cleanup_info(onerel, vacrelstats); + /* + * If passes through indexes exceed 1 add + * pages equal to rel_index_pages to the count of + * total pages to be scanned. + */ + if (vacrelstats->num_index_scans >= 1) + { + total_index_pages = total_index_pages + rel_index_pages; + total_pages = total_heap_pages + total_index_pages; + } + /* Remove index entries */ for (i = 0; i < nindexes; i++) + { lazy_vacuum_index(Irel[i], &indstats[i], vacrelstats); + scanned_index_pages += RelationGetNumberOfBlocks(Irel[i]); + scanned_total_pages = scanned_total_pages + RelationGetNumberOfBlocks(Irel[i]); + + /* Report progress to the statistics collector */ + progress_param[0] = total_pages; + progress_param[1] = scanned_total_pages; + progress_param[2] = total_heap_pages; + progress_param[3] = vacrelstats->scanned_pages; + progress_param[4] = total_index_pages; + progress_param[5] = scanned_index_pages; + + memcpy((char *) progress_message[0], schemaname, schemaname_len); + progress_message[0][schemaname_len] = '\0'; + strcat(progress_message[0],"."); + strcat(progress_message[0],relname); + + pgstat_report_progress(progress_param, 6, progress_param_float, + 0, progress_message, 1); + } /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); @@ -658,6 +723,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, UnlockReleaseBuffer(buf); vacrelstats->scanned_pages++; vacrelstats->pinskipped_pages++; + scanned_total_pages++; continue; } LockBuffer(buf, BUFFER_LOCK_UNLOCK); @@ -666,6 +732,7 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, } vacrelstats->scanned_pages++; + scanned_total_pages++; page = BufferGetPage(buf); @@ -1062,6 +1129,28 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, */ if (vacrelstats->num_dead_tuples == prev_dead_count) RecordPageWithFreeSpace(onerel, blkno, freespace); + + /* + * Reporting vacuum progress to statistics collector + */ + if (blkno == nblocks - 1 && vacrelstats->num_dead_tuples == 0 && nindexes != 0 + && vacrelstats->num_index_scans == 0) + total_pages = total_pages - total_index_pages; + + progress_param[0] = total_pages; + progress_param[1] = scanned_total_pages; + progress_param[2] = total_heap_pages; + progress_param[3] = vacrelstats->scanned_pages; + progress_param[4] = total_index_pages; + progress_param[5] = scanned_index_pages; + + memcpy((char *) progress_message[0], schemaname, schemaname_len); + progress_message[0][schemaname_len] = '\0'; + strcat(progress_message[0],"."); + strcat(progress_message[0],relname); + + pgstat_report_progress(progress_param, 6, progress_param_float, 0, + progress_message, 1); } pfree(frozen); @@ -1093,11 +1182,41 @@ lazy_scan_heap(Relation onerel, LVRelStats *vacrelstats, /* Log cleanup info before we touch indexes */ vacuum_log_cleanup_info(onerel, vacrelstats); + /* + * If passes through indexes exceed 1 add + * pages equal to rel_index_pages to the count of + * total pages to be scanned. + */ + if (vacrelstats->num_index_scans >= 1) + { + total_index_pages = total_index_pages + rel_index_pages; + total_pages = total_heap_pages + total_index_pages; + } + /* Remove index entries */ for (i = 0; i < nindexes; i++) + { lazy_vacuum_index(Irel[i], &indstats[i], vacrelstats); + scanned_index_pages += RelationGetNumberOfBlocks(Irel[i]); + scanned_total_pages = scanned_total_pages + RelationGetNumberOfBlocks(Irel[i]); + /* Report progress to the statistics collector */ + progress_param[0] = total_pages; + progress_param[1] = scanned_total_pages; + progress_param[2] = total_heap_pages; + progress_param[3] = vacrelstats->scanned_pages; + progress_param[4] = total_index_pages; + progress_param[5] = scanned_index_pages; + + memcpy((char *) progress_message[0], schemaname, schemaname_len); + progress_message[0][schemaname_len] = '\0'; + strcat(progress_message[0],"."); + strcat(progress_message[0],relname); + + pgstat_report_progress(progress_param, 6, progress_param_float, 0, + progress_message, 1); + } /* Remove tuples from heap */ lazy_vacuum_heap(onerel, vacrelstats); vacrelstats->num_index_scans++; diff --git a/src/backend/postmaster/pgstat.c b/src/backend/postmaster/pgstat.c index ab018c4..f97759e 100644 --- a/src/backend/postmaster/pgstat.c +++ b/src/backend/postmaster/pgstat.c @@ -2851,6 +2851,55 @@ pgstat_report_activity(BackendState state, const char *cmd_str) pgstat_increment_changecount_after(beentry); } +/* --------------------------------------------- + * Called from VACUUM after every heap page scan or index scan + * to report progress + * --------------------------------------------- + */ + +void +pgstat_report_progress(uint *param1, int num_of_int, double *param2, int num_of_float, + char param3[PROGRESS_MESSAGE_LENGTH][N_PROGRESS_PARAM], + int num_of_string) +{ + volatile PgBackendStatus *beentry = MyBEEntry; + int i; + + if (!beentry) + return; + + if (!pgstat_track_activities) + return; + + pgstat_increment_changecount_before(beentry); + + for(i = 0; i < num_of_int; i++) + { + beentry->progress_param[i] = param1[i]; + } + for (i = 0; i < num_of_float; i++) + { + beentry->progress_param_float[i] = param2[i]; + } + for (i = 0; i < num_of_string; i++) + { + strcpy((char *)beentry->progress_message[i], param3[i]); + } + pgstat_increment_changecount_after(beentry); +} + +void +pgstat_report_activityflag(activity_flag) +{ + PgBackendStatus *beentry = MyBEEntry; + beentry->flag_activity = activity_flag; +} +void +pgstat_reset_activityflag() +{ + PgBackendStatus *beentry = MyBEEntry; + beentry->flag_activity = 0; +} /* ---------- * pgstat_report_appname() - * diff --git a/src/backend/utils/adt/pgstatfuncs.c b/src/backend/utils/adt/pgstatfuncs.c index f7c9bf6..e113361 100644 --- a/src/backend/utils/adt/pgstatfuncs.c +++ b/src/backend/utils/adt/pgstatfuncs.c @@ -53,6 +53,7 @@ extern Datum pg_stat_get_function_self_time(PG_FUNCTION_ARGS); extern Datum pg_stat_get_backend_idset(PG_FUNCTION_ARGS); extern Datum pg_stat_get_activity(PG_FUNCTION_ARGS); +extern Datum pg_stat_get_vacuum_progress(PG_FUNCTION_ARGS); extern Datum pg_backend_pid(PG_FUNCTION_ARGS); extern Datum pg_stat_get_backend_pid(PG_FUNCTION_ARGS); extern Datum pg_stat_get_backend_dbid(PG_FUNCTION_ARGS); @@ -523,7 +524,130 @@ pg_stat_get_backend_idset(PG_FUNCTION_ARGS) SRF_RETURN_DONE(funcctx); } } +/* + * Returns VACUUM progress values stored by each backend + * executing VACUUM. + */ +Datum +pg_stat_get_vacuum_progress(PG_FUNCTION_ARGS) +{ +#define PG_STAT_GET_PROGRESS_COLS 30 + int num_backends = pgstat_fetch_stat_numbackends(); + int curr_backend; + TupleDesc tupdesc; + Tuplestorestate *tupstore; + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + MemoryContext per_query_ctx; + MemoryContext oldcontext; + + /* check to see if caller supports us returning a tuplestore */ + if (rsinfo == NULL || !IsA(rsinfo, ReturnSetInfo)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("set-valued function called in context that cannot accept a set"))); + if (!(rsinfo->allowedModes & SFRM_Materialize)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("materialize mode required, but it is not " \ + "allowed in this context"))); + + /* Build a tuple descriptor for our result type */ + if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) + elog(ERROR, "return type must be a row type"); + + per_query_ctx = rsinfo->econtext->ecxt_per_query_memory; + oldcontext = MemoryContextSwitchTo(per_query_ctx); + + tupstore = tuplestore_begin_heap(true, false, work_mem); + rsinfo->returnMode = SFRM_Materialize; + rsinfo->setResult = tupstore; + rsinfo->setDesc = tupdesc; + MemoryContextSwitchTo(oldcontext); + + for (curr_backend = 1; curr_backend <= num_backends; curr_backend++) + { + Datum values[PG_STAT_GET_PROGRESS_COLS]; + bool nulls[PG_STAT_GET_PROGRESS_COLS]; + LocalPgBackendStatus *local_beentry; + PgBackendStatus *beentry; + + MemSet(values, 0, sizeof(values)); + MemSet(nulls, 0, sizeof(nulls)); + + local_beentry = pgstat_fetch_stat_local_beentry(curr_backend); + if (!local_beentry) + continue; + beentry = &local_beentry->backendStatus; + + /* Report values for only those backends which are running VACUUM */ + if(!beentry || beentry->flag_activity != ACTIVITY_IS_VACUUM) + continue; + + values[0] = Int32GetDatum(beentry->st_procpid); + + /* Progress can only be viewed by role member */ + if (has_privs_of_role(GetUserId(), beentry->st_userid)) + { + if (beentry->progress_param[0] != 0) + values[1] = UInt32GetDatum(beentry->progress_param[0]); + else + nulls[1] = true; + + if (beentry->progress_param[1] != 0) + values[2] = UInt32GetDatum(beentry->progress_param[1]); + else + nulls[2] = true; + + if (beentry->progress_param[2] != 0) + values[3] = UInt32GetDatum(beentry->progress_param[2]); + else + nulls[3] = true; + + if (beentry->progress_param[3] != 0) + values[4] = UInt32GetDatum(beentry->progress_param[3]); + else + nulls[4] = true; + + if (beentry->progress_param[4] != 0) + values[5] = UInt32GetDatum(beentry->progress_param[4]); + else + nulls[5] = true; + + if (beentry->progress_param[5] != 0) + values[6] = UInt32GetDatum(beentry->progress_param[5]); + else + nulls[6] = true; + + if (beentry->progress_param[0] != 0) + values[7] = Float8GetDatum(beentry->progress_param[1] * 100 / beentry->progress_param[0]); + else + nulls[7] = true; + + if(beentry->progress_message[0]) + values[8] = CStringGetTextDatum(beentry->progress_message[0]); + else + nulls[8] = true; + } + else + { + values[1] = CStringGetTextDatum(""); + nulls[2] = true; + nulls[3] = true; + nulls[4] = true; + nulls[5] = true; + nulls[6] = true; + nulls[7] = true; + nulls[8] = true; + } + + tuplestore_putvalues(tupstore, tupdesc, values, nulls); + } + /* clean up and return the tuplestore */ + tuplestore_donestoring(tupstore); + + return (Datum) 0; +} /* * Returns activity of PG backends. */ diff --git a/src/include/catalog/pg_proc.h b/src/include/catalog/pg_proc.h index eb55b3a..a19f325 100644 --- a/src/include/catalog/pg_proc.h +++ b/src/include/catalog/pg_proc.h @@ -2781,6 +2781,8 @@ DATA(insert OID = 1936 ( pg_stat_get_backend_idset PGNSP PGUID 12 1 100 0 0 f DESCR("statistics: currently active backend IDs"); DATA(insert OID = 2022 ( pg_stat_get_activity PGNSP PGUID 12 1 100 0 0 f f f f f t s r 1 0 2249 "23" "{23,26,23,26,25,25,25,16,1184,1184,1184,1184,869,25,23,28,28,16,25,25,23,16,25}" "{i,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o}" "{pid,datid,pid,usesysid,application_name,state,query,waiting,xact_start,query_start,backend_start,state_change,client_addr,client_hostname,client_port,backend_xid,backend_xmin,ssl,sslversion,sslcipher,sslbits,sslcompression,sslclientdn}" _null_ _null_ pg_stat_get_activity _null_ _null_ _null_ )); DESCR("statistics: information about currently active backends"); +DATA(insert OID = 3308 ( pg_stat_get_vacuum_progress PGNSP PGUID 12 1 1 0 0 f f f f f t s r 0 0 2249 "" "{23,23,23,23,23,23,23,701,25}" "{o,o,o,o,o,o,o,o,o}" "{pid,total_pages,scanned_pages,total_heap_pages,scanned_heap_pages,total_index_pages,scanned_index_pages,percent_complete,table_name}" _null_ _null_ pg_stat_get_vacuum_progress _null_ _null_ _null_ )); +DESCR("statistics: information about progress of backends running VACUUM"); DATA(insert OID = 3099 ( pg_stat_get_wal_senders PGNSP PGUID 12 1 10 0 0 f f f f f t s r 0 0 2249 "" "{23,25,3220,3220,3220,3220,23,25}" "{o,o,o,o,o,o,o,o}" "{pid,state,sent_location,write_location,flush_location,replay_location,sync_priority,sync_state}" _null_ _null_ pg_stat_get_wal_senders _null_ _null_ _null_ )); DESCR("statistics: information about currently active replication"); DATA(insert OID = 2026 ( pg_backend_pid PGNSP PGUID 12 1 0 0 0 f f f f t f s r 0 0 23 "" _null_ _null_ _null_ _null_ _null_ pg_backend_pid _null_ _null_ _null_ )); diff --git a/src/include/pgstat.h b/src/include/pgstat.h index 9ecc163..4214b3d 100644 --- a/src/include/pgstat.h +++ b/src/include/pgstat.h @@ -20,6 +20,7 @@ #include "utils/hsearch.h" #include "utils/relcache.h" +#include "storage/block.h" /* ---------- * Paths for the statistics files (relative to installation's $PGDATA). @@ -205,6 +206,8 @@ typedef struct PgStat_MsgHdr #define PGSTAT_MAX_MSG_SIZE 1000 #define PGSTAT_MSG_PAYLOAD (PGSTAT_MAX_MSG_SIZE - sizeof(PgStat_MsgHdr)) +#define N_PROGRESS_PARAM 10 +#define PROGRESS_MESSAGE_LENGTH 30 /* ---------- * PgStat_MsgDummy A dummy message, ignored by the collector @@ -776,6 +779,12 @@ typedef struct PgBackendStatus /* current command string; MUST be null-terminated */ char *st_activity; + + uint32 flag_activity; + uint32 progress_param[N_PROGRESS_PARAM]; + double progress_param_float[N_PROGRESS_PARAM]; + char progress_message[PROGRESS_MESSAGE_LENGTH][N_PROGRESS_PARAM]; + } PgBackendStatus; /* @@ -815,6 +824,7 @@ typedef struct PgBackendStatus save_changecount = beentry->st_changecount; \ } while (0) +#define ACTIVITY_IS_VACUUM 0x01 /* ---------- * LocalPgBackendStatus * @@ -928,6 +938,12 @@ extern void pgstat_initialize(void); extern void pgstat_bestart(void); extern void pgstat_report_activity(BackendState state, const char *cmd_str); +extern void pgstat_report_activityflag(int activity_flag); +extern void pgstat_reset_activityflag(); +extern void pgstat_report_progress(uint *param1, int num_of_int, double *param2, + int int_num_float, + char param3[PROGRESS_MESSAGE_LENGTH][N_PROGRESS_PARAM], + int num_of_string); extern void pgstat_report_tempfile(size_t filesize); extern void pgstat_report_appname(const char *appname); extern void pgstat_report_xact_timestamp(TimestampTz tstamp); diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out index 80374e4..323002d 100644 --- a/src/test/regress/expected/rules.out +++ b/src/test/regress/expected/rules.out @@ -1848,6 +1848,16 @@ pg_stat_user_tables| SELECT pg_stat_all_tables.relid, pg_stat_all_tables.autoanalyze_count FROM pg_stat_all_tables WHERE ((pg_stat_all_tables.schemaname <> ALL (ARRAY['pg_catalog'::name, 'information_schema'::name])) AND (pg_stat_all_tables.schemaname !~ '^pg_toast'::text)); +pg_stat_vacuum_progress| SELECT s.pid, + s.total_pages, + s.scanned_pages, + s.total_heap_pages, + s.scanned_heap_pages, + s.total_index_pages, + s.scanned_index_pages, + s.percent_complete, + s.table_name + FROM pg_stat_get_vacuum_progress() s(pid, total_pages, scanned_pages, total_heap_pages, scanned_heap_pages, total_index_pages, scanned_index_pages, percent_complete, table_name); pg_stat_xact_all_tables| SELECT c.oid AS relid, n.nspname AS schemaname, c.relname,