Hi,
Am Donnerstag, den 28.03.2019, 10:08 +0100 schrieb Fabien COELHO:
> I marked it as ready,
Thanks!
> but I'd advise that you split it in (1) progress and (2) signal
> toggling so that the first part is more likely to make it before 12
> freeze.
Ok, done so in the attached.
Michael
--
Michael Banck
Projektleiter / Senior Berater
Tel.: +49 2166 9901-171
Fax: +49 2166 9901-100
Email: michael.ba...@credativ.de
credativ GmbH, HRB Mönchengladbach 12080
USt-ID-Nummer: DE204566209
Trompeterallee 108, 41189 Mönchengladbach
Geschäftsführung: Dr. Michael Meskes, Jörg Folz, Sascha Heuer
Unser Umgang mit personenbezogenen Daten unterliegt
folgenden Bestimmungen: https://www.credativ.de/datenschutz
From 0fe29a3c36e1f2b03e8a890996788539c637a5f0 Mon Sep 17 00:00:00 2001
From: Michael Banck <mba...@debian.org>
Date: Thu, 28 Mar 2019 10:47:54 +0100
Subject: [PATCH 1/2] Add progress reporting to pg_checksums.
This adds a new option -P/--progress that continuously reports the progress of
the operation, including percent done and rate (in MB/s).
Author: Bernd Helmle, Michael Banck
Reviewed-by: Fabien Coelho, Alvaro Herrera, Michael Paquier, Kyotaro HORIGUCHI
---
doc/src/sgml/ref/pg_checksums.sgml | 11 +++
src/bin/pg_checksums/pg_checksums.c | 138 +++++++++++++++++++++++++++++++++---
2 files changed, 141 insertions(+), 8 deletions(-)
diff --git a/doc/src/sgml/ref/pg_checksums.sgml b/doc/src/sgml/ref/pg_checksums.sgml
index 1f4d4ab8b4..0934426e58 100644
--- a/doc/src/sgml/ref/pg_checksums.sgml
+++ b/doc/src/sgml/ref/pg_checksums.sgml
@@ -136,6 +136,17 @@ PostgreSQL documentation
</varlistentry>
<varlistentry>
+ <term><option>-P</option></term>
+ <term><option>--progress</option></term>
+ <listitem>
+ <para>
+ Enable progress reporting. Turning this on will deliver a progress
+ report while checking or enabling checksums.
+ </para>
+ </listitem>
+ </varlistentry>
+
+ <varlistentry>
<term><option>-V</option></term>
<term><option>--version</option></term>
<listitem>
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index 5c185ebed8..d09cf8068d 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -15,6 +15,7 @@
#include "postgres_fe.h"
#include <dirent.h>
+#include <time.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -24,6 +25,7 @@
#include "common/file_utils.h"
#include "getopt_long.h"
#include "pg_getopt.h"
+#include "portability/instr_time.h"
#include "storage/bufpage.h"
#include "storage/checksum.h"
#include "storage/checksum_impl.h"
@@ -37,6 +39,7 @@ static ControlFileData *ControlFile;
static char *only_relfilenode = NULL;
static bool do_sync = true;
static bool verbose = false;
+static bool show_progress = false;
typedef enum
{
@@ -59,6 +62,14 @@ static PgChecksumMode mode = PG_MODE_CHECK;
static const char *progname;
+/*
+ * Progress status information.
+ */
+int64 total_size = 0;
+int64 current_size = 0;
+instr_time last_progress_update;
+instr_time scan_started;
+
static void
usage(void)
{
@@ -73,6 +84,7 @@ usage(void)
printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -v, --verbose output verbose messages\n"));
printf(_(" -r RELFILENODE check only relation with specified relfilenode\n"));
+ printf(_(" -P, --progress show progress information\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nIf no data directory (DATADIR) is specified, "
@@ -97,6 +109,66 @@ static const char *const skip[] = {
NULL,
};
+/*
+ * Report current progress status. Parts borrowed from
+ * PostgreSQLs' src/bin/pg_basebackup.c
+ */
+static void
+report_progress(bool force)
+{
+ instr_time now;
+ double elapsed;
+ double total_percent;
+ double current_speed;
+
+ char totalstr[32];
+ char currentstr[32];
+
+ INSTR_TIME_SET_CURRENT(now);
+
+ /* Make sure we report at most once every 250 milliseconds */
+ if ((INSTR_TIME_GET_MILLISEC(now) -
+ INSTR_TIME_GET_MILLISEC(last_progress_update) < 250) && !force)
+ return;
+
+ /* Save current time */
+ last_progress_update = now;
+
+ /* Elapsed time in milliseconds since start of scan */
+ elapsed = (INSTR_TIME_GET_MILLISEC(now) -
+ INSTR_TIME_GET_MILLISEC(scan_started));
+
+ /* Adjust total size if current_size is larger */
+ if (current_size > total_size)
+ total_size = current_size;
+
+ /* Calculate current percent done */
+ total_percent = total_size ? 100.0 * current_size / total_size : 0.0;
+
+#define MEGABYTES (1024 * 1024)
+
+ /*
+ * Calculate current speed, converting current_size from bytes to
+ * megabytes and elapsed from milliseconds to seconds.
+ */
+ current_speed = ((double) current_size / MEGABYTES) / (elapsed / 1000);
+
+ snprintf(totalstr, sizeof(totalstr), INT64_FORMAT,
+ total_size / MEGABYTES);
+ snprintf(currentstr, sizeof(currentstr), INT64_FORMAT,
+ current_size / MEGABYTES);
+
+ /*
+ * Print five blanks at the end so the end of previous lines which were
+ * longer don't remain partly visible.
+ */
+ fprintf(stderr, "%s/%s MB (%.0f%%, %.0f MB/s)%5s",
+ currentstr, totalstr, total_percent, current_speed, "");
+
+ /* Stay on the same line if reporting to a terminal */
+ fprintf(stderr, isatty(fileno(stderr)) ? "\r" : "\n");
+}
+
static bool
skipfile(const char *fn)
{
@@ -153,6 +225,7 @@ scan_file(const char *fn, BlockNumber segmentno)
continue;
csum = pg_checksum_page(buf.data, blockno + segmentno * RELSEG_SIZE);
+ current_size += r;
if (mode == PG_MODE_CHECK)
{
if (csum != header->pd_checksum)
@@ -183,6 +256,13 @@ scan_file(const char *fn, BlockNumber segmentno)
exit(1);
}
}
+
+ /*
+ * Call report_progress() every 1024 blocks if progress
+ * reporting is requested.
+ */
+ if (show_progress && (blockno % 1024 == 0))
+ report_progress(false);
}
if (verbose)
@@ -195,12 +275,17 @@ scan_file(const char *fn, BlockNumber segmentno)
_("%s: checksums enabled in file \"%s\"\n"), progname, fn);
}
+ /* Make sure progress is reported at least once per file */
+ if (show_progress)
+ report_progress(false);
+
close(f);
}
-static void
-scan_directory(const char *basedir, const char *subdir)
+static int64
+scan_directory(const char *basedir, const char *subdir, bool sizeonly)
{
+ int64 dirsize = 0;
char path[MAXPGPATH];
DIR *dir;
struct dirent *de;
@@ -279,16 +364,20 @@ scan_directory(const char *basedir, const char *subdir)
/* Relfilenode not to be included */
continue;
- scan_file(fn, segmentno);
+ dirsize += st.st_size;
+
+ if (!sizeonly)
+ scan_file(fn, segmentno);
}
#ifndef WIN32
else if (S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))
#else
else if (S_ISDIR(st.st_mode) || pgwin32_is_junction(fn))
#endif
- scan_directory(path, de->d_name);
+ dirsize += scan_directory(path, de->d_name, sizeonly);
}
closedir(dir);
+ return dirsize;
}
int
@@ -300,6 +389,7 @@ main(int argc, char *argv[])
{"disable", no_argument, NULL, 'd'},
{"enable", no_argument, NULL, 'e'},
{"no-sync", no_argument, NULL, 'N'},
+ {"progress", no_argument, NULL, 'P'},
{"verbose", no_argument, NULL, 'v'},
{NULL, 0, NULL, 0}
};
@@ -327,7 +417,7 @@ main(int argc, char *argv[])
}
}
- while ((c = getopt_long(argc, argv, "cD:deNr:v", long_options, &option_index)) != -1)
+ while ((c = getopt_long(argc, argv, "cD:deNPr:v", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -357,6 +447,9 @@ main(int argc, char *argv[])
}
only_relfilenode = pstrdup(optarg);
break;
+ case 'P':
+ show_progress = true;
+ break;
default:
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
@@ -436,6 +529,24 @@ main(int argc, char *argv[])
exit(1);
}
+ /*
+ * If progress status information is requested, we need to scan the
+ * directory tree(s) twice, once to get the idea how much data we need to
+ * scan and finally to do the real legwork.
+ */
+ if (show_progress)
+ {
+ total_size = scan_directory(DataDir, "global", true);
+ total_size += scan_directory(DataDir, "base", true);
+ total_size += scan_directory(DataDir, "pg_tblspc", true);
+
+ /*
+ * Remember start time. Required to calculate the current speed in
+ * report_progress().
+ */
+ INSTR_TIME_SET_CURRENT(scan_started);
+ }
+
if (ControlFile->data_checksum_version == 0 &&
mode == PG_MODE_DISABLE)
{
@@ -453,9 +564,20 @@ main(int argc, char *argv[])
/* Operate on all files if checking or enabling checksums */
if (mode == PG_MODE_CHECK || mode == PG_MODE_ENABLE)
{
- scan_directory(DataDir, "global");
- scan_directory(DataDir, "base");
- scan_directory(DataDir, "pg_tblspc");
+ scan_directory(DataDir, "global", false);
+ scan_directory(DataDir, "base", false);
+ scan_directory(DataDir, "pg_tblspc", false);
+
+ /*
+ * Done. Move to next line in case progress information was shown.
+ * Otherwise we clutter the summary output.
+ */
+ if (show_progress)
+ {
+ report_progress(true);
+ if (isatty(fileno(stderr)))
+ fprintf(stderr, "\n");
+ }
printf(_("Checksum operation completed\n"));
printf(_("Files scanned: %s\n"), psprintf(INT64_FORMAT, files));
--
2.11.0
From 78d551ec7e5151dceee2d182a8f258fdc4b59554 Mon Sep 17 00:00:00 2001
From: Michael Banck <mba...@debian.org>
Date: Thu, 28 Mar 2019 10:57:11 +0100
Subject: [PATCH 2/2] Support toggling of pg_checksums progress reporting via
SIGUSR1.
This feature does not work on Windows.
Author: Bernd Helmle, Michael Banck
Reviewed-by: Fabien Coelho, Alvaro Herrera, Michael Paquier, Kyotaro HORIGUCHI
---
doc/src/sgml/ref/pg_checksums.sgml | 4 ++-
src/bin/pg_checksums/pg_checksums.c | 51 +++++++++++++++++++++++++++----------
2 files changed, 40 insertions(+), 15 deletions(-)
diff --git a/doc/src/sgml/ref/pg_checksums.sgml b/doc/src/sgml/ref/pg_checksums.sgml
index 0934426e58..9f5cbc79ff 100644
--- a/doc/src/sgml/ref/pg_checksums.sgml
+++ b/doc/src/sgml/ref/pg_checksums.sgml
@@ -141,7 +141,9 @@ PostgreSQL documentation
<listitem>
<para>
Enable progress reporting. Turning this on will deliver a progress
- report while checking or enabling checksums.
+ report while checking or enabling checksums. Sending the
+ <systemitem>SIGUSR1</systemitem> signal will toggle progress reporting
+ on or off during the verification run (not available on Windows).
</para>
</listitem>
</varlistentry>
diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c
index d09cf8068d..bc7f29c3c4 100644
--- a/src/bin/pg_checksums/pg_checksums.c
+++ b/src/bin/pg_checksums/pg_checksums.c
@@ -15,6 +15,7 @@
#include "postgres_fe.h"
#include <dirent.h>
+#include <signal.h>
#include <time.h>
#include <sys/stat.h>
#include <unistd.h>
@@ -109,6 +110,23 @@ static const char *const skip[] = {
NULL,
};
+static void
+toggle_progress_report(int signum)
+{
+
+ /* we handle SIGUSR1 only, and toggle the value of show_progress */
+ if (signum == SIGUSR1)
+ {
+ /*
+ * Skip to the next line in order to avoid overwriting half of
+ * the progress report line with the final output.
+ */
+ if (show_progress && isatty(fileno(stderr)))
+ fprintf(stderr, "\n");
+ show_progress = !show_progress;
+ }
+}
+
/*
* Report current progress status. Parts borrowed from
* PostgreSQLs' src/bin/pg_basebackup.c
@@ -529,23 +547,28 @@ main(int argc, char *argv[])
exit(1);
}
+#ifndef WIN32
/*
- * If progress status information is requested, we need to scan the
- * directory tree(s) twice, once to get the idea how much data we need to
- * scan and finally to do the real legwork.
+ * Assign SIGUSR1 signal handler to toggle progress status information
*/
- if (show_progress)
- {
- total_size = scan_directory(DataDir, "global", true);
- total_size += scan_directory(DataDir, "base", true);
- total_size += scan_directory(DataDir, "pg_tblspc", true);
+ pqsignal(SIGUSR1, toggle_progress_report);
+#endif
- /*
- * Remember start time. Required to calculate the current speed in
- * report_progress().
- */
- INSTR_TIME_SET_CURRENT(scan_started);
- }
+ /*
+ * As progress status information may be requested even after start of
+ * operation, we need to scan the directory tree(s) twice, once to get
+ * the idea how much data we need to scan and finally to do the real
+ * legwork.
+ */
+ total_size = scan_directory(DataDir, "global", true);
+ total_size += scan_directory(DataDir, "base", true);
+ total_size += scan_directory(DataDir, "pg_tblspc", true);
+
+ /*
+ * Remember start time. Required to calculate the current speed in
+ * report_progress()
+ */
+ INSTR_TIME_SET_CURRENT(scan_started);
if (ControlFile->data_checksum_version == 0 &&
mode == PG_MODE_DISABLE)
--
2.11.0
