18.11.2013 07:53, Sawada Masahiko kirjoitti:
On 13 Nov 2013, at 20:51, Mika Eloranta <m...@ohmu.fi> wrote:
Prevent excessive progress reporting that can grow to gigabytes
of output with large databases.
I got error with following scenario.
$ initdb -D data -E UTF8 --no-locale
/* setting the replication parameters */
$ pg_basebackup -D 2data
Floating point exception
LOG: could not send data to client: Broken pipe
ERROR: base backup could not send data, aborting backup
FATAL: connection to client lost
Attached a rebased patch with a fix for division by zero error plus some
code style issues. I also moved the patch to the current commitfest.
/ Oskari
>From 1c54ffc5006320da1b021c2a07939f948ba9fdb1 Mon Sep 17 00:00:00 2001
From: Mika Eloranta <m...@ohmu.fi>
Date: Tue, 21 Jan 2014 00:15:27 +0200
Subject: [PATCH] pg_basebackup: progress report max once per second
Prevent excessive progress reporting that can grow to gigabytes
of output with large databases.
---
src/bin/pg_basebackup/pg_basebackup.c | 28 ++++++++++++++++++----------
1 file changed, 18 insertions(+), 10 deletions(-)
diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c
index 9d13d57..cae181c 100644
--- a/src/bin/pg_basebackup/pg_basebackup.c
+++ b/src/bin/pg_basebackup/pg_basebackup.c
@@ -15,6 +15,7 @@
#include "libpq-fe.h"
#include "pqexpbuffer.h"
#include "pgtar.h"
+#include "pgtime.h"
#include <unistd.h>
#include <dirent.h>
@@ -46,6 +47,7 @@ static bool streamwal = false;
static bool fastcheckpoint = false;
static bool writerecoveryconf = false;
static int standby_message_timeout = 10 * 1000; /* 10 sec = default */
+static pg_time_t last_progress_report = 0;
/* Progress counters */
static uint64 totalsize;
@@ -75,7 +77,7 @@ static PQExpBuffer recoveryconfcontents = NULL;
/* Function headers */
static void usage(void);
static void verify_dir_is_empty_or_create(char *dirname);
-static void progress_report(int tablespacenum, const char *filename);
+static void progress_report(int tablespacenum, const char *filename, bool force);
static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum);
static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum);
@@ -401,11 +403,18 @@ verify_dir_is_empty_or_create(char *dirname)
* is enabled, also print the current file name.
*/
static void
-progress_report(int tablespacenum, const char *filename)
+progress_report(int tablespacenum, const char *filename, bool force)
{
- int percent = (int) ((totaldone / 1024) * 100 / totalsize);
+ int percent;
char totaldone_str[32];
char totalsize_str[32];
+ pg_time_t now = time(NULL);
+
+ if (!showprogress || (now == last_progress_report && !force))
+ return; /* Max once per second */
+
+ last_progress_report = now;
+ percent = totalsize ? (int) ((totaldone / 1024) * 100 / totalsize) : 0;
/*
* Avoid overflowing past 100% or the full size. This may make the total
@@ -852,9 +861,9 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum)
}
}
totaldone += r;
- if (showprogress)
- progress_report(rownum, filename);
+ progress_report(rownum, filename, false);
} /* while (1) */
+ progress_report(rownum, filename, true);
if (copybuf != NULL)
PQfreemem(copybuf);
@@ -1079,8 +1088,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
disconnect_and_exit(1);
}
totaldone += r;
- if (showprogress)
- progress_report(rownum, filename);
+ progress_report(rownum, filename, false);
current_len_left -= r;
if (current_len_left == 0 && current_padding == 0)
@@ -1096,6 +1104,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum)
}
} /* continuing data in existing file */
} /* loop over all data blocks */
+ progress_report(rownum, filename, true);
if (file != NULL)
{
@@ -1456,8 +1465,7 @@ BaseBackup(void)
tablespacecount = PQntuples(res);
for (i = 0; i < PQntuples(res); i++)
{
- if (showprogress)
- totalsize += atol(PQgetvalue(res, i, 2));
+ totalsize += atol(PQgetvalue(res, i, 2));
/*
* Verify tablespace directories are empty. Don't bother with the
@@ -1504,7 +1512,7 @@ BaseBackup(void)
if (showprogress)
{
- progress_report(PQntuples(res), NULL);
+ progress_report(PQntuples(res), NULL, true);
fprintf(stderr, "\n"); /* Need to move to next line */
}
PQclear(res);
--
1.8.4.2
--
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers
