On Tue, Sep 5, 2017 at 10:19 PM, Peter Eisentraut
<peter.eisentr...@2ndquadrant.com> wrote:
> On 6/9/17 02:08, Michael Paquier wrote:
>> I have just played with that, and attached is a patch to implement the
>> so-said option with a basic set of tests, increasing code coverage of
>> pg_receivewal.c from 15% to 60%. I'll park that in the next CF of
>> September.
>
> Looks like a good idea.  A couple of thoughts:

Thanks.

> - I think the tests should be written in the style of
> $node->command_fails instead of just command_fails.  At least, that's
> how it's done in the pg_basebackup tests.

Good idea.

> - I think the tests will fail if libz support is not built.  Again,
> pg_basebackup doesn't have tests for that.  So maybe omit that and
> address it later.

Let's address it now. This can be countered by querying pg_config()
before running the command of pg_receivexwal which uses --compress.

> - The variable exit_on_stop seems redundant with time_to_abort.  They do
> the same thing, so maybe your patch could reuse it.

Yes, that's doable. time_to_abort does not seem a variable name
adapted to me though if both are combined, so I have renamed that to
time_to_stop, which maps more with the fact that stop can be also
willingly wanted without a SIGINT.

Attached is a new version of the patch.
-- 
Michael
From 7987b412b3b16f8995ccbe39c7de0e6762aa964d Mon Sep 17 00:00:00 2001
From: Michael Paquier <mich...@paquier.xyz>
Date: Wed, 6 Sep 2017 13:39:30 +0900
Subject: [PATCH] Implement pg_receivewal --endpos

This is primarily useful in making any regression test using this utility
more deterministic as pg_receivewal cannot be started as a deamon in TAP
tests.

While this is less useful than the equivalent of pg_recvlogical, users
can as well use it for example to enforce WAL streaming up to a
end-of-backup position, to save only a minimal amount of WAL, though the
end position of WAL is only known once the backup is finished, but some
users may want to get that done as a two-step process with one single WAL
receiver in use all the time.

Use this new option to stream WAL data in a deterministic way within a
new set of TAP tests.
---
 doc/src/sgml/ref/pg_receivewal.sgml          | 16 ++++++
 src/bin/pg_basebackup/pg_receivewal.c        | 38 +++++++++++---
 src/bin/pg_basebackup/t/020_pg_receivewal.pl | 75 +++++++++++++++++++++++++++-
 3 files changed, 121 insertions(+), 8 deletions(-)

diff --git a/doc/src/sgml/ref/pg_receivewal.sgml b/doc/src/sgml/ref/pg_receivewal.sgml
index 7c82e36c7c..6d192bd606 100644
--- a/doc/src/sgml/ref/pg_receivewal.sgml
+++ b/doc/src/sgml/ref/pg_receivewal.sgml
@@ -98,6 +98,22 @@ PostgreSQL documentation
       </listitem>
      </varlistentry>
 
+     <varlistentry>
+      <term><option>-E <replaceable>lsn</replaceable></option></term>
+      <term><option>--endpos=<replaceable>lsn</replaceable></option></term>
+      <listitem>
+       <para>
+        If specified, automatically stop replication and exit with normal
+        exit status 0 when receiving reaches the specified LSN.
+       </para>
+
+       <para>
+        If there is a record with LSN exactly equal to <replaceable>lsn</>,
+        the record will be considered.
+       </para>
+      </listitem>
+     </varlistentry>
+
      <varlistentry>
       <term><option>--if-not-exists</option></term>
       <listitem>
diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c
index 4a1a5658fb..93b396c10f 100644
--- a/src/bin/pg_basebackup/pg_receivewal.c
+++ b/src/bin/pg_basebackup/pg_receivewal.c
@@ -36,12 +36,13 @@ static int	verbose = 0;
 static int	compresslevel = 0;
 static int	noloop = 0;
 static int	standby_message_timeout = 10 * 1000;	/* 10 sec = default */
-static volatile bool time_to_abort = false;
+static volatile bool time_to_stop = false;
 static bool do_create_slot = false;
 static bool slot_exists_ok = false;
 static bool do_drop_slot = false;
 static bool synchronous = false;
 static char *replication_slot = NULL;
+static XLogRecPtr endpos = InvalidXLogRecPtr;
 
 
 static void usage(void);
@@ -78,6 +79,7 @@ usage(void)
 	printf(_("\nOptions:\n"));
 	printf(_("  -D, --directory=DIR    receive write-ahead log files into this directory\n"));
 	printf(_("      --if-not-exists    do not error if slot already exists when creating a slot\n"));
+	printf(_("  -E, --endpos=LSN       exit after receiving the specified LSN\n"));
 	printf(_("  -n, --no-loop          do not loop on connection lost\n"));
 	printf(_("  -s, --status-interval=SECS\n"
 			 "                         time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000));
@@ -112,6 +114,16 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
 				progname, (uint32) (xlogpos >> 32), (uint32) xlogpos,
 				timeline);
 
+	if (!XLogRecPtrIsInvalid(endpos) && endpos < xlogpos)
+	{
+		if (verbose)
+			fprintf(stderr, _("%s: stopped streaming at %X/%X (timeline %u)\n"),
+					progname, (uint32) (xlogpos >> 32), (uint32) xlogpos,
+					timeline);
+		time_to_stop = true;
+		return true;
+	}
+
 	/*
 	 * Note that we report the previous, not current, position here. After a
 	 * timeline switch, xlogpos points to the beginning of the segment because
@@ -128,7 +140,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished)
 	prevtimeline = timeline;
 	prevpos = xlogpos;
 
-	if (time_to_abort)
+	if (time_to_stop)
 	{
 		if (verbose)
 			fprintf(stderr, _("%s: received interrupt signal, exiting\n"),
@@ -448,7 +460,7 @@ StreamLog(void)
 static void
 sigint_handler(int signum)
 {
-	time_to_abort = true;
+	time_to_stop = true;
 }
 #endif
 
@@ -460,6 +472,7 @@ main(int argc, char **argv)
 		{"version", no_argument, NULL, 'V'},
 		{"directory", required_argument, NULL, 'D'},
 		{"dbname", required_argument, NULL, 'd'},
+		{"endpos", required_argument, NULL, 'E'},
 		{"host", required_argument, NULL, 'h'},
 		{"port", required_argument, NULL, 'p'},
 		{"username", required_argument, NULL, 'U'},
@@ -481,6 +494,7 @@ main(int argc, char **argv)
 	int			c;
 	int			option_index;
 	char	   *db_name;
+	uint32		hi, lo;
 
 	progname = get_progname(argv[0]);
 	set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup"));
@@ -500,7 +514,7 @@ main(int argc, char **argv)
 		}
 	}
 
-	while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nwWvZ:",
+	while ((c = getopt_long(argc, argv, "D:d:E:h:p:U:s:S:nwWvZ:",
 							long_options, &option_index)) != -1)
 	{
 		switch (c)
@@ -544,6 +558,16 @@ main(int argc, char **argv)
 			case 'S':
 				replication_slot = pg_strdup(optarg);
 				break;
+			case 'E':
+				if (sscanf(optarg, "%X/%X", &hi, &lo) != 2)
+				{
+					fprintf(stderr,
+							_("%s: could not parse end position \"%s\"\n"),
+							progname, optarg);
+					exit(1);
+				}
+				endpos = ((uint64) hi) << 32 | lo;
+				break;
 			case 'n':
 				noloop = 1;
 				break;
@@ -714,11 +738,11 @@ main(int argc, char **argv)
 	while (true)
 	{
 		StreamLog();
-		if (time_to_abort)
+		if (time_to_stop)
 		{
 			/*
-			 * We've been Ctrl-C'ed. That's not an error, so exit without an
-			 * errorcode.
+			 * We've been Ctrl-C'ed or end of streaming position has been
+			 * willingly reached, so exit without an error code.
 			 */
 			exit(0);
 		}
diff --git a/src/bin/pg_basebackup/t/020_pg_receivewal.pl b/src/bin/pg_basebackup/t/020_pg_receivewal.pl
index b4cb6f729d..2d14b8242f 100644
--- a/src/bin/pg_basebackup/t/020_pg_receivewal.pl
+++ b/src/bin/pg_basebackup/t/020_pg_receivewal.pl
@@ -1,8 +1,81 @@
 use strict;
 use warnings;
 use TestLib;
-use Test::More tests => 8;
+use PostgresNode;
+use Test::More tests => 15;
 
 program_help_ok('pg_receivewal');
 program_version_ok('pg_receivewal');
 program_options_handling_ok('pg_receivewal');
+
+my $primary = get_new_node('primary');
+$primary->init(allows_streaming => 1);
+$primary->start;
+
+my $stream_dir = $primary->basedir . '/archive_wal';
+mkdir($stream_dir);
+
+# Sanity checks for command line options.
+$primary->command_fails(['pg_receivewal'],
+	'pg_receivewal needs target directory specified');
+$primary->command_fails(['pg_receivewal', '-D', $stream_dir, '--create-slot',
+			   '--drop-slot'],
+	'failure if both --create-slot and --drop-slot specified');
+$primary->command_fails(['pg_receivewal', '-D', $stream_dir, '--create-slot'],
+	'failure if --create-slot defined without --slot');
+
+# Slot creation and drop
+my $slot_name = 'test';
+$primary->command_ok(['pg_receivewal', '--slot', $slot_name, '--create-slot' ],
+	'creation of replication slot');
+$primary->command_ok(['pg_receivewal', '--slot', $slot_name, '--drop-slot' ],
+	'drop of replication slot');
+
+# Generate some WAL using non-compression mode. Use --synchronous
+# at the same time to add more code coverage. Switch to the next
+# segment first so as subsequent restarts of pg_receivewal will
+# see this segment as full and non-compressed.
+$primary->psql('postgres', 'CREATE TABLE test_table(x integer);');
+$primary->psql('postgres', 'SELECT pg_switch_wal();');
+my $nextlsn =
+	$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+chomp($nextlsn);
+$primary->psql('postgres',
+	'INSERT INTO test_table VALUES (generate_series(1,100));');
+
+# Stream up to the given position.
+$primary->command_ok(
+	[   'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos',
+		$nextlsn, '--synchronous', '--no-loop' ],
+	'streaming some WAL with --synchronous and without compression');
+
+# Check if build supports compression with libz, in which case the following
+# command would pass or fail depending on its support.
+my $lz_support = $primary->safe_psql('postgres',
+	"SELECT setting ~ '-lz ' from pg_config WHERE name = 'LIBS'");
+chomp($lz_support);
+if ($lz_support eq 't')
+{
+	# Now generate more WAL, switch to a new segment and stream
+	# changes using the compression mode.
+	$primary->psql('postgres',
+		'INSERT INTO test_table VALUES (generate_series(1,100));');
+	$primary->psql('postgres', 'SELECT pg_switch_wal();');
+	$nextlsn =
+		$primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();');
+	chomp($nextlsn);
+	$primary->psql('postgres',
+		'INSERT INTO test_table VALUES (generate_series(1,100));');
+
+	$primary->command_ok(
+		[   'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos',
+			$nextlsn, '--compress', '1', '--no-loop' ],
+		'streaming some WAL with compression');
+}
+else
+{
+	$primary->command_fails(
+		[   'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos',
+			$nextlsn, '--compress', '1', '--no-loop' ],
+		'compression not supported');
+}
-- 
2.14.1

-- 
Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org)
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgsql-hackers

Reply via email to