On Tue, Sep 5, 2017 at 10:19 PM, Peter Eisentraut <peter.eisentr...@2ndquadrant.com> wrote: > On 6/9/17 02:08, Michael Paquier wrote: >> I have just played with that, and attached is a patch to implement the >> so-said option with a basic set of tests, increasing code coverage of >> pg_receivewal.c from 15% to 60%. I'll park that in the next CF of >> September. > > Looks like a good idea. A couple of thoughts:
Thanks. > - I think the tests should be written in the style of > $node->command_fails instead of just command_fails. At least, that's > how it's done in the pg_basebackup tests. Good idea. > - I think the tests will fail if libz support is not built. Again, > pg_basebackup doesn't have tests for that. So maybe omit that and > address it later. Let's address it now. This can be countered by querying pg_config() before running the command of pg_receivexwal which uses --compress. > - The variable exit_on_stop seems redundant with time_to_abort. They do > the same thing, so maybe your patch could reuse it. Yes, that's doable. time_to_abort does not seem a variable name adapted to me though if both are combined, so I have renamed that to time_to_stop, which maps more with the fact that stop can be also willingly wanted without a SIGINT. Attached is a new version of the patch. -- Michael
From 7987b412b3b16f8995ccbe39c7de0e6762aa964d Mon Sep 17 00:00:00 2001 From: Michael Paquier <mich...@paquier.xyz> Date: Wed, 6 Sep 2017 13:39:30 +0900 Subject: [PATCH] Implement pg_receivewal --endpos This is primarily useful in making any regression test using this utility more deterministic as pg_receivewal cannot be started as a deamon in TAP tests. While this is less useful than the equivalent of pg_recvlogical, users can as well use it for example to enforce WAL streaming up to a end-of-backup position, to save only a minimal amount of WAL, though the end position of WAL is only known once the backup is finished, but some users may want to get that done as a two-step process with one single WAL receiver in use all the time. Use this new option to stream WAL data in a deterministic way within a new set of TAP tests. --- doc/src/sgml/ref/pg_receivewal.sgml | 16 ++++++ src/bin/pg_basebackup/pg_receivewal.c | 38 +++++++++++--- src/bin/pg_basebackup/t/020_pg_receivewal.pl | 75 +++++++++++++++++++++++++++- 3 files changed, 121 insertions(+), 8 deletions(-) diff --git a/doc/src/sgml/ref/pg_receivewal.sgml b/doc/src/sgml/ref/pg_receivewal.sgml index 7c82e36c7c..6d192bd606 100644 --- a/doc/src/sgml/ref/pg_receivewal.sgml +++ b/doc/src/sgml/ref/pg_receivewal.sgml @@ -98,6 +98,22 @@ PostgreSQL documentation </listitem> </varlistentry> + <varlistentry> + <term><option>-E <replaceable>lsn</replaceable></option></term> + <term><option>--endpos=<replaceable>lsn</replaceable></option></term> + <listitem> + <para> + If specified, automatically stop replication and exit with normal + exit status 0 when receiving reaches the specified LSN. + </para> + + <para> + If there is a record with LSN exactly equal to <replaceable>lsn</>, + the record will be considered. + </para> + </listitem> + </varlistentry> + <varlistentry> <term><option>--if-not-exists</option></term> <listitem> diff --git a/src/bin/pg_basebackup/pg_receivewal.c b/src/bin/pg_basebackup/pg_receivewal.c index 4a1a5658fb..93b396c10f 100644 --- a/src/bin/pg_basebackup/pg_receivewal.c +++ b/src/bin/pg_basebackup/pg_receivewal.c @@ -36,12 +36,13 @@ static int verbose = 0; static int compresslevel = 0; static int noloop = 0; static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ -static volatile bool time_to_abort = false; +static volatile bool time_to_stop = false; static bool do_create_slot = false; static bool slot_exists_ok = false; static bool do_drop_slot = false; static bool synchronous = false; static char *replication_slot = NULL; +static XLogRecPtr endpos = InvalidXLogRecPtr; static void usage(void); @@ -78,6 +79,7 @@ usage(void) printf(_("\nOptions:\n")); printf(_(" -D, --directory=DIR receive write-ahead log files into this directory\n")); printf(_(" --if-not-exists do not error if slot already exists when creating a slot\n")); + printf(_(" -E, --endpos=LSN exit after receiving the specified LSN\n")); printf(_(" -n, --no-loop do not loop on connection lost\n")); printf(_(" -s, --status-interval=SECS\n" " time between status packets sent to server (default: %d)\n"), (standby_message_timeout / 1000)); @@ -112,6 +114,16 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) progname, (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); + if (!XLogRecPtrIsInvalid(endpos) && endpos < xlogpos) + { + if (verbose) + fprintf(stderr, _("%s: stopped streaming at %X/%X (timeline %u)\n"), + progname, (uint32) (xlogpos >> 32), (uint32) xlogpos, + timeline); + time_to_stop = true; + return true; + } + /* * Note that we report the previous, not current, position here. After a * timeline switch, xlogpos points to the beginning of the segment because @@ -128,7 +140,7 @@ stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) prevtimeline = timeline; prevpos = xlogpos; - if (time_to_abort) + if (time_to_stop) { if (verbose) fprintf(stderr, _("%s: received interrupt signal, exiting\n"), @@ -448,7 +460,7 @@ StreamLog(void) static void sigint_handler(int signum) { - time_to_abort = true; + time_to_stop = true; } #endif @@ -460,6 +472,7 @@ main(int argc, char **argv) {"version", no_argument, NULL, 'V'}, {"directory", required_argument, NULL, 'D'}, {"dbname", required_argument, NULL, 'd'}, + {"endpos", required_argument, NULL, 'E'}, {"host", required_argument, NULL, 'h'}, {"port", required_argument, NULL, 'p'}, {"username", required_argument, NULL, 'U'}, @@ -481,6 +494,7 @@ main(int argc, char **argv) int c; int option_index; char *db_name; + uint32 hi, lo; progname = get_progname(argv[0]); set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_basebackup")); @@ -500,7 +514,7 @@ main(int argc, char **argv) } } - while ((c = getopt_long(argc, argv, "D:d:h:p:U:s:S:nwWvZ:", + while ((c = getopt_long(argc, argv, "D:d:E:h:p:U:s:S:nwWvZ:", long_options, &option_index)) != -1) { switch (c) @@ -544,6 +558,16 @@ main(int argc, char **argv) case 'S': replication_slot = pg_strdup(optarg); break; + case 'E': + if (sscanf(optarg, "%X/%X", &hi, &lo) != 2) + { + fprintf(stderr, + _("%s: could not parse end position \"%s\"\n"), + progname, optarg); + exit(1); + } + endpos = ((uint64) hi) << 32 | lo; + break; case 'n': noloop = 1; break; @@ -714,11 +738,11 @@ main(int argc, char **argv) while (true) { StreamLog(); - if (time_to_abort) + if (time_to_stop) { /* - * We've been Ctrl-C'ed. That's not an error, so exit without an - * errorcode. + * We've been Ctrl-C'ed or end of streaming position has been + * willingly reached, so exit without an error code. */ exit(0); } diff --git a/src/bin/pg_basebackup/t/020_pg_receivewal.pl b/src/bin/pg_basebackup/t/020_pg_receivewal.pl index b4cb6f729d..2d14b8242f 100644 --- a/src/bin/pg_basebackup/t/020_pg_receivewal.pl +++ b/src/bin/pg_basebackup/t/020_pg_receivewal.pl @@ -1,8 +1,81 @@ use strict; use warnings; use TestLib; -use Test::More tests => 8; +use PostgresNode; +use Test::More tests => 15; program_help_ok('pg_receivewal'); program_version_ok('pg_receivewal'); program_options_handling_ok('pg_receivewal'); + +my $primary = get_new_node('primary'); +$primary->init(allows_streaming => 1); +$primary->start; + +my $stream_dir = $primary->basedir . '/archive_wal'; +mkdir($stream_dir); + +# Sanity checks for command line options. +$primary->command_fails(['pg_receivewal'], + 'pg_receivewal needs target directory specified'); +$primary->command_fails(['pg_receivewal', '-D', $stream_dir, '--create-slot', + '--drop-slot'], + 'failure if both --create-slot and --drop-slot specified'); +$primary->command_fails(['pg_receivewal', '-D', $stream_dir, '--create-slot'], + 'failure if --create-slot defined without --slot'); + +# Slot creation and drop +my $slot_name = 'test'; +$primary->command_ok(['pg_receivewal', '--slot', $slot_name, '--create-slot' ], + 'creation of replication slot'); +$primary->command_ok(['pg_receivewal', '--slot', $slot_name, '--drop-slot' ], + 'drop of replication slot'); + +# Generate some WAL using non-compression mode. Use --synchronous +# at the same time to add more code coverage. Switch to the next +# segment first so as subsequent restarts of pg_receivewal will +# see this segment as full and non-compressed. +$primary->psql('postgres', 'CREATE TABLE test_table(x integer);'); +$primary->psql('postgres', 'SELECT pg_switch_wal();'); +my $nextlsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); +chomp($nextlsn); +$primary->psql('postgres', + 'INSERT INTO test_table VALUES (generate_series(1,100));'); + +# Stream up to the given position. +$primary->command_ok( + [ 'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos', + $nextlsn, '--synchronous', '--no-loop' ], + 'streaming some WAL with --synchronous and without compression'); + +# Check if build supports compression with libz, in which case the following +# command would pass or fail depending on its support. +my $lz_support = $primary->safe_psql('postgres', + "SELECT setting ~ '-lz ' from pg_config WHERE name = 'LIBS'"); +chomp($lz_support); +if ($lz_support eq 't') +{ + # Now generate more WAL, switch to a new segment and stream + # changes using the compression mode. + $primary->psql('postgres', + 'INSERT INTO test_table VALUES (generate_series(1,100));'); + $primary->psql('postgres', 'SELECT pg_switch_wal();'); + $nextlsn = + $primary->safe_psql('postgres', 'SELECT pg_current_wal_insert_lsn();'); + chomp($nextlsn); + $primary->psql('postgres', + 'INSERT INTO test_table VALUES (generate_series(1,100));'); + + $primary->command_ok( + [ 'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos', + $nextlsn, '--compress', '1', '--no-loop' ], + 'streaming some WAL with compression'); +} +else +{ + $primary->command_fails( + [ 'pg_receivewal', '-D', $stream_dir, '--verbose', '--endpos', + $nextlsn, '--compress', '1', '--no-loop' ], + 'compression not supported'); +} -- 2.14.1
-- Sent via pgsql-hackers mailing list (pgsql-hackers@postgresql.org) To make changes to your subscription: http://www.postgresql.org/mailpref/pgsql-hackers