On Mon, Mar 02, 2020 at 08:59:49PM +0300, Alexey Kondratov wrote: > OK, sounds reasonable, but just to be clear. I will remove only a > possibility to bypass this sanity check (with 0), but leave expectedSize > argument intact. We still need it, since pg_rewind takes WalSegSz from > ControlFile and should pass it further, am I right?
We need to keep around the argument, but I think that we give any user of this routine a favor by making mandatory a file size. > pg_strip_crlf fits well, but would you mind if I also make pipe_read_line > external in this patch? I got to look at that more, and pipe_read_line() is a nice fit. > Actually, the verb 'construct' is used historically applied to > archive/restore commands (see also xlogarchive.c and pgarch.c), but it > should be 'build' in (fe_)archive.c, since we have BuildRestoreCommand there > now. > > All other remarks look clear for me, so I fix them in the next patch > version, thanks. Already done as per the attached, with a new routine named getRestoreCommand() and more done. There were a couple of extra things I noticed on the way: - Found some typos. - The translation of pg_rewind --help was incorrect, with a sentence cut in the middle (you used twice gettext). - I did not actually get why you don't check for a missing command when using wait_result_is_any_signal. In this case I'd think that it is better to exit immediately as follow-up calls would just fail. - The code was rather careless about error handling and RestoreArchivedWALFile(), and it seemed to me that it is rather pointless to report an extra message "could not restore file \"%s\" from archive" on top of the other error. - I could not resist to add more documentation for the new routines of src/common/.. - Used long options in the tests for readability, reworked the comments. On top of that, I have spent some time testing the reliability of the test, and tested the whole on Windows and Linux. This is in a rather committable shape now. -- Michael
diff --git a/src/include/common/archive.h b/src/include/common/archive.h
new file mode 100644
index 0000000000..e0260dd453
--- /dev/null
+++ b/src/include/common/archive.h
@@ -0,0 +1,28 @@
+/*-------------------------------------------------------------------------
+ *
+ * archive.h
+ * Common WAL archive routines
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/archive.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef ARCHIVE_H
+#define ARCHIVE_H
+
+/*
+ * Routine to build a restore command to retrieve WAL segments from a WAL
+ * archive. This uses the arguments given by the caller to replace the
+ * corresponding alias fields as defined in the GUC parameter
+ * restore_command.
+ */
+extern int BuildRestoreCommand(const char *restoreCommand,
+ const char *xlogpath, /* alias %p */
+ const char *xlogfname, /* alias %f */
+ const char *lastRestartPointFname, /* alias %r */
+ char *result);
+
+#endif /* ARCHIVE_H */
diff --git a/src/include/common/fe_archive.h b/src/include/common/fe_archive.h
new file mode 100644
index 0000000000..af7763ebac
--- /dev/null
+++ b/src/include/common/fe_archive.h
@@ -0,0 +1,19 @@
+/*-------------------------------------------------------------------------
+ *
+ * fe_archive.h
+ * Routines to access WAL archive from frontend
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/common/fe_archive.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef FE_ARCHIVE_H
+#define FE_ARCHIVE_H
+
+extern int RestoreArchivedWALFile(const char *path, const char *xlogfname,
+ off_t expectedSize, const char *restoreCommand);
+
+#endif /* FE_ARCHIVE_H */
diff --git a/src/include/port.h b/src/include/port.h
index 3be994b43c..0685ce3abc 100644
--- a/src/include/port.h
+++ b/src/include/port.h
@@ -102,10 +102,11 @@ extern void pgfnames_cleanup(char **filenames);
/* Portable locale initialization (in exec.c) */
extern void set_pglocale_pgservice(const char *argv0, const char *app);
-/* Portable way to find binaries (in exec.c) */
+/* Portable way to find and execute binaries (in exec.c) */
extern int find_my_exec(const char *argv0, char *retpath);
extern int find_other_exec(const char *argv0, const char *target,
const char *versionstr, char *retpath);
+extern char *pipe_read_line(char *cmd, char *line, int maxsize);
/* Doesn't belong here, but this is used with find_other_exec(), so... */
#define PG_BACKEND_VERSIONSTR "postgres (PostgreSQL) " PG_VERSION "\n"
diff --git a/src/backend/access/transam/xlogarchive.c b/src/backend/access/transam/xlogarchive.c
index 188b73e752..b90dbd44d8 100644
--- a/src/backend/access/transam/xlogarchive.c
+++ b/src/backend/access/transam/xlogarchive.c
@@ -21,6 +21,7 @@
#include "access/xlog.h"
#include "access/xlog_internal.h"
+#include "common/archive.h"
#include "miscadmin.h"
#include "postmaster/startup.h"
#include "replication/walsender.h"
@@ -55,9 +56,6 @@ RestoreArchivedFile(char *path, const char *xlogfname,
char xlogpath[MAXPGPATH];
char xlogRestoreCmd[MAXPGPATH];
char lastRestartPointFname[MAXPGPATH];
- char *dp;
- char *endp;
- const char *sp;
int rc;
struct stat stat_buf;
XLogSegNo restartSegNo;
@@ -65,8 +63,8 @@ RestoreArchivedFile(char *path, const char *xlogfname,
TimeLineID restartTli;
/*
- * Ignore restore_command when not in archive recovery (meaning
- * we are in crash recovery).
+ * Ignore restore_command when not in archive recovery (meaning we are in
+ * crash recovery).
*/
if (!ArchiveRecoveryRequested)
goto not_available;
@@ -149,58 +147,13 @@ RestoreArchivedFile(char *path, const char *xlogfname,
else
XLogFileName(lastRestartPointFname, 0, 0L, wal_segment_size);
- /*
- * construct the command to be executed
- */
- dp = xlogRestoreCmd;
- endp = xlogRestoreCmd + MAXPGPATH - 1;
- *endp = '\0';
+ /* Build the command to execute */
+ rc = BuildRestoreCommand(recoveryRestoreCommand, xlogpath, xlogfname,
+ lastRestartPointFname, xlogRestoreCmd);
- for (sp = recoveryRestoreCommand; *sp; sp++)
- {
- if (*sp == '%')
- {
- switch (sp[1])
- {
- case 'p':
- /* %p: relative path of target file */
- sp++;
- StrNCpy(dp, xlogpath, endp - dp);
- make_native_path(dp);
- dp += strlen(dp);
- break;
- case 'f':
- /* %f: filename of desired file */
- sp++;
- StrNCpy(dp, xlogfname, endp - dp);
- dp += strlen(dp);
- break;
- case 'r':
- /* %r: filename of last restartpoint */
- sp++;
- StrNCpy(dp, lastRestartPointFname, endp - dp);
- dp += strlen(dp);
- break;
- case '%':
- /* convert %% to a single % */
- sp++;
- if (dp < endp)
- *dp++ = *sp;
- break;
- default:
- /* otherwise treat the % as not special */
- if (dp < endp)
- *dp++ = *sp;
- break;
- }
- }
- else
- {
- if (dp < endp)
- *dp++ = *sp;
- }
- }
- *dp = '\0';
+ if (rc != 0)
+ elog(ERROR, "failed to build restore command \"%s\" due to incorrect parameters",
+ xlogRestoreCmd);
ereport(DEBUG3,
(errmsg_internal("executing restore command \"%s\"",
diff --git a/src/common/Makefile b/src/common/Makefile
index ce01df68b9..a97c723fbd 100644
--- a/src/common/Makefile
+++ b/src/common/Makefile
@@ -46,6 +46,7 @@ LIBS += $(PTHREAD_LIBS)
# If you add objects here, see also src/tools/msvc/Mkvcbuild.pm
OBJS_COMMON = \
+ archive.o \
base64.o \
config_info.o \
controldata_utils.o \
@@ -87,6 +88,7 @@ endif
# (Mkvcbuild.pm has a copy of this list, too)
OBJS_FRONTEND = \
$(OBJS_COMMON) \
+ fe_archive.o \
fe_memutils.o \
file_utils.o \
logging.o \
diff --git a/src/common/archive.c b/src/common/archive.c
new file mode 100644
index 0000000000..7152b96983
--- /dev/null
+++ b/src/common/archive.c
@@ -0,0 +1,106 @@
+/*-------------------------------------------------------------------------
+ *
+ * archive.c
+ * Common WAL archive routines
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/common/archive.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#include "postgres.h"
+#else
+#include "postgres_fe.h"
+#endif
+
+#include "common/archive.h"
+
+/*
+ * BuildRestoreCommand
+ *
+ * Builds a restore command to retrieve a WAL segment from WAL archives,
+ * replacing the supported aliases with values supplied by the caller:
+ * xlogpath for %p, xlogfname for %f and lastRestartPointFname for %r.
+ * Returns 0 if restore_command was successfully built.
+ *
+ * If any of the required arguments is NULL and that the corresponding alias
+ * is found in the command given by the caller, then -1 is returned.
+ */
+int
+BuildRestoreCommand(const char *restoreCommand,
+ const char *xlogpath,
+ const char *xlogfname,
+ const char *lastRestartPointFname,
+ char *result)
+{
+ char *dp,
+ *endp;
+ const char *sp;
+
+ /*
+ * Build the command to be executed.
+ */
+ dp = result;
+ endp = result + MAXPGPATH - 1;
+ *endp = '\0';
+
+ for (sp = restoreCommand; *sp; sp++)
+ {
+ if (*sp == '%')
+ {
+ switch (sp[1])
+ {
+ case 'p':
+ /* %p: relative path of target file */
+ if (xlogpath == NULL)
+ return -1;
+ sp++;
+ StrNCpy(dp, xlogpath, endp - dp);
+ make_native_path(dp);
+ dp += strlen(dp);
+ break;
+ case 'f':
+ /* %f: filename of desired file */
+ if (xlogfname == NULL)
+ return -1;
+ sp++;
+ StrNCpy(dp, xlogfname, endp - dp);
+ dp += strlen(dp);
+ break;
+ case 'r':
+ /* %r: filename of last restartpoint */
+ if (lastRestartPointFname == NULL)
+ return -1;
+ sp++;
+ StrNCpy(dp, lastRestartPointFname, endp - dp);
+ dp += strlen(dp);
+ break;
+ case '%':
+ /* convert %% to a single % */
+ sp++;
+ if (dp < endp)
+ *dp++ = *sp;
+ break;
+ default:
+ /* otherwise treat the % as not special */
+ if (dp < endp)
+ *dp++ = *sp;
+ break;
+ }
+ }
+ else
+ {
+ if (dp < endp)
+ *dp++ = *sp;
+ }
+ }
+ *dp = '\0';
+
+ return 0;
+}
diff --git a/src/common/exec.c b/src/common/exec.c
index 88400daa47..5321509b00 100644
--- a/src/common/exec.c
+++ b/src/common/exec.c
@@ -1,7 +1,7 @@
/*-------------------------------------------------------------------------
*
* exec.c
- * Functions for finding and validating executable files
+ * Functions for finding and validating from executables files
*
*
* Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
@@ -51,7 +51,6 @@
static int validate_exec(const char *path);
static int resolve_symlinks(char *path);
-static char *pipe_read_line(char *cmd, char *line, int maxsize);
#ifdef WIN32
static BOOL GetTokenUser(HANDLE hToken, PTOKEN_USER *ppTokenUser);
@@ -356,7 +355,7 @@ find_other_exec(const char *argv0, const char *target,
/*
* Execute a command in a pipe and read the first line from it.
*/
-static char *
+char *
pipe_read_line(char *cmd, char *line, int maxsize)
{
FILE *pgver;
diff --git a/src/common/fe_archive.c b/src/common/fe_archive.c
new file mode 100644
index 0000000000..3c688c4731
--- /dev/null
+++ b/src/common/fe_archive.c
@@ -0,0 +1,124 @@
+/*-------------------------------------------------------------------------
+ *
+ * fe_archive.c
+ * Routines to access WAL archive from frontend
+ *
+ * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ *
+ * IDENTIFICATION
+ * src/common/fe_archive.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef FRONTEND
+#error "This file is not expected to be compiled for backend code"
+#endif
+
+#include "postgres_fe.h"
+
+#include <unistd.h>
+#include <sys/stat.h>
+
+#include "access/xlog_internal.h"
+#include "common/archive.h"
+#include "common/fe_archive.h"
+#include "common/logging.h"
+
+
+/*
+ * RestoreArchivedWALFile
+ *
+ * Attempt to retrieve the specified file from off-line archival storage.
+ * If successful return a file descriptor of the restored WAL file, else
+ * return -1.
+ *
+ * The caller has to pass the expected size of the file restored from
+ * the archives via "expectedSize".
+ */
+int
+RestoreArchivedWALFile(const char *path, const char *xlogfname,
+ off_t expectedSize, const char *restoreCommand)
+{
+ char xlogpath[MAXPGPATH],
+ xlogRestoreCmd[MAXPGPATH];
+ int rc;
+ struct stat stat_buf;
+
+ Assert(expectedSize > 0);
+
+ snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);
+
+ rc = BuildRestoreCommand(restoreCommand, xlogpath, xlogfname, NULL, xlogRestoreCmd);
+
+ if (rc < 0)
+ {
+ pg_log_fatal("restore_command with %%r alias cannot be used.");
+ exit(1);
+ }
+
+ /*
+ * Execute restore_command, which should copy the missing WAL file from
+ * archival storage.
+ */
+ rc = system(xlogRestoreCmd);
+
+ if (rc == 0)
+ {
+ /*
+ * Command apparently succeeded, but let's make sure the file is
+ * really there now and has the correct size.
+ */
+ if (stat(xlogpath, &stat_buf) == 0)
+ {
+ if (stat_buf.st_size != expectedSize)
+ {
+ pg_log_error("unexpected WAL file size for \"%s\": %lu instead of %lu",
+ xlogfname, (unsigned long) stat_buf.st_size,
+ (unsigned long) expectedSize);
+ return -1;
+ }
+ else
+ {
+ int xlogfd = open(xlogpath, O_RDONLY | PG_BINARY, 0);
+
+ if (xlogfd < 0)
+ {
+ pg_log_error("could not open file \"%s\" restored from archive: %m",
+ xlogpath);
+ return -1;
+ }
+ else
+ return xlogfd;
+ }
+ }
+ else
+ {
+ pg_log_error("could not stat file \"%s\": %m",
+ xlogpath);
+ return -1;
+ }
+ }
+
+ /*
+ * If the failure was due to a signal, then it would be misleading to
+ * return with a failure at restoring the WAL file. So just bail out and
+ * exit.
+ */
+ if (wait_result_is_any_signal(rc, true))
+ {
+ pg_log_fatal("restore_command failed due to the signal: %s",
+ wait_result_to_str(rc));
+ exit(1);
+ }
+
+ /*
+ * The WAL file is not available, so just let the caller decide what to do
+ * next.
+ */
+ pg_log_error("could not restore file \"%s\" from archive",
+ xlogfname);
+ return -1;
+}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index eb61cb8803..d28096397c 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -19,6 +19,7 @@
#include "catalog/pg_control.h"
#include "catalog/storage_xlog.h"
#include "commands/dbcommands_xlog.h"
+#include "common/fe_archive.h"
#include "filemap.h"
#include "pg_rewind.h"
@@ -41,6 +42,7 @@ static char xlogfpath[MAXPGPATH];
typedef struct XLogPageReadPrivate
{
+ const char *restoreCommand;
int tliIndex;
} XLogPageReadPrivate;
@@ -55,7 +57,7 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
*/
void
extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
- XLogRecPtr endpoint)
+ XLogRecPtr endpoint, const char *restore_command)
{
XLogRecord *record;
XLogReaderState *xlogreader;
@@ -63,6 +65,7 @@ extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
XLogPageReadPrivate private;
private.tliIndex = tliIndex;
+ private.restoreCommand = restore_command;
xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -146,7 +149,7 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
void
findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo)
+ XLogRecPtr *lastchkptredo, const char *restoreCommand)
{
/* Walk backwards, starting from the given record */
XLogRecord *record;
@@ -170,6 +173,7 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
}
private.tliIndex = tliIndex;
+ private.restoreCommand = restoreCommand;
xlogreader = XLogReaderAllocate(WalSegSz, datadir, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -281,8 +285,29 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
- pg_log_error("could not open file \"%s\": %m", xlogfpath);
- return -1;
+ /*
+ * If we have no restore_command to execute, then exit.
+ */
+ if (private->restoreCommand == NULL)
+ {
+ pg_log_error("could not open file \"%s\": %m", xlogfpath);
+ return -1;
+ }
+
+ /*
+ * Since we have restore_command, then try to retrieve missing WAL
+ * file from the archive.
+ */
+ xlogreadfd = RestoreArchivedWALFile(xlogreader->segcxt.ws_dir,
+ xlogfname,
+ WalSegSz,
+ private->restoreCommand);
+
+ if (xlogreadfd < 0)
+ return -1;
+ else
+ pg_log_debug("using file \"%s\" restored from archive",
+ xlogfpath);
}
}
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index bf2d2983e7..aa5522e870 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -22,6 +22,7 @@
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "common/restricted_token.h"
+#include "common/string.h"
#include "fe_utils/recovery_gen.h"
#include "fetch.h"
#include "file_ops.h"
@@ -38,6 +39,7 @@ static void createBackupLabel(XLogRecPtr startpoint, TimeLineID starttli,
static void digestControlFile(ControlFileData *ControlFile, char *source,
size_t size);
static void syncTargetDirectory(void);
+static void getRestoreCommand(const char *argv0);
static void sanityChecks(void);
static void findCommonAncestorTimeline(XLogRecPtr *recptr, int *tliIndex);
static void ensureCleanShutdown(const char *argv0);
@@ -53,11 +55,13 @@ int WalSegSz;
char *datadir_target = NULL;
char *datadir_source = NULL;
char *connstr_source = NULL;
+char *restore_command = NULL;
static bool debug = false;
bool showprogress = false;
bool dry_run = false;
bool do_sync = true;
+bool restore_wal = false;
/* Target history */
TimeLineHistoryEntry *targetHistory;
@@ -74,6 +78,8 @@ usage(const char *progname)
printf(_("%s resynchronizes a PostgreSQL cluster with another copy of the cluster.\n\n"), progname);
printf(_("Usage:\n %s [OPTION]...\n\n"), progname);
printf(_("Options:\n"));
+ printf(_(" -c, --restore-target-wal use restore_command in target config\n"
+ " to retrieve WAL files from archive\n"));
printf(_(" -D, --target-pgdata=DIRECTORY existing data directory to modify\n"));
printf(_(" --source-pgdata=DIRECTORY source data directory to synchronize with\n"));
printf(_(" --source-server=CONNSTR source server to synchronize with\n"));
@@ -103,6 +109,7 @@ main(int argc, char **argv)
{"source-server", required_argument, NULL, 2},
{"no-ensure-shutdown", no_argument, NULL, 4},
{"version", no_argument, NULL, 'V'},
+ {"restore-target-wal", no_argument, NULL, 'c'},
{"dry-run", no_argument, NULL, 'n'},
{"no-sync", no_argument, NULL, 'N'},
{"progress", no_argument, NULL, 'P'},
@@ -144,7 +151,7 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "D:nNPR", long_options, &option_index)) != -1)
+ while ((c = getopt_long(argc, argv, "cD:nNPR", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -152,6 +159,10 @@ main(int argc, char **argv)
fprintf(stderr, _("Try \"%s --help\" for more information.\n"), progname);
exit(1);
+ case 'c':
+ restore_wal = true;
+ break;
+
case 'P':
showprogress = true;
break;
@@ -255,6 +266,8 @@ main(int argc, char **argv)
umask(pg_mode_mask);
+ getRestoreCommand(argv[0]);
+
atexit(disconnect_atexit);
/* Connect to remote server */
@@ -350,9 +363,8 @@ main(int argc, char **argv)
exit(0);
}
- findLastCheckpoint(datadir_target, divergerec,
- lastcommontliIndex,
- &chkptrec, &chkpttli, &chkptredo);
+ findLastCheckpoint(datadir_target, divergerec, lastcommontliIndex,
+ &chkptrec, &chkpttli, &chkptredo, restore_command);
pg_log_info("rewinding from last common checkpoint at %X/%X on timeline %u",
(uint32) (chkptrec >> 32), (uint32) chkptrec,
chkpttli);
@@ -378,7 +390,7 @@ main(int argc, char **argv)
if (showprogress)
pg_log_info("reading WAL in target");
extractPageMap(datadir_target, chkptrec, lastcommontliIndex,
- ControlFile_target.checkPoint);
+ ControlFile_target.checkPoint, restore_command);
filemap_finalize();
if (showprogress)
@@ -804,6 +816,67 @@ syncTargetDirectory(void)
fsync_pgdata(datadir_target, PG_VERSION_NUM);
}
+/*
+ */
+static void
+getRestoreCommand(const char *argv0)
+{
+ int rc;
+ char postgres_exec_path[MAXPGPATH],
+ postgres_cmd[MAXPGPATH],
+ cmd_output[MAXPGPATH];
+
+ if (!restore_wal)
+ return;
+
+ /* find postgres executable */
+ rc = find_other_exec(argv0, "postgres",
+ PG_BACKEND_VERSIONSTR,
+ postgres_exec_path);
+
+ if (rc < 0)
+ {
+ char full_path[MAXPGPATH];
+
+ if (find_my_exec(argv0, full_path) < 0)
+ strlcpy(full_path, progname, sizeof(full_path));
+
+ if (rc == -1)
+ pg_log_error("The program \"postgres\" is needed by %s but was not found in the\n"
+ "same directory as \"%s\".\n"
+ "Check your installation.",
+ progname, full_path);
+ else
+ pg_log_error("The program \"postgres\" was found by \"%s\"\n"
+ "but was not the same version as %s.\n"
+ "Check your installation.",
+ full_path, progname);
+ exit(1);
+ }
+
+ /*
+ * Build a command able to retrieve the value of GUC parameter
+ * restore_command, if set.
+ */
+ snprintf(postgres_cmd, sizeof(postgres_cmd),
+ "\"%s\" -D \"%s\" -C restore_command",
+ postgres_exec_path, datadir_target);
+
+ if (!pipe_read_line(postgres_cmd, cmd_output, sizeof(cmd_output)))
+ exit(1);
+
+ (void) pg_strip_crlf(cmd_output);
+
+ if (strcmp(cmd_output, "") == 0)
+ pg_fatal("restore_command is not set on the target cluster");
+
+ restore_command = pg_strdup(cmd_output);
+
+ pg_log_debug("using for rewind restore_command = \'%s\'",
+ restore_command);
+}
+
+
/*
* Ensure clean shutdown of target instance by launching single-user mode
* postgres to do crash recovery.
diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h
index e4e8d23c32..b122ae43e5 100644
--- a/src/bin/pg_rewind/pg_rewind.h
+++ b/src/bin/pg_rewind/pg_rewind.h
@@ -42,11 +42,13 @@ extern uint64 fetch_done;
/* in parsexlog.c */
extern void extractPageMap(const char *datadir, XLogRecPtr startpoint,
- int tliIndex, XLogRecPtr endpoint);
+ int tliIndex, XLogRecPtr endpoint,
+ const char *restoreCommand);
extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr,
int tliIndex,
XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo);
+ XLogRecPtr *lastchkptredo,
+ const char *restoreCommand);
extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr,
int tliIndex);
diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl
index 95d8ccfced..d97e437741 100644
--- a/src/bin/pg_rewind/t/001_basic.pl
+++ b/src/bin/pg_rewind/t/001_basic.pl
@@ -1,7 +1,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 15;
+use Test::More tests => 20;
use FindBin;
use lib $FindBin::RealBin;
@@ -171,5 +171,6 @@ in master, before promotion
# Run the test in both modes
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/bin/pg_rewind/t/RewindTest.pm b/src/bin/pg_rewind/t/RewindTest.pm
index 82fa220ac8..577a4294f3 100644
--- a/src/bin/pg_rewind/t/RewindTest.pm
+++ b/src/bin/pg_rewind/t/RewindTest.pm
@@ -38,6 +38,7 @@ use File::Copy;
use File::Path qw(rmtree);
use IPC::Run qw(run);
use PostgresNode;
+use RecursiveCopy;
use TestLib;
use Test::More;
@@ -227,10 +228,23 @@ sub run_pg_rewind
# Append the rewind-specific role to the connection string.
$standby_connstr = "$standby_connstr user=rewind_user";
- # Stop the master and be ready to perform the rewind. The cluster
- # needs recovery to finish once, and pg_rewind makes sure that it
- # happens automatically.
- $node_master->stop('immediate');
+ if ($test_mode eq 'archive')
+ {
+ # We test pg_rewind with restore_command by simply moving all WAL files
+ # to another location. It leads to failed ensureCleanShutdown
+ # execution. Since it is difficult to emulate a situation, when
+ # keeping the last WAL segment is enough for startup recovery, but
+ # not enough for successful pg_rewind run, we run these modes with
+ # --no-ensure-shutdown. So stop the master gracefully.
+ $node_master->stop;
+ }
+ else
+ {
+ # Stop the master and be ready to perform the rewind. The cluster
+ # needs recovery to finish once, and pg_rewind makes sure that it
+ # happens automatically.
+ $node_master->stop('immediate');
+ }
# At this point, the rewind processing is ready to run.
# We now have a very simple scenario with a few diverged WAL record.
@@ -284,6 +298,51 @@ sub run_pg_rewind
$node_standby->safe_psql('postgres',
"ALTER ROLE rewind_user WITH REPLICATION;");
}
+ elsif ($test_mode eq "archive")
+ {
+
+ # Do rewind using a local pgdata as source and specified
+ # directory with target WAL archive. The old master has
+ # to be stopped at this point.
+
+ # First, remove all the content in the archive directory,
+ # as RecursiveCopy::copypath does not support copying to
+ # existing directories.
+ rmtree($node_master->archive_dir);
+
+ # Move all WAL segments from the old master to the archives.
+ # These will be used by pg_rewind.
+ RecursiveCopy::copypath($node_master->data_dir . "/pg_wal",
+ $node_master->archive_dir);
+
+ # Fast way to remove entire directory content
+ rmtree($node_master->data_dir . "/pg_wal");
+ mkdir($node_master->data_dir . "/pg_wal");
+
+ # Make sure that directories have the right umask as this is
+ # required by a follow-up check on permissions, and better
+ # safe than sorry.
+ chmod(0700, $node_master->archive_dir);
+ chmod(0700, $node_master->data_dir . "/pg_wal");
+
+ # Add appropriate restore_command to the target cluster
+ $node_master->enable_restoring($node_master, 0);
+
+ # Stop the new master and be ready to perform the rewind.
+ $node_standby->stop;
+
+ command_ok(
+ [
+ 'pg_rewind',
+ "--debug",
+ "--source-pgdata=$standby_pgdata",
+ "--target-pgdata=$master_pgdata",
+ "--no-sync",
+ "--no-ensure-shutdown",
+ "--restore-target-wal"
+ ],
+ 'pg_rewind archive');
+ }
else
{
diff --git a/doc/src/sgml/ref/pg_rewind.sgml b/doc/src/sgml/ref/pg_rewind.sgml
index 42d29edd4e..180b05ddb6 100644
--- a/doc/src/sgml/ref/pg_rewind.sgml
+++ b/doc/src/sgml/ref/pg_rewind.sgml
@@ -66,11 +66,11 @@ PostgreSQL documentation
can be found either on the target timeline, the source timeline, or their common
ancestor. In the typical failover scenario where the target cluster was
shut down soon after the divergence, this is not a problem, but if the
- target cluster ran for a long time after the divergence, the old WAL
- files might no longer be present. In that case, they can be manually
- copied from the WAL archive to the <filename>pg_wal</filename> directory, or
- fetched on startup by configuring <xref linkend="guc-primary-conninfo"/> or
- <xref linkend="guc-restore-command"/>. The use of
+ target cluster ran for a long time after the divergence, its old WAL
+ files might no longer be present. In this case, you can manually copy them
+ from the WAL archive to the <filename>pg_wal</filename> directory, or run
+ <application>pg_rewind</application> with the <literal>-c</literal> option to
+ automatically retrieve them from the WAL archive. The use of
<application>pg_rewind</application> is not limited to failover, e.g. a standby
server can be promoted, run some write transactions, and then rewinded
to become a standby again.
@@ -232,6 +232,19 @@ PostgreSQL documentation
</listitem>
</varlistentry>
+ <varlistentry>
+ <term><option>-c</option></term>
+ <term><option>--restore-target-wal</option></term>
+ <listitem>
+ <para>
+ Use <varname>restore_command</varname> defined in the target cluster
+ configuration to retrieve WAL files from the WAL archive if these
+ files are no longer available in the <filename>pg_wal</filename>
+ directory.
+ </para>
+ </listitem>
+ </varlistentry>
+
<varlistentry>
<term><option>--debug</option></term>
<listitem>
@@ -318,7 +331,10 @@ GRANT EXECUTE ON function pg_catalog.pg_read_binary_file(text, bigint, bigint, b
history forked off from the target cluster. For each WAL record,
record each data block that was touched. This yields a list of all
the data blocks that were changed in the target cluster, after the
- source cluster forked off.
+ source cluster forked off. If some of the WAL files are no longer
+ available, try re-running <application>pg_rewind</application> with
+ the <option>-c</option> option to search for the missing files in
+ the WAL archive.
</para>
</step>
<step>
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index 834c2c39d1..25da1fa8ec 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -119,8 +119,8 @@ sub mkvcbuild
}
our @pgcommonallfiles = qw(
- base64.c config_info.c controldata_utils.c d2s.c encnames.c exec.c
- f2s.c file_perm.c hashfn.c ip.c jsonapi.c
+ archive.c base64.c config_info.c controldata_utils.c d2s.c encnames.c
+ exec.c f2s.c file_perm.c hashfn.c ip.c jsonapi.c
keywords.c kwlookup.c link-canary.c md5.c
pg_lzcompress.c pgfnames.c psprintf.c relpath.c rmtree.c
saslprep.c scram-common.c string.c stringinfo.c unicode_norm.c username.c
@@ -137,8 +137,8 @@ sub mkvcbuild
}
our @pgcommonfrontendfiles = (
- @pgcommonallfiles, qw(fe_memutils.c file_utils.c
- logging.c restricted_token.c));
+ @pgcommonallfiles, qw(fe_archive.c fe_memutils.c
+ file_utils.c logging.c restricted_token.c));
our @pgcommonbkndfiles = @pgcommonallfiles;
signature.asc
Description: PGP signature
