On 30.10.2018 06:01, Michael Paquier wrote:
On Mon, Oct 29, 2018 at 12:09:21PM +0300, Alexey Kondratov wrote:
Currently in the patch, with dry-run option (-n) pg_rewind only fetches
missing WALs to be able to build file map, while doesn't touch any data
files. So I guess it behaves exactly as you described and we do not need a
separate tool.
Makes sense perhaps. Fetching only WAL segments which are needed for
the file map is critical, as you don't want to spend bandwidth for
nothing. Now, I look at your patch, and I can see things to complain
about, at least three at short glance:
- The TAP test added will fail on Windows.
Thank you for this. Build on Windows has been broken as well. I fixed it
in the new version of patch, please find attached.
- Simply copy-pasting RestoreArchivedWAL() from the backend code to
pg_rewind is not an acceptable option. You don't care about %r either
in this case.
According to the docs [1] %r is a valid alias and may be used in
restore_command too, so if we take restore_command from recovery.conf it
might be there. If we just drop it, then restore_command may stop
working. Though I do not know real life examples of restore_command with
%r, we should treat it in expected way (as backend does), of course if
we want an option to take it from recovery.conf.
- Reusing the GUC parser is something I would avoid as well. Not worth
the complexity.
Yes, I don't like it either. I will try to make guc-file.l frontend safe.
[1] https://www.postgresql.org/docs/11/archive-recovery-settings.html
--
Alexey Kondratov
Postgres Professional https://www.postgrespro.com
Russian Postgres Company
diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile
index 2bcfcc61af2e..a22fef1352b9 100644
--- a/src/bin/pg_rewind/Makefile
+++ b/src/bin/pg_rewind/Makefile
@@ -20,6 +20,7 @@ LDFLAGS_INTERNAL += $(libpq_pgport)
OBJS = pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \
fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o logging.o \
+ guc-file-fe.o \
$(WIN32RES)
EXTRA_CLEAN = xlogreader.c
diff --git a/src/bin/pg_rewind/RewindTest.pm b/src/bin/pg_rewind/RewindTest.pm
index 1dce56d0352e..a5499c7027b1 100644
--- a/src/bin/pg_rewind/RewindTest.pm
+++ b/src/bin/pg_rewind/RewindTest.pm
@@ -39,7 +39,9 @@ use Carp;
use Config;
use Exporter 'import';
use File::Copy;
-use File::Path qw(rmtree);
+use File::Glob ':bsd_glob';
+use File::Path qw(remove_tree make_path);
+use File::Spec::Functions 'catpath';
use IPC::Run qw(run);
use PostgresNode;
use TestLib;
@@ -249,6 +251,48 @@ sub run_pg_rewind
],
'pg_rewind remote');
}
+ elsif ($test_mode eq "archive")
+ {
+
+ # Do rewind using a local pgdata as source and
+ # specified directory with target WALs archive.
+ my $wals_archive_dir = catpath(${TestLib::tmp_check}, 'master_wals_archive');
+ my $test_master_datadir = $node_master->data_dir;
+ my @wal_files = bsd_glob catpath($test_master_datadir, 'pg_wal', '0000000*');
+ my $restore_command;
+
+ remove_tree($wals_archive_dir);
+ make_path($wals_archive_dir) or die;
+
+ # Move all old master WAL files to the archive.
+ # Old master should be stopped at this point.
+ foreach my $wal_file (@wal_files)
+ {
+ move($wal_file, "$wals_archive_dir/") or die;
+ }
+
+ if ($windows_os)
+ {
+ $restore_command = "copy $wals_archive_dir\\\%f \%p";
+ }
+ else
+ {
+ $restore_command = "cp $wals_archive_dir/\%f \%p";
+ }
+
+ # Stop the new master and be ready to perform the rewind.
+ $node_standby->stop;
+ command_ok(
+ [
+ 'pg_rewind',
+ "--debug",
+ "--source-pgdata=$standby_pgdata",
+ "--target-pgdata=$master_pgdata",
+ "--no-sync",
+ "-R", $restore_command
+ ],
+ 'pg_rewind archive');
+ }
else
{
diff --git a/src/bin/pg_rewind/guc-file-fe.h b/src/bin/pg_rewind/guc-file-fe.h
new file mode 100644
index 000000000000..cf480b806ae5
--- /dev/null
+++ b/src/bin/pg_rewind/guc-file-fe.h
@@ -0,0 +1,40 @@
+#ifndef PG_REWIND_GUC_FILE_FE_H
+#define PG_REWIND_GUC_FILE_FE_H
+
+#include "c.h"
+
+#define RECOVERY_COMMAND_FILE "recovery.conf"
+
+/*
+ * Parsing the configuration file(s) will return a list of name-value pairs
+ * with source location info. We also abuse this data structure to carry
+ * error reports about the config files. An entry reporting an error will
+ * have errmsg != NULL, and might have NULLs for name, value, and/or filename.
+ *
+ * If "ignore" is true, don't attempt to apply the item (it might be an error
+ * report, or an item we determined to be duplicate). "applied" is set true
+ * if we successfully applied, or could have applied, the setting.
+ */
+typedef struct ConfigVariable
+{
+ char *name;
+ char *value;
+ char *errmsg;
+ char *filename;
+ int sourceline;
+ bool ignore;
+ bool applied;
+ struct ConfigVariable *next;
+} ConfigVariable;
+
+extern bool ParseConfigFile(const char *config_file, bool strict,
+ const char *calling_file, int calling_lineno,
+ int depth, int elevel,
+ ConfigVariable **head_p, ConfigVariable **tail_p);
+
+extern bool ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,
+ ConfigVariable **head_p, ConfigVariable **tail_p);
+
+extern void FreeConfigVariables(ConfigVariable *list);
+
+#endif /* PG_REWIND_GUC_FILE_FE_H */
diff --git a/src/bin/pg_rewind/guc-file-fe.l b/src/bin/pg_rewind/guc-file-fe.l
new file mode 100644
index 000000000000..b7ef47111a79
--- /dev/null
+++ b/src/bin/pg_rewind/guc-file-fe.l
@@ -0,0 +1,776 @@
+/* -*-pgsql-c-*- */
+/*
+ * Configuration files parser for usage in frontend.
+ *
+ * Copyright (c) 2000-2018, PostgreSQL Global Development Group
+ *
+ * src/bin/pg_rewind/guc-file-fe.l
+ * Modified version of src/backend/utils/misc/guc-file.l
+ */
+
+%{
+
+#include "postgres_fe.h"
+
+#include <ctype.h>
+#include <dirent.h>
+#include <unistd.h>
+#include <setjmp.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#include "guc-file-fe.h"
+#include "logging.h"
+
+#include "common/fe_memutils.h"
+
+/*
+ * flex emits a yy_fatal_error() function that it calls in response to
+ * critical errors like malloc failure, file I/O errors, and detection of
+ * internal inconsistency. That function prints a message and calls exit().
+ * Mutate it to instead call our handler, which jumps out of the parser.
+ */
+#undef fprintf
+#define fprintf(file, fmt, msg) GUC_flex_fatal(msg)
+
+enum
+{
+ GUC_ID = 1,
+ GUC_STRING = 2,
+ GUC_INTEGER = 3,
+ GUC_REAL = 4,
+ GUC_EQUALS = 5,
+ GUC_UNQUOTED_STRING = 6,
+ GUC_QUALIFIED_ID = 7,
+ GUC_EOL = 99,
+ GUC_ERROR = 100
+};
+
+static unsigned int ConfigFileLineno;
+static const char *GUC_flex_fatal_errmsg;
+static sigjmp_buf *GUC_flex_fatal_jmp;
+
+static void FreeConfigVariable(ConfigVariable *item);
+
+static void record_config_file_error(const char *errmsg,
+ const char *config_file,
+ int lineno,
+ ConfigVariable **head_p,
+ ConfigVariable **tail_p);
+
+static int GUC_flex_fatal(const char *msg);
+static char *GUC_scanstr(const char *s);
+static bool ParseConfigDirectory(const char *includedir,
+ const char *calling_file, int calling_lineno,
+ int depth, int elevel,
+ ConfigVariable **head_p,
+ ConfigVariable **tail_p);
+
+/* LCOV_EXCL_START */
+
+%}
+
+%option 8bit
+%option never-interactive
+%option nodefault
+%option noinput
+%option nounput
+%option noyywrap
+%option warn
+%option prefix="GUC_yy"
+
+
+SIGN ("-"|"+")
+DIGIT [0-9]
+HEXDIGIT [0-9a-fA-F]
+
+UNIT_LETTER [a-zA-Z]
+
+INTEGER {SIGN}?({DIGIT}+|0x{HEXDIGIT}+){UNIT_LETTER}*
+
+EXPONENT [Ee]{SIGN}?{DIGIT}+
+REAL {SIGN}?{DIGIT}*"."{DIGIT}*{EXPONENT}?
+
+LETTER [A-Za-z_\200-\377]
+LETTER_OR_DIGIT [A-Za-z_0-9\200-\377]
+
+ID {LETTER}{LETTER_OR_DIGIT}*
+QUALIFIED_ID {ID}"."{ID}
+
+UNQUOTED_STRING {LETTER}({LETTER_OR_DIGIT}|[-._:/])*
+STRING \'([^'\\\n]|\\.|\'\')*\'
+
+%%
+
+\n ConfigFileLineno++; return GUC_EOL;
+[ \t\r]+ /* eat whitespace */
+#.* /* eat comment (.* matches anything until newline) */
+
+{ID} return GUC_ID;
+{QUALIFIED_ID} return GUC_QUALIFIED_ID;
+{STRING} return GUC_STRING;
+{UNQUOTED_STRING} return GUC_UNQUOTED_STRING;
+{INTEGER} return GUC_INTEGER;
+{REAL} return GUC_REAL;
+= return GUC_EQUALS;
+
+. return GUC_ERROR;
+
+%%
+
+/* LCOV_EXCL_STOP */
+
+/*
+ * the bare comparison function for GUC names
+ */
+static int
+guc_name_compare(const char *namea, const char *nameb)
+{
+ /*
+ * The temptation to use strcasecmp() here must be resisted, because the
+ * array ordering has to remain stable across setlocale() calls. So, build
+ * our own with a simple ASCII-only downcasing.
+ */
+ while (*namea && *nameb)
+ {
+ char cha = *namea++;
+ char chb = *nameb++;
+
+ if (cha >= 'A' && cha <= 'Z')
+ cha += 'a' - 'A';
+ if (chb >= 'A' && chb <= 'Z')
+ chb += 'a' - 'A';
+ if (cha != chb)
+ return cha - chb;
+ }
+ if (*namea)
+ return 1; /* a is longer */
+ if (*nameb)
+ return -1; /* b is longer */
+ return 0;
+}
+
+/*
+ * Given a configuration file or directory location that may be a relative
+ * path, return an absolute one. We consider the location to be relative to
+ * the directory holding the calling file.
+ */
+static char *
+AbsoluteConfigLocation(const char *location, const char *calling_file)
+{
+ char abs_path[MAXPGPATH];
+
+ if (is_absolute_path(location))
+ return pstrdup(location);
+ else
+ {
+ strlcpy(abs_path, calling_file, sizeof(abs_path));
+ get_parent_directory(abs_path);
+ join_path_components(abs_path, abs_path, location);
+ canonicalize_path(abs_path);
+
+ return pstrdup(abs_path);
+ }
+}
+
+/*
+ * Read and parse a single configuration file. This function recurses
+ * to handle "include" directives.
+ *
+ * If "strict" is true, treat failure to open the config file as an error,
+ * otherwise just skip the file.
+ *
+ * calling_file/calling_lineno identify the source of the request.
+ * Pass NULL/0 if not recursing from an inclusion request.
+ *
+ * See ParseConfigFp for further details. This one merely adds opening the
+ * config file rather than working from a caller-supplied file descriptor,
+ * and absolute-ifying the path name if necessary.
+ */
+bool
+ParseConfigFile(const char *config_file, bool strict,
+ const char *calling_file, int calling_lineno,
+ int depth, int elevel,
+ ConfigVariable **head_p,
+ ConfigVariable **tail_p)
+{
+ char *abs_path;
+ bool OK = true;
+ FILE *fp;
+
+ /*
+ * Reject too-deep include nesting depth. This is just a safety check to
+ * avoid dumping core due to stack overflow if an include file loops back
+ * to itself. The maximum nesting depth is pretty arbitrary.
+ */
+ if (depth > 10)
+ {
+ pg_log(PG_WARNING,
+ "could not open configuration file \"%s\": maximum nesting depth exceeded\n",
+ config_file);
+ record_config_file_error("nesting depth exceeded",
+ calling_file, calling_lineno,
+ head_p, tail_p);
+ return false;
+ }
+
+ abs_path = AbsoluteConfigLocation(config_file, calling_file);
+ fp = fopen(abs_path, "r");
+ if (!fp)
+ {
+ if (strict)
+ {
+ pg_log(PG_WARNING,
+ "could not open configuration file \"%s\"\n",
+ abs_path);
+ record_config_file_error(psprintf("could not open file \"%s\"",
+ abs_path),
+ calling_file, calling_lineno,
+ head_p, tail_p);
+ OK = false;
+ }
+ else
+ {
+ pg_log(PG_WARNING,
+ "skipping missing configuration file \"%s\"\n",
+ abs_path);
+ }
+ goto cleanup;
+ }
+
+ OK = ParseConfigFp(fp, abs_path, depth, elevel, head_p, tail_p);
+
+cleanup:
+ if (fp)
+ fclose(fp);
+ pfree(abs_path);
+
+ return OK;
+}
+
+/*
+ * Capture an error message in the ConfigVariable list returned by
+ * config file parsing.
+ */
+static void
+record_config_file_error(const char *errmsg,
+ const char *config_file,
+ int lineno,
+ ConfigVariable **head_p,
+ ConfigVariable **tail_p)
+{
+ ConfigVariable *item;
+
+ item = palloc(sizeof *item);
+ item->name = NULL;
+ item->value = NULL;
+ item->errmsg = pstrdup(errmsg);
+ item->filename = config_file ? pstrdup(config_file) : NULL;
+ item->sourceline = lineno;
+ item->ignore = true;
+ item->applied = false;
+ item->next = NULL;
+ if (*head_p == NULL)
+ *head_p = item;
+ else
+ (*tail_p)->next = item;
+ *tail_p = item;
+}
+
+/*
+ * Flex fatal errors bring us here. Stash the error message and jump back to
+ * ParseConfigFp(). Assume all msg arguments point to string constants; this
+ * holds for flex 2.5.31 (earliest we support) and flex 2.5.35 (latest as of
+ * this writing). Otherwise, we would need to copy the message.
+ *
+ * We return "int" since this takes the place of calls to fprintf().
+*/
+static int
+GUC_flex_fatal(const char *msg)
+{
+ GUC_flex_fatal_errmsg = msg;
+ siglongjmp(*GUC_flex_fatal_jmp, 1);
+ return 0; /* keep compiler quiet */
+}
+
+/*
+ * Read and parse a single configuration file. This function recurses
+ * to handle "include" directives.
+ *
+ * Input parameters:
+ * fp: file pointer from AllocateFile for the configuration file to parse
+ * config_file: absolute or relative path name of the configuration file
+ * depth: recursion depth (should be 0 in the outermost call)
+ * elevel: error logging level to use
+ * Input/Output parameters:
+ * head_p, tail_p: head and tail of linked list of name/value pairs
+ *
+ * *head_p and *tail_p must be initialized, either to NULL or valid pointers
+ * to a ConfigVariable list, before calling the outer recursion level. Any
+ * name-value pairs read from the input file(s) will be appended to the list.
+ * Error reports will also be appended to the list, if elevel < ERROR.
+ *
+ * Returns TRUE if successful, FALSE if an error occurred.
+ *
+ * Note: this function is used to parse not only postgresql.conf, but
+ * various other configuration files that use the same "name = value"
+ * syntax. Hence, do not do anything here or in the subsidiary routines
+ * ParseConfigFile/ParseConfigDirectory that assumes we are processing
+ * GUCs specifically.
+ */
+bool
+ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,
+ ConfigVariable **head_p, ConfigVariable **tail_p)
+{
+ volatile bool OK = true;
+ unsigned int save_ConfigFileLineno = ConfigFileLineno;
+ sigjmp_buf *save_GUC_flex_fatal_jmp = GUC_flex_fatal_jmp;
+ sigjmp_buf flex_fatal_jmp;
+ volatile YY_BUFFER_STATE lex_buffer = NULL;
+ int errorcount;
+ int token;
+
+ if (sigsetjmp(flex_fatal_jmp, 1) == 0)
+ GUC_flex_fatal_jmp = &flex_fatal_jmp;
+ else
+ {
+ /*
+ * Regain control after a fatal, internal flex error. It may have
+ * corrupted parser state. Consequently, abandon the file, but trust
+ * that the state remains sane enough for yy_delete_buffer().
+ */
+ pg_log(PG_WARNING, "%s at file \"%s\" line %u\n", GUC_flex_fatal_errmsg,
+ config_file, ConfigFileLineno);
+ record_config_file_error(GUC_flex_fatal_errmsg,
+ config_file, ConfigFileLineno,
+ head_p, tail_p);
+ OK = false;
+ goto cleanup;
+ }
+
+ /*
+ * Parse
+ */
+ ConfigFileLineno = 1;
+ errorcount = 0;
+
+ lex_buffer = yy_create_buffer(fp, YY_BUF_SIZE);
+ yy_switch_to_buffer(lex_buffer);
+
+ /* This loop iterates once per logical line */
+ while ((token = yylex()))
+ {
+ char *opt_name = NULL;
+ char *opt_value = NULL;
+ ConfigVariable *item;
+
+ if (token == GUC_EOL) /* empty or comment line */
+ continue;
+
+ /* first token on line is option name */
+ if (token != GUC_ID && token != GUC_QUALIFIED_ID)
+ goto parse_error;
+ opt_name = pstrdup(yytext);
+
+ /* next we have an optional equal sign; discard if present */
+ token = yylex();
+ if (token == GUC_EQUALS)
+ token = yylex();
+
+ /* now we must have the option value */
+ if (token != GUC_ID &&
+ token != GUC_STRING &&
+ token != GUC_INTEGER &&
+ token != GUC_REAL &&
+ token != GUC_UNQUOTED_STRING)
+ goto parse_error;
+ if (token == GUC_STRING) /* strip quotes and escapes */
+ opt_value = GUC_scanstr(yytext);
+ else
+ opt_value = pstrdup(yytext);
+
+ /* now we'd like an end of line, or possibly EOF */
+ token = yylex();
+ if (token != GUC_EOL)
+ {
+ if (token != 0)
+ goto parse_error;
+ /* treat EOF like \n for line numbering purposes, cf bug 4752 */
+ ConfigFileLineno++;
+ }
+
+ /* OK, process the option name and value */
+ if (guc_name_compare(opt_name, "include_dir") == 0)
+ {
+ /*
+ * An include_dir directive isn't a variable and should be
+ * processed immediately.
+ */
+ if (!ParseConfigDirectory(opt_value,
+ config_file, ConfigFileLineno - 1,
+ depth + 1, elevel,
+ head_p, tail_p))
+ OK = false;
+ yy_switch_to_buffer(lex_buffer);
+ pfree(opt_name);
+ pfree(opt_value);
+ }
+ else if (guc_name_compare(opt_name, "include_if_exists") == 0)
+ {
+ /*
+ * An include_if_exists directive isn't a variable and should be
+ * processed immediately.
+ */
+ if (!ParseConfigFile(opt_value, false,
+ config_file, ConfigFileLineno - 1,
+ depth + 1, elevel,
+ head_p, tail_p))
+ OK = false;
+ yy_switch_to_buffer(lex_buffer);
+ pfree(opt_name);
+ pfree(opt_value);
+ }
+ else if (guc_name_compare(opt_name, "include") == 0)
+ {
+ /*
+ * An include directive isn't a variable and should be processed
+ * immediately.
+ */
+ if (!ParseConfigFile(opt_value, true,
+ config_file, ConfigFileLineno - 1,
+ depth + 1, elevel,
+ head_p, tail_p))
+ OK = false;
+ yy_switch_to_buffer(lex_buffer);
+ pfree(opt_name);
+ pfree(opt_value);
+ }
+ else
+ {
+ /* ordinary variable, append to list */
+ item = palloc(sizeof *item);
+ item->name = opt_name;
+ item->value = opt_value;
+ item->errmsg = NULL;
+ item->filename = pstrdup(config_file);
+ item->sourceline = ConfigFileLineno - 1;
+ item->ignore = false;
+ item->applied = false;
+ item->next = NULL;
+ if (*head_p == NULL)
+ *head_p = item;
+ else
+ (*tail_p)->next = item;
+ *tail_p = item;
+ }
+
+ /* break out of loop if read EOF, else loop for next line */
+ if (token == 0)
+ break;
+ continue;
+
+parse_error:
+ /* release storage if we allocated any on this line */
+ if (opt_name)
+ pfree(opt_name);
+ if (opt_value)
+ pfree(opt_value);
+
+ /* report the error */
+ if (token == GUC_EOL || token == 0)
+ {
+ pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near end of line\n",
+ config_file, ConfigFileLineno - 1);
+ record_config_file_error("syntax error",
+ config_file, ConfigFileLineno - 1,
+ head_p, tail_p);
+ }
+ else
+ {
+ pg_log(PG_WARNING, "syntax error in file \"%s\" line %u, near token \"%s\"\n",
+ config_file, ConfigFileLineno, yytext);
+ record_config_file_error("syntax error",
+ config_file, ConfigFileLineno,
+ head_p, tail_p);
+ }
+ OK = false;
+ errorcount++;
+
+ /*
+ * To avoid producing too much noise when fed a totally bogus file,
+ * give up after 100 syntax errors per file (an arbitrary number).
+ */
+ if (errorcount >= 100)
+ {
+ pg_log(PG_WARNING, "too many syntax errors found, abandoning file \"%s\"\n",
+ config_file);
+ break;
+ }
+
+ /* resync to next end-of-line or EOF */
+ while (token != GUC_EOL && token != 0)
+ token = yylex();
+ /* break out of loop on EOF */
+ if (token == 0)
+ break;
+ }
+
+cleanup:
+ yy_delete_buffer(lex_buffer);
+ /* Each recursion level must save and restore these static variables. */
+ ConfigFileLineno = save_ConfigFileLineno;
+ GUC_flex_fatal_jmp = save_GUC_flex_fatal_jmp;
+ return OK;
+}
+
+/*
+ * Read and parse all config files in a subdirectory in alphabetical order
+ *
+ * includedir is the absolute or relative path to the subdirectory to scan.
+ *
+ * calling_file/calling_lineno identify the source of the request.
+ * Pass NULL/0 if not recursing from an inclusion request.
+ *
+ * See ParseConfigFp for further details.
+ */
+bool
+ParseConfigDirectory(const char *includedir,
+ const char *calling_file, int calling_lineno,
+ int depth, int elevel,
+ ConfigVariable **head_p,
+ ConfigVariable **tail_p)
+{
+ char *directory;
+ DIR *d;
+ struct dirent *de;
+ char **filenames;
+ int num_filenames;
+ int size_filenames;
+ bool status;
+
+ directory = AbsoluteConfigLocation(includedir, calling_file);
+ d = opendir(directory);
+ if (d == NULL)
+ {
+ pg_log(PG_WARNING, "could not open configuration directory \"%s\"\n",
+ directory);
+ record_config_file_error(psprintf("could not open directory \"%s\"",
+ directory),
+ calling_file, calling_lineno,
+ head_p, tail_p);
+ status = false;
+ goto cleanup;
+ }
+
+ /*
+ * Read the directory and put the filenames in an array, so we can sort
+ * them prior to processing the contents.
+ */
+ size_filenames = 32;
+ filenames = (char **) palloc(size_filenames * sizeof(char *));
+ num_filenames = 0;
+
+ while ((de = readdir(d)) != NULL)
+ {
+ struct stat st;
+ char filename[MAXPGPATH];
+
+ /*
+ * Only parse files with names ending in ".conf". Explicitly reject
+ * files starting with ".". This excludes things like "." and "..",
+ * as well as typical hidden files, backup files, and editor debris.
+ */
+ if (strlen(de->d_name) < 6)
+ continue;
+ if (de->d_name[0] == '.')
+ continue;
+ if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0)
+ continue;
+
+ join_path_components(filename, directory, de->d_name);
+ canonicalize_path(filename);
+ if (stat(filename, &st) == 0)
+ {
+ if (!S_ISDIR(st.st_mode))
+ {
+ /* Add file to array, increasing its size in blocks of 32 */
+ if (num_filenames >= size_filenames)
+ {
+ size_filenames += 32;
+ filenames = (char **) repalloc(filenames,
+ size_filenames * sizeof(char *));
+ }
+ filenames[num_filenames] = pstrdup(filename);
+ num_filenames++;
+ }
+ }
+ else
+ {
+ /*
+ * stat does not care about permissions, so the most likely reason
+ * a file can't be accessed now is if it was removed between the
+ * directory listing and now.
+ */
+ pg_log(PG_WARNING, "could not stat file \"%s\"\n", filename);
+ record_config_file_error(psprintf("could not stat file \"%s\"",
+ filename),
+ calling_file, calling_lineno,
+ head_p, tail_p);
+ status = false;
+ goto cleanup;
+ }
+ }
+
+ if (num_filenames > 0)
+ {
+ int i;
+
+ qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp);
+ for (i = 0; i < num_filenames; i++)
+ {
+ if (!ParseConfigFile(filenames[i], true,
+ calling_file, calling_lineno,
+ depth, elevel,
+ head_p, tail_p))
+ {
+ status = false;
+ goto cleanup;
+ }
+ }
+ }
+ status = true;
+
+cleanup:
+ if (d)
+ closedir(d);
+ pfree(directory);
+ return status;
+}
+
+/*
+ * Free a list of ConfigVariables, including the names and the values
+ */
+void
+FreeConfigVariables(ConfigVariable *list)
+{
+ ConfigVariable *item;
+
+ item = list;
+ while (item)
+ {
+ ConfigVariable *next = item->next;
+
+ FreeConfigVariable(item);
+ item = next;
+ }
+}
+
+/*
+ * Free a single ConfigVariable
+ */
+static void
+FreeConfigVariable(ConfigVariable *item)
+{
+ if (item->name)
+ pfree(item->name);
+ if (item->value)
+ pfree(item->value);
+ if (item->errmsg)
+ pfree(item->errmsg);
+ if (item->filename)
+ pfree(item->filename);
+ pfree(item);
+}
+
+
+/*
+ * scanstr
+ *
+ * Strip the quotes surrounding the given string, and collapse any embedded
+ * '' sequences and backslash escapes.
+ *
+ * the string returned is palloc'd and should eventually be pfree'd by the
+ * caller.
+ */
+static char *
+GUC_scanstr(const char *s)
+{
+ char *newStr;
+ int len,
+ i,
+ j;
+
+ Assert(s != NULL && s[0] == '\'');
+ len = strlen(s);
+ Assert(len >= 2);
+ Assert(s[len - 1] == '\'');
+
+ /* Skip the leading quote; we'll handle the trailing quote below */
+ s++, len--;
+
+ /* Since len still includes trailing quote, this is enough space */
+ newStr = palloc(len);
+
+ for (i = 0, j = 0; i < len; i++)
+ {
+ if (s[i] == '\\')
+ {
+ i++;
+ switch (s[i])
+ {
+ case 'b':
+ newStr[j] = '\b';
+ break;
+ case 'f':
+ newStr[j] = '\f';
+ break;
+ case 'n':
+ newStr[j] = '\n';
+ break;
+ case 'r':
+ newStr[j] = '\r';
+ break;
+ case 't':
+ newStr[j] = '\t';
+ break;
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ {
+ int k;
+ long octVal = 0;
+
+ for (k = 0;
+ s[i + k] >= '0' && s[i + k] <= '7' && k < 3;
+ k++)
+ octVal = (octVal << 3) + (s[i + k] - '0');
+ i += k - 1;
+ newStr[j] = ((char) octVal);
+ }
+ break;
+ default:
+ newStr[j] = s[i];
+ break;
+ } /* switch */
+ }
+ else if (s[i] == '\'' && s[i + 1] == '\'')
+ {
+ /* doubled quote becomes just one quote */
+ newStr[j] = s[++i];
+ }
+ else
+ newStr[j] = s[i];
+ j++;
+ }
+
+ /* We copied the ending quote to newStr, so replace with \0 */
+ Assert(j > 0 && j <= len);
+ newStr[--j] = '\0';
+
+ return newStr;
+}
diff --git a/src/bin/pg_rewind/parsexlog.c b/src/bin/pg_rewind/parsexlog.c
index 40028471bf6e..11a9c26cd23a 100644
--- a/src/bin/pg_rewind/parsexlog.c
+++ b/src/bin/pg_rewind/parsexlog.c
@@ -12,6 +12,7 @@
#include "postgres_fe.h"
#include <unistd.h>
+#include <sys/stat.h>
#include "pg_rewind.h"
#include "filemap.h"
@@ -45,7 +46,10 @@ static char xlogfpath[MAXPGPATH];
typedef struct XLogPageReadPrivate
{
const char *datadir;
+ const char *restoreCommand;
int tliIndex;
+ XLogRecPtr oldrecptr;
+ TimeLineID oldtli;
} XLogPageReadPrivate;
static int SimpleXLogPageRead(XLogReaderState *xlogreader,
@@ -53,6 +57,10 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
int reqLen, XLogRecPtr targetRecPtr, char *readBuf,
TimeLineID *pageTLI);
+static bool RestoreArchivedWAL(const char *path, const char *xlogfname,
+ off_t expectedSize, const char *restoreCommand,
+ const char *lastRestartPointFname);
+
/*
* Read WAL from the datadir/pg_wal, starting from 'startpoint' on timeline
* index 'tliIndex' in target timeline history, until 'endpoint'. Make note of
@@ -60,15 +68,19 @@ static int SimpleXLogPageRead(XLogReaderState *xlogreader,
*/
void
extractPageMap(const char *datadir, XLogRecPtr startpoint, int tliIndex,
- XLogRecPtr endpoint)
+ ControlFileData *targetCF, const char *restore_command)
{
XLogRecord *record;
+ XLogRecPtr endpoint = targetCF->checkPoint;
XLogReaderState *xlogreader;
char *errormsg;
XLogPageReadPrivate private;
private.datadir = datadir;
private.tliIndex = tliIndex;
+ private.restoreCommand = restore_command;
+ private.oldrecptr = targetCF->checkPointCopy.redo;
+ private.oldtli = targetCF->checkPointCopy.ThisTimeLineID;
xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -154,9 +166,9 @@ readOneRecord(const char *datadir, XLogRecPtr ptr, int tliIndex)
* Find the previous checkpoint preceding given WAL location.
*/
void
-findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
+findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr forkptr, int tliIndex,
XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo)
+ XLogRecPtr *lastchkptredo, const char *restoreCommand)
{
/* Walk backwards, starting from the given record */
XLogRecord *record;
@@ -181,6 +193,9 @@ findLastCheckpoint(const char *datadir, XLogRecPtr forkptr, int tliIndex,
private.datadir = datadir;
private.tliIndex = tliIndex;
+ private.restoreCommand = restoreCommand;
+ private.oldrecptr = targetCF->checkPointCopy.redo;
+ private.oldtli = targetCF->checkPointCopy.ThisTimeLineID;
xlogreader = XLogReaderAllocate(WalSegSz, &SimpleXLogPageRead,
&private);
if (xlogreader == NULL)
@@ -291,9 +306,53 @@ SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr,
if (xlogreadfd < 0)
{
- printf(_("could not open file \"%s\": %s\n"), xlogfpath,
+ bool restore_ok;
+ char lastRestartPointFname[MAXFNAMELEN];
+ XLogSegNo restartSegNo;
+
+ /*
+ * If we have no restore_command to execute, then exit.
+ */
+ if (private->restoreCommand == NULL)
+ {
+ printf(_("could not open file \"%s\": %s\n"), xlogfpath,
strerror(errno));
- return -1;
+ return -1;
+ }
+
+ XLByteToSeg(private->oldrecptr, restartSegNo, WalSegSz);
+ XLogFileName(lastRestartPointFname, private->oldtli, restartSegNo,
+ WalSegSz);
+
+ /*
+ * Since we have restore_command to execute, then try to retreive
+ * missing WAL file from the archive.
+ */
+ restore_ok = RestoreArchivedWAL(private->datadir,
+ xlogfname,
+ WalSegSz,
+ private->restoreCommand,
+ lastRestartPointFname);
+
+ if (restore_ok)
+ {
+ xlogreadfd = open(xlogfpath, O_RDONLY | PG_BINARY, 0);
+
+ if (xlogreadfd < 0)
+ {
+ printf(_("could not open restored from archive file \"%s\": %s\n"), xlogfpath,
+ strerror(errno));
+ return -1;
+ }
+ else
+ pg_log(PG_DEBUG, "using restored from archive version of file \"%s\"\n", xlogfpath);
+ }
+ else
+ {
+ printf(_("could not restore file \"%s\" from archive: %s\n"), xlogfname,
+ strerror(errno));
+ return -1;
+ }
}
}
@@ -409,3 +468,116 @@ extractPageInfo(XLogReaderState *record)
process_block_change(forknum, rnode, blkno);
}
}
+
+/*
+ * Attempt to retrieve the specified file from off-line archival storage.
+ * If successful return true.
+ *
+ * For fixed-size files, the caller may pass the expected size as an
+ * additional crosscheck on successful recovery. If the file size is not
+ * known, set expectedSize = 0.
+ *
+ * This is an adapted to frontend version of
+ * RestoreArchivedFile function from transam/xlogarchive.c
+ */
+bool
+RestoreArchivedWAL(const char *path, const char *xlogfname, off_t expectedSize,
+ const char *restoreCommand, const char *lastRestartPointFname)
+{
+ char xlogpath[MAXPGPATH];
+ char xlogRestoreCmd[MAXPGPATH];
+ char *dp;
+ char *endp;
+ const char *sp;
+ int rc;
+ struct stat stat_buf;
+
+ snprintf(xlogpath, MAXPGPATH, "%s/" XLOGDIR "/%s", path, xlogfname);
+
+ /*
+ * Construct the command to be executed.
+ */
+ dp = xlogRestoreCmd;
+ endp = xlogRestoreCmd + MAXPGPATH - 1;
+ *endp = '\0';
+
+ for (sp = restoreCommand; *sp; sp++)
+ {
+ if (*sp == '%')
+ {
+ switch (sp[1])
+ {
+ case 'p':
+ /* %p: relative path of target file */
+ sp++;
+ StrNCpy(dp, xlogpath, endp - dp);
+ make_native_path(dp);
+ dp += strlen(dp);
+ break;
+ case 'f':
+ /* %f: filename of desired file */
+ sp++;
+ StrNCpy(dp, xlogfname, endp - dp);
+ dp += strlen(dp);
+ break;
+ case 'r':
+ /* %r: filename of last restartpoint */
+ sp++;
+ StrNCpy(dp, lastRestartPointFname, endp - dp);
+ dp += strlen(dp);
+ break;
+ case '%':
+ /* convert %% to a single % */
+ sp++;
+ if (dp < endp)
+ *dp++ = *sp;
+ break;
+ default:
+ /* otherwise treat the % as not special */
+ if (dp < endp)
+ *dp++ = *sp;
+ break;
+ }
+ }
+ else
+ {
+ if (dp < endp)
+ *dp++ = *sp;
+ }
+ }
+ *dp = '\0';
+
+ /*
+ * Execute restore_command, which should copy
+ * the missing WAL file from archival storage.
+ */
+ rc = system(xlogRestoreCmd);
+
+ if (rc == 0)
+ {
+ /*
+ * Command apparently succeeded, but let's make sure the file is
+ * really there now and has the correct size.
+ */
+ if (stat(xlogpath, &stat_buf) == 0)
+ {
+ if (expectedSize > 0 && stat_buf.st_size != expectedSize)
+ {
+ printf(_("archive file \"%s\" has wrong size: %lu instead of %lu, %s"),
+ xlogfname, (unsigned long) stat_buf.st_size, (unsigned long) expectedSize,
+ strerror(errno));
+ }
+ else
+ return true;
+ }
+ else
+ {
+ /* Stat failed */
+ printf(_("could not stat file \"%s\": %s"),
+ xlogpath,
+ strerror(errno));
+ }
+ }
+
+ return false;
+}
diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c
index 96531063869c..3acac817eb4f 100644
--- a/src/bin/pg_rewind/pg_rewind.c
+++ b/src/bin/pg_rewind/pg_rewind.c
@@ -19,11 +19,13 @@
#include "file_ops.h"
#include "filemap.h"
#include "logging.h"
+#include "guc-file-fe.h"
#include "access/timeline.h"
#include "access/xlog_internal.h"
#include "catalog/catversion.h"
#include "catalog/pg_control.h"
+#include "common/controldata_utils.h"
#include "common/file_perm.h"
#include "common/file_utils.h"
#include "common/restricted_token.h"
@@ -52,11 +54,13 @@ int WalSegSz;
char *datadir_target = NULL;
char *datadir_source = NULL;
char *connstr_source = NULL;
+char *restore_command = NULL;
bool debug = false;
bool showprogress = false;
bool dry_run = false;
bool do_sync = true;
+bool restore_wals = false;
/* Target history */
TimeLineHistoryEntry *targetHistory;
@@ -75,6 +79,9 @@ usage(const char *progname)
printf(_(" -N, --no-sync do not wait for changes to be written\n"));
printf(_(" safely to disk\n"));
printf(_(" -P, --progress write progress messages\n"));
+ printf(_(" -r, --use-recovery-conf use restore_command in the recovery.conf to\n"));
+ printf(_(" retreive WALs from archive\n"));
+ printf(_(" -R, --restore-command=COMMAND restore command\n"));
printf(_(" --debug write a lot of debug messages\n"));
printf(_(" -V, --version output version information, then exit\n"));
printf(_(" -?, --help show this help, then exit\n"));
@@ -94,9 +101,12 @@ main(int argc, char **argv)
{"dry-run", no_argument, NULL, 'n'},
{"no-sync", no_argument, NULL, 'N'},
{"progress", no_argument, NULL, 'P'},
+ {"use-recovery-conf", no_argument, NULL, 'r'},
+ {"restore-command", required_argument, NULL, 'R'},
{"debug", no_argument, NULL, 3},
{NULL, 0, NULL, 0}
};
+ char recfile_fullpath[MAXPGPATH];
int option_index;
int c;
XLogRecPtr divergerec;
@@ -129,7 +139,7 @@ main(int argc, char **argv)
}
}
- while ((c = getopt_long(argc, argv, "D:nNP", long_options, &option_index)) != -1)
+ while ((c = getopt_long(argc, argv, "DR:nNPr", long_options, &option_index)) != -1)
{
switch (c)
{
@@ -140,6 +150,10 @@ main(int argc, char **argv)
case 'P':
showprogress = true;
break;
+
+ case 'r':
+ restore_wals = true;
+ break;
case 'n':
dry_run = true;
@@ -157,6 +171,10 @@ main(int argc, char **argv)
datadir_target = pg_strdup(optarg);
break;
+ case 'R':
+ restore_command = pg_strdup(optarg);
+ break;
+
case 1: /* --source-pgdata */
datadir_source = pg_strdup(optarg);
break;
@@ -223,6 +241,71 @@ main(int argc, char **argv)
umask(pg_mode_mask);
+ if (restore_command != NULL)
+ {
+ pg_log(PG_DEBUG, "using command line restore_command=\'%s\'.\n", restore_command);
+ }
+ else if (restore_wals)
+ {
+ FILE *recovery_conf_file;
+
+ /*
+ * Look for recovery.conf in the target data directory and
+ * try to get restore_command from there.
+ */
+ snprintf(recfile_fullpath, sizeof(recfile_fullpath), "%s/%s", datadir_target, RECOVERY_COMMAND_FILE);
+ recovery_conf_file = fopen(recfile_fullpath, "r");
+
+ if (recovery_conf_file == NULL)
+ {
+ fprintf(stderr, _("%s: option -r/--use-recovery-conf is specified, but recovery.conf is absent in the target directory\n"),
+ progname);
+ fprintf(stdout, _("You have to add recovery.conf or pass restore_command with -R/--restore-command option.\n"));
+ exit(1);
+ }
+ else
+ {
+ ConfigVariable *item,
+ *head = NULL,
+ *tail = NULL;
+ bool config_is_parsed;
+
+ /*
+ * We pass a fullpath to the recovery.conf as calling_file here, since
+ * parser will use its parent directory as base for all further includes
+ * if any exist.
+ */
+ config_is_parsed = ParseConfigFile(RECOVERY_COMMAND_FILE, true,
+ recfile_fullpath, 0, 0,
+ PG_WARNING, &head, &tail);
+ fclose(recovery_conf_file);
+
+ if (config_is_parsed)
+ {
+ for (item = head; item; item = item->next)
+ {
+ if (strcmp(item->name, "restore_command") == 0)
+ {
+ if (restore_command != NULL)
+ {
+ pfree(restore_command);
+ restore_command = NULL;
+ }
+ restore_command = pstrdup(item->value);
+ pg_log(PG_DEBUG, "using restore_command=\'%s\' from recovery.conf.\n", restore_command);
+ }
+ }
+
+ if (restore_command == NULL)
+ pg_fatal("could not find restore_command in recovery.conf file %s\n", recfile_fullpath);
+ }
+ else
+ pg_fatal("could not parse recovery.conf file %s\n", recfile_fullpath);
+
+ FreeConfigVariables(head);
+ }
+ }
+
/* Connect to remote server */
if (connstr_source)
libpqConnect(connstr_source);
@@ -294,9 +377,9 @@ main(int argc, char **argv)
exit(0);
}
- findLastCheckpoint(datadir_target, divergerec,
+ findLastCheckpoint(datadir_target, &ControlFile_target, divergerec,
lastcommontliIndex,
- &chkptrec, &chkpttli, &chkptredo);
+ &chkptrec, &chkpttli, &chkptredo, restore_command);
printf(_("rewinding from last common checkpoint at %X/%X on timeline %u\n"),
(uint32) (chkptrec >> 32), (uint32) chkptrec,
chkpttli);
@@ -319,7 +402,7 @@ main(int argc, char **argv)
*/
pg_log(PG_PROGRESS, "reading WAL in target\n");
extractPageMap(datadir_target, chkptrec, lastcommontliIndex,
- ControlFile_target.checkPoint);
+ &ControlFile_target, restore_command);
filemap_finalize();
if (showprogress)
diff --git a/src/bin/pg_rewind/pg_rewind.h b/src/bin/pg_rewind/pg_rewind.h
index 3f4ba7a26760..ba9d13e9bd04 100644
--- a/src/bin/pg_rewind/pg_rewind.h
+++ b/src/bin/pg_rewind/pg_rewind.h
@@ -14,6 +14,7 @@
#include "datapagemap.h"
#include "access/timeline.h"
+#include "catalog/pg_control.h"
#include "storage/block.h"
#include "storage/relfilenode.h"
@@ -32,11 +33,10 @@ extern int targetNentries;
/* in parsexlog.c */
extern void extractPageMap(const char *datadir, XLogRecPtr startpoint,
- int tliIndex, XLogRecPtr endpoint);
-extern void findLastCheckpoint(const char *datadir, XLogRecPtr searchptr,
- int tliIndex,
- XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
- XLogRecPtr *lastchkptredo);
+ int tliIndex, ControlFileData *targetCF, const char *restoreCommand);
+extern void findLastCheckpoint(const char *datadir, ControlFileData *targetCF, XLogRecPtr searchptr,
+ int tliIndex, XLogRecPtr *lastchkptrec, TimeLineID *lastchkpttli,
+ XLogRecPtr *lastchkptredo, const char *restoreCommand);
extern XLogRecPtr readOneRecord(const char *datadir, XLogRecPtr ptr,
int tliIndex);
diff --git a/src/bin/pg_rewind/t/001_basic.pl b/src/bin/pg_rewind/t/001_basic.pl
index 53dbf45be29d..22777bff82d4 100644
--- a/src/bin/pg_rewind/t/001_basic.pl
+++ b/src/bin/pg_rewind/t/001_basic.pl
@@ -1,7 +1,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 10;
+use Test::More tests => 15;
use RewindTest;
@@ -103,5 +103,6 @@ sub run_test
# Run the test in both modes
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/bin/pg_rewind/t/002_databases.pl b/src/bin/pg_rewind/t/002_databases.pl
index 2c9e42783113..10c835efc136 100644
--- a/src/bin/pg_rewind/t/002_databases.pl
+++ b/src/bin/pg_rewind/t/002_databases.pl
@@ -1,7 +1,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 6;
+use Test::More tests => 9;
use RewindTest;
@@ -59,5 +59,6 @@ sub run_test
# Run the test in both modes.
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/bin/pg_rewind/t/003_extrafiles.pl b/src/bin/pg_rewind/t/003_extrafiles.pl
index 496f38c4570c..f94ce329f473 100644
--- a/src/bin/pg_rewind/t/003_extrafiles.pl
+++ b/src/bin/pg_rewind/t/003_extrafiles.pl
@@ -3,7 +3,7 @@
use strict;
use warnings;
use TestLib;
-use Test::More tests => 4;
+use Test::More tests => 6;
use File::Find;
@@ -87,5 +87,6 @@ sub run_test
# Run the test in both modes.
run_test('local');
run_test('remote');
+run_test('archive');
exit(0);
diff --git a/src/tools/msvc/Mkvcbuild.pm b/src/tools/msvc/Mkvcbuild.pm
index b562044fa716..674e4a1df5ac 100644
--- a/src/tools/msvc/Mkvcbuild.pm
+++ b/src/tools/msvc/Mkvcbuild.pm
@@ -73,6 +73,7 @@ my $frontend_extraincludes = {
};
my $frontend_extrasource = {
'psql' => ['src/bin/psql/psqlscanslash.l'],
+ 'pg_rewind' => ['src/bin/pg_rewind/guc-file-fe.l'],
'pgbench' =>
[ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ]
};
