diff -r e638ba9c3c11 -r e37682f34fe1 doc/src/sgml/ref/pg_rewind.sgml --- a/doc/src/sgml/ref/pg_rewind.sgml Sun Oct 29 12:58:40 2017 +0530 +++ b/doc/src/sgml/ref/pg_rewind.sgml Mon Oct 30 14:09:56 2017 +0100 @@ -48,17 +48,27 @@ - The result is equivalent to replacing the target data directory with the - source one. Only changed blocks from relation files are copied; - all other files are copied in full, including configuration files. The - advantage of pg_rewind over taking a new base backup, or - tools like rsync, is that pg_rewind does - not require reading through unchanged blocks in the cluster. This makes + The result is equivalent to replacing the data-related files in the target + data directory with the source one. Only changed blocks from relation files + are copied; all other files relating to control or WAL information are copied + in full. The advantage of pg_rewind over taking a new base + backup, or tools like rsync, is that pg_rewind + does not require reading through unchanged blocks in the cluster. This makes it a lot faster when the database is large and only a small fraction of blocks differ between the clusters. + A second advantage is predictability. pg_rewind is aware of + what directories are relevant to restoring replication and which ones are not. + The result is that you get something of a guaranteed state at the end. Log + files on the source are unlikely to clobber those on the client. The + postgresql.conf.auto is unlikely to be copied over. Replication + slot information is removed (and must be added again), and so forth. Your + system is as it had been before, but the data is synchronized from the master. + + + pg_rewind examines the timeline histories of the source and target clusters to determine the point where they diverged, and expects to find WAL in the target cluster's pg_wal directory @@ -84,7 +94,9 @@ pg_rewind session, it must be made available when the target server is started. This can be done by creating a recovery.conf file in the target data directory with a - suitable restore_command. + suitable restore_command. Alternatively replication slots can + be set just before promotion to ensure that the wal files have not been + removed. @@ -206,7 +218,7 @@ The basic idea is to copy all file system-level changes from the source - cluster to the target cluster: + cluster to the target cluster if they implicate the data stored: @@ -229,9 +241,7 @@ - Copy all other files such as pg_xact and - configuration files from the source cluster to the target cluster - (everything except the relation files). + Copy wal-related files such as those in pg_xact. diff -r e638ba9c3c11 -r e37682f34fe1 src/bin/pg_rewind/copy_fetch.c --- a/src/bin/pg_rewind/copy_fetch.c Sun Oct 29 12:58:40 2017 +0530 +++ b/src/bin/pg_rewind/copy_fetch.c Mon Oct 30 14:09:56 2017 +0100 @@ -26,6 +26,28 @@ static void recurse_dir(const char *datadir, const char *path, process_file_callback_t callback); +/* List of directories to synchronize: + * base data dirs (and ablespaces) + * wal/transaction data + * and that is it. + * + * This array is null-terminated to make + * it easy to expand + */ + +const char *rewind_dirs[] = { + "base", // Default tablespace + "global", // global tablespace + "pg_commit_ts", // In case we need to do PITR before up to sync + "pg_logical", // WAL related and no good reason to exclude + "pg_multixact", // WAL related and may need for vacuum-related reasons + "pg_tblspc", // Pther tablespaces + "pg_twophase", // mostly to *clear* + "pg_wal", // WAL + "pg_xact", // Commits of transactions + NULL +}; + static void execute_pagemap(datapagemap_t *pagemap, const char *path); /* @@ -33,13 +55,16 @@ * for each file. */ void -traverse_datadir(const char *datadir, process_file_callback_t callback) +traverse_rewinddirs(const char *datadir, process_file_callback_t callback) { - recurse_dir(datadir, NULL, callback); + int i; + for(i = 0; rewind_dirs[i] != NULL; i++){ + recurse_dir(datadir, rewind_dirs[i], callback); + } } /* - * recursive part of traverse_datadir + * recursive part of traverse_rewinddirs * * parentpath is the current subdirectory's path relative to datadir, * or NULL at the top level. diff -r e638ba9c3c11 -r e37682f34fe1 src/bin/pg_rewind/fetch.c --- a/src/bin/pg_rewind/fetch.c Sun Oct 29 12:58:40 2017 +0530 +++ b/src/bin/pg_rewind/fetch.c Mon Oct 30 14:09:56 2017 +0100 @@ -28,7 +28,7 @@ fetchSourceFileList(void) { if (datadir_source) - traverse_datadir(datadir_source, &process_source_file); + traverse_rewinddirs(datadir_source, &process_source_file); else libpqProcessFileList(); } diff -r e638ba9c3c11 -r e37682f34fe1 src/bin/pg_rewind/fetch.h --- a/src/bin/pg_rewind/fetch.h Sun Oct 29 12:58:40 2017 +0530 +++ b/src/bin/pg_rewind/fetch.h Mon Oct 30 14:09:56 2017 +0100 @@ -36,9 +36,11 @@ extern XLogRecPtr libpqGetCurrentXlogInsertLocation(void); /* in copy_fetch.c */ +extern const char *rewind_dirs[]; extern void copy_executeFileMap(filemap_t *map); typedef void (*process_file_callback_t) (const char *path, file_type_t type, size_t size, const char *link_target); extern void traverse_datadir(const char *datadir, process_file_callback_t callback); +extern void traverse_rewinddirs(const char *datadir, process_file_callback_t callback); #endif /* FETCH_H */ diff -r e638ba9c3c11 -r e37682f34fe1 src/bin/pg_rewind/libpq_fetch.c --- a/src/bin/pg_rewind/libpq_fetch.c Sun Oct 29 12:58:40 2017 +0530 +++ b/src/bin/pg_rewind/libpq_fetch.c Mon Oct 30 14:09:56 2017 +0100 @@ -147,7 +147,7 @@ { PGresult *res; const char *sql; - int i; + int i, p; /* * Create a recursive directory listing of the whole data directory. @@ -163,7 +163,8 @@ "WITH RECURSIVE files (path, filename, size, isdir) AS (\n" " SELECT '' AS path, filename, size, isdir FROM\n" " (SELECT pg_ls_dir('.', true, false) AS filename) AS fn,\n" - " pg_stat_file(fn.filename, true) AS this\n" + " LATERAL pg_stat_file(fn.filename, true) AS this\n" + " WHERE filename = $1\n" " UNION ALL\n" " SELECT parent.path || parent.filename || '/' AS path,\n" " fn, this.size, this.isdir\n" @@ -177,44 +178,56 @@ "FROM files\n" "LEFT OUTER JOIN pg_tablespace ON files.path = 'pg_tblspc/'\n" " AND oid::text = files.filename\n"; - res = PQexec(conn, sql); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - pg_fatal("could not fetch file list: %s", + /* Going through the directories in a loop. Doing it this way + * makes it easier to add more inclusions later. + * + * Note that the query filters out on top-level directories before + * recursion so this will not give us problems in terms of listing + * lots of files many times. + */ + for (p = 0; rewind_dirs[p] != NULL; ++p) + { + const char *paths[1]; + paths[0] = rewind_dirs[p]; + res = PQexecParams(conn, sql, 1, NULL, paths, NULL, NULL, 0); + + if (PQresultStatus(res) != PGRES_TUPLES_OK) + pg_fatal("could not fetch file list: %s", PQresultErrorMessage(res)); - /* sanity check the result set */ - if (PQnfields(res) != 4) - pg_fatal("unexpected result set while fetching file list\n"); + /* sanity check the result set */ + if (PQnfields(res) != 4) + pg_fatal("unexpected result set while fetching file list\n"); - /* Read result to local variables */ - for (i = 0; i < PQntuples(res); i++) - { - char *path = PQgetvalue(res, i, 0); - int64 filesize = atol(PQgetvalue(res, i, 1)); - bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0); - char *link_target = PQgetvalue(res, i, 3); - file_type_t type; - - if (PQgetisnull(res, 0, 1)) + /* Read result to local variables */ + for (i = 0; i < PQntuples(res); i++) { - /* - * The file was removed from the server while the query was - * running. Ignore it. - */ - continue; - } + char *path = PQgetvalue(res, i, 0); + int64 filesize = atol(PQgetvalue(res, i, 1)); + bool isdir = (strcmp(PQgetvalue(res, i, 2), "t") == 0); + char *link_target = PQgetvalue(res, i, 3); + file_type_t type; - if (link_target[0]) - type = FILE_TYPE_SYMLINK; - else if (isdir) - type = FILE_TYPE_DIRECTORY; - else - type = FILE_TYPE_REGULAR; + if (PQgetisnull(res, 0, 1)) + { + /* + * The file was removed from the server while the query was + * running. Ignore it. + */ + continue; + } - process_source_file(path, type, filesize, link_target); + if (link_target[0]) + type = FILE_TYPE_SYMLINK; + else if (isdir) + type = FILE_TYPE_DIRECTORY; + else + type = FILE_TYPE_REGULAR; + process_source_file(path, type, filesize, link_target); + } + PQclear(res); } - PQclear(res); } /*---- diff -r e638ba9c3c11 -r e37682f34fe1 src/bin/pg_rewind/pg_rewind.c --- a/src/bin/pg_rewind/pg_rewind.c Sun Oct 29 12:58:40 2017 +0530 +++ b/src/bin/pg_rewind/pg_rewind.c Mon Oct 30 14:09:56 2017 +0100 @@ -287,7 +287,7 @@ pg_log(PG_PROGRESS, "reading source file list\n"); fetchSourceFileList(); pg_log(PG_PROGRESS, "reading target file list\n"); - traverse_datadir(datadir_target, &process_target_file); + traverse_rewinddirs(datadir_target, &process_target_file); /* * Read the target WAL from last checkpoint before the point of fork, to diff -r e638ba9c3c11 -r e37682f34fe1 src/bin/pg_rewind/t/003_extrafiles.pl --- a/src/bin/pg_rewind/t/003_extrafiles.pl Sun Oct 29 12:58:40 2017 +0530 +++ b/src/bin/pg_rewind/t/003_extrafiles.pl Mon Oct 30 14:09:56 2017 +0100 @@ -71,12 +71,12 @@ "$test_master_datadir/tst_both_dir/both_file2", "$test_master_datadir/tst_both_dir/both_subdir", "$test_master_datadir/tst_both_dir/both_subdir/both_file3", - "$test_master_datadir/tst_standby_dir", - "$test_master_datadir/tst_standby_dir/standby_file1", - "$test_master_datadir/tst_standby_dir/standby_file2", - "$test_master_datadir/tst_standby_dir/standby_subdir", -"$test_master_datadir/tst_standby_dir/standby_subdir/standby_file3" ], - "file lists match"); + "$test_master_datadir/tst_master_dir", + "$test_master_datadir/tst_master_dir/master_file1", + "$test_master_datadir/tst_master_dir/master_file2", + "$test_master_datadir/tst_master_dir/master_subdir", + "$test_master_datadir/tst_master_dir/master_subdir/master_file3", ], + "file lists match") or (diag("Files found:"), diag(explain(\@paths))); RewindTest::clean_rewind_test(); }