From 865827daa52d69fed40e48171e0b4d33383f51fa Mon Sep 17 00:00:00 2001 From: Paul Guo Date: Thu, 18 Apr 2019 18:55:09 +0800 Subject: [PATCH v2 1/2] Extact common functions from pg_basebackup into separate files for pg_rewind use. --- src/bin/pg_basebackup/Makefile | 2 +- src/bin/pg_basebackup/backup_common.c | 181 ++++++++++++++++++++++++++ src/bin/pg_basebackup/backup_common.h | 33 +++++ src/bin/pg_basebackup/pg_basebackup.c | 179 +------------------------ src/bin/pg_basebackup/streamutil.c | 1 - src/bin/pg_rewind/Makefile | 14 +- src/bin/pg_rewind/libpq_fetch.c | 2 +- src/bin/pg_rewind/pg_rewind.c | 4 + 8 files changed, 234 insertions(+), 182 deletions(-) create mode 100644 src/bin/pg_basebackup/backup_common.c create mode 100644 src/bin/pg_basebackup/backup_common.h diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index d7a081b9bb..2ed77bd0d6 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -21,7 +21,7 @@ include $(top_builddir)/src/Makefile.global override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) -OBJS=receivelog.o streamutil.o walmethods.o $(WIN32RES) +OBJS=receivelog.o streamutil.o walmethods.o backup_common.o $(WIN32RES) all: pg_basebackup pg_receivewal pg_recvlogical diff --git a/src/bin/pg_basebackup/backup_common.c b/src/bin/pg_basebackup/backup_common.c new file mode 100644 index 0000000000..7a2accea12 --- /dev/null +++ b/src/bin/pg_basebackup/backup_common.c @@ -0,0 +1,181 @@ +/*------------------------------------------------------------------------- + * + * backup_common.c - common code that is used in both pg_basebackup and pg_rewind. + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#include "postgres_fe.h" +#include "fe_utils/logging.h" +#include "fe_utils/string_utils.h" +#include "backup_common.h" + +/* Contents of configuration file to be generated */ +PQExpBuffer recoveryconfcontents = NULL; + +bool writerecoveryconf = false; +char *replication_slot = NULL; +PGconn *conn = NULL; + +void +disconnect_atexit(void) +{ + if (conn != NULL) + PQfinish(conn); +} + +/* + * Escape a string so that it can be used as a value in a key-value pair + * a configuration file. + */ +static char * +escape_quotes(const char *src) +{ + char *result = escape_single_quotes_ascii(src); + + if (!result) + { + pg_log_error("out of memory"); + exit(1); + } + return result; +} + +/* + * Create a configuration file in memory using a PQExpBuffer + */ +void +GenerateRecoveryConf(void) +{ + PQconninfoOption *connOptions; + PQconninfoOption *option; + PQExpBufferData conninfo_buf; + char *escaped; + + recoveryconfcontents = createPQExpBuffer(); + if (!recoveryconfcontents) + { + pg_log_error("out of memory"); + exit(1); + } + + /* + * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by + * standby.signal to trigger a standby state at recovery. + */ + if (conn && PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) + appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n"); + + connOptions = PQconninfo(conn); + if (connOptions == NULL) + { + pg_log_error("out of memory"); + exit(1); + } + + initPQExpBuffer(&conninfo_buf); + for (option = connOptions; option && option->keyword; option++) + { + /* + * Do not emit this setting if: - the setting is "replication", + * "dbname" or "fallback_application_name", since these would be + * overridden by the libpqwalreceiver module anyway. - not set or + * empty. + */ + if (strcmp(option->keyword, "replication") == 0 || + strcmp(option->keyword, "dbname") == 0 || + strcmp(option->keyword, "fallback_application_name") == 0 || + (option->val == NULL) || + (option->val != NULL && option->val[0] == '\0')) + continue; + + /* Separate key-value pairs with spaces */ + if (conninfo_buf.len != 0) + appendPQExpBufferChar(&conninfo_buf, ' '); + + /* + * Write "keyword=value" pieces, the value string is escaped and/or + * quoted if necessary. + */ + appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword); + appendConnStrVal(&conninfo_buf, option->val); + } + + /* + * Escape the connection string, so that it can be put in the config file. + * Note that this is different from the escaping of individual connection + * options above! + */ + escaped = escape_quotes(conninfo_buf.data); + appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped); + free(escaped); + + if (replication_slot) + { + escaped = escape_quotes(replication_slot); + appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n", replication_slot); + free(escaped); + } + + if (PQExpBufferBroken(recoveryconfcontents) || + PQExpBufferDataBroken(conninfo_buf)) + { + pg_log_error("out of memory"); + exit(1); + } + + termPQExpBuffer(&conninfo_buf); + + PQconninfoFree(connOptions); +} + +/* + * Write the configuration file into the directory specified in basedir, + * with the contents already collected in memory appended. Then write + * the signal file into the basedir. If the server does not support + * recovery parameters as GUCs, the signal file is not necessary, and + * configuration is written to recovery.conf. + */ +void +WriteRecoveryConf(char *target_dir) +{ + char filename[MAXPGPATH]; + FILE *cf; + bool is_recovery_guc_supported = true; + + if (conn && PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) + is_recovery_guc_supported = false; + + snprintf(filename, MAXPGPATH, "%s/%s", target_dir, + is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf"); + + cf = fopen(filename, is_recovery_guc_supported ? "a" : "w"); + if (cf == NULL) + { + pg_log_error("could not open file \"%s\": %m", filename); + exit(1); + } + + if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1) + { + pg_log_error("could not write to file \"%s\": %m", filename); + exit(1); + } + + fclose(cf); + + if (is_recovery_guc_supported) + { + snprintf(filename, MAXPGPATH, "%s/%s", target_dir, "standby.signal"); + cf = fopen(filename, "w"); + if (cf == NULL) + { + pg_log_error("could not create file \"%s\": %m", filename); + exit(1); + } + + fclose(cf); + } +} diff --git a/src/bin/pg_basebackup/backup_common.h b/src/bin/pg_basebackup/backup_common.h new file mode 100644 index 0000000000..8f9f860485 --- /dev/null +++ b/src/bin/pg_basebackup/backup_common.h @@ -0,0 +1,33 @@ +/*------------------------------------------------------------------------- + * + * backup_common.h - common code that is used in both pg_basebackup and pg_rewind. + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * + *------------------------------------------------------------------------- + */ + +#ifndef BACKUP_COMMON_H +#define BACKUP_COMMON_H + +#include "libpq-fe.h" +#include "pqexpbuffer.h" + +/* Contents of configuration file to be generated */ +extern PQExpBuffer recoveryconfcontents; + +extern bool writerecoveryconf; +extern char *replication_slot; +PGconn *conn; + +extern void disconnect_atexit(void); +extern void GenerateRecoveryConf(void); +extern void WriteRecoveryConf(char *target_dir); + +/* + * recovery.conf is integrated into postgresql.conf from version 12. + */ +#define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000 + + +#endif /* BACKUP_COMMON_H */ diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 1a735b8046..21653144bc 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -37,6 +37,7 @@ #include "pqexpbuffer.h" #include "pgtar.h" #include "pgtime.h" +#include "backup_common.h" #include "receivelog.h" #include "replication/basebackup.h" #include "streamutil.h" @@ -67,11 +68,6 @@ typedef struct TablespaceList */ #define MINIMUM_VERSION_FOR_TEMP_SLOTS 100000 -/* - * recovery.conf is integrated into postgresql.conf from version 12. - */ -#define MINIMUM_VERSION_FOR_RECOVERY_GUC 120000 - /* * Different ways to include WAL */ @@ -95,12 +91,10 @@ static int verbose = 0; static int compresslevel = 0; static IncludeWal includewal = STREAM_WAL; static bool fastcheckpoint = false; -static bool writerecoveryconf = false; static bool do_sync = true; static int standby_message_timeout = 10 * 1000; /* 10 sec = default */ static pg_time_t last_progress_report = 0; static int32 maxrate = 0; /* no limit by default */ -static char *replication_slot = NULL; static bool temp_replication_slot = true; static bool create_slot = false; static bool no_slot = false; @@ -137,9 +131,6 @@ static int has_xlogendptr = 0; static volatile LONG has_xlogendptr = 0; #endif -/* Contents of configuration file to be generated */ -static PQExpBuffer recoveryconfcontents = NULL; - /* Function headers */ static void usage(void); static void verify_dir_is_empty_or_create(char *dirname, bool *created, bool *found); @@ -147,8 +138,6 @@ static void progress_report(int tablespacenum, const char *filename, bool force) static void ReceiveTarFile(PGconn *conn, PGresult *res, int rownum); static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum); -static void GenerateRecoveryConf(PGconn *conn); -static void WriteRecoveryConf(void); static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, @@ -205,13 +194,6 @@ cleanup_directories_atexit(void) pg_log_info("changes to tablespace directories will not be undone"); } -static void -disconnect_atexit(void) -{ - if (conn != NULL) - PQfinish(conn); -} - #ifndef WIN32 /* * On windows, our background thread dies along with the process. But on @@ -1621,7 +1603,7 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) PQfreemem(copybuf); if (basetablespace && writerecoveryconf) - WriteRecoveryConf(); + WriteRecoveryConf(basedir); /* * No data is synced here, everything is done for all tablespaces at the @@ -1629,161 +1611,6 @@ ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum) */ } -/* - * Escape a string so that it can be used as a value in a key-value pair - * a configuration file. - */ -static char * -escape_quotes(const char *src) -{ - char *result = escape_single_quotes_ascii(src); - - if (!result) - { - pg_log_error("out of memory"); - exit(1); - } - return result; -} - -/* - * Create a configuration file in memory using a PQExpBuffer - */ -static void -GenerateRecoveryConf(PGconn *conn) -{ - PQconninfoOption *connOptions; - PQconninfoOption *option; - PQExpBufferData conninfo_buf; - char *escaped; - - recoveryconfcontents = createPQExpBuffer(); - if (!recoveryconfcontents) - { - pg_log_error("out of memory"); - exit(1); - } - - /* - * In PostgreSQL 12 and newer versions, standby_mode is gone, replaced by - * standby.signal to trigger a standby state at recovery. - */ - if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) - appendPQExpBufferStr(recoveryconfcontents, "standby_mode = 'on'\n"); - - connOptions = PQconninfo(conn); - if (connOptions == NULL) - { - pg_log_error("out of memory"); - exit(1); - } - - initPQExpBuffer(&conninfo_buf); - for (option = connOptions; option && option->keyword; option++) - { - /* - * Do not emit this setting if: - the setting is "replication", - * "dbname" or "fallback_application_name", since these would be - * overridden by the libpqwalreceiver module anyway. - not set or - * empty. - */ - if (strcmp(option->keyword, "replication") == 0 || - strcmp(option->keyword, "dbname") == 0 || - strcmp(option->keyword, "fallback_application_name") == 0 || - (option->val == NULL) || - (option->val != NULL && option->val[0] == '\0')) - continue; - - /* Separate key-value pairs with spaces */ - if (conninfo_buf.len != 0) - appendPQExpBufferChar(&conninfo_buf, ' '); - - /* - * Write "keyword=value" pieces, the value string is escaped and/or - * quoted if necessary. - */ - appendPQExpBuffer(&conninfo_buf, "%s=", option->keyword); - appendConnStrVal(&conninfo_buf, option->val); - } - - /* - * Escape the connection string, so that it can be put in the config file. - * Note that this is different from the escaping of individual connection - * options above! - */ - escaped = escape_quotes(conninfo_buf.data); - appendPQExpBuffer(recoveryconfcontents, "primary_conninfo = '%s'\n", escaped); - free(escaped); - - if (replication_slot) - { - escaped = escape_quotes(replication_slot); - appendPQExpBuffer(recoveryconfcontents, "primary_slot_name = '%s'\n", replication_slot); - free(escaped); - } - - if (PQExpBufferBroken(recoveryconfcontents) || - PQExpBufferDataBroken(conninfo_buf)) - { - pg_log_error("out of memory"); - exit(1); - } - - termPQExpBuffer(&conninfo_buf); - - PQconninfoFree(connOptions); -} - - -/* - * Write the configuration file into the directory specified in basedir, - * with the contents already collected in memory appended. Then write - * the signal file into the basedir. If the server does not support - * recovery parameters as GUCs, the signal file is not necessary, and - * configuration is written to recovery.conf. - */ -static void -WriteRecoveryConf(void) -{ - char filename[MAXPGPATH]; - FILE *cf; - bool is_recovery_guc_supported = true; - - if (PQserverVersion(conn) < MINIMUM_VERSION_FOR_RECOVERY_GUC) - is_recovery_guc_supported = false; - - snprintf(filename, MAXPGPATH, "%s/%s", basedir, - is_recovery_guc_supported ? "postgresql.auto.conf" : "recovery.conf"); - - cf = fopen(filename, is_recovery_guc_supported ? "a" : "w"); - if (cf == NULL) - { - pg_log_error("could not open file \"%s\": %m", filename); - exit(1); - } - - if (fwrite(recoveryconfcontents->data, recoveryconfcontents->len, 1, cf) != 1) - { - pg_log_error("could not write to file \"%s\": %m", filename); - exit(1); - } - - fclose(cf); - - if (is_recovery_guc_supported) - { - snprintf(filename, MAXPGPATH, "%s/%s", basedir, "standby.signal"); - cf = fopen(filename, "w"); - if (cf == NULL) - { - pg_log_error("could not create file \"%s\": %m", filename); - exit(1); - } - - fclose(cf); - } -} - static void BaseBackup(void) @@ -1840,7 +1667,7 @@ BaseBackup(void) * Build contents of configuration file if requested */ if (writerecoveryconf) - GenerateRecoveryConf(conn); + GenerateRecoveryConf(); /* * Run IDENTIFY_SYSTEM so we can get the timeline diff --git a/src/bin/pg_basebackup/streamutil.c b/src/bin/pg_basebackup/streamutil.c index ab2e55d695..528d06e358 100644 --- a/src/bin/pg_basebackup/streamutil.c +++ b/src/bin/pg_basebackup/streamutil.c @@ -53,7 +53,6 @@ char *dbname = NULL; int dbgetpassword = 0; /* 0=auto, -1=never, 1=always */ static bool have_password = false; static char password[100]; -PGconn *conn = NULL; /* * Connect to the server. Returns a valid PGconn pointer if connected, diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile index 019e19986e..7793702b20 100644 --- a/src/bin/pg_rewind/Makefile +++ b/src/bin/pg_rewind/Makefile @@ -19,16 +19,24 @@ override CPPFLAGS := -I$(libpq_srcdir) -DFRONTEND $(CPPFLAGS) LDFLAGS_INTERNAL += -L$(top_builddir)/src/fe_utils -lpgfeutils $(libpq_pgport) OBJS = pg_rewind.o parsexlog.o xlogreader.o datapagemap.o timeline.o \ - fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o logging.o \ + fetch.o file_ops.o copy_fetch.o libpq_fetch.o filemap.o logging.o backup_common.o \ $(WIN32RES) -EXTRA_CLEAN = xlogreader.c +EXTRA_CLEAN = xlogreader.c backup_common.c backup_common.h all: pg_rewind pg_rewind: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils $(CC) $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +pg_rewind.o: backup_common.h + +backup_common.c: % : $(top_srcdir)/src/bin/pg_basebackup/% + rm -f $@ && $(LN_S) $< . + +backup_common.h: % : $(top_srcdir)/src/bin/pg_basebackup/% + rm -f $@ && $(LN_S) $< . + xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% rm -f $@ && $(LN_S) $< . @@ -42,7 +50,7 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/pg_rewind$(X)' clean distclean maintainer-clean: - rm -f pg_rewind$(X) $(OBJS) xlogreader.c + rm -f pg_rewind$(X) $(OBJS) xlogreader.c backup_common.c backup_common.h rm -rf tmp_check check: diff --git a/src/bin/pg_rewind/libpq_fetch.c b/src/bin/pg_rewind/libpq_fetch.c index 11ec045b8e..3c601bdf3c 100644 --- a/src/bin/pg_rewind/libpq_fetch.c +++ b/src/bin/pg_rewind/libpq_fetch.c @@ -27,7 +27,7 @@ #include "fe_utils/logging.h" #include "port/pg_bswap.h" -static PGconn *conn = NULL; +extern PGconn *conn; /* * Files are fetched max CHUNKSIZE bytes at a time. diff --git a/src/bin/pg_rewind/pg_rewind.c b/src/bin/pg_rewind/pg_rewind.c index b7e6aca97f..7337bdd833 100644 --- a/src/bin/pg_rewind/pg_rewind.c +++ b/src/bin/pg_rewind/pg_rewind.c @@ -31,6 +31,7 @@ #include "fe_utils/logging.h" #include "getopt_long.h" #include "storage/bufpage.h" +#include "backup_common.h" static void usage(const char *progname); @@ -228,7 +229,10 @@ main(int argc, char **argv) /* Connect to remote server */ if (connstr_source) + { libpqConnect(connstr_source); + atexit(disconnect_atexit); + } /* * Ok, we have all the options and we're ready to start. Read in all the -- 2.17.2