From e09305b999028c7accfc0f30285de0ed69e256ae Mon Sep 17 00:00:00 2001 From: Suraj Kharage Date: Thu, 12 Dec 2019 16:13:20 +0530 Subject: [PATCH v4 1/2] Backup manifest with file names, sizes, timestamps, optional checksums. Original patch from Robert Haas to create the backup manifest file. Fix review comments, making checksum optional and provide user to choose the checksum algorithm by Rushabh Lathia. Reviewed by Jeevan Chalke and Rushabh Lathia. Further refactored by me. --- doc/src/sgml/protocol.sgml | 24 ++- doc/src/sgml/ref/pg_basebackup.sgml | 12 ++ src/backend/access/transam/xlog.c | 3 +- src/backend/replication/basebackup.c | 289 ++++++++++++++++++++++++++++++--- src/backend/replication/repl_gram.y | 6 + src/backend/replication/repl_scanner.l | 1 + src/backend/utils/adt/encode.c | 17 +- src/backend/utils/adt/varlena.c | 1 + src/bin/pg_basebackup/pg_basebackup.c | 130 ++++++++++++++- src/common/Makefile | 2 + src/common/checksum_utils.c | 109 +++++++++++++ src/common/encode.c | 38 +++++ src/include/common/checksum_utils.h | 46 ++++++ src/include/common/encode.h | 20 +++ src/include/replication/basebackup.h | 15 +- src/include/utils/builtins.h | 1 - src/tools/pgindent/typedefs.list | 3 + 17 files changed, 667 insertions(+), 50 deletions(-) create mode 100644 src/common/checksum_utils.c create mode 100644 src/common/encode.c create mode 100644 src/include/common/checksum_utils.h create mode 100644 src/include/common/encode.h diff --git a/doc/src/sgml/protocol.sgml b/doc/src/sgml/protocol.sgml index 8027521..bdff6f5 100644 --- a/doc/src/sgml/protocol.sgml +++ b/doc/src/sgml/protocol.sgml @@ -2466,15 +2466,19 @@ The commands accepted in replication mode are: - BASE_BACKUP [ LABEL 'label' ] [ PROGRESS ] [ FAST ] [ WAL ] [ NOWAIT ] [ MAX_RATE rate ] [ TABLESPACE_MAP ] [ NOVERIFY_CHECKSUMS ] + BASE_BACKUP [ LABEL 'label' ] [ PROGRESS ] [ FAST ] [ WAL ] [ NOWAIT ] [ MAX_RATE rate ] [ TABLESPACE_MAP ] [ NOVERIFY_CHECKSUMS ] [ MANIFEST_CHECKSUMS 'algorithm'] BASE_BACKUP Instructs the server to start streaming a base backup. The system will automatically be put in backup mode before the backup - is started, and taken out of it when the backup is complete. The - following options are accepted: + is started, and taken out of it when the backup is complete. This also + store a manifest as part of each backup under the backup folder in a + file named backup_manifest. This file contains the list of files, and + the lengths of those files, file modified time and optional checksum + for each file as well as for a manifest file.The following options are + accepted: LABEL 'label' @@ -2576,6 +2580,20 @@ The commands accepted in replication mode are: + + + MANIFEST_CHECKSUMS + + + By default, checksum for each backup file in backup manifest file is + off. Specifying MANIFEST_CHECKSUMS when a checksum + algorithm name, enables the checksum for each backup file in backup + manifest file as well a for manifest file itself. Currently it supports + SHA256 and CRC32C as checksum algorithm. + + + + diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index fc9e222..af7c731 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -536,6 +536,18 @@ PostgreSQL documentation + + + + + + Enables a checksum for the each backup file in manifest file as well as + for manifest file itself and will choose the given algorithm to generate + the checksum value. + + + + diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index edee0c0..0e1fe38 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -10501,7 +10501,8 @@ do_pg_start_backup(const char *backupidstr, bool fast, TimeLineID *starttli_p, ti->oid = pstrdup(de->d_name); ti->path = pstrdup(buflinkpath.data); ti->rpath = relpath ? pstrdup(relpath) : NULL; - ti->size = infotbssize ? sendTablespace(fullpath, true) : -1; + ti->size = infotbssize ? + sendTablespace(fullpath, ti->oid, true, NULL) : -1; if (tablespaces) *tablespaces = lappend(*tablespaces, ti); diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 1bb72a0..c8d94a8 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -18,6 +18,7 @@ #include "access/xlog_internal.h" /* for pg_start/stop_backup */ #include "catalog/pg_type.h" +#include "common/encode.h" #include "common/file_perm.h" #include "lib/stringinfo.h" #include "libpq/libpq.h" @@ -51,20 +52,30 @@ typedef struct bool includewal; uint32 maxrate; bool sendtblspcmapfile; + ChecksumAlgorithm checksumAlgo; } basebackup_options; static int64 sendDir(const char *path, int basepathlen, bool sizeonly, - List *tablespaces, bool sendtblspclinks); + List *tablespaces, bool sendtblspclinks, + manifestinfo *manifestInfo, const char *tsoid); static bool sendFile(const char *readfilename, const char *tarfilename, - struct stat *statbuf, bool missing_ok, Oid dboid); -static void sendFileWithContent(const char *filename, const char *content); + struct stat *statbuf, bool missing_ok, Oid dboid, + manifestinfo *manifestInfo, const char *tsoid); +static void sendFileWithContent(const char *filename, const char *content, + manifestinfo *manifestInfo); static int64 _tarWriteHeader(const char *filename, const char *linktarget, struct stat *statbuf, bool sizeonly); static int64 _tarWriteDir(const char *pathbuf, int basepathlen, struct stat *statbuf, bool sizeonly); static void send_int8_string(StringInfoData *buf, int64 intval); static void SendBackupHeader(List *tablespaces); +static void InitializeManifest(manifestinfo *manifestInfo, + ChecksumAlgorithm checksumAlgo); +static void AddFileToManifest(manifestinfo *manifestInfo, const char *tsoid, + const char *filename, size_t size, time_t mtime); +static void SendBackupManifest(manifestinfo *manifestInfo); +static char *escape_field_for_manifest(const char *s); static void perform_base_backup(basebackup_options *opt); static void parse_basebackup_options(List *options, basebackup_options *opt); static void SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli); @@ -231,6 +242,7 @@ perform_base_backup(basebackup_options *opt) StringInfo tblspc_map_file = NULL; int datadirpathlen; List *tablespaces = NIL; + manifestinfo manifestInfo; datadirpathlen = strlen(DataDir); @@ -238,6 +250,7 @@ perform_base_backup(basebackup_options *opt) labelfile = makeStringInfo(); tblspc_map_file = makeStringInfo(); + InitializeManifest(&manifestInfo, opt->checksumAlgo); total_checksum_failures = 0; @@ -274,7 +287,10 @@ perform_base_backup(basebackup_options *opt) /* Add a node for the base directory at the end */ ti = palloc0(sizeof(tablespaceinfo)); - ti->size = opt->progress ? sendDir(".", 1, true, tablespaces, true) : -1; + if (opt->progress) + ti->size = sendDir(".", 1, true, tablespaces, true, NULL, NULL); + else + ti->size = -1; tablespaces = lappend(tablespaces, ti); /* Send tablespace header */ @@ -321,7 +337,8 @@ perform_base_backup(basebackup_options *opt) struct stat statbuf; /* In the main tar, include the backup_label first... */ - sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data); + sendFileWithContent(BACKUP_LABEL_FILE, labelfile->data, + &manifestInfo); /* * Send tablespace_map file if required and then the bulk of @@ -329,11 +346,14 @@ perform_base_backup(basebackup_options *opt) */ if (tblspc_map_file && opt->sendtblspcmapfile) { - sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data); - sendDir(".", 1, false, tablespaces, false); + sendFileWithContent(TABLESPACE_MAP, tblspc_map_file->data, + &manifestInfo); + sendDir(".", 1, false, tablespaces, + false, &manifestInfo, NULL); } else - sendDir(".", 1, false, tablespaces, true); + sendDir(".", 1, false, tablespaces, + true, &manifestInfo, NULL); /* ... and pg_control after everything else. */ if (lstat(XLOG_CONTROL_FILE, &statbuf) != 0) @@ -341,10 +361,11 @@ perform_base_backup(basebackup_options *opt) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", XLOG_CONTROL_FILE))); - sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, false, InvalidOid); + sendFile(XLOG_CONTROL_FILE, XLOG_CONTROL_FILE, &statbuf, + false, InvalidOid, &manifestInfo, NULL); } else - sendTablespace(ti->path, false); + sendTablespace(ti->path, ti->oid, false, &manifestInfo); /* * If we're including WAL, and this is the main data directory we @@ -563,7 +584,7 @@ perform_base_backup(basebackup_options *opt) * complete segment. */ StatusFilePath(pathbuf, walFileName, ".done"); - sendFileWithContent(pathbuf, ""); + sendFileWithContent(pathbuf, "", &manifestInfo); } /* @@ -586,16 +607,20 @@ perform_base_backup(basebackup_options *opt) (errcode_for_file_access(), errmsg("could not stat file \"%s\": %m", pathbuf))); - sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid); + sendFile(pathbuf, pathbuf, &statbuf, false, InvalidOid, + &manifestInfo, NULL); /* unconditionally mark file as archived */ StatusFilePath(pathbuf, fname, ".done"); - sendFileWithContent(pathbuf, ""); + sendFileWithContent(pathbuf, "", &manifestInfo); } /* Send CopyDone message for the last tar file */ pq_putemptymessage('c'); } + + SendBackupManifest(&manifestInfo); + SendXlogRecPtrResult(endptr, endtli); if (total_checksum_failures) @@ -639,6 +664,7 @@ parse_basebackup_options(List *options, basebackup_options *opt) bool o_maxrate = false; bool o_tablespace_map = false; bool o_noverify_checksums = false; + bool o_manifest_checksums = false; MemSet(opt, 0, sizeof(*opt)); foreach(lopt, options) @@ -727,6 +753,24 @@ parse_basebackup_options(List *options, basebackup_options *opt) noverify_checksums = true; o_noverify_checksums = true; } + else if (strcmp(defel->defname, "manifest_checksums") == 0) + { + char *manifest_checksum_algo = NULL; + + if (o_manifest_checksums) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("duplicate option \"%s\"", defel->defname))); + manifest_checksum_algo = strVal(defel->arg); + + if (!parse_checksum_algorithm(manifest_checksum_algo, + &opt->checksumAlgo)) + ereport(ERROR, + (errcode(ERRCODE_SYNTAX_ERROR), + errmsg("invalid manifest_checksums option \"%s\"", + manifest_checksum_algo))); + } + else elog(ERROR, "option \"%s\" not recognized", defel->defname); @@ -848,6 +892,186 @@ SendBackupHeader(List *tablespaces) pq_puttextmessage('C', "SELECT"); } +static void +InitializeManifest(manifestinfo *manifestInfo, ChecksumAlgorithm checksumAlgo) +{ + Assert(manifestInfo != NULL); + + MemSet(manifestInfo, 0, sizeof(*manifestInfo)); + + manifestInfo->checksumAlgo = checksumAlgo; + manifestInfo->manifest = makeStringInfo(); + appendStringInfoString(manifestInfo->manifest, "PostgreSQL-Backup-Manifest-Version 1\n"); + + switch (manifestInfo->checksumAlgo) + { + case MC_SHA256: + strcpy(manifestInfo->checksum_label, "SHA256:"); + break; + case MC_CRC32C: + strcpy(manifestInfo->checksum_label, "CRC32C:"); + break; + case MC_NONE: + break; + } +} + +/* + * Add an entry to the backup manifest for a file. + */ +static void +AddFileToManifest(manifestinfo *manifestInfo, const char *tsoid, + const char *filename, size_t size, time_t mtime) +{ + char pathbuf[MAXPGPATH]; + char *escaped_filename; + static char timebuf[128]; + static char checksumbuf[256]; + char encode_checksumbuf[256]; + struct pg_tm *tm; + char *checksumlabel = manifestInfo->checksum_label; + int checksumbuflen; + ChecksumCtx *cCtx = &manifestInfo->cCtx; + StringInfo manifest = manifestInfo->manifest; + + /* + * If this file is part of a tablespace, the filename passed to this + * function will be relative to the tar file that contains it. We want the + * pathname relative to the data directory (ignoring the intermediate + * symlink traversal). + */ + if (tsoid != NULL) + { + snprintf(pathbuf, sizeof(pathbuf), "pg_tblspc/%s/%s", tsoid, filename); + filename = pathbuf; + } + + /* Escape filename, if necessary. */ + escaped_filename = escape_field_for_manifest(filename); + + /* + * Convert time to a string. Since it's not clear what time zone to use + * and since time zone definitions can change, possibly causing confusion, + * use GMT always. + */ + tm = pg_gmtime(&mtime); + if (tm == NULL) + elog(ERROR, "could not convert epoch to timestamp: %m"); + pg_strftime(timebuf, sizeof(timebuf), "%Y-%m-%d %H:%M:%S %Z", tm); + + /* Generate final checksum and Convert it to hexadecimal. */ + if (manifestInfo->checksumAlgo != MC_NONE) + { + checksumbuflen = finalize_checksum(cCtx, manifestInfo->checksumAlgo, + checksumbuf); + checksumbuflen = hex_encode(checksumbuf, checksumbuflen, + encode_checksumbuf); + encode_checksumbuf[checksumbuflen] = '\0'; + } + + /* Add to manifest. */ + appendStringInfo(manifest, "File\t%s\t%zu\t%s\t%s%s\n", + escaped_filename == NULL ? filename : escaped_filename, + size, timebuf, checksumlabel ? checksumlabel : "", + manifestInfo->checksumAlgo != MC_NONE ? encode_checksumbuf : "-"); + + /* Avoid leaking memory. */ + if (escaped_filename != NULL) + pfree(escaped_filename); +} + +/* + * Finalize the backup manifest, and send it to the client. + */ +static void +SendBackupManifest(manifestinfo *manifestInfo) +{ + char checksumbuf[256]; + StringInfoData protobuf; + int checksumbuflen; + ChecksumCtx *cCtx = &manifestInfo->cCtx; + StringInfo manifest = manifestInfo->manifest; + + /* Checksum the manifest. */ + if (manifestInfo->checksumAlgo != MC_NONE) + { + initialize_checksum(cCtx, manifestInfo->checksumAlgo); + update_checksum(cCtx, manifestInfo->checksumAlgo, + manifest->data, manifest->len); + checksumbuflen = finalize_checksum(cCtx, + manifestInfo->checksumAlgo, + (char *) checksumbuf); + appendStringInfoString(manifest, "Manifest-Checksum\t"); + appendStringInfoString(manifest, manifestInfo->checksum_label); + enlargeStringInfo(manifest, checksumbuflen * 2); + checksumbuflen = hex_encode(checksumbuf, checksumbuflen, + manifest->data + manifest->len); + manifest->len += checksumbuflen; + appendStringInfoChar(manifest, '\n'); + } + + /* Send CopyOutResponse message */ + pq_beginmessage(&protobuf, 'H'); + pq_sendbyte(&protobuf, 0); /* overall format */ + pq_sendint16(&protobuf, 0); /* natts */ + pq_endmessage(&protobuf); + + /* Send CopyData message */ + pq_putmessage('d', manifest->data, manifest->len); + + /* And finally CopyDone message */ + pq_putemptymessage('c'); +} + +/* + * Escape a field for inclusion in a manifest. + * + * We use the following escaping rule: If a field contains \t, \r, or \n, + * the field must be surrounded by double-quotes, and any internal double + * quotes must be doubled. Otherwise, no escaping is required. + * + * The return value is a new palloc'd string with escaping added, or NULL + * if no escaping is required. + */ +static char * +escape_field_for_manifest(const char *s) +{ + bool escaping_required = false; + int escaped_length = 2; + const char *t; + char *result; + char *r; + + for (t = s; *t != '\0'; ++t) + { + if (*t == '\t' || *t == '\r' || *t == '\n') + escaping_required = true; + if (*t == '"') + ++escaped_length; + ++escaped_length; + } + + if (!escaping_required) + return NULL; + + result = palloc(escaped_length + 1); + result[0] = '"'; + result[escaped_length - 1] = '"'; + result[escaped_length] = '\0'; + r = result + 1; + + for (t = s; *t != '\0'; ++t) + { + *(r++) = *t; + if (*t == '"') + *(r++) = *t; + } + + Assert(r == &result[escaped_length - 1]); + + return result; +} + /* * Send a single resultset containing just a single * XLogRecPtr record (in text format) @@ -908,7 +1132,8 @@ SendXlogRecPtrResult(XLogRecPtr ptr, TimeLineID tli) * Inject a file with given name and content in the output tar stream. */ static void -sendFileWithContent(const char *filename, const char *content) +sendFileWithContent(const char *filename, const char *content, + manifestinfo *manifestInfo) { struct stat statbuf; int pad, @@ -945,6 +1170,11 @@ sendFileWithContent(const char *filename, const char *content) MemSet(buf, 0, pad); pq_putmessage('d', buf, pad); } + + initialize_checksum(&manifestInfo->cCtx, manifestInfo->checksumAlgo); + update_checksum(&manifestInfo->cCtx, manifestInfo->checksumAlgo, + content, len); + AddFileToManifest(manifestInfo, NULL, filename, len, statbuf.st_mtime); } /* @@ -955,7 +1185,7 @@ sendFileWithContent(const char *filename, const char *content) * Only used to send auxiliary tablespaces, not PGDATA. */ int64 -sendTablespace(char *path, bool sizeonly) +sendTablespace(char *path, char *oid, bool sizeonly, manifestinfo *manifestInfo) { int64 size; char pathbuf[MAXPGPATH]; @@ -988,7 +1218,8 @@ sendTablespace(char *path, bool sizeonly) sizeonly); /* Send all the files in the tablespace version directory */ - size += sendDir(pathbuf, strlen(path), sizeonly, NIL, true); + size += sendDir(pathbuf, strlen(path), sizeonly, NIL, + true, manifestInfo, oid); return size; } @@ -1007,7 +1238,7 @@ sendTablespace(char *path, bool sizeonly) */ static int64 sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, - bool sendtblspclinks) + bool sendtblspclinks, manifestinfo *manifestInfo, const char *tsoid) { DIR *dir; struct dirent *de; @@ -1283,7 +1514,8 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, skip_this_dir = true; if (!skip_this_dir) - size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, sendtblspclinks); + size += sendDir(pathbuf, basepathlen, sizeonly, tablespaces, + sendtblspclinks, manifestInfo, tsoid); } else if (S_ISREG(statbuf.st_mode)) { @@ -1291,7 +1523,8 @@ sendDir(const char *path, int basepathlen, bool sizeonly, List *tablespaces, if (!sizeonly) sent = sendFile(pathbuf, pathbuf + basepathlen + 1, &statbuf, - true, isDbDir ? pg_atoi(lastDir + 1, sizeof(Oid), 0) : InvalidOid); + true, isDbDir ? pg_atoi(lastDir + 1, sizeof(Oid), 0) : InvalidOid, + manifestInfo, tsoid); if (sent || sizeonly) { @@ -1354,8 +1587,9 @@ is_checksummed_file(const char *fullpath, const char *filename) * and the file did not exist. */ static bool -sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf, - bool missing_ok, Oid dboid) +sendFile(const char *readfilename, const char *tarfilename, + struct stat *statbuf, bool missing_ok, Oid dboid, + manifestinfo *manifestInfo, const char *tsoid) { FILE *fp; BlockNumber blkno = 0; @@ -1373,6 +1607,8 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf char *segmentpath; bool verify_checksum = false; + initialize_checksum(&manifestInfo->cCtx, manifestInfo->checksumAlgo); + fp = AllocateFile(readfilename, "rb"); if (fp == NULL) { @@ -1541,6 +1777,10 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf ereport(ERROR, (errmsg("base backup could not send data, aborting backup"))); + /* Also feed it to the checksum machinery. */ + update_checksum(&manifestInfo->cCtx, manifestInfo->checksumAlgo, + buf, cnt); + len += cnt; throttle(cnt); @@ -1565,6 +1805,8 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf { cnt = Min(sizeof(buf), statbuf->st_size - len); pq_putmessage('d', buf, cnt); + update_checksum(&manifestInfo->cCtx, manifestInfo->checksumAlgo, + buf, cnt); len += cnt; throttle(cnt); } @@ -1572,7 +1814,8 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf /* * Pad to 512 byte boundary, per tar format requirements. (This small - * piece of data is probably not worth throttling.) + * piece of data is probably not worth throttling, and is not checksummed + * because it's not actually part of the file.) */ pad = ((len + 511) & ~511) - len; if (pad > 0) @@ -1595,6 +1838,8 @@ sendFile(const char *readfilename, const char *tarfilename, struct stat *statbuf } total_checksum_failures += checksum_failures; + AddFileToManifest(manifestInfo, tsoid, tarfilename, statbuf->st_size, + statbuf->st_mtime); return true; } diff --git a/src/backend/replication/repl_gram.y b/src/backend/replication/repl_gram.y index c4e11cc..e527dd2 100644 --- a/src/backend/replication/repl_gram.y +++ b/src/backend/replication/repl_gram.y @@ -87,6 +87,7 @@ static SQLCmd *make_sqlcmd(void); %token K_EXPORT_SNAPSHOT %token K_NOEXPORT_SNAPSHOT %token K_USE_SNAPSHOT +%token K_MANIFEST_CHECKSUMS %type command %type base_backup start_replication start_logical_replication @@ -214,6 +215,11 @@ base_backup_opt: $$ = makeDefElem("noverify_checksums", (Node *)makeInteger(true), -1); } + | K_MANIFEST_CHECKSUMS SCONST + { + $$ = makeDefElem("manifest_checksums", + (Node *)makeString($2), -1); + } ; create_replication_slot: diff --git a/src/backend/replication/repl_scanner.l b/src/backend/replication/repl_scanner.l index 380faeb..1b73f1e 100644 --- a/src/backend/replication/repl_scanner.l +++ b/src/backend/replication/repl_scanner.l @@ -107,6 +107,7 @@ EXPORT_SNAPSHOT { return K_EXPORT_SNAPSHOT; } NOEXPORT_SNAPSHOT { return K_NOEXPORT_SNAPSHOT; } USE_SNAPSHOT { return K_USE_SNAPSHOT; } WAIT { return K_WAIT; } +MANIFEST_CHECKSUMS { return K_MANIFEST_CHECKSUMS; } "," { return ','; } ";" { return ';'; } diff --git a/src/backend/utils/adt/encode.c b/src/backend/utils/adt/encode.c index 7293d66..aad1880 100644 --- a/src/backend/utils/adt/encode.c +++ b/src/backend/utils/adt/encode.c @@ -15,6 +15,7 @@ #include +#include "common/encode.h" #include "utils/builtins.h" @@ -109,8 +110,6 @@ binary_decode(PG_FUNCTION_ARGS) * HEX */ -static const char hextbl[] = "0123456789abcdef"; - static const int8 hexlookup[128] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -122,20 +121,6 @@ static const int8 hexlookup[128] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, }; -unsigned -hex_encode(const char *src, unsigned len, char *dst) -{ - const char *end = src + len; - - while (src < end) - { - *dst++ = hextbl[(*src >> 4) & 0xF]; - *dst++ = hextbl[*src & 0xF]; - src++; - } - return len * 2; -} - static inline char get_hex(char c) { diff --git a/src/backend/utils/adt/varlena.c b/src/backend/utils/adt/varlena.c index 69165eb..3232687 100644 --- a/src/backend/utils/adt/varlena.c +++ b/src/backend/utils/adt/varlena.c @@ -20,6 +20,7 @@ #include "access/detoast.h" #include "catalog/pg_collation.h" #include "catalog/pg_type.h" +#include "common/encode.h" #include "common/int.h" #include "lib/hyperloglog.h" #include "libpq/pqformat.h" diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 16886fb..7d5ed0d 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -88,6 +88,12 @@ typedef struct UnpackTarState FILE *file; } UnpackTarState; +typedef struct WriteManifestState +{ + char filename[MAXPGPATH]; + FILE *file; +} WriteManifestState; + typedef void (*WriteDataCallback) (size_t nbytes, char *buf, void *callback_data); @@ -135,6 +141,7 @@ static bool temp_replication_slot = true; static bool create_slot = false; static bool no_slot = false; static bool verify_checksums = true; +static char *manifest_checksums = NULL; static bool success = false; static bool made_new_pgdata = false; @@ -180,6 +187,12 @@ static void ReceiveTarCopyChunk(size_t r, char *copybuf, void *callback_data); static void ReceiveAndUnpackTarFile(PGconn *conn, PGresult *res, int rownum); static void ReceiveTarAndUnpackCopyChunk(size_t r, char *copybuf, void *callback_data); +static void ReceiveBackupManifest(PGconn *conn); +static void ReceiveBackupManifestChunk(size_t r, char *copybuf, + void *callback_data); +static void ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf); +static void ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, + void *callback_data); static void BaseBackup(void); static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, @@ -386,6 +399,8 @@ usage(void) printf(_(" --no-slot prevent creation of temporary replication slot\n")); printf(_(" --no-verify-checksums\n" " do not verify checksums\n")); + printf(_(" --manifest-checksums=SHA256|CRC32C|NONE\n" + " calculate checksums for manifest files using provided algorithm\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nConnection options:\n")); printf(_(" -d, --dbname=CONNSTR connection string\n")); @@ -924,8 +939,8 @@ ReceiveCopyData(PGconn *conn, WriteDataCallback callback, res = PQgetResult(conn); if (PQresultStatus(res) != PGRES_COPY_OUT) { - pg_log_error("could not get COPY data stream: %s", - PQerrorMessage(conn)); + pg_log_error("could not get COPY data stream: %s [%s]", + PQerrorMessage(conn), PQresStatus(PQresultStatus(res))); exit(1); } PQclear(res); @@ -1170,6 +1185,31 @@ ReceiveTarFile(PGconn *conn, PGresult *res, int rownum) } } + /* + * Normally, we emit the backup manifest as a separate file, but when + * we're writing a tarfile to stdout, we don't have that option, so + * include it in the one tarfile we've got. + */ + if (strcmp(basedir, "-") == 0) + { + char header[512]; + PQExpBufferData buf; + + initPQExpBuffer(&buf); + ReceiveBackupManifestInMemory(conn, &buf); + if (PQExpBufferDataBroken(buf)) + { + pg_log_error("out of memory"); + exit(1); + } + tarCreateHeader(header, "backup_manifest", NULL, buf.len, + pg_file_create_mode, 04000, 02000, + time(NULL)); + writeTarData(&state, header, sizeof(header)); + writeTarData(&state, buf.data, buf.len); + termPQExpBuffer(&buf); + } + /* 2 * 512 bytes empty data at end of file */ writeTarData(&state, zerobuf, sizeof(zerobuf)); @@ -1417,6 +1457,64 @@ get_tablespace_mapping(const char *dir) /* + * Receive the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifest(PGconn *conn) +{ + WriteManifestState state; + + snprintf(state.filename, sizeof(state.filename), + "%s/backup_manifest", basedir); + state.file = fopen(state.filename, "wb"); + if (state.file == NULL) + { + pg_log_error("could not create file \"%s\": %m", state.filename); + exit(1); + } + + ReceiveCopyData(conn, ReceiveBackupManifestChunk, &state); + + fclose(state.file); +} + +/* + * Receive one chunk of the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifestChunk(size_t r, char *copybuf, void *callback_data) +{ + WriteManifestState *state = callback_data; + + if (fwrite(copybuf, r, 1, state->file) != 1) + { + pg_log_error("could not write to file \"%s\": %m", state->filename); + exit(1); + } +} + +/* + * Receive the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifestInMemory(PGconn *conn, PQExpBuffer buf) +{ + ReceiveCopyData(conn, ReceiveBackupManifestInMemoryChunk, buf); +} + +/* + * Receive one chunk of the backup manifest file and write it out to a file. + */ +static void +ReceiveBackupManifestInMemoryChunk(size_t r, char *copybuf, + void *callback_data) +{ + PQExpBuffer buf = callback_data; + + appendPQExpBuffer(buf, copybuf, r); +} + +/* * Receive a tar format stream from the connection to the server, and unpack * the contents of it into a directory. Only files, directories and * symlinks are supported, no other kinds of special files. @@ -1658,6 +1756,7 @@ BaseBackup(void) maxServerMajor; int serverVersion, serverMajor; + int writing_to_stdout; Assert(conn != NULL); @@ -1725,7 +1824,7 @@ BaseBackup(void) } basebkp = - psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s", + psprintf("BASE_BACKUP LABEL '%s' %s %s %s %s %s %s %s MANIFEST_CHECKSUMS '%s'", escaped_label, showprogress ? "PROGRESS" : "", includewal == FETCH_WAL ? "WAL" : "", @@ -1733,7 +1832,8 @@ BaseBackup(void) includewal == NO_WAL ? "" : "NOWAIT", maxrate_clause ? maxrate_clause : "", format == 't' ? "TABLESPACE_MAP" : "", - verify_checksums ? "" : "NOVERIFY_CHECKSUMS"); + verify_checksums ? "" : "NOVERIFY_CHECKSUMS", + manifest_checksums ? manifest_checksums : "NONE"); if (PQsendQuery(conn, basebkp) == 0) { @@ -1821,7 +1921,8 @@ BaseBackup(void) /* * When writing to stdout, require a single tablespace */ - if (format == 't' && strcmp(basedir, "-") == 0 && PQntuples(res) > 1) + writing_to_stdout = format == 't' && strcmp(basedir, "-") == 0; + if (writing_to_stdout && PQntuples(res) > 1) { pg_log_error("can only write single tablespace to stdout, database has %d", PQntuples(res)); @@ -1850,6 +1951,19 @@ BaseBackup(void) ReceiveAndUnpackTarFile(conn, res, i); } /* Loop over all tablespaces */ + /* + * Now receive backup manifest, if appropriate. + * + * If we're writing a tarfile to stdout, ReceiveTarFile will have already + * processed the backup manifest and included it in the output tarfile. + * Such a configuration doesn't allow for writing multiple files. + * + * If we're talking to an older server, it won't send a backup manifest, + * so don't try to receive one. + */ + if (!writing_to_stdout && serverMajor >= 1300) + ReceiveBackupManifest(conn); + if (showprogress) { progress_report(PQntuples(res), NULL, true); @@ -2052,6 +2166,7 @@ main(int argc, char **argv) {"waldir", required_argument, NULL, 1}, {"no-slot", no_argument, NULL, 2}, {"no-verify-checksums", no_argument, NULL, 3}, + {"manifest-checksums", required_argument, NULL, 'm'}, {NULL, 0, NULL, 0} }; int c; @@ -2079,7 +2194,7 @@ main(int argc, char **argv) atexit(cleanup_directories_atexit); - while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvP", + while ((c = getopt_long(argc, argv, "CD:F:r:RS:T:X:l:nNzZ:d:c:h:p:U:s:wWkvPm:", long_options, &option_index)) != -1) { switch (c) @@ -2220,6 +2335,9 @@ main(int argc, char **argv) case 3: verify_checksums = false; break; + case 'm': + manifest_checksums = pg_strdup(optarg); + break; default: /* diff --git a/src/common/Makefile b/src/common/Makefile index ffb0f6e..5fa2e23 100644 --- a/src/common/Makefile +++ b/src/common/Makefile @@ -48,9 +48,11 @@ LIBS += $(PTHREAD_LIBS) OBJS_COMMON = \ base64.o \ + checksum_utils.o \ config_info.o \ controldata_utils.o \ d2s.o \ + encode.o \ exec.o \ f2s.o \ file_perm.o \ diff --git a/src/common/checksum_utils.c b/src/common/checksum_utils.c new file mode 100644 index 0000000..82f0e2a --- /dev/null +++ b/src/common/checksum_utils.c @@ -0,0 +1,109 @@ +/*------------------------------------------------------------------------- + * + * checksum_utils.c + * checksum handling helpers + * + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/common/checksum_utils.c + * + *------------------------------------------------------------------------- + */ + + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include "common/checksum_utils.h" + +/* + * Initialize the checksum context according to the provided algorithm. + */ +void +initialize_checksum(ChecksumCtx * cCtx, ChecksumAlgorithm checksumAlgo) +{ + switch (checksumAlgo) + { + case MC_SHA256: + pg_sha256_init(&cCtx->sha256_ctx); + break; + case MC_CRC32C: + INIT_CRC32C(cCtx->crc_ctx); + break; + case MC_NONE: + break; + } +} + +void +update_checksum(ChecksumCtx * cCtx, ChecksumAlgorithm checksumAlgo, + const char *buf, off_t cnt) +{ + switch (checksumAlgo) + { + case MC_SHA256: + pg_sha256_update(&cCtx->sha256_ctx, (uint8 *) buf, cnt); + break; + case MC_CRC32C: + COMP_CRC32C(cCtx->crc_ctx, buf, cnt); + break; + case MC_NONE: + break; + } +} + +/* + * Function calculate the final checksum for the provided context and returns + * the length of checksum. + */ +int +finalize_checksum(ChecksumCtx * cCtx, ChecksumAlgorithm checksumAlgo, + char *checksumbuf) +{ + int checksumlen = 0; + + switch (checksumAlgo) + { + case MC_SHA256: + pg_sha256_final(&cCtx->sha256_ctx, (uint8 *) checksumbuf); + checksumlen = PG_SHA256_DIGEST_LENGTH; + break; + case MC_CRC32C: + FIN_CRC32C(cCtx->crc_ctx); + sprintf(checksumbuf, "%u", cCtx->crc_ctx); + checksumlen = strlen(checksumbuf); + break; + case MC_NONE: + break; + } + return checksumlen; +} + +bool +parse_checksum_algorithm(char *name, ChecksumAlgorithm * checksumAlgo) +{ + if (pg_strcasecmp(name, "SHA256") == 0) + { + *checksumAlgo = MC_SHA256; + return true; + } + else if (pg_strcasecmp(name, "CRC32C") == 0) + { + *checksumAlgo = MC_CRC32C; + return true; + } + else if (pg_strcasecmp(name, "NONE") == 0) + { + *checksumAlgo = MC_NONE; + return true; + } + + return false; +} diff --git a/src/common/encode.c b/src/common/encode.c new file mode 100644 index 0000000..a450c53 --- /dev/null +++ b/src/common/encode.c @@ -0,0 +1,38 @@ +/*------------------------------------------------------------------------- + * + * encode.c + * data encoding/decoding things. + * + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * + * IDENTIFICATION + * src/common/encode.c + * + *------------------------------------------------------------------------- + */ + + +#ifndef FRONTEND +#include "postgres.h" +#else +#include "postgres_fe.h" +#endif + +#include "common/encode.h" + +unsigned +hex_encode(const char *src, unsigned len, char *dst) +{ + const char *end = src + len; + + while (src < end) + { + *dst++ = hextbl[(*src >> 4) & 0xF]; + *dst++ = hextbl[*src & 0xF]; + src++; + } + return len * 2; +} diff --git a/src/include/common/checksum_utils.h b/src/include/common/checksum_utils.h new file mode 100644 index 0000000..c931070 --- /dev/null +++ b/src/include/common/checksum_utils.h @@ -0,0 +1,46 @@ +/*------------------------------------------------------------------------- + * checksum_utils.h + * checksum handling helpers + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/common/checksum_utils.h + * + *------------------------------------------------------------------------- + */ +#ifndef COMMON_CHECKSUM_UTILS_H +#define COMMON_CHECKSUM_UTILS_H + +#include "common/sha2.h" +#include "port/pg_crc32c.h" +#include "lib/stringinfo.h" + +/* Checksum algorithm option for manifest */ +typedef enum ChecksumAlgorithm +{ + MC_NONE = 0, + MC_SHA256, + MC_CRC32C +} ChecksumAlgorithm; + +/* checksum algorithm context */ +typedef union checksumCtx +{ + pg_sha256_ctx sha256_ctx; + pg_crc32c crc_ctx; +} ChecksumCtx; + +extern void initialize_checksum(ChecksumCtx * cCtx, + ChecksumAlgorithm checksumAlgo); +extern void update_checksum(ChecksumCtx * cCtx, + ChecksumAlgorithm checksumAlgo, + const char *buf, off_t cnt); +extern int finalize_checksum(ChecksumCtx * cCtx, + ChecksumAlgorithm checksumAlgo, + char *checksumbuf); +extern bool parse_checksum_algorithm(char *name, + ChecksumAlgorithm * checksumAlgo); + +#endif /* COMMON_CHECKSUM_UTILS_H */ diff --git a/src/include/common/encode.h b/src/include/common/encode.h new file mode 100644 index 0000000..63328bc --- /dev/null +++ b/src/include/common/encode.h @@ -0,0 +1,20 @@ +/*------------------------------------------------------------------------- + * encode.h + * data encoding/decoding things. + * + * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * + * IDENTIFICATION + * src/include/common/encode.h + * + *------------------------------------------------------------------------- + */ +#ifndef COMMON_encode_H +#define COMMON_encode_H + +static const char hextbl[] = "0123456789abcdef"; + +extern unsigned hex_encode(const char *src, unsigned len, char *dst); + +#endif /* COMMON_ENCODE_H */ diff --git a/src/include/replication/basebackup.h b/src/include/replication/basebackup.h index 503a5b9..07748de 100644 --- a/src/include/replication/basebackup.h +++ b/src/include/replication/basebackup.h @@ -12,6 +12,9 @@ #ifndef _BASEBACKUP_H #define _BASEBACKUP_H +#include "common/checksum_utils.h" +#include "common/sha2.h" +#include "lib/stringinfo.h" #include "nodes/replnodes.h" /* @@ -29,8 +32,18 @@ typedef struct int64 size; } tablespaceinfo; +/* Backup manifest info */ +typedef struct +{ + ChecksumAlgorithm checksumAlgo; + char checksum_label[10]; + ChecksumCtx cCtx; + StringInfo manifest; +} manifestinfo; + extern void SendBaseBackup(BaseBackupCmd *cmd); -extern int64 sendTablespace(char *path, bool sizeonly); +extern int64 sendTablespace(char *path, char *oid, bool sizeonly, + manifestinfo * manifestInfo); #endif /* _BASEBACKUP_H */ diff --git a/src/include/utils/builtins.h b/src/include/utils/builtins.h index 937ddb7..6d2b0a6 100644 --- a/src/include/utils/builtins.h +++ b/src/include/utils/builtins.h @@ -30,7 +30,6 @@ extern int errdatatype(Oid datatypeOid); extern int errdomainconstraint(Oid datatypeOid, const char *conname); /* encode.c */ -extern unsigned hex_encode(const char *src, unsigned len, char *dst); extern unsigned hex_decode(const char *src, unsigned len, char *dst); /* int.c */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index caf6b86..87556f6 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -329,6 +329,8 @@ CheckPointStmt CheckpointStatsData CheckpointerRequest CheckpointerShmemStruct +ChecksumAlgorithm +ChecksumCtx Chromosome CkptSortItem CkptTsStatus @@ -1351,6 +1353,7 @@ MultiXactOffset MultiXactStateData MultiXactStatus MyData +manifestinfo NDBOX NODE NUMCacheEntry -- 1.8.3.1