From bdaa60fed1d2b6f9001c74d21197f15078cc64f2 Mon Sep 17 00:00:00 2001 From: Suraj Kharage Date: Tue, 17 Dec 2019 10:56:42 +0530 Subject: [PATCH v3 2/3] Implementation of backup validator Patch by Suraj Kharage, inputs from Robert Haas, review from Jeevan Chalke, and Robert Haas. --- doc/src/sgml/ref/pg_basebackup.sgml | 12 + src/bin/pg_basebackup/pg_basebackup.c | 416 ++++++++++++++++++++++++++++++++++ src/common/encode.c | 18 ++ src/include/common/encode.h | 1 + src/tools/pgindent/typedefs.list | 1 + 5 files changed, 448 insertions(+) diff --git a/doc/src/sgml/ref/pg_basebackup.sgml b/doc/src/sgml/ref/pg_basebackup.sgml index af7c731..043ee39 100644 --- a/doc/src/sgml/ref/pg_basebackup.sgml +++ b/doc/src/sgml/ref/pg_basebackup.sgml @@ -548,6 +548,18 @@ PostgreSQL documentation + + + + + Validate the given backup directory and detect the modification if any + without restarting the server. For plain backup, provide the backup + directory path with option. Tar format + backups can be verified after untarring. + + + + diff --git a/src/bin/pg_basebackup/pg_basebackup.c b/src/bin/pg_basebackup/pg_basebackup.c index 7d5ed0d..c3c3c85 100644 --- a/src/bin/pg_basebackup/pg_basebackup.c +++ b/src/bin/pg_basebackup/pg_basebackup.c @@ -27,9 +27,12 @@ #endif #include "access/xlog_internal.h" +#include "common/checksum_utils.h" +#include "common/encode.h" #include "common/file_perm.h" #include "common/file_utils.h" #include "common/logging.h" +#include "common/sha2.h" #include "common/string.h" #include "fe_utils/recovery_gen.h" #include "fe_utils/string_utils.h" @@ -44,6 +47,8 @@ #define ERRCODE_DATA_CORRUPTED "XX001" +#define CHECKSUM_LENGTH 256 + typedef struct TablespaceListCell { struct TablespaceListCell *next; @@ -97,6 +102,30 @@ typedef struct WriteManifestState typedef void (*WriteDataCallback) (size_t nbytes, char *buf, void *callback_data); +typedef struct DataDirectoryFileInfo +{ + char filetype[10]; + char *filename; + int filesize; + char checksum[CHECKSUM_LENGTH]; + bool matched; + uint32 status; /* hash status */ +} DataDirectoryFileInfo; + +typedef struct manifesthash_hash *hashtab; + +#define SH_PREFIX manifesthash +#define SH_ELEMENT_TYPE DataDirectoryFileInfo +#define SH_KEY_TYPE char* +#define SH_KEY filename +#define SH_HASH_KEY(tb, key) string_hash_sdbm(key) +#define SH_EQUAL(tb, a, b) (strcmp(a, b) == 0) +#define SH_SCOPE static inline +#define SH_RAW_ALLOCATOR pg_malloc +#define SH_DECLARE +#define SH_DEFINE +#include "lib/simplehash.h" + /* * pg_xlog has been renamed to pg_wal in version 10. This version number * should be compared with PQserverVersion(). @@ -142,6 +171,7 @@ static bool create_slot = false; static bool no_slot = false; static bool verify_checksums = true; static char *manifest_checksums = NULL; +static enum ChecksumAlgorithm checksum_type = MC_NONE; static bool success = false; static bool made_new_pgdata = false; @@ -201,6 +231,13 @@ static bool reached_end_position(XLogRecPtr segendpos, uint32 timeline, static const char *get_tablespace_mapping(const char *dir); static void tablespace_list_append(const char *arg); +static void VerifyBackup(void); +static manifesthash_hash *create_manifest_hash(char manifest_path[MAXPGPATH]); +static void scan_data_directory(char *basedir, const char *subdirpath, + manifesthash_hash *hashtab); +static void verify_file(struct dirent *de, char fn[MAXPGPATH], + struct stat st, char relative_path[MAXPGPATH], + manifesthash_hash *hashtab); static void cleanup_directories_atexit(void) @@ -401,6 +438,7 @@ usage(void) " do not verify checksums\n")); printf(_(" --manifest-checksums=SHA256|CRC32C|NONE\n" " calculate checksums for manifest files using provided algorithm\n")); + printf(_(" --verify-backup validate the backup\n")); printf(_(" -?, --help show this help, then exit\n")); printf(_("\nConnection options:\n")); printf(_(" -d, --dbname=CONNSTR connection string\n")); @@ -2167,11 +2205,13 @@ main(int argc, char **argv) {"no-slot", no_argument, NULL, 2}, {"no-verify-checksums", no_argument, NULL, 3}, {"manifest-checksums", required_argument, NULL, 'm'}, + {"verify-backup", no_argument, NULL, 4}, {NULL, 0, NULL, 0} }; int c; int option_index; + bool verify_backup = false; pg_logging_init(argv[0]); progname = get_progname(argv[0]); @@ -2338,6 +2378,9 @@ main(int argc, char **argv) case 'm': manifest_checksums = pg_strdup(optarg); break; + case 4: + verify_backup = true; + break; default: /* @@ -2460,6 +2503,12 @@ main(int argc, char **argv) } #endif + if(verify_backup) + { + VerifyBackup(); + return 0; + } + /* connection in replication mode to server */ conn = GetConnection(); if (!conn) @@ -2524,3 +2573,370 @@ main(int argc, char **argv) success = true; return 0; } + +/* + * Read the backup_manifest file and generate the hash table, then scan data + * directroy and verify each file. Finally, iterate on hash table to find + * out missing files. + */ +static void +VerifyBackup(void) +{ + char manifest_path[MAXPGPATH]; + manifesthash_hash *hashtab; + manifesthash_iterator i; + DataDirectoryFileInfo *entry; + + snprintf(manifest_path, sizeof(manifest_path), "%s/%s", basedir, + "backup_manifest"); + + /* create hash table */ + hashtab = create_manifest_hash(manifest_path); + + scan_data_directory(basedir, NULL, hashtab); + + manifesthash_start_iterate(hashtab, &i); + while ((entry = manifesthash_iterate(hashtab, &i)) != NULL) + { + if (!entry->matched) + { + pg_log_info("missing file: %s", entry->filename); + } + } +} + +/* + * Given a file path, read that file and generate the hash table for same. + * Also generate the checksum for the records that are read from file and + * compare that with checksum written in backup_manifest file. If both + * checksums are identical then proceed, otherwise throw an error and abort. + */ +static manifesthash_hash * +create_manifest_hash(char manifest_path[MAXPGPATH]) +{ + manifesthash_hash *hashtab; + FILE *file; + DataDirectoryFileInfo *entry; + PQExpBuffer manifest; + char file_checksum[256]; + char header[1024]; + long pos = 0; + + manifest = createPQExpBuffer(); + if (!manifest) + { + pg_log_error("out of memory"); + exit(1); + } + + file = fopen(manifest_path, "r"); + + if (!file) + { + pg_log_error("could not open backup_manifest"); + exit(1); + } + + /* read the file header */ + if (fscanf(file, "%1023[^\n]\n", header) != 1) + { + pg_log_error("error while reading the header from backup_manifest"); + exit(1); + } + + appendPQExpBufferStr(manifest, header); + appendPQExpBufferStr(manifest, "\n"); + + hashtab = manifesthash_create(1024, NULL); + + while (!feof(file)) + { + char *filename; + char filetype[10]; + int filesize = 0; + char mtime[24]; + char checksum_with_type[CHECKSUM_LENGTH]; + bool found; + char checksum[CHECKSUM_LENGTH]; + + filename = (char*) pg_malloc(MAXPGPATH); + + if (fscanf(file, "%s\t%s\t%d\t%23[^\t] %s\n", filetype, filename, + &filesize, mtime, checksum_with_type) != 5) + { + /* + * On failure, re-read the last line of record and check if it is + * a last line where manifest checksum is written. If yes, then + * parse it. + */ + if (fseek(file, pos, SEEK_SET) == -1) + { + pg_log_error("error while reading the backup_manifest file"); + exit(1); + } + + if (fscanf(file, "Manifest-Checksum\t%s\n", file_checksum) != 1) + { + pg_log_error("error while reading the backup_manifest file"); + exit(1); + } + + if (feof(file)) + break; + } + + pos = ftell(file); + + entry = manifesthash_insert(hashtab, filename, &found); + + if (found) + { + pg_log_info("duplicate file present: %s\n", filename); + pg_free(filename); + } + else + { + memcpy(entry->filetype, filetype, strlen(filetype)); + entry->filetype[strlen(filetype)+1] = '\0'; + entry->filesize = filesize; + } + + if (strcmp(checksum_with_type, "-") == 0) + { + checksum_type = MC_NONE; + } + else + { + if (strncmp(checksum_with_type, "SHA256", 6) == 0) + { + checksum_type = MC_SHA256; + snprintf(checksum, CHECKSUM_LENGTH, "%s", + &checksum_with_type[strlen("SHA256:")]); + } + else if (strncmp(checksum_with_type, "CRC32C", 6) == 0) + { + checksum_type = MC_CRC32C; + snprintf(checksum, CHECKSUM_LENGTH, "%s", + &checksum_with_type[strlen("CRC32C:")]); + } + else + { + pg_log_error("unknown checksum method"); + exit(1); + } + + snprintf(entry->checksum, CHECKSUM_LENGTH, "%s", checksum); + } + + appendPQExpBuffer(manifest, "File\t%s\t%d\t%s\t%s\n", filename, + filesize, mtime, checksum_with_type); + + } + + /* + * Once read all the records from backup_manifest file, generate the + * backup manifest checksum and compare it with the backup menifest + * checksum written in manifest file. + */ + if (checksum_type != MC_NONE) + { + + char checksumbuf[CHECKSUM_LENGTH]; + int checksumbuflen; + ChecksumCtx cCtx; + char encoded_checksum[CHECKSUM_LENGTH]; + char manifest_checksum[CHECKSUM_LENGTH]; + + initialize_checksum(&cCtx, checksum_type); + update_checksum(&cCtx, checksum_type, manifest->data, + manifest->len); + checksumbuflen = finalize_checksum(&cCtx, checksum_type, + (char *) checksumbuf); + appendPQExpBuffer(manifest, "Manifest-Checksum\t"); + + switch (checksum_type) + { + case MC_SHA256: + appendPQExpBuffer(manifest, "SHA256:"); + snprintf(manifest_checksum, CHECKSUM_LENGTH, "%s", + &file_checksum[strlen("SHA256:")]); + break; + case MC_CRC32C: + appendPQExpBuffer(manifest, "CRC32C:"); + snprintf(manifest_checksum, CHECKSUM_LENGTH, "%s", + &file_checksum[strlen("CRC32C:")]); + break; + case MC_NONE: + break; + } + + checksumbuflen = hex_encode(checksumbuf, checksumbuflen, + encoded_checksum); + encoded_checksum[checksumbuflen] = '\0'; + + /* + * Compare the both checksums, if they are not same that means + * backup_manifest file is changed. Throw an error and abort. + */ + if (strcmp(encoded_checksum, manifest_checksum) != 0) + { + pg_log_error("backup manifest checksum difference. Aborting"); + exit(1); + } + } + + return hashtab; +} + +/* + * Scan the data directory and check whether each file entry present in hash + * table with the correct details, i.e. filesize and checksum. + */ +static void +scan_data_directory(char *basedir, const char *subdirpath, + manifesthash_hash *hashtab) +{ + char path[MAXPGPATH]; + char relative_path[MAXPGPATH] = ""; + DIR *dir; + struct dirent *de; + + if (subdirpath) + { + snprintf(path, sizeof(path), "%s/%s", basedir, + subdirpath); + snprintf(relative_path, sizeof(relative_path), "%s/", subdirpath); + + } + else + snprintf(path, sizeof(path), "%s", basedir); + + dir = opendir(path); + if (!dir) + { + pg_log_error("could not open directory \"%s\": %m", path); + exit(1); + } + + while ((de = readdir(dir)) != NULL) + { + char fn[MAXPGPATH]; + struct stat st; + + if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0 || + strcmp(de->d_name, "pg_wal") == 0) + continue; + + snprintf(fn, sizeof(fn), "%s/%s", path, de->d_name); + if (stat(fn, &st) < 0) + { + pg_log_error("could not stat file \"%s\": %m", fn); + exit(1); + } + if (S_ISREG(st.st_mode)) + { + verify_file(de, fn, st, relative_path, hashtab); + } + else if (S_ISDIR(st.st_mode)) + { + char newsubdirpath[MAXPGPATH]; + + if (subdirpath) + snprintf(newsubdirpath, MAXPGPATH, "%s/%s", subdirpath, + de->d_name); + else + snprintf(newsubdirpath, MAXPGPATH, "%s", de->d_name); + + scan_data_directory(basedir, newsubdirpath, hashtab); + } + } + closedir(dir); +} + +/* + * Given the file and its details, check whether it is present in hash table + * and if yes, then compare its details with hash table entry. + */ +static void +verify_file(struct dirent *de, char fn[MAXPGPATH], struct stat st, + char relative_path[MAXPGPATH], manifesthash_hash *hashtab) +{ + PQExpBuffer filename = NULL; + DataDirectoryFileInfo *record; + + /* Skip backup manifest file. */ + if (strcmp(de->d_name, "backup_manifest") == 0) + return; + + filename = createPQExpBuffer(); + if (!filename) + { + pg_log_error("out of memory"); + exit(1); + } + + appendPQExpBuffer(filename, "%s%s", relative_path, de->d_name); + + /* + * Compare the hash and if record found then we match the file size and + * checksum (if enabled). Modified time cannot be compared with the + * file in the backup directory and its entry in the manifest as + * manifest entry gives mtime from server file whereas the same file in + * the backup will have different mtime. + */ + record = manifesthash_lookup(hashtab, filename->data);; + if (record) + { + record->matched = true; + if (record->filesize != st.st_size) + pg_log_info("size changed for file: %s, original size: %d, current size: %zu", + filename->data, record->filesize, st.st_size); + + /* + * Read the file and generate the checksum based on checksum method + * and compare that with the checksum present in hash entry. + */ + if (checksum_type != MC_NONE) + { + FILE *fp; + char buf[1048576]; // 1MB chunk + pgoff_t len = 0; + off_t cnt; + char checksumbuf[CHECKSUM_LENGTH]; + char encode_checksumbuf[CHECKSUM_LENGTH]; + int checksumbuflen; + ChecksumCtx cCtx; + + initialize_checksum(&cCtx, checksum_type); + + fp = fopen(fn, "r"); + if (!fp) + { + pg_log_error("could not open file \"%s\": %m", de->d_name); + exit(1); + } + + /* Read file in chunks [1 MB each chunk]*/ + while ((cnt = fread(buf, 1, Min(sizeof(buf), st.st_size - len), fp)) > 0) + { + update_checksum(&cCtx, checksum_type, buf, cnt); + len += cnt; + } + + checksumbuflen = finalize_checksum(&cCtx, checksum_type, + checksumbuf); + + /* Convert checksum to hexadecimal. */ + checksumbuflen = hex_encode(checksumbuf, checksumbuflen, + encode_checksumbuf); + encode_checksumbuf[checksumbuflen] = '\0'; + + fclose(fp); + + if (strcmp(record->checksum, encode_checksumbuf) != 0) + pg_log_info("checksum difference for file: %s", filename->data); + } + } + else + pg_log_info("extra file found: %s", filename->data); +} diff --git a/src/common/encode.c b/src/common/encode.c index a450c53..14f2ec2 100644 --- a/src/common/encode.c +++ b/src/common/encode.c @@ -36,3 +36,21 @@ hex_encode(const char *src, unsigned len, char *dst) } return len * 2; } + +/* + * Simple string hash function from http://www.cse.yorku.ca/~oz/hash.html + * + * The backend uses a more sophisticated function for hashing strings, + * but we don't really need that complexity here. + */ +uint32 +string_hash_sdbm(const char *key) +{ + uint32 hash = 0; + int c; + + while ((c = *key++)) + hash = c + (hash << 6) + (hash << 16) - hash; + + return hash; +} diff --git a/src/include/common/encode.h b/src/include/common/encode.h index 63328bc..44a062f 100644 --- a/src/include/common/encode.h +++ b/src/include/common/encode.h @@ -16,5 +16,6 @@ static const char hextbl[] = "0123456789abcdef"; extern unsigned hex_encode(const char *src, unsigned len, char *dst); +extern uint32 string_hash_sdbm(const char *key); #endif /* COMMON_ENCODE_H */ diff --git a/src/tools/pgindent/typedefs.list b/src/tools/pgindent/typedefs.list index 8b2dae6..982ba29 100644 --- a/src/tools/pgindent/typedefs.list +++ b/src/tools/pgindent/typedefs.list @@ -486,6 +486,7 @@ DR_sqlfunction DR_transientrel DSA DWORD +DataDirectoryFileInfo DataDumperPtr DataPageDeleteStack DateADT -- 1.8.3.1