Hi,
I took a quick look at the remaining part adding copy_file_range to
pg_combinebackup. The patch no longer applies, so I had to rebase it.
Most of the issues were trivial, but I had to fix a couple missing
prototypes - I added them to copy_file.h/c, mostly.
0001 is the minimal rebase + those fixes
0002 has a couple review comments in copy_file, and it also undoes a lot
of unnecessary formatting changes (already pointed out by Peter a couple
days ago).
A couple review comments:
1) AFAIK opt_errinfo() returns pointer to the local "buf" variable.
2) I wonder if we even need opt_errinfo(). I'm not sure it actually
makes anything simpler.
3) I think it'd be nice to make CopyFileMethod more consistent with
transferMode in pg_upgrade.h (I mean, it seems wise to make the naming
more consistent, it's probably not worth unifying this somehow).
4) I wonder how we came up with copying the files by 50 blocks, but I
now realize it's been like this before this patch. I only noticed
because the patch adds a comment before buffer_size calculation.
5) I dislike the renaming of copy_file_blocks to pg_copyfile. The new
name is way more generic / less descriptive - it's clear it copies the
file block by block (well, in chunks). pg_copyfile is pretty vague.
6) This leaves behind copy_file_copyfile, which is now unused.
7) The patch reworks how combinebackup deals with alternative copy
implementations - instead of setting strategy_implementation and calling
that, the decisions now happen in pg_copyfile_offload with a lot of
conditions / ifdef / defined ... I find it pretty hard to understand and
reason about. I liked the strategy_implementation approach, as it forces
us to keep each method in a separate function.
Perhaps there's a reason why that doesn't work for copy_file_range? But
in that case this needs much clearer comments.
regards
--
Tomas Vondra
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
From 39f42eee4c6f50d106672afe108294ee59082500 Mon Sep 17 00:00:00 2001
From: Tomas Vondra <to...@2ndquadrant.com>
Date: Tue, 19 Mar 2024 15:34:18 +0100
Subject: [PATCH v20240319 2/2] review and cleanup
---
src/bin/pg_combinebackup/copy_file.c | 3 +
src/bin/pg_combinebackup/copy_file.h | 1 +
src/bin/pg_combinebackup/pg_combinebackup.c | 197 +++++++++++---------
src/bin/pg_combinebackup/reconstruct.c | 105 ++++++-----
src/bin/pg_combinebackup/reconstruct.h | 19 +-
5 files changed, 190 insertions(+), 135 deletions(-)
diff --git a/src/bin/pg_combinebackup/copy_file.c b/src/bin/pg_combinebackup/copy_file.c
index 16e26b4f573..f45670dd47c 100644
--- a/src/bin/pg_combinebackup/copy_file.c
+++ b/src/bin/pg_combinebackup/copy_file.c
@@ -77,6 +77,8 @@ opt_errinfo(const char *addon_errmsg)
return "";
strcpy(buf, " ");
+
+ /* XXX isn't this broken? this returns pointer to local variable */
return strncat(buf, addon_errmsg, sizeof(buf) - 2);
}
@@ -93,6 +95,7 @@ pg_copyfile(const char *src, const char *dest, const char *addon_errmsg,
int dest_fd;
uint8 *buffer;
+ /* XXX where does the 50 blocks come from? larger/smaller? */
/* copy in fairly large chunks for best efficiency */
const int buffer_size = 50 * BLCKSZ;
diff --git a/src/bin/pg_combinebackup/copy_file.h b/src/bin/pg_combinebackup/copy_file.h
index 2797a340055..f4d0ac47d0e 100644
--- a/src/bin/pg_combinebackup/copy_file.h
+++ b/src/bin/pg_combinebackup/copy_file.h
@@ -15,6 +15,7 @@
#include "common/checksum_helper.h"
#include "common/file_utils.h"
+/* XXX do we even want this? how does pg_upgrade to this? */
typedef enum CopyFileMethod
{
PG_COPYFILE_FALLBACK = 0x1,
diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c
index 1455360d81c..8fa7827c563 100644
--- a/src/bin/pg_combinebackup/pg_combinebackup.c
+++ b/src/bin/pg_combinebackup/pg_combinebackup.c
@@ -99,10 +99,15 @@ static void cleanup_directories_atexit(void);
static void create_output_directory(char *dirname, cb_options *opt);
static void help(const char *progname);
static bool parse_oid(char *s, Oid *result);
-static void process_directory_recursively(
- Oid tsoid, char *input_directory, char *output_directory,
- char *relative_path, int n_prior_backups, char **prior_backup_dirs,
- manifest_data **manifests, manifest_writer *mwriter, cb_options *opt);
+static void process_directory_recursively(Oid tsoid,
+ char *input_directory,
+ char *output_directory,
+ char *relative_path,
+ int n_prior_backups,
+ char **prior_backup_dirs,
+ manifest_data **manifests,
+ manifest_writer *mwriter,
+ cb_options *opt);
static int read_pg_version_file(char *directory);
static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
static void reset_directory_cleanup_list(void);
@@ -156,8 +161,8 @@ main(int argc, char *argv[])
opt.copy_method = 0;
/* process command-line options */
- while ((c = getopt_long(argc, argv, "dnNPo:T:", long_options, &optindex)) !=
- -1)
+ while ((c = getopt_long(argc, argv, "dnNPo:T:",
+ long_options, &optindex)) != -1)
{
switch (c)
{
@@ -178,8 +183,10 @@ main(int argc, char *argv[])
add_tablespace_mapping(&opt, optarg);
break;
case 1:
- if (!pg_checksum_parse_type(optarg, &opt.manifest_checksums))
- pg_fatal("unrecognized checksum algorithm: \"%s\"", optarg);
+ if (!pg_checksum_parse_type(optarg,
+ &opt.manifest_checksums))
+ pg_fatal("unrecognized checksum algorithm: \"%s\"",
+ optarg);
break;
case 2:
opt.no_manifest = true;
@@ -295,8 +302,7 @@ main(int argc, char *argv[])
* won't have the WAL ranges for the resulting manifest.
*/
if (manifests[n_prior_backups] == NULL)
- pg_fatal("can't generate a manifest because no manifest is available for "
- "the final input backup");
+ pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
}
else
mwriter = NULL;
@@ -308,15 +314,15 @@ main(int argc, char *argv[])
{
pg_log_debug("generating \"%s/backup_label\"", opt.output);
last_backup_label->cursor = 0;
- write_backup_label(opt.output, last_backup_label, opt.manifest_checksums,
- mwriter);
+ write_backup_label(opt.output, last_backup_label,
+ opt.manifest_checksums, mwriter);
}
/* Process everything that's not part of a user-defined tablespace. */
pg_log_debug("processing backup directory \"%s\"", last_input_dir);
- process_directory_recursively(InvalidOid, last_input_dir, opt.output, NULL,
- n_prior_backups, prior_backup_dirs, manifests,
- mwriter, &opt);
+ process_directory_recursively(InvalidOid, last_input_dir, opt.output,
+ NULL, n_prior_backups, prior_backup_dirs,
+ manifests, mwriter, &opt);
/* Process user-defined tablespaces. */
for (ts = tablespaces; ts != NULL; ts = ts->next)
@@ -332,15 +338,16 @@ main(int argc, char *argv[])
{
char linkpath[MAXPGPATH];
- snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output, ts->oid);
+ snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
+ ts->oid);
if (opt.dry_run)
pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
linkpath, ts->new_dir);
else
{
- pg_log_debug("creating symbolic link from \"%s\" to \"%s\"", linkpath,
- ts->new_dir);
+ pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
+ linkpath, ts->new_dir);
if (symlink(ts->new_dir, linkpath) != 0)
pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
linkpath, ts->new_dir);
@@ -354,19 +361,21 @@ main(int argc, char *argv[])
{
pg_log_debug("creating directory \"%s\"", ts->new_dir);
if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
- pg_fatal("could not create directory \"%s\": %m", ts->new_dir);
+ pg_fatal("could not create directory \"%s\": %m",
+ ts->new_dir);
}
}
/* OK, now handle the directory contents. */
- process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir, NULL,
- n_prior_backups, prior_backup_dirs, manifests,
- mwriter, &opt);
+ process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
+ NULL, n_prior_backups, prior_backup_dirs,
+ manifests, mwriter, &opt);
}
/* Finalize the backup_manifest, if we're generating one. */
if (mwriter != NULL)
- finalize_manifest(mwriter, manifests[n_prior_backups]->first_wal_range);
+ finalize_manifest(mwriter,
+ manifests[n_prior_backups]->first_wal_range);
/* fsync that output directory unless we've been told not to do so */
if (!opt.no_sync)
@@ -422,9 +431,7 @@ add_tablespace_mapping(cb_options *opt, char *arg)
*dst_ptr++ = *arg_ptr;
}
if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
- pg_fatal(
- "invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"",
- arg);
+ pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
/*
* All tablespaces are created with absolute directories, so specifying a
@@ -496,8 +503,8 @@ check_backup_label_files(int n_backups, char **backup_dirs)
pg_fatal("could not close \"%s\": %m", pathbuf);
/* Parse the file contents. */
- parse_backup_label(pathbuf, buf, &start_tli, &start_lsn, &previous_tli,
- &previous_lsn);
+ parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
+ &previous_tli, &previous_lsn);
/*
* Sanity checks.
@@ -508,19 +515,18 @@ check_backup_label_files(int n_backups, char **backup_dirs)
* we don't have that information.
*/
if (i > 0 && previous_tli == 0)
- pg_fatal("backup at \"%s\" is a full backup, but only the first backup "
- "should be a full backup",
+ pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
backup_dirs[i]);
if (i == 0 && previous_tli != 0)
- pg_fatal("backup at \"%s\" is an incremental backup, but the first "
- "backup should be a full backup",
+ pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
backup_dirs[i]);
if (i < n_backups - 1 && start_tli != check_tli)
pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
backup_dirs[i], start_tli, check_tli);
if (i < n_backups - 1 && start_lsn != check_lsn)
pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
- backup_dirs[i], LSN_FORMAT_ARGS(start_lsn),
+ backup_dirs[i],
+ LSN_FORMAT_ARGS(start_lsn),
LSN_FORMAT_ARGS(check_lsn));
check_tli = previous_tli;
check_lsn = previous_lsn;
@@ -572,7 +578,8 @@ check_control_files(int n_backups, char **backup_dirs)
/* Can't interpret control file if not current version. */
if (control_file->pg_control_version != PG_CONTROL_VERSION)
- pg_fatal("%s: unexpected control file version", controlpath);
+ pg_fatal("%s: unexpected control file version",
+ controlpath);
/* System identifiers should all match. */
if (i == n_backups - 1)
@@ -698,23 +705,16 @@ help(const char *progname)
printf(_("\nOptions:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
printf(_(" -n, --dry-run don't actually do anything\n"));
- printf(_(" -N, --no-sync do not wait for changes to be written "
- "safely to disk\n"));
+ printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
printf(_(" -o, --output output directory\n"));
- printf(_(
- " -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
+ printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
" relocate tablespace in OLDDIR to NEWDIR\n"));
- printf(
- _(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
+ printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
" use algorithm for manifest checksums\n"));
- printf(_(
- " --no-manifest suppress generation of backup manifest\n"));
- printf(
- _(" --sync-method=METHOD set method for syncing files to disk\n"));
- printf(_(" --clone clone (reflink) instead of copying "
- "files\n"));
- printf(
- _(" --copy-file-range copy using copy_file_range() syscall\n"));
+ printf(_(" --no-manifest suppress generation of backup manifest\n"));
+ printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --clone clone (reflink) instead of copying files\n"));
+ printf(_(" --copy-file-range copy using copy_file_range() syscall\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
@@ -761,10 +761,15 @@ parse_oid(char *s, Oid *result)
* the locations of those previous backups.
*/
static void
-process_directory_recursively(
- Oid tsoid, char *input_directory, char *output_directory,
- char *relative_path, int n_prior_backups, char **prior_backup_dirs,
- manifest_data **manifests, manifest_writer *mwriter, cb_options *opt)
+process_directory_recursively(Oid tsoid,
+ char *input_directory,
+ char *output_directory,
+ char *relative_path,
+ int n_prior_backups,
+ char **prior_backup_dirs,
+ manifest_data **manifests,
+ manifest_writer *mwriter,
+ cb_options *opt)
{
char ifulldir[MAXPGPATH];
char ofulldir[MAXPGPATH];
@@ -817,11 +822,13 @@ process_directory_recursively(
}
else
{
- snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory, relative_path);
- snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory, relative_path);
+ snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
+ relative_path);
+ snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
+ relative_path);
if (OidIsValid(tsoid))
- snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/", tsoid,
- relative_path);
+ snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
+ tsoid, relative_path);
else
snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
}
@@ -857,7 +864,8 @@ process_directory_recursively(
pg_checksum_context checksum_ctx;
/* Ignore "." and ".." entries. */
- if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
+ if (strcmp(de->d_name, ".") == 0 ||
+ strcmp(de->d_name, "..") == 0)
continue;
/* Construct input path. */
@@ -893,9 +901,11 @@ process_directory_recursively(
de->d_name);
/* And recurse. */
- process_directory_recursively(tsoid, input_directory, output_directory,
- new_relative_path, n_prior_backups,
- prior_backup_dirs, manifests, mwriter, opt);
+ process_directory_recursively(tsoid,
+ input_directory, output_directory,
+ new_relative_path,
+ n_prior_backups, prior_backup_dirs,
+ manifests, mwriter, opt);
continue;
}
@@ -913,37 +923,47 @@ process_directory_recursively(
* Skip the backup_label and backup_manifest files; they require
* special handling and are handled elsewhere.
*/
- if (relative_path == NULL && (strcmp(de->d_name, "backup_label") == 0 ||
- strcmp(de->d_name, "backup_manifest") == 0))
+ if (relative_path == NULL &&
+ (strcmp(de->d_name, "backup_label") == 0 ||
+ strcmp(de->d_name, "backup_manifest") == 0))
continue;
/*
* If it's an incremental file, hand it off to the reconstruction
* code, which will figure out what to do.
*/
- if (strncmp(de->d_name, INCREMENTAL_PREFIX, INCREMENTAL_PREFIX_LENGTH) ==
- 0)
+ if (strncmp(de->d_name, INCREMENTAL_PREFIX,
+ INCREMENTAL_PREFIX_LENGTH) == 0)
{
/* Output path should not include "INCREMENTAL." prefix. */
snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
de->d_name + INCREMENTAL_PREFIX_LENGTH);
+
/* Manifest path likewise omits incremental prefix. */
snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
de->d_name + INCREMENTAL_PREFIX_LENGTH);
/* Reconstruction logic will do the rest. */
- reconstruct_from_incremental_file(
- ifullpath, ofullpath, relative_path,
- de->d_name + INCREMENTAL_PREFIX_LENGTH, n_prior_backups,
- prior_backup_dirs, manifests, manifest_path, checksum_type,
- &checksum_length, &checksum_payload, opt->debug, opt->dry_run,
+ reconstruct_from_incremental_file(ifullpath, ofullpath,
+ relative_path,
+ de->d_name + INCREMENTAL_PREFIX_LENGTH,
+ n_prior_backups,
+ prior_backup_dirs,
+ manifests,
+ manifest_path,
+ checksum_type,
+ &checksum_length,
+ &checksum_payload,
+ opt->debug,
+ opt->dry_run,
opt->copy_method);
}
else
{
/* Construct the path that the backup_manifest will use. */
- snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix, de->d_name);
+ snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
+ de->d_name);
/*
* It's not an incremental file, so we need to copy the entire
@@ -953,11 +973,13 @@ process_directory_recursively(
* backup_manifest for the final input directory, we can save some
* work by reusing that checksum instead of computing a new one.
*/
- if (checksum_type != CHECKSUM_TYPE_NONE && latest_manifest != NULL)
+ if (checksum_type != CHECKSUM_TYPE_NONE &&
+ latest_manifest != NULL)
{
manifest_file *mfile;
- mfile = manifest_files_lookup(latest_manifest->files, manifest_path);
+ mfile = manifest_files_lookup(latest_manifest->files,
+ manifest_path);
if (mfile == NULL)
{
char *bmpath;
@@ -966,9 +988,10 @@ process_directory_recursively(
* The directory is out of sync with the backup_manifest,
* so emit a warning.
*/
- bmpath = psprintf("%s/%s", input_directory, "backup_manifest");
- pg_log_warning("\"%s\" contains no entry for \"%s\"", bmpath,
- manifest_path);
+ bmpath = psprintf("%s/%s", input_directory,
+ "backup_manifest");
+ pg_log_warning("\"%s\" contains no entry for \"%s\"",
+ bmpath, manifest_path);
pfree(bmpath);
}
else if (mfile->checksum_type == checksum_type)
@@ -1001,7 +1024,8 @@ process_directory_recursively(
if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
{
checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
- checksum_length = pg_checksum_final(&checksum_ctx, checksum_payload);
+ checksum_length = pg_checksum_final(&checksum_ctx,
+ checksum_payload);
}
}
@@ -1027,8 +1051,10 @@ process_directory_recursively(
pg_fatal("could not stat file \"%s\": %m", ofullpath);
/* OK, now do the work. */
- add_file_to_manifest(mwriter, manifest_path, sb.st_size, sb.st_mtime,
- checksum_type, checksum_length, checksum_payload);
+ add_file_to_manifest(mwriter, manifest_path,
+ sb.st_size, sb.st_mtime,
+ checksum_type, checksum_length,
+ checksum_payload);
}
/* Avoid leaking memory. */
@@ -1136,8 +1162,7 @@ reset_directory_cleanup_list(void)
* final backup in the backup chain.
*/
static cb_tablespace *
-scan_for_existing_tablespaces(char *pathname,
- cb_options *opt)
+scan_for_existing_tablespaces(char *pathname, cb_options *opt)
{
char pg_tblspc[MAXPGPATH];
DIR *dir;
@@ -1170,8 +1195,7 @@ scan_for_existing_tablespaces(char *pathname,
/* Ignore any file name that doesn't look like a proper OID. */
if (!parse_oid(de->d_name, &oid))
{
- pg_log_debug(
- "skipping \"%s\" because the filename is not a legal tablespace OID",
+ pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
tblspcdir);
continue;
}
@@ -1182,8 +1206,7 @@ scan_for_existing_tablespaces(char *pathname,
exit(1);
if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
{
- pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor "
- "a directory",
+ pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
tblspcdir);
continue;
}
@@ -1203,7 +1226,8 @@ scan_for_existing_tablespaces(char *pathname,
/* Read the link target. */
link_length = readlink(tblspcdir, link_target, sizeof(link_target));
if (link_length < 0)
- pg_fatal("could not read symbolic link \"%s\": %m", tblspcdir);
+ pg_fatal("could not read symbolic link \"%s\": %m",
+ tblspcdir);
if (link_length >= sizeof(link_target))
pg_fatal("symbolic link \"%s\" is too long", tblspcdir);
link_target[link_length] = '\0';
@@ -1230,7 +1254,8 @@ scan_for_existing_tablespaces(char *pathname,
/* Every non-in-place tablespace must be mapped. */
if (tsmap == NULL)
- pg_fatal("tablespace at \"%s\" has no tablespace mapping", link_target);
+ pg_fatal("tablespace at \"%s\" has no tablespace mapping",
+ link_target);
}
else
{
diff --git a/src/bin/pg_combinebackup/reconstruct.c b/src/bin/pg_combinebackup/reconstruct.c
index 4daff9c77be..c37cceba030 100644
--- a/src/bin/pg_combinebackup/reconstruct.c
+++ b/src/bin/pg_combinebackup/reconstruct.c
@@ -46,16 +46,20 @@ typedef struct rfile
off_t highest_offset_read;
} rfile;
-static void debug_reconstruction(int n_source, rfile **sources, bool dry_run);
+static void debug_reconstruction(int n_source,
+ rfile **sources,
+ bool dry_run);
static unsigned find_reconstructed_block_length(rfile *s);
static rfile *make_incremental_rfile(char *filename);
static rfile *make_rfile(char *filename, bool missing_ok);
static void write_reconstructed_file(char *input_filename,
char *output_filename,
- unsigned block_length, rfile **sourcemap,
+ unsigned block_length,
+ rfile **sourcemap,
off_t *offsetmap,
pg_checksum_context *checksum_ctx,
- bool debug, bool dry_run);
+ bool debug,
+ bool dry_run);
static void read_bytes(rfile *rf, void *buffer, unsigned length);
/*
@@ -74,12 +78,19 @@ static void read_bytes(rfile *rf, void *buffer, unsigned length);
* an array of pathnames where those backups can be found.
*/
void
-reconstruct_from_incremental_file(
- char *input_filename, char *output_filename, char *relative_path,
- char *bare_file_name, int n_prior_backups, char **prior_backup_dirs,
- manifest_data **manifests, char *manifest_path,
- pg_checksum_type checksum_type, int *checksum_length,
- uint8 **checksum_payload, bool debug, bool dry_run,
+reconstruct_from_incremental_file(char *input_filename,
+ char *output_filename,
+ char *relative_path,
+ char *bare_file_name,
+ int n_prior_backups,
+ char **prior_backup_dirs,
+ manifest_data **manifests,
+ char *manifest_path,
+ pg_checksum_type checksum_type,
+ int *checksum_length,
+ uint8 **checksum_payload,
+ bool debug,
+ bool dry_run,
CopyFileMethod copy_method)
{
rfile **source;
@@ -157,8 +168,8 @@ reconstruct_from_incremental_file(
* Look for the full file in the previous backup. If not found, then
* look for an incremental file instead.
*/
- snprintf(source_filename, MAXPGPATH, "%s/%s/%s", prior_backup_dirs[sidx],
- relative_path, bare_file_name);
+ snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
+ prior_backup_dirs[sidx], relative_path, bare_file_name);
if ((s = make_rfile(source_filename, true)) == NULL)
{
snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
@@ -221,7 +232,8 @@ reconstruct_from_incremental_file(
{
uint64 expected_length;
- expected_length = (uint64) latest_source->truncation_block_length;
+ expected_length =
+ (uint64) latest_source->truncation_block_length;
expected_length *= BLCKSZ;
if (expected_length == sb.st_size)
{
@@ -242,7 +254,8 @@ reconstruct_from_incremental_file(
{
BlockNumber b = s->relative_block_numbers[i];
- if (b < latest_source->truncation_block_length && sourcemap[b] == NULL)
+ if (b < latest_source->truncation_block_length &&
+ sourcemap[b] == NULL)
{
sourcemap[b] = s;
offsetmap[b] = s->header_length + (i * BLCKSZ);
@@ -271,16 +284,16 @@ reconstruct_from_incremental_file(
manifest_path);
if (mfile == NULL)
{
- char *path =
- psprintf("%s/backup_manifest", prior_backup_dirs[copy_source_index]);
+ char *path = psprintf("%s/backup_manifest",
+ prior_backup_dirs[copy_source_index]);
/*
* The directory is out of sync with the backup_manifest, so emit
* a warning.
*/
- /*- translator: the first %s is a backup manifest file, the second is a
- * file absent therein */
- pg_log_warning("\"%s\" contains no entry for \"%s\"", path,
+ /*- translator: the first %s is a backup manifest file, the second is a file absent therein */
+ pg_log_warning("\"%s\" contains no entry for \"%s\"",
+ path,
manifest_path);
pfree(path);
}
@@ -288,7 +301,8 @@ reconstruct_from_incremental_file(
{
*checksum_length = mfile->checksum_length;
*checksum_payload = pg_malloc(*checksum_length);
- memcpy(*checksum_payload, mfile->checksum_payload, *checksum_length);
+ memcpy(*checksum_payload, mfile->checksum_payload,
+ *checksum_length);
checksum_type = CHECKSUM_TYPE_NONE;
}
}
@@ -305,13 +319,13 @@ reconstruct_from_incremental_file(
* Otherwise, reconstruct.
*/
if (copy_source != NULL)
- copy_file(copy_source->filename, output_filename, &checksum_ctx, dry_run,
- copy_method);
+ copy_file(copy_source->filename, output_filename,
+ &checksum_ctx, dry_run, copy_method);
else
{
- write_reconstructed_file(input_filename, output_filename, block_length,
- sourcemap, offsetmap, &checksum_ctx, debug,
- dry_run);
+ write_reconstructed_file(input_filename, output_filename,
+ block_length, sourcemap, offsetmap,
+ &checksum_ctx, debug, dry_run);
debug_reconstruction(n_prior_backups + 1, source, dry_run);
}
@@ -319,7 +333,8 @@ reconstruct_from_incremental_file(
if (checksum_type != CHECKSUM_TYPE_NONE)
{
*checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
- *checksum_length = pg_checksum_final(&checksum_ctx, *checksum_payload);
+ *checksum_length = pg_checksum_final(&checksum_ctx,
+ *checksum_payload);
}
/*
@@ -364,11 +379,11 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run)
/* Debug logging. */
if (dry_run)
- pg_log_debug("would have read %u blocks from \"%s\"", s->num_blocks_read,
- s->filename);
+ pg_log_debug("would have read %u blocks from \"%s\"",
+ s->num_blocks_read, s->filename);
else
- pg_log_debug("read %u blocks from \"%s\"", s->num_blocks_read,
- s->filename);
+ pg_log_debug("read %u blocks from \"%s\"",
+ s->num_blocks_read, s->filename);
/*
* In dry-run mode, we don't actually try to read data from the file,
@@ -387,7 +402,8 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run)
pg_fatal("could not stat \"%s\": %m", s->filename);
if (sb.st_size < s->highest_offset_read)
pg_fatal("file \"%s\" is too short: expected %llu, found %llu",
- s->filename, (unsigned long long) s->highest_offset_read,
+ s->filename,
+ (unsigned long long) s->highest_offset_read,
(unsigned long long) sb.st_size);
}
}
@@ -440,8 +456,7 @@ make_incremental_rfile(char *filename)
read_bytes(rf, &rf->truncation_block_length,
sizeof(rf->truncation_block_length));
if (rf->truncation_block_length > RELSEG_SIZE)
- pg_fatal("file \"%s\" has truncation block length %u in excess of segment "
- "size %u",
+ pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
filename, rf->truncation_block_length, RELSEG_SIZE);
/* Read block numbers if there are any. */
@@ -508,10 +523,12 @@ read_bytes(rfile *rf, void *buffer, unsigned length)
static void
write_reconstructed_file(char *input_filename,
char *output_filename,
- unsigned block_length, rfile **sourcemap,
+ unsigned block_length,
+ rfile **sourcemap,
off_t *offsetmap,
pg_checksum_context *checksum_ctx,
- bool debug, bool dry_run)
+ bool debug,
+ bool dry_run)
{
int wfd = -1;
unsigned i;
@@ -554,8 +571,8 @@ write_reconstructed_file(char *input_filename,
if (current_block == start_of_range)
appendStringInfo(&debug_buf, " %u:zero", current_block);
else
- appendStringInfo(&debug_buf, " %u-%u:zero", start_of_range,
- current_block);
+ appendStringInfo(&debug_buf, " %u-%u:zero",
+ start_of_range, current_block);
}
else
{
@@ -587,7 +604,8 @@ write_reconstructed_file(char *input_filename,
/* Open the output file, except in dry_run mode. */
if (!dry_run &&
- (wfd = open(output_filename, O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
+ (wfd = open(output_filename,
+ O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
pg_file_create_mode)) < 0)
pg_fatal("could not open file \"%s\": %m", output_filename);
@@ -604,8 +622,8 @@ write_reconstructed_file(char *input_filename,
else
{
s->num_blocks_read++;
- s->highest_offset_read =
- Max(s->highest_offset_read, offsetmap[i] + BLCKSZ);
+ s->highest_offset_read = Max(s->highest_offset_read,
+ offsetmap[i] + BLCKSZ);
}
/* Skip the rest of this in dry-run mode. */
@@ -632,9 +650,9 @@ write_reconstructed_file(char *input_filename,
if (rb < 0)
pg_fatal("could not read file \"%s\": %m", s->filename);
else
- pg_fatal("could not read file \"%s\": read only %d of %d bytes at "
- "offset %llu",
- s->filename, rb, BLCKSZ, (unsigned long long) offsetmap[i]);
+ pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu",
+ s->filename, rb, BLCKSZ,
+ (unsigned long long) offsetmap[i]);
}
}
@@ -650,7 +668,8 @@ write_reconstructed_file(char *input_filename,
/* Update the checksum computation. */
if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
- pg_fatal("could not update checksum of file \"%s\"", output_filename);
+ pg_fatal("could not update checksum of file \"%s\"",
+ output_filename);
}
/* Debugging output. */
diff --git a/src/bin/pg_combinebackup/reconstruct.h b/src/bin/pg_combinebackup/reconstruct.h
index 1fa734011bd..8d19dbf7e50 100644
--- a/src/bin/pg_combinebackup/reconstruct.h
+++ b/src/bin/pg_combinebackup/reconstruct.h
@@ -18,12 +18,19 @@
#include "common/file_utils.h"
#include "load_manifest.h"
-extern void reconstruct_from_incremental_file(
- char *input_filename, char *output_filename, char *relative_path,
- char *bare_file_name, int n_prior_backups, char **prior_backup_dirs,
- manifest_data **manifests, char *manifest_path,
- pg_checksum_type checksum_type, int *checksum_length,
- uint8 **checksum_payload, bool debug, bool dry_run,
+extern void reconstruct_from_incremental_file(char *input_filename,
+ char *output_filename,
+ char *relative_path,
+ char *bare_file_name,
+ int n_prior_backups,
+ char **prior_backup_dirs,
+ manifest_data **manifests,
+ char *manifest_path,
+ pg_checksum_type checksum_type,
+ int *checksum_length,
+ uint8 **checksum_payload,
+ bool debug,
+ bool dry_run,
CopyFileMethod copy_method);
#endif
--
2.44.0
From b1183fbae8ed0123d7385a8501f1a843f0d9aa85 Mon Sep 17 00:00:00 2001
From: Tomas Vondra <to...@2ndquadrant.com>
Date: Tue, 19 Mar 2024 15:16:29 +0100
Subject: [PATCH v20240319 1/2] rebased patch
---
src/bin/pg_combinebackup/copy_file.c | 227 +++++++++++++-------
src/bin/pg_combinebackup/copy_file.h | 14 +-
src/bin/pg_combinebackup/pg_combinebackup.c | 220 ++++++++++---------
src/bin/pg_combinebackup/reconstruct.c | 106 ++++-----
src/bin/pg_combinebackup/reconstruct.h | 22 +-
5 files changed, 328 insertions(+), 261 deletions(-)
diff --git a/src/bin/pg_combinebackup/copy_file.c b/src/bin/pg_combinebackup/copy_file.c
index e6d2423278a..16e26b4f573 100644
--- a/src/bin/pg_combinebackup/copy_file.c
+++ b/src/bin/pg_combinebackup/copy_file.c
@@ -10,19 +10,21 @@
*/
#include "postgres_fe.h"
-#ifdef HAVE_COPYFILE_H
-#include <copyfile.h>
-#endif
#include <fcntl.h>
+#include <limits.h>
#include <sys/stat.h>
#include <unistd.h>
#include "common/file_perm.h"
+#include "common/file_utils.h"
#include "common/logging.h"
#include "copy_file.h"
-static void copy_file_blocks(const char *src, const char *dst,
- pg_checksum_context *checksum_ctx);
+static void pg_copyfile(const char *src, const char *dest, const char *addon_errmsg,
+ pg_checksum_context *ctx);
+
+static void pg_copyfile_offload(const char *src, const char *dest,
+ const char *addon_errmsg, CopyFileMethod flags);
#ifdef WIN32
static void copy_file_copyfile(const char *src, const char *dst);
@@ -35,7 +37,8 @@ static void copy_file_copyfile(const char *src, const char *dst);
*/
void
copy_file(const char *src, const char *dst,
- pg_checksum_context *checksum_ctx, bool dry_run)
+ pg_checksum_context *checksum_ctx, bool dry_run,
+ CopyFileMethod copy_strategy)
{
/*
* In dry-run mode, we don't actually copy anything, nor do we read any
@@ -49,6 +52,8 @@ copy_file(const char *src, const char *dst,
pg_fatal("could not open \"%s\": %m", src);
if (close(fd) < 0)
pg_fatal("could not close \"%s\": %m", src);
+
+ return;
}
/*
@@ -56,104 +61,180 @@ copy_file(const char *src, const char *dst,
* operating system primitives that we know about to copy the file; this
* may be quicker than a naive block copy.
*/
- if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
- {
- char *strategy_name = NULL;
- void (*strategy_implementation) (const char *, const char *) = NULL;
+ if (checksum_ctx->type == CHECKSUM_TYPE_NONE && copy_strategy != 0)
+ pg_copyfile_offload(src, dst, NULL, copy_strategy);
+ else
+ pg_copyfile(src, dst, NULL, checksum_ctx);
+}
-#ifdef WIN32
- strategy_name = "CopyFile";
- strategy_implementation = copy_file_copyfile;
-#endif
+/* Helper function to optionally prepend error string */
+static inline char *
+opt_errinfo(const char *addon_errmsg)
+{
+ char buf[128];
- if (strategy_name != NULL)
- {
- if (dry_run)
- pg_log_debug("would copy \"%s\" to \"%s\" using strategy %s",
- src, dst, strategy_name);
- else
- {
- pg_log_debug("copying \"%s\" to \"%s\" using strategy %s",
- src, dst, strategy_name);
- (*strategy_implementation) (src, dst);
- }
- return;
- }
- }
+ if (addon_errmsg == NULL)
+ return "";
- /*
- * Fall back to the simple approach of reading and writing all the blocks,
- * feeding them into the checksum context as we go.
- */
- if (dry_run)
- {
- if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
- pg_log_debug("would copy \"%s\" to \"%s\"",
- src, dst);
- else
- pg_log_debug("would copy \"%s\" to \"%s\" and checksum with %s",
- src, dst, pg_checksum_type_name(checksum_ctx->type));
- }
- else
- {
- if (checksum_ctx->type == CHECKSUM_TYPE_NONE)
- pg_log_debug("copying \"%s\" to \"%s\"",
- src, dst);
- else
- pg_log_debug("copying \"%s\" to \"%s\" and checksumming with %s",
- src, dst, pg_checksum_type_name(checksum_ctx->type));
- copy_file_blocks(src, dst, checksum_ctx);
- }
+ strcpy(buf, " ");
+ return strncat(buf, addon_errmsg, sizeof(buf) - 2);
}
/*
- * Copy a file block by block, and optionally compute a checksum as we go.
+ * Copies a relation file from src to dest. addon_errmsg is an optional
+ * addon error message (can be NULL or include schema/relName)
*/
static void
-copy_file_blocks(const char *src, const char *dst,
- pg_checksum_context *checksum_ctx)
+pg_copyfile(const char *src, const char *dest, const char *addon_errmsg,
+ pg_checksum_context *ctx)
{
+#ifndef WIN32
int src_fd;
int dest_fd;
uint8 *buffer;
+
+ /* copy in fairly large chunks for best efficiency */
const int buffer_size = 50 * BLCKSZ;
- ssize_t rb;
- unsigned offset = 0;
if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
- pg_fatal("could not open file \"%s\": %m", src);
+ pg_fatal("error while copying%s: could not open file \"%s\": %s",
+ opt_errinfo(addon_errmsg), src, strerror(errno));
- if ((dest_fd = open(dst, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY,
+ if ((dest_fd = open(dest, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
pg_file_create_mode)) < 0)
- pg_fatal("could not open file \"%s\": %m", dst);
+ pg_fatal("error while copying%s: could not create file \"%s\": %s",
+ opt_errinfo(addon_errmsg), dest, strerror(errno));
buffer = pg_malloc(buffer_size);
- while ((rb = read(src_fd, buffer, buffer_size)) > 0)
+ /* perform data copying i.e read src source, write to destination */
+ while (true)
{
- ssize_t wb;
+ ssize_t nbytes = read(src_fd, buffer, buffer_size);
- if ((wb = write(dest_fd, buffer, rb)) != rb)
+ if (nbytes < 0)
+ pg_fatal("error while copying%s: could not read file "
+ "\"%s\": %s",
+ opt_errinfo(addon_errmsg), src, strerror(errno));
+
+ if (nbytes == 0)
+ break;
+
+ errno = 0;
+ if (write(dest_fd, buffer, nbytes) != nbytes)
{
- if (wb < 0)
- pg_fatal("could not write file \"%s\": %m", dst);
- else
- pg_fatal("could not write file \"%s\": wrote only %d of %d bytes at offset %u",
- dst, (int) wb, (int) rb, offset);
+ /*
+ * if write didn't set errno, assume problem is no disk space
+ */
+ if (errno == 0)
+ errno = ENOSPC;
+ pg_fatal("error while copying%s: could not write file \"%s\": %s",
+ opt_errinfo(addon_errmsg), dest, strerror(errno));
}
- if (pg_checksum_update(checksum_ctx, buffer, rb) < 0)
- pg_fatal("could not update checksum of file \"%s\"", dst);
+ if (pg_checksum_update(ctx, buffer, nbytes) < 0)
+ pg_fatal("could not calculate checksum of file \"%s\"", dest);
+ }
+
+ pg_free(buffer);
+ close(src_fd);
+ close(dest_fd);
+
+#else /* WIN32 */
+ if (CopyFile(src, dest, true) == 0)
+ {
+ _dosmaperr(GetLastError());
+ pg_fatal("error while copying%s (\"%s\" to \"%s\"): %s", addon_errmsg,
+ opt_errinfo(addon_errmsg), src, dest, strerror(errno));
+ }
+#endif /* WIN32 */
+}
+
+/*
+ * pg_copyfile_offload()
+ *
+ * Clones/reflinks a relation file from src to dest using variety of methods
+ *
+ * addon_errmsg can be used to pass additional information in case of errors.
+ * flags, see PG_COPYFILE_* enum in file_utils.h
+ */
+static void
+pg_copyfile_offload(const char *src, const char *dest,
+ const char *addon_errmsg, CopyFileMethod flags)
+{
- offset += rb;
+#ifdef WIN32
+ /* on WIN32 we ignore flags, we have no other choice */
+ if (CopyFile(src, dest, true) == 0)
+ {
+ _dosmaperr(GetLastError());
+ pg_fatal("error while copying%s (\"%s\" to \"%s\"): %s", addon_errmsg,
+ opt_errinfo(addon_errmsg), src, dest, strerror(errno));
}
+#elif defined(HAVE_COPYFILE) && defined(COPYFILE_CLONE_FORCE)
+ /* on MacOS we ignore flags, we have no other choice */
+ if (copyfile(src, dest, NULL, COPYFILE_CLONE_FORCE) < 0)
+ pg_fatal("error while cloning%s: (\"%s\" to \"%s\"): %s",
+ opt_errinfo(addon_errmsg), src, dest, strerror(errno));
+
+#elif defined(HAVE_COPY_FILE_RANGE) || defined(FICLONE)
+ int src_fd;
+ int dest_fd;
+ ssize_t nbytes;
- if (rb < 0)
- pg_fatal("could not read file \"%s\": %m", dst);
+ if ((src_fd = open(src, O_RDONLY | PG_BINARY, 0)) < 0)
+ pg_fatal("error while copying%s: could not open file \"%s\": %s",
+ opt_errinfo(addon_errmsg), src, strerror(errno));
+
+ if ((dest_fd = open(dest, O_RDWR | O_CREAT | O_EXCL | PG_BINARY,
+ pg_file_create_mode)) < 0)
+ pg_fatal("error while copying%s: could not create file \"%s\": %s",
+ opt_errinfo(addon_errmsg), dest, strerror(errno));
+
+ if (flags & PG_COPYFILE_COPY_FILE_RANGE)
+ {
+#ifdef HAVE_COPY_FILE_RANGE
+ do
+ {
+ nbytes = copy_file_range(src_fd, NULL, dest_fd, NULL, SSIZE_MAX, 0);
+ if (nbytes < 0 && errno != EINTR)
+ pg_fatal("error while copying%s: could not copy_file_range()"
+ "from \"%s\" to \"%s\": %s",
+ opt_errinfo(addon_errmsg), src, dest, strerror(errno));
+ } while (nbytes > 0);
+#else
+ pg_fatal("copy file accelaration via copy_file_range() is not supported on "
+ "this platform");
+#endif
+ }
+ else if (flags & PG_COPYFILE_IOCTL_FICLONE)
+ {
+#if defined(__linux__) && defined(FICLONE)
+ if (ioctl(dest_fd, FICLONE, src_fd) < 0)
+ {
+ int save_errno = errno;
+
+ unlink(dest);
+
+ pg_fatal("error while cloning%s: (\"%s\" to \"%s\"): %s",
+ opt_errinfo(addon_errmsg), src, dest, strerror(save_errno));
+ }
+#else
+ pg_fatal("clone file accelaration via ioctl(FICLONE) is not supported on "
+ "this platform");
+#endif
+ }
- pg_free(buffer);
close(src_fd);
close(dest_fd);
+
+#else
+ if (flags & PG_COPYFILE_FALLBACK)
+ pg_copyfile(src, dest, addon_errmsg);
+ else
+ pg_fatal("none of the copy file acceleration methods are supported on this "
+ "platform");
+#endif
}
#ifdef WIN32
diff --git a/src/bin/pg_combinebackup/copy_file.h b/src/bin/pg_combinebackup/copy_file.h
index 0f6bc09403f..2797a340055 100644
--- a/src/bin/pg_combinebackup/copy_file.h
+++ b/src/bin/pg_combinebackup/copy_file.h
@@ -11,9 +11,21 @@
#ifndef COPY_FILE_H
#define COPY_FILE_H
+#include "c.h"
#include "common/checksum_helper.h"
+#include "common/file_utils.h"
+
+typedef enum CopyFileMethod
+{
+ PG_COPYFILE_FALLBACK = 0x1,
+ PG_COPYFILE_IOCTL_FICLONE = 0x2, /* Linux */
+ PG_COPYFILE_COPY_FILE_RANGE = 0x4, /* FreeBSD & Linux >= 4.5 */
+ PG_COPYFILE_COPYFILE_CLONE_FORCE = 0x8 /* MacOS */
+} CopyFileMethod;
+#define PG_COPYFILE_ANY_WITH_FALLBACK (2 << 4) - 1
extern void copy_file(const char *src, const char *dst,
- pg_checksum_context *checksum_ctx, bool dry_run);
+ pg_checksum_context *checksum_ctx, bool dry_run,
+ CopyFileMethod copy_strategy);
#endif /* COPY_FILE_H */
diff --git a/src/bin/pg_combinebackup/pg_combinebackup.c b/src/bin/pg_combinebackup/pg_combinebackup.c
index 74f8be9eeac..1455360d81c 100644
--- a/src/bin/pg_combinebackup/pg_combinebackup.c
+++ b/src/bin/pg_combinebackup/pg_combinebackup.c
@@ -69,6 +69,7 @@ typedef struct cb_options
pg_checksum_type manifest_checksums;
bool no_manifest;
DataDirSyncMethod sync_method;
+ CopyFileMethod copy_method;
} cb_options;
/*
@@ -98,15 +99,10 @@ static void cleanup_directories_atexit(void);
static void create_output_directory(char *dirname, cb_options *opt);
static void help(const char *progname);
static bool parse_oid(char *s, Oid *result);
-static void process_directory_recursively(Oid tsoid,
- char *input_directory,
- char *output_directory,
- char *relative_path,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- manifest_writer *mwriter,
- cb_options *opt);
+static void process_directory_recursively(
+ Oid tsoid, char *input_directory, char *output_directory,
+ char *relative_path, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, manifest_writer *mwriter, cb_options *opt);
static int read_pg_version_file(char *directory);
static void remember_to_cleanup_directory(char *target_path, bool rmtopdir);
static void reset_directory_cleanup_list(void);
@@ -129,8 +125,9 @@ main(int argc, char *argv[])
{"manifest-checksums", required_argument, NULL, 1},
{"no-manifest", no_argument, NULL, 2},
{"sync-method", required_argument, NULL, 3},
- {NULL, 0, NULL, 0}
- };
+ {"clone", no_argument, NULL, 4},
+ {"copy-file-range", no_argument, NULL, 5},
+ {NULL, 0, NULL, 0}};
const char *progname;
char *last_input_dir;
@@ -156,10 +153,11 @@ main(int argc, char *argv[])
memset(&opt, 0, sizeof(opt));
opt.manifest_checksums = CHECKSUM_TYPE_CRC32C;
opt.sync_method = DATA_DIR_SYNC_METHOD_FSYNC;
+ opt.copy_method = 0;
/* process command-line options */
- while ((c = getopt_long(argc, argv, "dnNPo:T:",
- long_options, &optindex)) != -1)
+ while ((c = getopt_long(argc, argv, "dnNPo:T:", long_options, &optindex)) !=
+ -1)
{
switch (c)
{
@@ -180,10 +178,8 @@ main(int argc, char *argv[])
add_tablespace_mapping(&opt, optarg);
break;
case 1:
- if (!pg_checksum_parse_type(optarg,
- &opt.manifest_checksums))
- pg_fatal("unrecognized checksum algorithm: \"%s\"",
- optarg);
+ if (!pg_checksum_parse_type(optarg, &opt.manifest_checksums))
+ pg_fatal("unrecognized checksum algorithm: \"%s\"", optarg);
break;
case 2:
opt.no_manifest = true;
@@ -192,6 +188,12 @@ main(int argc, char *argv[])
if (!parse_sync_method(optarg, &opt.sync_method))
exit(1);
break;
+ case 4:
+ opt.copy_method = PG_COPYFILE_IOCTL_FICLONE;
+ break;
+ case 5:
+ opt.copy_method = PG_COPYFILE_COPY_FILE_RANGE;
+ break;
default:
/* getopt_long already emitted a complaint */
pg_log_error_hint("Try \"%s --help\" for more information.", progname);
@@ -213,6 +215,14 @@ main(int argc, char *argv[])
if (opt.no_manifest)
opt.manifest_checksums = CHECKSUM_TYPE_NONE;
+ /*
+ * We cannot provide file copy/clone offload in case when we need to
+ * calculate checksums
+ */
+ if (opt.copy_method != 0 && opt.manifest_checksums != CHECKSUM_TYPE_NONE)
+ pg_fatal("unable to use accelerated copy when manifest checksums "
+ "are to be calculated. Use --no-manifest");
+
/* Read the server version from the final backup. */
version = read_pg_version_file(argv[argc - 1]);
@@ -285,7 +295,8 @@ main(int argc, char *argv[])
* won't have the WAL ranges for the resulting manifest.
*/
if (manifests[n_prior_backups] == NULL)
- pg_fatal("can't generate a manifest because no manifest is available for the final input backup");
+ pg_fatal("can't generate a manifest because no manifest is available for "
+ "the final input backup");
}
else
mwriter = NULL;
@@ -297,15 +308,15 @@ main(int argc, char *argv[])
{
pg_log_debug("generating \"%s/backup_label\"", opt.output);
last_backup_label->cursor = 0;
- write_backup_label(opt.output, last_backup_label,
- opt.manifest_checksums, mwriter);
+ write_backup_label(opt.output, last_backup_label, opt.manifest_checksums,
+ mwriter);
}
/* Process everything that's not part of a user-defined tablespace. */
pg_log_debug("processing backup directory \"%s\"", last_input_dir);
- process_directory_recursively(InvalidOid, last_input_dir, opt.output,
- NULL, n_prior_backups, prior_backup_dirs,
- manifests, mwriter, &opt);
+ process_directory_recursively(InvalidOid, last_input_dir, opt.output, NULL,
+ n_prior_backups, prior_backup_dirs, manifests,
+ mwriter, &opt);
/* Process user-defined tablespaces. */
for (ts = tablespaces; ts != NULL; ts = ts->next)
@@ -321,16 +332,15 @@ main(int argc, char *argv[])
{
char linkpath[MAXPGPATH];
- snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output,
- ts->oid);
+ snprintf(linkpath, MAXPGPATH, "%s/pg_tblspc/%u", opt.output, ts->oid);
if (opt.dry_run)
pg_log_debug("would create symbolic link from \"%s\" to \"%s\"",
linkpath, ts->new_dir);
else
{
- pg_log_debug("creating symbolic link from \"%s\" to \"%s\"",
- linkpath, ts->new_dir);
+ pg_log_debug("creating symbolic link from \"%s\" to \"%s\"", linkpath,
+ ts->new_dir);
if (symlink(ts->new_dir, linkpath) != 0)
pg_fatal("could not create symbolic link from \"%s\" to \"%s\": %m",
linkpath, ts->new_dir);
@@ -344,21 +354,19 @@ main(int argc, char *argv[])
{
pg_log_debug("creating directory \"%s\"", ts->new_dir);
if (pg_mkdir_p(ts->new_dir, pg_dir_create_mode) == -1)
- pg_fatal("could not create directory \"%s\": %m",
- ts->new_dir);
+ pg_fatal("could not create directory \"%s\": %m", ts->new_dir);
}
}
/* OK, now handle the directory contents. */
- process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir,
- NULL, n_prior_backups, prior_backup_dirs,
- manifests, mwriter, &opt);
+ process_directory_recursively(ts->oid, ts->old_dir, ts->new_dir, NULL,
+ n_prior_backups, prior_backup_dirs, manifests,
+ mwriter, &opt);
}
/* Finalize the backup_manifest, if we're generating one. */
if (mwriter != NULL)
- finalize_manifest(mwriter,
- manifests[n_prior_backups]->first_wal_range);
+ finalize_manifest(mwriter, manifests[n_prior_backups]->first_wal_range);
/* fsync that output directory unless we've been told not to do so */
if (!opt.no_sync)
@@ -414,7 +422,9 @@ add_tablespace_mapping(cb_options *opt, char *arg)
*dst_ptr++ = *arg_ptr;
}
if (!tsmap->old_dir[0] || !tsmap->new_dir[0])
- pg_fatal("invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"", arg);
+ pg_fatal(
+ "invalid tablespace mapping format \"%s\", must be \"OLDDIR=NEWDIR\"",
+ arg);
/*
* All tablespaces are created with absolute directories, so specifying a
@@ -486,8 +496,8 @@ check_backup_label_files(int n_backups, char **backup_dirs)
pg_fatal("could not close \"%s\": %m", pathbuf);
/* Parse the file contents. */
- parse_backup_label(pathbuf, buf, &start_tli, &start_lsn,
- &previous_tli, &previous_lsn);
+ parse_backup_label(pathbuf, buf, &start_tli, &start_lsn, &previous_tli,
+ &previous_lsn);
/*
* Sanity checks.
@@ -498,18 +508,19 @@ check_backup_label_files(int n_backups, char **backup_dirs)
* we don't have that information.
*/
if (i > 0 && previous_tli == 0)
- pg_fatal("backup at \"%s\" is a full backup, but only the first backup should be a full backup",
+ pg_fatal("backup at \"%s\" is a full backup, but only the first backup "
+ "should be a full backup",
backup_dirs[i]);
if (i == 0 && previous_tli != 0)
- pg_fatal("backup at \"%s\" is an incremental backup, but the first backup should be a full backup",
+ pg_fatal("backup at \"%s\" is an incremental backup, but the first "
+ "backup should be a full backup",
backup_dirs[i]);
if (i < n_backups - 1 && start_tli != check_tli)
pg_fatal("backup at \"%s\" starts on timeline %u, but expected %u",
backup_dirs[i], start_tli, check_tli);
if (i < n_backups - 1 && start_lsn != check_lsn)
pg_fatal("backup at \"%s\" starts at LSN %X/%X, but expected %X/%X",
- backup_dirs[i],
- LSN_FORMAT_ARGS(start_lsn),
+ backup_dirs[i], LSN_FORMAT_ARGS(start_lsn),
LSN_FORMAT_ARGS(check_lsn));
check_tli = previous_tli;
check_lsn = previous_lsn;
@@ -561,8 +572,7 @@ check_control_files(int n_backups, char **backup_dirs)
/* Can't interpret control file if not current version. */
if (control_file->pg_control_version != PG_CONTROL_VERSION)
- pg_fatal("%s: unexpected control file version",
- controlpath);
+ pg_fatal("%s: unexpected control file version", controlpath);
/* System identifiers should all match. */
if (i == n_backups - 1)
@@ -688,14 +698,23 @@ help(const char *progname)
printf(_("\nOptions:\n"));
printf(_(" -d, --debug generate lots of debugging output\n"));
printf(_(" -n, --dry-run don't actually do anything\n"));
- printf(_(" -N, --no-sync do not wait for changes to be written safely to disk\n"));
+ printf(_(" -N, --no-sync do not wait for changes to be written "
+ "safely to disk\n"));
printf(_(" -o, --output output directory\n"));
- printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
+ printf(_(
+ " -T, --tablespace-mapping=OLDDIR=NEWDIR\n"
" relocate tablespace in OLDDIR to NEWDIR\n"));
- printf(_(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
+ printf(
+ _(" --manifest-checksums=SHA{224,256,384,512}|CRC32C|NONE\n"
" use algorithm for manifest checksums\n"));
- printf(_(" --no-manifest suppress generation of backup manifest\n"));
- printf(_(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(
+ " --no-manifest suppress generation of backup manifest\n"));
+ printf(
+ _(" --sync-method=METHOD set method for syncing files to disk\n"));
+ printf(_(" --clone clone (reflink) instead of copying "
+ "files\n"));
+ printf(
+ _(" --copy-file-range copy using copy_file_range() syscall\n"));
printf(_(" -?, --help show this help, then exit\n"));
printf(_("\nReport bugs to <%s>.\n"), PACKAGE_BUGREPORT);
@@ -742,15 +761,10 @@ parse_oid(char *s, Oid *result)
* the locations of those previous backups.
*/
static void
-process_directory_recursively(Oid tsoid,
- char *input_directory,
- char *output_directory,
- char *relative_path,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- manifest_writer *mwriter,
- cb_options *opt)
+process_directory_recursively(
+ Oid tsoid, char *input_directory, char *output_directory,
+ char *relative_path, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, manifest_writer *mwriter, cb_options *opt)
{
char ifulldir[MAXPGPATH];
char ofulldir[MAXPGPATH];
@@ -803,13 +817,11 @@ process_directory_recursively(Oid tsoid,
}
else
{
- snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory,
- relative_path);
- snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory,
- relative_path);
+ snprintf(ifulldir, MAXPGPATH, "%s/%s", input_directory, relative_path);
+ snprintf(ofulldir, MAXPGPATH, "%s/%s", output_directory, relative_path);
if (OidIsValid(tsoid))
- snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/",
- tsoid, relative_path);
+ snprintf(manifest_prefix, MAXPGPATH, "pg_tblspc/%u/%s/", tsoid,
+ relative_path);
else
snprintf(manifest_prefix, MAXPGPATH, "%s/", relative_path);
}
@@ -845,8 +857,7 @@ process_directory_recursively(Oid tsoid,
pg_checksum_context checksum_ctx;
/* Ignore "." and ".." entries. */
- if (strcmp(de->d_name, ".") == 0 ||
- strcmp(de->d_name, "..") == 0)
+ if (strcmp(de->d_name, ".") == 0 || strcmp(de->d_name, "..") == 0)
continue;
/* Construct input path. */
@@ -882,11 +893,9 @@ process_directory_recursively(Oid tsoid,
de->d_name);
/* And recurse. */
- process_directory_recursively(tsoid,
- input_directory, output_directory,
- new_relative_path,
- n_prior_backups, prior_backup_dirs,
- manifests, mwriter, opt);
+ process_directory_recursively(tsoid, input_directory, output_directory,
+ new_relative_path, n_prior_backups,
+ prior_backup_dirs, manifests, mwriter, opt);
continue;
}
@@ -904,46 +913,37 @@ process_directory_recursively(Oid tsoid,
* Skip the backup_label and backup_manifest files; they require
* special handling and are handled elsewhere.
*/
- if (relative_path == NULL &&
- (strcmp(de->d_name, "backup_label") == 0 ||
- strcmp(de->d_name, "backup_manifest") == 0))
+ if (relative_path == NULL && (strcmp(de->d_name, "backup_label") == 0 ||
+ strcmp(de->d_name, "backup_manifest") == 0))
continue;
/*
* If it's an incremental file, hand it off to the reconstruction
* code, which will figure out what to do.
*/
- if (strncmp(de->d_name, INCREMENTAL_PREFIX,
- INCREMENTAL_PREFIX_LENGTH) == 0)
+ if (strncmp(de->d_name, INCREMENTAL_PREFIX, INCREMENTAL_PREFIX_LENGTH) ==
+ 0)
{
/* Output path should not include "INCREMENTAL." prefix. */
snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir,
de->d_name + INCREMENTAL_PREFIX_LENGTH);
-
/* Manifest path likewise omits incremental prefix. */
snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
de->d_name + INCREMENTAL_PREFIX_LENGTH);
/* Reconstruction logic will do the rest. */
- reconstruct_from_incremental_file(ifullpath, ofullpath,
- relative_path,
- de->d_name + INCREMENTAL_PREFIX_LENGTH,
- n_prior_backups,
- prior_backup_dirs,
- manifests,
- manifest_path,
- checksum_type,
- &checksum_length,
- &checksum_payload,
- opt->debug,
- opt->dry_run);
+ reconstruct_from_incremental_file(
+ ifullpath, ofullpath, relative_path,
+ de->d_name + INCREMENTAL_PREFIX_LENGTH, n_prior_backups,
+ prior_backup_dirs, manifests, manifest_path, checksum_type,
+ &checksum_length, &checksum_payload, opt->debug, opt->dry_run,
+ opt->copy_method);
}
else
{
/* Construct the path that the backup_manifest will use. */
- snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix,
- de->d_name);
+ snprintf(manifest_path, MAXPGPATH, "%s%s", manifest_prefix, de->d_name);
/*
* It's not an incremental file, so we need to copy the entire
@@ -953,13 +953,11 @@ process_directory_recursively(Oid tsoid,
* backup_manifest for the final input directory, we can save some
* work by reusing that checksum instead of computing a new one.
*/
- if (checksum_type != CHECKSUM_TYPE_NONE &&
- latest_manifest != NULL)
+ if (checksum_type != CHECKSUM_TYPE_NONE && latest_manifest != NULL)
{
manifest_file *mfile;
- mfile = manifest_files_lookup(latest_manifest->files,
- manifest_path);
+ mfile = manifest_files_lookup(latest_manifest->files, manifest_path);
if (mfile == NULL)
{
char *bmpath;
@@ -968,10 +966,9 @@ process_directory_recursively(Oid tsoid,
* The directory is out of sync with the backup_manifest,
* so emit a warning.
*/
- bmpath = psprintf("%s/%s", input_directory,
- "backup_manifest");
- pg_log_warning("\"%s\" contains no entry for \"%s\"",
- bmpath, manifest_path);
+ bmpath = psprintf("%s/%s", input_directory, "backup_manifest");
+ pg_log_warning("\"%s\" contains no entry for \"%s\"", bmpath,
+ manifest_path);
pfree(bmpath);
}
else if (mfile->checksum_type == checksum_type)
@@ -993,7 +990,8 @@ process_directory_recursively(Oid tsoid,
/* Actually copy the file. */
snprintf(ofullpath, MAXPGPATH, "%s/%s", ofulldir, de->d_name);
- copy_file(ifullpath, ofullpath, &checksum_ctx, opt->dry_run);
+ copy_file(ifullpath, ofullpath, &checksum_ctx, opt->dry_run,
+ opt->copy_method);
/*
* If copy_file() performed a checksum calculation for us, then
@@ -1003,8 +1001,7 @@ process_directory_recursively(Oid tsoid,
if (checksum_ctx.type != CHECKSUM_TYPE_NONE && !opt->dry_run)
{
checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
- checksum_length = pg_checksum_final(&checksum_ctx,
- checksum_payload);
+ checksum_length = pg_checksum_final(&checksum_ctx, checksum_payload);
}
}
@@ -1030,10 +1027,8 @@ process_directory_recursively(Oid tsoid,
pg_fatal("could not stat file \"%s\": %m", ofullpath);
/* OK, now do the work. */
- add_file_to_manifest(mwriter, manifest_path,
- sb.st_size, sb.st_mtime,
- checksum_type, checksum_length,
- checksum_payload);
+ add_file_to_manifest(mwriter, manifest_path, sb.st_size, sb.st_mtime,
+ checksum_type, checksum_length, checksum_payload);
}
/* Avoid leaking memory. */
@@ -1141,7 +1136,8 @@ reset_directory_cleanup_list(void)
* final backup in the backup chain.
*/
static cb_tablespace *
-scan_for_existing_tablespaces(char *pathname, cb_options *opt)
+scan_for_existing_tablespaces(char *pathname,
+ cb_options *opt)
{
char pg_tblspc[MAXPGPATH];
DIR *dir;
@@ -1174,7 +1170,8 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
/* Ignore any file name that doesn't look like a proper OID. */
if (!parse_oid(de->d_name, &oid))
{
- pg_log_debug("skipping \"%s\" because the filename is not a legal tablespace OID",
+ pg_log_debug(
+ "skipping \"%s\" because the filename is not a legal tablespace OID",
tblspcdir);
continue;
}
@@ -1185,7 +1182,8 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
exit(1);
if (type != PGFILETYPE_LNK && type != PGFILETYPE_DIR)
{
- pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor a directory",
+ pg_log_debug("skipping \"%s\" because it is neither a symbolic link nor "
+ "a directory",
tblspcdir);
continue;
}
@@ -1205,8 +1203,7 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
/* Read the link target. */
link_length = readlink(tblspcdir, link_target, sizeof(link_target));
if (link_length < 0)
- pg_fatal("could not read symbolic link \"%s\": %m",
- tblspcdir);
+ pg_fatal("could not read symbolic link \"%s\": %m", tblspcdir);
if (link_length >= sizeof(link_target))
pg_fatal("symbolic link \"%s\" is too long", tblspcdir);
link_target[link_length] = '\0';
@@ -1233,8 +1230,7 @@ scan_for_existing_tablespaces(char *pathname, cb_options *opt)
/* Every non-in-place tablespace must be mapped. */
if (tsmap == NULL)
- pg_fatal("tablespace at \"%s\" has no tablespace mapping",
- link_target);
+ pg_fatal("tablespace at \"%s\" has no tablespace mapping", link_target);
}
else
{
diff --git a/src/bin/pg_combinebackup/reconstruct.c b/src/bin/pg_combinebackup/reconstruct.c
index 41f06bb26b5..4daff9c77be 100644
--- a/src/bin/pg_combinebackup/reconstruct.c
+++ b/src/bin/pg_combinebackup/reconstruct.c
@@ -46,20 +46,16 @@ typedef struct rfile
off_t highest_offset_read;
} rfile;
-static void debug_reconstruction(int n_source,
- rfile **sources,
- bool dry_run);
+static void debug_reconstruction(int n_source, rfile **sources, bool dry_run);
static unsigned find_reconstructed_block_length(rfile *s);
static rfile *make_incremental_rfile(char *filename);
static rfile *make_rfile(char *filename, bool missing_ok);
static void write_reconstructed_file(char *input_filename,
char *output_filename,
- unsigned block_length,
- rfile **sourcemap,
+ unsigned block_length, rfile **sourcemap,
off_t *offsetmap,
pg_checksum_context *checksum_ctx,
- bool debug,
- bool dry_run);
+ bool debug, bool dry_run);
static void read_bytes(rfile *rf, void *buffer, unsigned length);
/*
@@ -78,19 +74,13 @@ static void read_bytes(rfile *rf, void *buffer, unsigned length);
* an array of pathnames where those backups can be found.
*/
void
-reconstruct_from_incremental_file(char *input_filename,
- char *output_filename,
- char *relative_path,
- char *bare_file_name,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- char *manifest_path,
- pg_checksum_type checksum_type,
- int *checksum_length,
- uint8 **checksum_payload,
- bool debug,
- bool dry_run)
+reconstruct_from_incremental_file(
+ char *input_filename, char *output_filename, char *relative_path,
+ char *bare_file_name, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, char *manifest_path,
+ pg_checksum_type checksum_type, int *checksum_length,
+ uint8 **checksum_payload, bool debug, bool dry_run,
+ CopyFileMethod copy_method)
{
rfile **source;
rfile *latest_source = NULL;
@@ -167,8 +157,8 @@ reconstruct_from_incremental_file(char *input_filename,
* Look for the full file in the previous backup. If not found, then
* look for an incremental file instead.
*/
- snprintf(source_filename, MAXPGPATH, "%s/%s/%s",
- prior_backup_dirs[sidx], relative_path, bare_file_name);
+ snprintf(source_filename, MAXPGPATH, "%s/%s/%s", prior_backup_dirs[sidx],
+ relative_path, bare_file_name);
if ((s = make_rfile(source_filename, true)) == NULL)
{
snprintf(source_filename, MAXPGPATH, "%s/%s/INCREMENTAL.%s",
@@ -231,8 +221,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
uint64 expected_length;
- expected_length =
- (uint64) latest_source->truncation_block_length;
+ expected_length = (uint64) latest_source->truncation_block_length;
expected_length *= BLCKSZ;
if (expected_length == sb.st_size)
{
@@ -253,8 +242,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
BlockNumber b = s->relative_block_numbers[i];
- if (b < latest_source->truncation_block_length &&
- sourcemap[b] == NULL)
+ if (b < latest_source->truncation_block_length && sourcemap[b] == NULL)
{
sourcemap[b] = s;
offsetmap[b] = s->header_length + (i * BLCKSZ);
@@ -283,16 +271,16 @@ reconstruct_from_incremental_file(char *input_filename,
manifest_path);
if (mfile == NULL)
{
- char *path = psprintf("%s/backup_manifest",
- prior_backup_dirs[copy_source_index]);
+ char *path =
+ psprintf("%s/backup_manifest", prior_backup_dirs[copy_source_index]);
/*
* The directory is out of sync with the backup_manifest, so emit
* a warning.
*/
- /*- translator: the first %s is a backup manifest file, the second is a file absent therein */
- pg_log_warning("\"%s\" contains no entry for \"%s\"",
- path,
+ /*- translator: the first %s is a backup manifest file, the second is a
+ * file absent therein */
+ pg_log_warning("\"%s\" contains no entry for \"%s\"", path,
manifest_path);
pfree(path);
}
@@ -300,8 +288,7 @@ reconstruct_from_incremental_file(char *input_filename,
{
*checksum_length = mfile->checksum_length;
*checksum_payload = pg_malloc(*checksum_length);
- memcpy(*checksum_payload, mfile->checksum_payload,
- *checksum_length);
+ memcpy(*checksum_payload, mfile->checksum_payload, *checksum_length);
checksum_type = CHECKSUM_TYPE_NONE;
}
}
@@ -318,13 +305,13 @@ reconstruct_from_incremental_file(char *input_filename,
* Otherwise, reconstruct.
*/
if (copy_source != NULL)
- copy_file(copy_source->filename, output_filename,
- &checksum_ctx, dry_run);
+ copy_file(copy_source->filename, output_filename, &checksum_ctx, dry_run,
+ copy_method);
else
{
- write_reconstructed_file(input_filename, output_filename,
- block_length, sourcemap, offsetmap,
- &checksum_ctx, debug, dry_run);
+ write_reconstructed_file(input_filename, output_filename, block_length,
+ sourcemap, offsetmap, &checksum_ctx, debug,
+ dry_run);
debug_reconstruction(n_prior_backups + 1, source, dry_run);
}
@@ -332,8 +319,7 @@ reconstruct_from_incremental_file(char *input_filename,
if (checksum_type != CHECKSUM_TYPE_NONE)
{
*checksum_payload = pg_malloc(PG_CHECKSUM_MAX_LENGTH);
- *checksum_length = pg_checksum_final(&checksum_ctx,
- *checksum_payload);
+ *checksum_length = pg_checksum_final(&checksum_ctx, *checksum_payload);
}
/*
@@ -378,11 +364,11 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run)
/* Debug logging. */
if (dry_run)
- pg_log_debug("would have read %u blocks from \"%s\"",
- s->num_blocks_read, s->filename);
+ pg_log_debug("would have read %u blocks from \"%s\"", s->num_blocks_read,
+ s->filename);
else
- pg_log_debug("read %u blocks from \"%s\"",
- s->num_blocks_read, s->filename);
+ pg_log_debug("read %u blocks from \"%s\"", s->num_blocks_read,
+ s->filename);
/*
* In dry-run mode, we don't actually try to read data from the file,
@@ -401,8 +387,7 @@ debug_reconstruction(int n_source, rfile **sources, bool dry_run)
pg_fatal("could not stat \"%s\": %m", s->filename);
if (sb.st_size < s->highest_offset_read)
pg_fatal("file \"%s\" is too short: expected %llu, found %llu",
- s->filename,
- (unsigned long long) s->highest_offset_read,
+ s->filename, (unsigned long long) s->highest_offset_read,
(unsigned long long) sb.st_size);
}
}
@@ -455,7 +440,8 @@ make_incremental_rfile(char *filename)
read_bytes(rf, &rf->truncation_block_length,
sizeof(rf->truncation_block_length));
if (rf->truncation_block_length > RELSEG_SIZE)
- pg_fatal("file \"%s\" has truncation block length %u in excess of segment size %u",
+ pg_fatal("file \"%s\" has truncation block length %u in excess of segment "
+ "size %u",
filename, rf->truncation_block_length, RELSEG_SIZE);
/* Read block numbers if there are any. */
@@ -522,12 +508,10 @@ read_bytes(rfile *rf, void *buffer, unsigned length)
static void
write_reconstructed_file(char *input_filename,
char *output_filename,
- unsigned block_length,
- rfile **sourcemap,
+ unsigned block_length, rfile **sourcemap,
off_t *offsetmap,
pg_checksum_context *checksum_ctx,
- bool debug,
- bool dry_run)
+ bool debug, bool dry_run)
{
int wfd = -1;
unsigned i;
@@ -570,8 +554,8 @@ write_reconstructed_file(char *input_filename,
if (current_block == start_of_range)
appendStringInfo(&debug_buf, " %u:zero", current_block);
else
- appendStringInfo(&debug_buf, " %u-%u:zero",
- start_of_range, current_block);
+ appendStringInfo(&debug_buf, " %u-%u:zero", start_of_range,
+ current_block);
}
else
{
@@ -603,8 +587,7 @@ write_reconstructed_file(char *input_filename,
/* Open the output file, except in dry_run mode. */
if (!dry_run &&
- (wfd = open(output_filename,
- O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
+ (wfd = open(output_filename, O_RDWR | PG_BINARY | O_CREAT | O_EXCL,
pg_file_create_mode)) < 0)
pg_fatal("could not open file \"%s\": %m", output_filename);
@@ -621,8 +604,8 @@ write_reconstructed_file(char *input_filename,
else
{
s->num_blocks_read++;
- s->highest_offset_read = Max(s->highest_offset_read,
- offsetmap[i] + BLCKSZ);
+ s->highest_offset_read =
+ Max(s->highest_offset_read, offsetmap[i] + BLCKSZ);
}
/* Skip the rest of this in dry-run mode. */
@@ -649,9 +632,9 @@ write_reconstructed_file(char *input_filename,
if (rb < 0)
pg_fatal("could not read file \"%s\": %m", s->filename);
else
- pg_fatal("could not read file \"%s\": read only %d of %d bytes at offset %llu",
- s->filename, rb, BLCKSZ,
- (unsigned long long) offsetmap[i]);
+ pg_fatal("could not read file \"%s\": read only %d of %d bytes at "
+ "offset %llu",
+ s->filename, rb, BLCKSZ, (unsigned long long) offsetmap[i]);
}
}
@@ -667,8 +650,7 @@ write_reconstructed_file(char *input_filename,
/* Update the checksum computation. */
if (pg_checksum_update(checksum_ctx, buffer, BLCKSZ) < 0)
- pg_fatal("could not update checksum of file \"%s\"",
- output_filename);
+ pg_fatal("could not update checksum of file \"%s\"", output_filename);
}
/* Debugging output. */
diff --git a/src/bin/pg_combinebackup/reconstruct.h b/src/bin/pg_combinebackup/reconstruct.h
index 8e33a8a95a0..1fa734011bd 100644
--- a/src/bin/pg_combinebackup/reconstruct.h
+++ b/src/bin/pg_combinebackup/reconstruct.h
@@ -13,21 +13,17 @@
#ifndef RECONSTRUCT_H
#define RECONSTRUCT_H
+#include "c.h"
#include "common/checksum_helper.h"
+#include "common/file_utils.h"
#include "load_manifest.h"
-extern void reconstruct_from_incremental_file(char *input_filename,
- char *output_filename,
- char *relative_path,
- char *bare_file_name,
- int n_prior_backups,
- char **prior_backup_dirs,
- manifest_data **manifests,
- char *manifest_path,
- pg_checksum_type checksum_type,
- int *checksum_length,
- uint8 **checksum_payload,
- bool debug,
- bool dry_run);
+extern void reconstruct_from_incremental_file(
+ char *input_filename, char *output_filename, char *relative_path,
+ char *bare_file_name, int n_prior_backups, char **prior_backup_dirs,
+ manifest_data **manifests, char *manifest_path,
+ pg_checksum_type checksum_type, int *checksum_length,
+ uint8 **checksum_payload, bool debug, bool dry_run,
+ CopyFileMethod copy_method);
#endif
--
2.44.0
