Add --align=SIZE for create, append, and update operations. For regular files at least SIZE bytes long, pad the POSIX extended header so the file data begins at a SIZE-byte boundary in the uncompressed archive stream. This makes the resulting layout suitable for efficient extent sharing during extraction.
Require SIZE to be a nonzero power of two, a multiple of the 512-byte tar block size, and no larger than 1 GiB. The option implies POSIX format, rejects multi-volume archives, and prevents append or update from adding POSIX alignment records to an existing non-POSIX archive. Store padding in an ignorable, protected GNU.pad PAX record. Account for existing extended-header records and avoid emitting an unnecessary header when the file data is already naturally aligned. Alignment is applied before compression, so compressed archives retain the layout after decompression. Document the option and add tests for size validation, ordinary and compressed archives, natural and long-name extended headers, command and format restrictions, and append behavior. Signed-off-by: Daan De Meyer <[email protected]> --- NEWS | 8 ++++ doc/tar.texi | 23 ++++++++++ src/common.h | 5 ++ src/create.c | 20 +++++++- src/tar.c | 36 ++++++++++++++- src/update.c | 7 +++ src/xheader.c | 69 ++++++++++++++++++++++++++++ tests/Makefile.am | 1 + tests/align.at | 111 +++++++++++++++++++++++++++++++++++++++++++++ tests/testsuite.at | 1 + 10 files changed, 278 insertions(+), 3 deletions(-) create mode 100644 tests/align.at diff --git a/NEWS b/NEWS index 8e18040d..2dbdc243 100644 --- a/NEWS +++ b/NEWS @@ -5,6 +5,14 @@ version 1.35.90 (git) * New manual section "Reproducibility", for reproducible tarballs. +* New option: --align=SIZE + +When creating a POSIX archive, pad extended headers so that the data of +each regular file at least SIZE bytes long begins at a SIZE-byte boundary +in the uncompressed archive stream. SIZE must be a power of two and a +multiple of 512, and cannot exceed 1 GiB. This can make extraction via +reflinks or copy_file_range(2) more efficient. + * New options: --set-mtime-command and --set-mtime-format Both options are valid when archiving files. diff --git a/doc/tar.texi b/doc/tar.texi index f5b09cab..91632552 100644 --- a/doc/tar.texi +++ b/doc/tar.texi @@ -2470,6 +2470,14 @@ Enable POSIX ACLs support. @xref{Extended File Attributes, acls}. (See @option{--newer}, @pxref{after}) +@opsummary{align} +@item --align=@var{size}[@var{suf}] + +When creating a POSIX archive, align the data of sufficiently large regular +files to @var{size}-byte boundaries in the uncompressed archive stream. +The argument can use a @ref{size-suffixes, size suffix}. It must be a power +of two, a multiple of 512, and no larger than 1 GiB. @xref{posix}. + @opsummary{anchored} @item --anchored A pattern must match an initial subsequence of the name's components. @@ -10655,6 +10663,21 @@ was given @option{--format=posix} (@option{--format=pax}) option. No special option is required to read and extract from a @acronym{POSIX} archive. +@opindex align +The @option{--align=@var{size}} option pads extended headers as needed so that +the data of every regular file whose size is at least @var{size} begins at a +multiple of @var{size} bytes in the uncompressed archive stream. It can be +used only with @option{--create}, @option{--append}, or @option{--update}, and +cannot be combined with multi-volume archives. It implies +@option{--format=posix}. The size must be a power of two, a multiple of 512, +and no larger than 1 GiB; it can have a @ref{size-suffixes, size suffix}. + +Alignment is applied before compression, so decompressing a compressed +archive preserves the aligned layout. This can let extraction tools share +file contents from an uncompressed archive using reflinks or +@code{copy_file_range}, at the cost of additional padding. The padding is +stored in an ignorable @code{GNU.pad} extended-header record. + @menu * PAX keywords:: Controlling Extended Header Keywords. @end menu diff --git a/src/common.h b/src/common.h index e835d888..f4d31be8 100644 --- a/src/common.h +++ b/src/common.h @@ -102,6 +102,10 @@ extern enum archive_format archive_format; extern idx_t blocking_factor; extern idx_t record_size; +/* If nonzero, align sufficiently large regular file data to this many + bytes in POSIX archives. */ +extern idx_t alignment_option; + extern bool absolute_names_option; /* Display file times in UTC */ @@ -936,6 +940,7 @@ void xheader_write (char type, char *name, time_t t, struct xheader *xhdr); void xheader_write_global (struct xheader *xhdr); void xheader_forbid_global (void); void xheader_finish (struct xheader *hdr); +void xheader_finish_aligned (struct xheader *hdr, uintmax_t off, idx_t align); void xheader_destroy (struct xheader *hdr); char *xheader_xhdr_name (struct tar_stat_info *st); char *xheader_ghdr_name (void); diff --git a/src/create.c b/src/create.c index bf9f9838..0ea3a1cc 100644 --- a/src/create.c +++ b/src/create.c @@ -666,10 +666,26 @@ write_extended (bool global, struct tar_stat_info *st, union block *old_header) int type; time_t t; - if (st->xhdr.buffer || st->xhdr.stk == NULL) + bool align = (!global && alignment_option + && S_ISREG (st->stat.st_mode) + && st->stat.st_size >= alignment_option); + + if (st->xhdr.buffer || (st->xhdr.stk == NULL && !align)) return old_header; - xheader_finish (&st->xhdr); + if (align) + { + xheader_finish_aligned (&st->xhdr, + (uintmax_t) current_block_ordinal () * BLOCKSIZE, + alignment_option); + if (st->xhdr.size == 0) + { + xheader_destroy (&st->xhdr); + return old_header; + } + } + else + xheader_finish (&st->xhdr); memcpy (hp.buffer, old_header, sizeof (hp)); if (global) { diff --git a/src/tar.c b/src/tar.c index a920d9c2..bc1b0fbf 100644 --- a/src/tar.c +++ b/src/tar.c @@ -36,6 +36,7 @@ enum subcommand subcommand_option; enum archive_format archive_format; idx_t blocking_factor; idx_t record_size; +idx_t alignment_option; bool absolute_names_option; bool utc_option; bool full_time_option; @@ -353,7 +354,8 @@ tar_set_quoting_style (char *arg) enum { - ACLS_OPTION = CHAR_MAX + 1, + ALIGN_OPTION = CHAR_MAX + 1, + ACLS_OPTION, ATIME_PRESERVE_OPTION, BACKUP_OPTION, CHECK_DEVICE_OPTION, @@ -774,6 +776,10 @@ static struct argp_option options[] = { {"format", 'H', N_("FORMAT"), 0, N_("create archive of the given format"), GRID_FORMAT }, + {"align", ALIGN_OPTION, N_("SIZE"), 0, + N_("align data of regular files at least SIZE bytes long"), + GRID_FORMAT_OPT }, + {NULL, 0, NULL, 0, N_("FORMAT is one of the following:"), GRDOC_FORMAT }, {" v7", 0, NULL, OPTION_DOC|OPTION_NO_TRANS, N_("old V7 tar format"), GRDOC_FORMAT }, @@ -1429,6 +1435,21 @@ parse_opt (int key, char *arg, struct argp_state *state) switch (key) { + case ALIGN_OPTION: + { + uintmax_t u; + + if (! (xstrtoumax (arg, NULL, 10, &u, TAR_SIZE_SUFFIXES) == LONGINT_OK + && 0 < u && u <= (UINTMAX_C (1) << 30) + && u % BLOCKSIZE == 0 + && (u & (u - 1)) == 0)) + paxusage (_("--align value must be a power of two, a multiple of %d," + " and at most 1G"), BLOCKSIZE); + alignment_option = u; + set_archive_format ("posix"); + } + break; + case ARGP_KEY_INIT: if (state->root_argp->children) for (idx_t i = 0; state->root_argp->children[i].argp; i++) @@ -2619,6 +2640,19 @@ decode_options (int argc, char **argv) && !is_subcommand_class (SUBCL_READ)) paxusage (_("--pax-option can be used only on POSIX archives")); + if (alignment_option + && subcommand_option != CREATE_SUBCOMMAND + && subcommand_option != APPEND_SUBCOMMAND + && subcommand_option != UPDATE_SUBCOMMAND) + paxusage (_("--align can be used only with --create, --append, or" + " --update")); + + if (alignment_option && archive_format != POSIX_FORMAT) + paxusage (_("--align can be used only on POSIX archives")); + + if (alignment_option && multi_volume_option) + paxusage (_("--align cannot be used with multi-volume archives")); + /* star creates non-POSIX typed archives with xattr support, so allow the extra headers when reading */ if ((acls_option > 0) diff --git a/src/update.c b/src/update.c index 872e701f..2e26acec 100644 --- a/src/update.c +++ b/src/update.c @@ -206,6 +206,13 @@ update_archive (void) } reset_eof (); + + /* The format requested on the command line can be replaced above by the + format detected in an existing archive. Do not create a mixed-format + archive by adding POSIX alignment headers to a non-POSIX archive. */ + if (alignment_option && archive_format != POSIX_FORMAT) + paxusage (_("--align can be used only on POSIX archives")); + time_to_start_writing = true; output_start = charptr (current_block); diff --git a/src/xheader.c b/src/xheader.c index 05f905ed..76fc7b96 100644 --- a/src/xheader.c +++ b/src/xheader.c @@ -881,6 +881,59 @@ xheader_print (struct xheader *xhdr, char const *keyword, char const *value) xheader_print_n (xhdr, keyword, value, strlen (value)); } +/* Append a GNU.pad record that makes the extended header body exactly SIZE + bytes long. The record is ignored by readers; its only purpose is to move + the following member data to a requested boundary. */ +static void +xheader_print_padding (struct xheader *xhdr, idx_t size) +{ + static char const keyword[] = "GNU.pad"; + char nbuf[INTMAX_STRSIZE_BOUND]; + char const *np = imaxtostr (size - xhdr->size, nbuf); + idx_t n = nbuf + sizeof nbuf - 1 - np; + idx_t overhead = n + 1 + sizeof keyword - 1 + 1 + 1; + idx_t value_size = size - xhdr->size - overhead; + + x_obstack_grow (xhdr, np, n); + x_obstack_1grow (xhdr, ' '); + x_obstack_grow (xhdr, keyword, sizeof keyword - 1); + x_obstack_1grow (xhdr, '='); + x_obstack_blank (xhdr, value_size); + memset (obstack_next_free (xhdr->stk) - value_size, 'X', value_size); + x_obstack_1grow (xhdr, '\n'); +} + +/* Pad XHDR so that data following its extended-header header, body and the + member header starts at an ALIGN boundary. OFF is the byte offset of the + first of those headers. */ +static void +xheader_align (struct xheader *xhdr, uintmax_t off, idx_t align) +{ + uintmax_t mask = align - 1; + uintmax_t base = off + 2 * BLOCKSIZE; + uintmax_t natural; + + if (xhdr->size == 0) + natural = off + BLOCKSIZE; + else + { + idx_t rounded; + if (ckd_add (&rounded, xhdr->size, BLOCKSIZE - 1)) + xalloc_die (); + natural = base + (rounded & - (uintmax_t) BLOCKSIZE); + } + + if ((natural & mask) != 0) + { + idx_t min_size, size; + if (ckd_add (&min_size, xhdr->size, 64)) + xalloc_die (); + if (ckd_add (&size, min_size, (- base - min_size) & mask)) + xalloc_die (); + xheader_print_padding (xhdr, size); + } +} + void xheader_finish (struct xheader *xhdr) { @@ -892,6 +945,19 @@ xheader_finish (struct xheader *xhdr) xhdr->buffer = obstack_finish (xhdr->stk); } +void +xheader_finish_aligned (struct xheader *xhdr, uintmax_t off, idx_t align) +{ + struct keyword_list *kp; + + xheader_init (xhdr); + for (kp = keyword_override_list; kp; kp = kp->next) + code_string (kp->value, kp->pattern, xhdr); + + xheader_align (xhdr, off, align); + xhdr->buffer = obstack_finish (xhdr->stk); +} + void xheader_destroy (struct xheader *xhdr) { @@ -1706,6 +1772,9 @@ struct xhdr_tab const xhdr_tab[] = { { "GNU.dumpdir", dumpdir_coder, dumpdir_decoder, XHDR_PROTECTED, false }, + /* Ignorable padding used by --align. */ + { "GNU.pad", dummy_coder, dummy_decoder, XHDR_PROTECTED, false }, + /* Keeps the tape/volume label. May be present only in the global headers. Equivalent to GNUTYPE_VOLHDR. */ { "GNU.volume.label", volume_label_coder, volume_label_decoder, diff --git a/tests/Makefile.am b/tests/Makefile.am index 4f666b09..b06abbfa 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -64,6 +64,7 @@ TESTSUITE_AT = \ acls02.at\ acls03.at\ add-file.at\ + align.at\ append.at\ append01.at\ append02.at\ diff --git a/tests/align.at b/tests/align.at new file mode 100644 index 00000000..d3415b95 --- /dev/null +++ b/tests/align.at @@ -0,0 +1,111 @@ +# Test --align. -*- Autotest -*- + +# Copyright 2026 Free Software Foundation, Inc. + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3, or (at your option) +# any later version. + +AT_SETUP([align regular file data]) +AT_KEYWORDS([align pax posix]) + +AT_CHECK([ +genfile --file=big1 --length=5000 +genfile --file=big2 --length=4096 +genfile --file=small --length=100 + +# The first two payloads start at offsets 4096 and 12288 respectively. +tar --align=4K -cf archive.tar big1 big2 small || exit 1 +dd if=archive.tar bs=4096 skip=1 count=1 2>/dev/null > got1 +dd if=big1 bs=4096 count=1 2>/dev/null > exp1 +dd if=archive.tar bs=4096 skip=3 count=1 2>/dev/null > got2 +cmp got1 exp1 || exit 1 +cmp got2 big2 || exit 1 + +# Padding remains transparent, including to GNU tar's warning machinery. +mkdir out +tar -xf archive.tar -C out || exit 1 +cmp big1 out/big1 || exit 1 +cmp big2 out/big2 || exit 1 +cmp small out/small || exit 1 +], +[0], [], []) + +AT_CHECK([tar --align=0 -cf invalid.tar small], [2], [], [ignore]) +AT_CHECK([tar --align=1536 -cf invalid.tar small], [2], [], [ignore]) +AT_CHECK([tar --align=2G -cf invalid.tar small], [2], [], [ignore]) +AT_CHECK([tar --align=1G -cf maximum.tar small], [0], [], []) +AT_CHECK([tar --align=512 -cf minimum.tar small], [0], [], []) + +AT_CLEANUP + +AT_SETUP([align compressed archive data]) +AT_KEYWORDS([align pax posix gzip]) + +AT_CHECK([ +AT_GZIP_PREREQ +genfile --file=big1 --length=5000 +genfile --file=big2 --length=4096 + +# Alignment describes the stream before compression. +tar --align=4096 -czf archive.tar.gz big1 big2 || exit 1 +gzip -dc archive.tar.gz > plain.tar || exit 1 +dd if=plain.tar bs=4096 skip=1 count=1 2>/dev/null > got1 +dd if=big1 bs=4096 count=1 2>/dev/null > exp1 +dd if=plain.tar bs=4096 skip=3 count=1 2>/dev/null > got2 +cmp got1 exp1 || exit 1 +cmp got2 big2 +], +[0], [], []) + +AT_CLEANUP + +AT_SETUP([align without a natural extended header]) +AT_KEYWORDS([align pax posix]) + +AT_CHECK([ +for i in 1 2 3 4 5 6 7; do + genfile --file=empty$i --length=0 || exit 1 +done +genfile --file=big --length=4096 + +# Seven ordinary headers put BIG's member header at byte 3584. Its data is +# naturally aligned, so --align must not introduce an unnecessary xheader. +tar --align=4096 \ + --pax-option=delete=mtime,delete=atime,delete=ctime \ + -cf archive.tar empty1 empty2 empty3 empty4 empty5 empty6 empty7 big || exit 1 +dd if=archive.tar bs=4096 skip=1 count=1 2>/dev/null > got +cmp got big || exit 1 + +# A long name forces a natural path record before the alignment record. +long=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +genfile --file=$long --length=4096 || exit 1 +tar --align=4096 -cf long.tar $long || exit 1 +dd if=long.tar bs=4096 skip=1 count=1 2>/dev/null > got-long +cmp got-long $long +], +[0], [], []) + +AT_CLEANUP + +AT_SETUP([align command and format restrictions]) +AT_KEYWORDS([align pax posix options append update multivolume]) + +AT_CHECK([ +genfile --file=small --length=100 +genfile --file=big --length=4096 +tar -Hgnu -cf gnu.tar small || exit 1 +tar -Hposix -cf posix.tar small || exit 1 +], [0], [], []) + +AT_CHECK([tar --align=4096 -tf posix.tar], [2], [], [ignore]) +AT_CHECK([tar --align=4096 -Af posix.tar posix.tar], [2], [], [ignore]) +AT_CHECK([tar --align=4096 -M -L10 -cf multi.tar big], [2], [], [ignore]) +AT_CHECK([tar --align=4096 -rf gnu.tar big], [2], [], [ignore]) +AT_CHECK([tar --align=4096 -rf posix.tar big], [0], [], []) +AT_CHECK([tar -tf posix.tar], [0], [small +big +]) + +AT_CLEANUP diff --git a/tests/testsuite.at b/tests/testsuite.at index ab30c2ab..f046adb6 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -421,6 +421,7 @@ m4_include([link03.at]) m4_include([link04.at]) AT_BANNER([Specific archive formats]) +m4_include([align.at]) m4_include([longv7.at]) m4_include([long01.at]) m4_include([lustar01.at]) -- 2.54.0
