On Sun, Feb 28, 2016 at 9:31 PM, Jim Meyering <j...@meyering.net> wrote: > I want to make a release soon. > The preceding release (gzip-1.6) was 2.5 years ago. > > Here's a small admin change I've just pushed. > I suspect I'm the only one who noticed, because it > is triggered only via "make distcheck" when also > using automake built from automake.git's master branch.
Finally, here is a long-overdue upstream-ready --rsyncable patch for gzip. I've Cc'd Rusty Russel, who is listed as the author. I'll push this in the next day or so, so any improvements people can suggest in the mean time would be most welcome. Most of the changes came from some distro patches (not sure if it was debian or fedora, since I began this long ago). I added the test, a few comments and the commit log.
From 82ab83b1c48762ac769e2076e1fa23f33e105495 Mon Sep 17 00:00:00 2001 From: Rusty Russell <ru...@rustcorp.com.au> Date: Sat, 5 Sep 2015 12:08:28 -0700 Subject: [PATCH] gzip: support the --rsyncable option * deflate.c: Include verify.h. (RSYNC_WIN, RSYNC_SUM_MATCH): Define. (rsync_sum, rsync_chunk_end): Declare file-scoped globals. (lm_init): Initialize globals. (fill_window): Update rsync_chunk_end. (rsync_roll): New function. (RSYNC_ROLL): New macro. (FLUSH_BLOCK): Update for new "pad" parameter. (deflate_fast): Use RSYNC_ROLL and flush/pad. (deflate): Likewise. * trees.c (flush_block): Add "pad" parameter. * gzip.c (rsync): New global. (RSYNCABLE_OPTION, longopts, help): Add the option. (main): Set the new global. * gzip.h (rsync): Declare new global. (flush_block): Update prototype. * doc/gzip.texi: Document it. * gzip.1: Likewise. * bootstrap.conf: Use verify module. * NEWS (New feature): Mention it. * Makefile.am (check-local): Add tests and use AM_V__* command- hiding opions. https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=118118 --- Makefile.am | 21 ++++++++------ NEWS | 10 +++++++ bootstrap.conf | 1 + deflate.c | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++---- doc/gzip.texi | 8 ++++++ gzip.1 | 5 ++++ gzip.c | 9 +++++- gzip.h | 3 +- trees.c | 7 ++++- 9 files changed, 132 insertions(+), 18 deletions(-) diff --git a/Makefile.am b/Makefile.am index 9d06d98..b01ddee 100644 --- a/Makefile.am +++ b/Makefile.am @@ -108,16 +108,19 @@ gen-ChangeLog: FILES_TO_CHECK = $(bin_SCRIPTS) \ $(top_srcdir)/ChangeLog $(top_srcdir)/configure $(top_srcdir)/gzip.c check-local: $(FILES_TO_CHECK) $(bin_PROGRAMS) gzip.doc.gz - { test '$(srcdir)' != . || ./zdiff --__bindir . -c gzip.doc.gz; } - ./zdiff --__bindir . -c $(srcdir)/gzip.doc $(srcdir)/gzip.doc - ./zdiff --__bindir . $(srcdir)/gzip.doc gzip.doc.gz - ./zdiff --__bindir . -c - $(srcdir)/gzip.doc <gzip.doc.gz - ./zdiff --__bindir . -c gzip.doc.gz gzip.doc.gz - ./zgrep --__bindir . -iV >/dev/null - for file in $(FILES_TO_CHECK); do \ - ./gzip -cv -- "$$file" | ./gzip -d | cmp - "$$file" || exit 1; \ + $(AM_V_GEN){ test '$(srcdir)' != . \ + || ./zdiff --__bindir . -c gzip.doc.gz; } + $(AM_V_at)./zdiff --__bindir . -c $(srcdir)/gzip.doc $(srcdir)/gzip.doc + $(AM_V_at)./zdiff --__bindir . $(srcdir)/gzip.doc gzip.doc.gz + $(AM_V_at)./zdiff --__bindir . -c - $(srcdir)/gzip.doc <gzip.doc.gz + $(AM_V_at)./zdiff --__bindir . -c gzip.doc.gz gzip.doc.gz + $(AM_V_at)./zgrep --__bindir . -iV >/dev/null + $(AM_V_at)for opt in --rsyncable '' -1 -9; do \ + for file in $(FILES_TO_CHECK); do \ + ./gzip $$opt -c -- "$$file" \ + | ./gzip -d | cmp - "$$file" || exit 1; \ + done; \ done - @echo 'Test succeeded.' install-exec-hook: remove-installed-links install-exec-hook remove-installed-links: diff --git a/NEWS b/NEWS index a3989af..b68c403 100644 --- a/NEWS +++ b/NEWS @@ -2,6 +2,16 @@ GNU gzip NEWS -*- outline -*- * Noteworthy changes in release ?.? (????-??-??) [?] +** New features + + gzip now accepts the --rsyncable option. This option is accepted in + all modes, but has effect only when compressing: it makes the resulting + output more amenable to efficient use of rsync. For example, when a + large input file gets a small change, a gzip --rsyncable image of + that file will remain largely unchanged, too. Without --rsyncable, + even a tiny change in the input could result in a totally different + gzip-compressed output file. + ** Changes in behavior The GZIP environment variable is now obsolescent; gzip now warns if diff --git a/bootstrap.conf b/bootstrap.conf index 308dc5e..25acaac 100644 --- a/bootstrap.conf +++ b/bootstrap.conf @@ -62,6 +62,7 @@ unistd-safer unlinkat update-copyright utimens +verify xalloc yesno ' diff --git a/deflate.c b/deflate.c index 020810d..6e235f3 100644 --- a/deflate.c +++ b/deflate.c @@ -80,6 +80,7 @@ #include "tailor.h" #include "gzip.h" #include "lzw.h" /* just for consistency checking */ +#include "verify.h" /* =========================================================================== * Configuration parameters @@ -131,6 +132,14 @@ #endif /* Matches of length 3 are discarded if their distance exceeds TOO_FAR */ +#ifndef RSYNC_WIN +# define RSYNC_WIN 4096 +#endif +verify(RSYNC_WIN < MAX_DIST); + +#define RSYNC_SUM_MATCH(sum) ((sum) % RSYNC_WIN == 0) +/* Whether window sum matches magic value */ + /* =========================================================================== * Local data used by the "longest match" routines. */ @@ -212,6 +221,8 @@ local int compr_level; unsigned good_match; /* Use a faster search when the previous match is longer than this */ +local ulg rsync_sum; /* rolling sum of rsync window */ +local ulg rsync_chunk_end; /* next rsync sequence point */ /* Values for max_lazy_match, good_match and max_chain_length, depending on * the desired pack level (0..9). The values given below have been tuned to @@ -314,6 +325,10 @@ void lm_init (pack_level, flags) #endif /* prev will be initialized on the fly */ + /* rsync params */ + rsync_chunk_end = 0xFFFFFFFFUL; + rsync_sum = 0; + /* Set the default configuration parameters: */ max_lazy_match = configuration_table[pack_level].max_lazy; @@ -550,6 +565,8 @@ local void fill_window() memcpy((char*)window, (char*)window+WSIZE, (unsigned)WSIZE); match_start -= WSIZE; strstart -= WSIZE; /* we now have strstart >= MAX_DIST: */ + if (rsync_chunk_end != 0xFFFFFFFFUL) + rsync_chunk_end -= WSIZE; block_start -= (long) WSIZE; @@ -579,13 +596,47 @@ local void fill_window() } } +/* With an initial offset of START, advance rsync's rolling checksum + by NUM bytes. */ +local void rsync_roll(unsigned int start, unsigned int num) +{ + unsigned i; + + if (start < RSYNC_WIN) { + /* before window fills. */ + for (i = start; i < RSYNC_WIN; i++) { + if (i == start + num) + return; + rsync_sum += (ulg)window[i]; + } + num -= (RSYNC_WIN - start); + start = RSYNC_WIN; + } + + /* buffer after window full */ + for (i = start; i < start+num; i++) { + /* New character in */ + rsync_sum += (ulg)window[i]; + /* Old character out */ + rsync_sum -= (ulg)window[i - RSYNC_WIN]; + if (rsync_chunk_end == 0xFFFFFFFFUL && RSYNC_SUM_MATCH(rsync_sum)) + rsync_chunk_end = i; + } +} + +/* =========================================================================== + * Set rsync_chunk_end if window sum matches magic value. + */ +#define RSYNC_ROLL(s, n) \ + do { if (rsync) rsync_roll((s), (n)); } while(0) + /* =========================================================================== * Flush the current block, with given end-of-file flag. * IN assertion: strstart is set to the end of the current match. */ #define FLUSH_BLOCK(eof) \ flush_block(block_start >= 0L ? (char*)&window[(unsigned)block_start] : \ - (char*)NULL, (long)strstart - block_start, (eof)) + (char*)NULL, (long)strstart - block_start, flush-1, (eof)) /* =========================================================================== * Processes a new input file and return its compressed length. This @@ -596,7 +647,7 @@ local void fill_window() local off_t deflate_fast() { IPos hash_head; /* head of the hash chain */ - int flush; /* set if current block must be flushed */ + int flush = 0; /* set if current block must be flushed, 2=>and padded */ unsigned match_length = 0; /* length of best match */ prev_length = MIN_MATCH-1; @@ -626,6 +677,7 @@ local off_t deflate_fast() lookahead -= match_length; + RSYNC_ROLL(strstart, match_length); /* Insert new strings in the hash table only if the match length * is not too large. This saves time but degrades compression. */ @@ -654,9 +706,14 @@ local off_t deflate_fast() /* No match, output a literal byte */ Tracevv((stderr,"%c",window[strstart])); flush = ct_tally (0, window[strstart]); + RSYNC_ROLL(strstart, 1); lookahead--; strstart++; } + if (rsync && strstart > rsync_chunk_end) { + rsync_chunk_end = 0xFFFFFFFFUL; + flush = 2; + } if (flush) FLUSH_BLOCK(0), block_start = strstart; /* Make sure that we always have enough lookahead, except @@ -679,7 +736,7 @@ off_t deflate() { IPos hash_head; /* head of hash chain */ IPos prev_match; /* previous match */ - int flush; /* set if current block must be flushed */ + int flush = 0; /* set if current block must be flushed */ int match_available = 0; /* set if previous match exists */ register unsigned match_length = MIN_MATCH-1; /* length of best match */ @@ -730,6 +787,7 @@ off_t deflate() */ lookahead -= prev_length-1; prev_length -= 2; + RSYNC_ROLL(strstart, prev_length+1); do { strstart++; INSERT_STRING(strstart, hash_head); @@ -742,24 +800,40 @@ off_t deflate() match_available = 0; match_length = MIN_MATCH-1; strstart++; - if (flush) FLUSH_BLOCK(0), block_start = strstart; + if (rsync && strstart > rsync_chunk_end) { + rsync_chunk_end = 0xFFFFFFFFUL; + flush = 2; + } + if (flush) FLUSH_BLOCK(0), block_start = strstart; } else if (match_available) { /* If there was no match at the previous position, output a * single literal. If there was a match but the current match * is longer, truncate the previous match to a single literal. */ Tracevv((stderr,"%c",window[strstart-1])); - if (ct_tally (0, window[strstart-1])) { - FLUSH_BLOCK(0), block_start = strstart; + flush = ct_tally (0, window[strstart-1]); + if (rsync && strstart > rsync_chunk_end) { + rsync_chunk_end = 0xFFFFFFFFUL; + flush = 2; } + if (flush) FLUSH_BLOCK(0), block_start = strstart; + RSYNC_ROLL(strstart, 1); strstart++; lookahead--; } else { /* There is no previous match to compare with, wait for * the next step to decide. */ + if (rsync && strstart > rsync_chunk_end) { + /* Reset huffman tree */ + rsync_chunk_end = 0xFFFFFFFFUL; + flush = 2; + FLUSH_BLOCK(0), block_start = strstart; + } + match_available = 1; + RSYNC_ROLL(strstart, 1); strstart++; lookahead--; } diff --git a/doc/gzip.texi b/doc/gzip.texi index fa94b84..80a5a17 100644 --- a/doc/gzip.texi +++ b/doc/gzip.texi @@ -360,6 +360,14 @@ specified on the command line are directories, @command{gzip} will descend into the directory and compress all the files it finds there (or decompress them in the case of @command{gunzip}). +@item --rsyncable +While compressing, synchronize the output occasionally, based on the +input. This reduces compression by about one percent in most cases, but +means that the @code{rsync} program can take advantage of similarities +in the uncompressed input when synchronizing two files compressed with +this flag. @code{gunzip} cannot tell the difference between a +compressed file created with this option, and one created without it. + @item --suffix @var{suf} @itemx -S @var{suf} Use suffix @var{suf} instead of @samp{.gz}. Any suffix can be diff --git a/gzip.1 b/gzip.1 index 3262a87..e6aa278 100644 --- a/gzip.1 +++ b/gzip.1 @@ -328,6 +328,11 @@ indicates the slowest compression method (best compression). The default compression level is .BR \-6 (that is, biased towards high compression at expense of speed). +.TP +.B \-\-rsyncable +When you synchronize a compressed file between two computers, this option allows rsync to transfer only files that were changed in the archive instead of the entire archive. +Normally, after a change is made to any file in the archive, the compression algorithm can generate a new version of the archive that does not match the previous version of the archive. In this case, rsync transfers the entire new version of the archive to the remote computer. +With this option, rsync can transfer only the changed files as well as a small amount of metadata that is required to update the archive structure in the area that was changed. .SH "ADVANCED USAGE" Multiple compressed files can be concatenated. In this case, .I gunzip diff --git a/gzip.c b/gzip.c index d9cdfaa..e243f2e 100644 --- a/gzip.c +++ b/gzip.c @@ -220,6 +220,7 @@ static int dfd = -1; /* output directory file descriptor */ unsigned insize; /* valid bytes in inbuf */ unsigned inptr; /* index of next byte to be processed in inbuf */ unsigned outcnt; /* bytes in output buffer */ +int rsync = 0; /* make ryncable chunks */ static int handled_sig[] = { @@ -248,6 +249,7 @@ static int handled_sig[] = enum { PRESUME_INPUT_TTY_OPTION = CHAR_MAX + 1, + RSYNCABLE_OPTION, SYNCHRONOUS_OPTION, /* A value greater than all valid long options, used as a flag to @@ -288,7 +290,7 @@ static const struct option longopts[] = {"best", 0, 0, '9'}, /* compress better */ {"lzw", 0, 0, 'Z'}, /* make output compatible with old compress */ {"bits", 1, 0, 'b'}, /* max number of bits per code (implies -Z) */ - + {"rsyncable", 0, 0, RSYNCABLE_OPTION}, /* make rsync-friendly archive */ { 0, 0, 0, 0 } }; @@ -373,6 +375,7 @@ local void help() " -Z, --lzw produce output compatible with old compress", " -b, --bits=BITS max number of bits per code (implies -Z)", #endif + " --rsyncable Make rsync-friendly archive", "", "With no FILE, or when FILE is -, read standard input.", "", @@ -555,6 +558,10 @@ int main (int argc, char **argv) recursive = 1; #endif break; + + case RSYNCABLE_OPTION: + rsync = 1; + break; case 'S': #ifdef NO_MULTIPLE_DOTS if (*optarg == '.') optarg++; diff --git a/gzip.h b/gzip.h index bb4000a..f298b47 100644 --- a/gzip.h +++ b/gzip.h @@ -140,6 +140,7 @@ EXTERN(uch, window); /* Sliding window and suffix table (unlzw) */ extern unsigned insize; /* valid bytes in inbuf */ extern unsigned inptr; /* index of next byte to be processed in inbuf */ extern unsigned outcnt; /* bytes in output buffer */ +extern int rsync; /* deflate into rsyncable chunks */ extern off_t bytes_in; /* number of input bytes */ extern off_t bytes_out; /* number of output bytes */ @@ -287,7 +288,7 @@ extern off_t deflate (void); /* in trees.c */ extern void ct_init (ush *attr, int *method); extern int ct_tally (int dist, int lc); -extern off_t flush_block (char *buf, ulg stored_len, int eof); +extern off_t flush_block (char *buf, ulg stored_len, int pad, int eof); /* in bits.c */ extern void bi_init (file_t zipfile); diff --git a/trees.c b/trees.c index f314e57..025d5ba 100644 --- a/trees.c +++ b/trees.c @@ -856,9 +856,10 @@ local void send_all_trees(lcodes, dcodes, blcodes) * trees or store, and output the encoded block to the zip file. This function * returns the total compressed length for the file so far. */ -off_t flush_block(buf, stored_len, eof) +off_t flush_block(buf, stored_len, pad, eof) char *buf; /* input block, or NULL if too old */ ulg stored_len; /* length of input block */ + int pad; /* pad output to byte boundary */ int eof; /* true if this is the last block for a file */ { ulg opt_lenb, static_lenb; /* opt_len and static_len in bytes */ @@ -951,6 +952,10 @@ off_t flush_block(buf, stored_len, eof) Assert (input_len == bytes_in, "bad input size"); bi_windup(); compressed_len += 7; /* align on byte boundary */ + } else if (pad && (compressed_len % 8) != 0) { + send_bits((STORED_BLOCK<<1)+eof, 3); /* send block type */ + compressed_len = (compressed_len + 3 + 7) & ~7L; + copy_block(buf, 0, 1); /* with header */ } return compressed_len >> 3; -- 2.6.4
