Hello,
I've created a patch for tar, which adds support for lz4 and zstd
compression. Both compression tools have become increasingly popular and
offer high-speed compression and decompression, making them interesting
alternatives to previous compression tools.
lz4 is similar to lzo, but offers faster decompression. zstd is similar
to gzip, but offers much faster compression and decompression.
See also:
http://lz4.github.io/lz4/
http://facebook.github.io/zstd/
For example, creating an archive from a directory of 1.12 GB containing
binary and ASCII files ("/usr/local/gnu/"):
Compression
name time size ratio
gnu.tar.gz 57.2s 534077354 44.3%
gnu.tar.bz2 1:53.9s 509401491 42.3%
gnu.tar.xz 7:15.6s 423886520 35.2%
gnu.tar.lz 7:37.2s 425162846 35.3%
gnu.tar.lzo 4.7s 676569472 56.2%
gnu.tar.lz4 4.9s 676600069 56.2%
gnu.tar.zst 10.7s 517897416 43.0%
Decompression
name time
gnu.tar.gz 7.1s
gnu.tar.bz2 45.2s
gnu.tar.xz 26.3s
gnu.tar.lz 30.4s
gnu.tar.lzo 3.5s
gnu.tar.lz4 2.0s
gnu.tar.zst 3.2s
A second example using the Linux kernel source code (4.12.4), containing
697 MB of data:
Compression
name time size ratio
linux.tar.gz 28.2s 156988489 21.4%
linux.tar.bz2 1:05.5s 120659282 16.5%
linux.tar.xz 4:26.9s 102214892 13.9%
linux.tar.lz 4:18.4s 104105244 14.2%
linux.tar.lzo 3.6s 261745137 35.7%
linux.tar.lz4 3.9s 252707015 34.5%
linux.tar.zst 5.9s 150448272 20.5%
Decompression
name time
linux.tar.gz 3.6s
linux.tar.bz2 19.4s
linux.tar.xz 7.2s
linux.tar.lz 8.1s
linux.tar.lzo 2.5s
linux.tar.lz4 1.5s
linux.tar.zst 2.1s
(The tools were used with their default arguments. The hardware used was
an AMD FX8350 4GHz.)
Attached files are:
patch-part-1 - diffs of buffer.c config.h.in configure.ac suffix.c tar.c
patch-part-2 - diffs of tar.texi tar.1 cs.po de.po fi.po ga.po id.po
ru.po sl.po zh_CN.po
Cheers
--- tar-1.29/src/buffer.c 2016-03-14 20:58:16.000000000 +0000
+++ tar-1.29.1/src/buffer.c 2017-08-05 11:51:57.457112307 +0100
@@ -270,6 +270,8 @@
ct_lzip,
ct_lzma,
ct_lzop,
+ ct_lz4,
+ ct_zstd,
ct_xz
};
@@ -298,6 +300,8 @@
{ ct_lzip, 4, "LZIP" },
{ ct_lzma, 6, "\xFFLZMA" },
{ ct_lzop, 4, "\211LZO" },
+ { ct_lz4, 4, "\x04\x22\x4D\x18" },
+ { ct_zstd, 4, "\x28\xB5\x2F\xFD" },
{ ct_xz, 6, "\xFD" "7zXZ" },
};
@@ -313,6 +317,8 @@
{ ct_lzma, LZMA_PROGRAM, "--lzma" },
{ ct_lzma, XZ_PROGRAM, "-J" },
{ ct_lzop, LZOP_PROGRAM, "--lzop" },
+ { ct_lz4, LZ4_PROGRAM, "--lz4" },
+ { ct_zstd, ZSTD_PROGRAM, "--zstd" },
{ ct_xz, XZ_PROGRAM, "-J" },
{ ct_none }
};
--- tar-1.29/config.h.in 2016-05-16 09:52:47.000000000 +0100
+++ tar-1.29.1/config.h.in 2017-08-05 13:17:22.327294420 +0100
@@ -2190,6 +2190,9 @@
slash. */
#undef LSTAT_FOLLOWS_SLASHED_SYMLINK
+/* Define to the program name of lz4 compressor program */
+#undef LZ4_PROGRAM
+
/* Define to the program name of lzip compressor program */
#undef LZIP_PROGRAM
@@ -2454,6 +2457,9 @@
/* Define to the program name of xz compressor program */
#undef XZ_PROGRAM
+/* Define to the program name of zstd compressor program */
+#undef ZSTD_PROGRAM
+
/* Enable large inode numbers on Mac OS X 10.5. */
#undef _DARWIN_USE_64_BIT_INODE
--- tar-1.29/configure.ac 2016-05-16 09:51:12.000000000 +0100
+++ tar-1.29.1/configure.ac 2017-08-05 12:02:08.206629003 +0100
@@ -249,6 +249,8 @@
TAR_COMPR_PROGRAM(lzip)
TAR_COMPR_PROGRAM(lzma)
TAR_COMPR_PROGRAM(lzop)
+TAR_COMPR_PROGRAM(lz4)
+TAR_COMPR_PROGRAM(zstd)
TAR_COMPR_PROGRAM(xz)
AC_MSG_CHECKING(for default archive format)
--- tar-1.29/src/suffix.c 2016-01-20 09:26:32.000000000 +0000
+++ tar-1.29.1/src/suffix.c 2017-08-05 12:57:33.987681075 +0100
@@ -43,6 +43,8 @@
{ S(lzma, LZMA) },
{ S(tlz, LZMA) },
{ S(lzo, LZOP) },
+ { S(lz4, LZ4) },
+ { S(zst, ZSTD) },
{ S(xz, XZ) },
{ S(txz, XZ) }, /* Slackware */
{ NULL }
--- tar-1.29/src/tar.c 2016-03-24 05:42:14.000000000 +0000
+++ tar-1.29.1/src/tar.c 2017-08-05 11:31:35.793518014 +0100
@@ -293,6 +293,8 @@
LZIP_OPTION,
LZMA_OPTION,
LZOP_OPTION,
+ LZ4_OPTION,
+ ZSTD_OPTION,
MODE_OPTION,
MTIME_OPTION,
NEWER_MTIME_OPTION,
@@ -681,6 +683,8 @@
{"lzip", LZIP_OPTION, 0, 0, NULL, GRID+1 },
{"lzma", LZMA_OPTION, 0, 0, NULL, GRID+1 },
{"lzop", LZOP_OPTION, 0, 0, NULL, GRID+1 },
+ {"lz4", LZ4_OPTION, 0, 0, NULL, GRID+1 },
+ {"zstd", ZSTD_OPTION, 0, 0, NULL, GRID+1 },
{"xz", 'J', 0, 0, NULL, GRID+1 },
#undef GRID
@@ -1125,6 +1129,14 @@
case LZOP_OPTION:
s = xasprintf (_("filter the archive through %s"), LZOP_PROGRAM);
+ case LZ4_OPTION:
+ s = xasprintf (_("filter the archive through %s"), LZ4_PROGRAM);
+ break;
+
+ case ZSTD_OPTION:
+ s = xasprintf (_("filter the archive through %s"), ZSTD_PROGRAM);
+ break;
+
case 'J':
s = xasprintf (_("filter the archive through %s"), XZ_PROGRAM);
break;
@@ -1496,6 +1508,14 @@
set_use_compress_program_option (LZOP_PROGRAM, args->loc);
break;
+ case LZ4_OPTION:
+ set_use_compress_program_option (LZ4_PROGRAM, args->loc);
+ break;
+
+ case ZSTD_OPTION:
+ set_use_compress_program_option (ZSTD_PROGRAM, args->loc);
+ break;
+
case 'm':
touch_option = true;
break;
--- tar-1.29/doc/tar.texi 2016-04-14 09:50:55.000000000 +0100
+++ tar-1.29.1/doc/tar.texi 2017-08-05 13:11:30.062498623 +0100
@@ -2971,6 +2971,16 @@
This option tells @command{tar} to read or write archives through
@command{lzop}. @xref{gzip}.
+@item --lz4
+
+This option tells @command{tar} to read or write archives through
+@command{lz4}. @xref{gzip}.
+
+@item --zstd
+
+This option tells @command{tar} to read or write archives through
+@command{zstd}. @xref{gzip}.
+
@opsummary{mode}
@item --mode=@var{permissions}
@@ -9554,14 +9564,17 @@
@cindex lzip
@cindex lzma
@cindex lzop
+@cindex lz4
+@cindex zstd
@cindex compress
@GNUTAR{} is able to create and read compressed archives. It supports
a wide variety of compression programs, namely: @command{gzip},
@command{bzip2}, @command{lzip}, @command{lzma}, @command{lzop},
-@command{xz} and traditional @command{compress}. The latter is
-supported mostly for backward compatibility, and we recommend
-against using it, because it is by far less effective than the other
-compression programs@footnote{It also had patent problems in the past.}.
+@command{lz4}, @command{zstd}, @command{xz} and traditional
+@command{compress}. The latter is supported mostly for backward
+compatibility, and we recommend against using it, because it is by far
+less effective than the other compression programs@footnote{It also
+had patent problems in the past.}.
Creating a compressed archive is simple: you just specify a
@dfn{compression option} along with the usual archive creation
@@ -9570,10 +9583,11 @@
(@option{--bzip2}) to create a @command{bzip2} compressed archive,
@option{--lzip} to create an @asis{lzip} compressed archive,
@option{-J} (@option{--xz}) to create an @asis{XZ} archive,
-@option{--lzma} to create an @asis{LZMA} compressed
-archive, @option{--lzop} to create an @asis{LSOP} archive, and
-@option{-Z} (@option{--compress}) to use @command{compress} program.
-For example:
+@option{--lzma} to create an @asis{LZMA} compressed archive,
+@option{--lzop} to create an @asis{LSOP} archive, @option{--lz4} to
+create an @asis{LZ4} archive, @option{--zstd} to create an @asis{ZST}
+archive, and @option{-Z} (@option{--compress}) to use
+@command{compress} program. For example:
@smallexample
$ @kbd{tar czf archive.tar.gz .}
@@ -9693,6 +9707,14 @@
@item --lzop
Filter the archive through @command{lzop}.
+@opindex lz4
+@item --lz4
+Filter the archive through @command{lz4}.
+
+@opindex zstd
+@item --zstd
+Filter the archive through @command{zstd}.
+
@opindex compress
@opindex uncompress
@item -Z
@@ -9764,6 +9786,8 @@
@item @samp{.lzma} @tab @command{lzma}
@item @samp{.tlz} @tab @command{lzma}
@item @samp{.lzo} @tab @command{lzop}
+@item @samp{.lz4} @tab @command{lz4}
+@item @samp{.zst} @tab @command{zstd}
@item @samp{.xz} @tab @command{xz}
@end multitable
--- tar-1.29/doc/tar.1 2016-03-23 14:34:31.000000000 +0000
+++ tar-1.29.1/doc/tar.1 2017-08-05 12:21:04.050226023 +0100
@@ -13,7 +13,7 @@
.\"
.\" You should have received a copy of the GNU General Public License
.\" along with this program. If not, see <http://www.gnu.org/licenses/>.
-.TH TAR 1 "March 23, 2016" "TAR" "GNU TAR Manual"
+.TH TAR 1 "August 5, 2017" "TAR" "GNU TAR Manual"
.SH NAME
tar \- an archiving utility
.SH SYNOPSIS
@@ -815,6 +815,14 @@
Filter the archive through
.BR lzop (1).
.TP
+\fB\-\-lz4\fR
+Filter the archive through
+.BR lz4 (1).
+.TP
+\fB\-\-zstd\fR
+Filter the archive through
+.BR zstd (1).
+.TP
\fB\-\-no\-auto\-compress\fR
Do not use archive suffix to determine the compression program.
.TP
@@ -1282,6 +1290,8 @@
.BR gzip (1),
.BR lzma (1),
.BR lzop (1),
+.BR lz4 (1),
+.BR zstd (1),
.BR rmt (8),
.BR symlink (7),
.BR tar (5),
--- tar-1.29/po/cs.po 2016-05-16 07:55:11.000000000 +0100
+++ tar-1.29.1/po/cs.po 2017-08-05 12:45:28.982138853 +0100
@@ -3085,6 +3085,12 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "archiv protáhne skrze lzop"
+#~ msgid "filter the archive through lz4"
+#~ msgstr "archiv protáhne skrze lz4"
+
+#~ msgid "filter the archive through zstd"
+#~ msgstr "archiv protáhne skrze zstd"
+
#~ msgid "block size"
#~ msgstr "velikost bloku"
--- tar-1.29/po/de.po 2016-05-16 07:55:11.000000000 +0100
+++ tar-1.29.1/po/de.po 2017-08-05 12:38:34.827664336 +0100
@@ -3101,6 +3101,12 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "Archiv durch lzop filtern"
+#~ msgid "filter the archive through lz4"
+#~ msgstr "Archiv durch lz4 filtern"
+
+#~ msgid "filter the archive through zstd"
+#~ msgstr "Archiv durch zstd filtern"
+
#~ msgid "Warning: the -I option is not supported; perhaps you meant -j or -T?"
#~ msgstr ""
#~ "Warnung: Die Option -I ist nicht unterstützt, meinen Sie -j oder -T?"
--- tar-1.29/po/fi.po 2016-05-16 07:55:12.000000000 +0100
+++ tar-1.29.1/po/fi.po 2017-08-05 12:41:31.745588733 +0100
@@ -3019,6 +3019,14 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "ohjaa arkisto gzip-ohjelman läpi"
+#, fuzzy
+#~ msgid "filter the archive through lz4"
+#~ msgstr "ohjaa arkisto lz4-ohjelman läpi"
+
+#, fuzzy
+#~ msgid "filter the archive through zstd"
+#~ msgstr "ohjaa arkisto zstd-ohjelman läpi"
+
#~ msgid "rmtd: Cannot allocate buffer space\n"
#~ msgstr "rmtd: Puskuritilaa ei voi varata\n"
--- tar-1.29/po/ga.po 2016-05-16 07:55:12.000000000 +0100
+++ tar-1.29.1/po/ga.po 2017-08-05 12:39:14.036426387 +0100
@@ -3122,6 +3122,12 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "scag an chartlann le lzop"
+#~ msgid "filter the archive through lz4"
+#~ msgstr "scag an chartlann le lz4"
+
+#~ msgid "filter the archive through zstd"
+#~ msgstr "scag an chartlann le zstd"
+
#~ msgid "%s: Invalid group"
#~ msgstr "%s: Grúpa neamhbhailí"
--- tar-1.29/po/id.po 2016-05-16 07:55:12.000000000 +0100
+++ tar-1.29.1/po/id.po 2017-08-05 12:39:49.570210535 +0100
@@ -3061,6 +3061,12 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "saring archive melalui lzop"
+#~ msgid "filter the archive through lz4"
+#~ msgstr "saring archive melalui lz4"
+
+#~ msgid "filter the archive through zstd"
+#~ msgstr "saring archive melalui zstd"
+
#~ msgid "Warning: the -I option is not supported; perhaps you meant -j or -T?"
#~ msgstr ""
#~ "Peringatan: option -l tidak didukung; mungkin maksud anda -j atau -T?"
--- tar-1.29/po/ru.po 2016-05-16 07:55:13.000000000 +0100
+++ tar-1.29.1/po/ru.po 2017-08-05 12:45:00.316314491 +0100
@@ -3097,3 +3097,9 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "пропустить архив через lzop"
+
+#~ msgid "filter the archive through lz4"
+#~ msgstr "пропустить архив через lz4"
+
+#~ msgid "filter the archive through zstd"
+#~ msgstr "пропустить архив через zstd"
--- tar-1.29/po/sl.po 2016-05-16 07:55:13.000000000 +0100
+++ tar-1.29.1/po/sl.po 2017-08-05 12:44:13.786599325 +0100
@@ -3095,6 +3095,14 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "filtriraj arhiv skozi gzip"
+#, fuzzy
+#~ msgid "filter the archive through lz4"
+#~ msgstr "filtriraj arhiv skozi lz4"
+
+#, fuzzy
+#~ msgid "filter the archive through zstd"
+#~ msgstr "filtriraj arhiv skozi zstd"
+
#~ msgid "[.]NUMBER"
#~ msgstr "[.]N"
--- tar-1.29/po/zh_CN.po 2016-05-16 07:55:14.000000000 +0100
+++ tar-1.29.1/po/zh_CN.po 2017-08-05 12:46:07.721901805 +0100
@@ -2982,6 +2982,12 @@
#~ msgid "filter the archive through lzop"
#~ msgstr "通过 lzop 过滤归档"
+#~ msgid "filter the archive through lz4"
+#~ msgstr "通过 lz4 过滤归档"
+
+#~ msgid "filter the archive through zstd"
+#~ msgstr "通过 zstd 过滤归档"
+
#~ msgid "Warning: the -I option is not supported; perhaps you meant -j or -T?"
#~ msgstr "警告：不支持 -I 选项，或许您的意思是 -j 或者 -T ？"
