This patch adds zstd for backup/restore. It also factors out the common parts on the decompression tools. Sadly tar 1.31 (includes zstd) was not available at the time of writing this patch.
Signed-off-by: Alwin Antreich <[email protected]> --- PVE/Storage.pm | 124 ++++++++++++++++++++++++++++++++++++++++---------- PVE/Storage/Plugin.pm | 2 +- 2 files changed, 102 insertions(+), 24 deletions(-) diff --git a/PVE/Storage.pm b/PVE/Storage.pm index 588e775..f3c50ca 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -510,7 +510,7 @@ sub path_to_volume_id { } elsif ($path =~ m!^$privatedir/(\d+)$!) { my $vmid = $1; return ('rootdir', "$sid:rootdir/$vmid"); - } elsif ($path =~ m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) { + } elsif ($path =~ m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!) { my $name = $1; return ('iso', "$sid:backup/$name"); } @@ -862,7 +862,7 @@ sub template_list { $info = { volid => "$sid:vztmpl/$1", format => "t$2" }; } elsif ($tt eq 'backup') { - next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!; + next if $fn !~ m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst))$!; $info = { volid => "$sid:backup/$1", format => $2 }; } @@ -1362,36 +1362,112 @@ sub foreach_volid { } } -sub extract_vzdump_config_tar { - my ($archive, $conf_re) = @_; - - die "ERROR: file '$archive' does not exist\n" if ! -f $archive; - - my $pid = open(my $fh, '-|', 'tar', 'tf', $archive) || - die "unable to open file '$archive'\n"; +sub decompressor_info { + my ($archive, $comp, $noerr) = @_; + my $format; + if (!defined($comp)) { + my $volid = basename($archive); + if ($volid =~ /vzdump-(lxc|openvz|qemu)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/) { + if ($8 eq 'tgz') { + $format = 'tar'; + $comp = 'gz'; + } else { + $format = $10; + $comp = $12 if defined($12); + } + } else { + die "ERROR: couldn't determine format and compression type\n" if !$noerr; + } + } - my $file; - while (defined($file = <$fh>)) { - if ($file =~ $conf_re) { - $file = $1; # untaint - last; + my $cmd; + if (defined($comp)) { + if ($comp eq 'gz') { + $cmd = ["zcat", $archive]; + } elsif ($comp eq 'lzo') { + $cmd = ["lzop", "-d", "-c", $archive]; + } elsif ($comp eq 'zst') { + $cmd = ["zstd", "-q", "-d", "-c", $archive]; + } else { + die "unknown compression method '$comp'\n" if !$noerr; } + } else { + die "compression type not set\n" if !$noerr; } - kill 15, $pid; - waitpid $pid, 0; - close $fh; + pop(@$cmd) if !defined($archive); - die "ERROR: archive contains no configuration file\n" if !$file; - chomp $file; + return $cmd; +} +sub extract_from_archive { + my ($cmd) = @_; my $raw = ''; my $out = sub { my $output = shift; $raw .= "$output\n"; }; + # in some cases, lzop|zcat|zstd exits with 1 when its stdout pipe is closed + # early, detect this and ignore the exit code later + my $broken_pipe; + my $errstring; + my $err = sub { + my $output = shift; + if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/Error 70 : Write error :/i) { + $broken_pipe = 1; + } elsif (!defined ($errstring) && $output !~ m/^\s*$/) { + $errstring = "Failed to extract config from archive: $output\n"; + } + }; + + # in other cases, the pipeline will exit with exit code 141 + # because of the broken pipe, handle / ignore this as well + my $rc; + eval { + $rc = PVE::Tools::run_command($cmd, outfunc => $out, errfunc => $err, noerr => 1); + }; + my $rerr = $@; + + # use exit code if no stderr output and not just broken pipe + if (!$errstring && !$broken_pipe && $rc != 0 && $rc != 141) { + die "$rerr\n" if $rerr; + die "config extraction failed with exit code $rc\n"; + } + die "$errstring\n" if $errstring; + + return $raw; + +} +sub extract_vzdump_config_tar { + my ($archive, $conf_re) = @_; + + die "ERROR: file '$archive' does not exist\n" if ! -f $archive; + + my $decomp = decompressor_info($archive, undef, 1); - PVE::Tools::run_command(['tar', '-xpOf', $archive, $file, '--occurrence'], outfunc => $out); + my $file; + my $file_list = sub { + my (@flst) = shift; + + foreach my $line (@flst) { + if ($line =~ $conf_re) { + $file = $1; # untaint + last; + } + } + + die "ERROR: archive contains no configuration file\n" if !$file; + }; + + my $raw; + + if ($decomp) { + PVE::Tools::run_command([$decomp, ['tar', '-tf', '-']], outfunc => $file_list); + $raw = extract_from_archive([$decomp, ['tar', '-xpO', "$file", '--occurrence']]); + } else { + PVE::Tools::run_command(['tar', '-tf', "$archive"], outfunc => $file_list); + $raw = extract_from_archive(['tar', '-xpO', "$file", '--occurrence', '-f', "$archive"]); + } return wantarray ? ($raw, $file) : $raw; } @@ -1413,6 +1489,8 @@ sub extract_vzdump_config_vma { $uncomp = ["zcat", $archive]; } elsif ($comp eq 'lzo') { $uncomp = ["lzop", "-d", "-c", $archive]; + } elsif ($comp eq 'zst') { + $uncomp = ["zstd", "-q", "-d", "-c", $archive]; } else { die "unknown compression method '$comp'\n"; } @@ -1424,7 +1502,7 @@ sub extract_vzdump_config_vma { my $errstring; my $err = sub { my $output = shift; - if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/) { + if ($output =~ m/lzop: Broken pipe: <stdout>/ || $output =~ m/gzip: stdout: Broken pipe/ || $output =~ m/Error 70 : Write error :/) { $broken_pipe = 1; } elsif (!defined ($errstring) && $output !~ m/^\s*$/) { $errstring = "Failed to extract config from VMA archive: $output\n"; @@ -1458,9 +1536,9 @@ sub extract_vzdump_config { my $archive = abs_filesystem_path($cfg, $volid); - if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/) { + if ($volid =~ /vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo|zst))?))$/) { return extract_vzdump_config_tar($archive, qr!^(\./etc/vzdump/(pct|vps)\.conf)$!); - } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/) { + } elsif ($volid =~ /vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo|zst))?))$/) { my $format; my $comp; if ($7 eq 'tgz') { diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 255c643..cdb9bd8 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -421,7 +421,7 @@ sub parse_volname { return ('vztmpl', $1); } elsif ($volname =~ m!^rootdir/(\d+)$!) { return ('rootdir', $1, $1); - } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) { + } elsif ($volname =~ m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tar\.zst|tgz|vma|vma\.gz|vma\.lzo|vma\.zst)))$!) { my $fn = $1; if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) { return ('backup', $fn, $2); -- 2.11.0 _______________________________________________ pve-devel mailing list [email protected] https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
