[pve-devel] [PATCH manager] vzdump: test: add first tests to the guest include logic

2020-04-28 Thread Aaron Lauterer 
Signed-off-by: Aaron Lauterer 
---

I forgot the makefile, please ignore the other patch and use this one.


 test/Makefile  |   5 +-
 test/vzdump_guest_included_test.pl | 190 +
 2 files changed, 194 insertions(+), 1 deletion(-)
 create mode 100755 test/vzdump_guest_included_test.pl

diff --git a/test/Makefile b/test/Makefile
index c26e16b1..44f3b0d7 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -5,7 +5,7 @@ all:
 export PERLLIB=..
 
 .PHONY: check
-check: replication_test balloon_test mail_test
+check: replication_test balloon_test mail_test vzdump_test
 
 balloon_test:
./balloontest.pl
@@ -21,6 +21,9 @@ replication_test:
 mail_test:
./mail_test.pl
 
+vzdump_test:
+   ./vzdump_guest_included_test.pl
+
 .PHONY: install
 install:
 
diff --git a/test/vzdump_guest_included_test.pl 
b/test/vzdump_guest_included_test.pl
new file mode 100755
index ..31c520fa
--- /dev/null
+++ b/test/vzdump_guest_included_test.pl
@@ -0,0 +1,190 @@
+#!/usr/bin/perl
+
+# Some of the tests can only be applied once the whole include logic is moved
+# into one single method. Right now parts of it, (all, exclude)  are in the
+# PVE::VZDump->exec_backup() method.
+
+use strict;
+use warnings;
+
+use lib '..';
+
+use Test::More tests => 7;
+use Test::MockModule;
+
+use PVE::VZDump;
+
+use Data::Dumper;
+
+my $vmlist = {
+'ids' => {
+   100 => {
+   'type' => 'qemu',
+   'node' => 'node1',
+   },
+   101 => {
+   'type' => 'qemu',
+   'node' => 'node1',
+   },
+   112 => {
+   'type' => 'lxc',
+   'node' => 'node1',
+   },
+   113 => {
+   'type' => 'lxc',
+   'node' => 'node1',
+   },
+   200 => {
+   'type' => 'qemu',
+   'node' => 'node2',
+   },
+   201 => {
+   'type' => 'qemu',
+   'node' => 'node2',
+   },
+   212 => {
+   'type' => 'lxc',
+   'node' => 'node2',
+   },
+   213 => {
+   'type' => 'lxc',
+   'node' => 'node2',
+   },
+}
+};
+
+my $pve_cluster_module = Test::MockModule->new('PVE::Cluster');
+$pve_cluster_module->mock(
+get_vmlist => sub {
+   return $vmlist;
+}
+);
+
+my $pve_inotify = Test::MockModule->new('PVE::INotify');
+$pve_inotify->mock(
+nodename => sub {
+   return 'node1';
+}
+);
+
+my $pve_api2tools = Test::MockModule->new('PVE::API2Tools');
+$pve_api2tools->mock(
+get_resource_pool_guest_members => sub {
+   return [100, 101, 200, 201];
+}
+);
+
+
+
+my $tests = {};
+
+# is handled in the PVE::VZDump->exec_backup() method for now
+# $tests->{'Test all guests'} = {
+# expected_vmids => [ 100, 101, 112, 113, 200, 201, 212, 213 ],
+# expected_skiplist => [ ],
+# param => {
+#  all => 1,
+# }
+# };
+
+# is handled in the PVE::VZDump->exec_backup() method for now
+# $tests->{'Test all guests with cluster node limit'} = {
+# expected_vmids => [ 100, 101, 112, 113, 200, 201, 212, 213 ],
+# expected_skiplist => [],
+# param => {
+#  all => 1,
+#  node => 'node2',
+# }
+# };
+
+# is handled in the PVE::VZDump->exec_backup() method for now
+# $tests->{'Test all guests with local node limit'} = {
+# expected_vmids => [ 100, 101, 112, 113 ],
+# expected_skiplist => [ 200, 201, 212, 213 ],
+# param => {
+#  all => 1,
+#  node => 'node1',
+# }
+# };
+#
+# TODO: all test variants with exclude
+
+$tests->{'Test pool members'} = {
+expected_vmids => [ 100, 101 ],
+expected_skiplist => [ 200, 201 ],
+param => {
+   pool => 'testpool',
+}
+};
+
+$tests->{'Test pool members with cluster node limit'} = {
+expected_vmids => [ 100, 101, 200, 201 ],
+expected_skiplist => [],
+param => {
+   pool => 'testpool',
+   node => 'node2',
+}
+};
+
+$tests->{'Test pool members with local node limit'} = {
+expected_vmids => [ 100, 101 ],
+expected_skiplist => [ 200, 201 ],
+param => {
+   pool => 'testpool',
+   node => 'node1',
+}
+};
+
+$tests->{'Test selected VMIDs'} = {
+expected_vmids => [ 100 ],
+expected_skiplist => [ 201, 212 ],
+param => {
+   vmid => '100, 201, 212',
+}
+};
+
+
+$tests->{'Test selected VMIDs with cluster node limit'} = {
+expected_vmids => [ 100, 201, 212 ],
+expected_skiplist => [],
+param => {
+   vmid => '100, 201, 212',
+   node => 'node2',
+}
+};
+
+$tests->{'Test selected VMIDs with local node limit'} = {
+expected_vmids => [ 100 ],
+expected_skiplist => [ 201, 212 ],
+param => {
+   vmid => '100, 201, 212',
+   node => 'node1',
+}
+};
+
+$tests->{'Test selected VMIDs on other nodes'} = {
+expected_vmids => [],
+expected_skiplist => [ 201, 212 ],
+param => {
+   vmid => '201, 212',
+   node => 'node1',
+}
+};
+
+
+
+foreach my $testname (sort keys %{$tests}) {
+my $test

[pve-devel] [PATCH v2 common] print_text_table: handle undefined values in comparision

2020-04-28 Thread Fabian Ebner 
by introducing a safe_compare helper. Fixes warnings, e.g.
pvesh get /nodes//network
would print "use of uninitialized"-warnings if there are inactive
network interfaces, because for those, 'active' is undef.

Signed-off-by: Fabian Ebner 
---

Changes from v1:
* don't change sort_key depending on data
* instead handle undefined values directly
  in comparision

 src/PVE/CLIFormatter.pm | 11 +++
 src/PVE/Tools.pm|  9 +
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/src/PVE/CLIFormatter.pm b/src/PVE/CLIFormatter.pm
index 0e9cbe6..4f18fa9 100644
--- a/src/PVE/CLIFormatter.pm
+++ b/src/PVE/CLIFormatter.pm
@@ -150,8 +150,7 @@ sub data_to_text {
 # $props_to_print - ordered list of properties to print
 # $options
 # - sort_key: can be used to sort after a specific column, if it isn't set we 
sort
-#   after the leftmost column (with no undef value in $data) this can be
-#   turned off by passing 0 as sort_key
+#   after the leftmost column. This can be turned off by passing 0 as sort_key
 # - noborder: print without asciiart border
 # - noheader: print without table header
 # - columns: limit output width (if > 0)
@@ -174,11 +173,15 @@ sub print_text_table {
 
 if (defined($sort_key) && $sort_key ne 0) {
my $type = $returnprops->{$sort_key}->{type} // 'string';
+   my $cmpfn;
if ($type eq 'integer' || $type eq 'number') {
-   @$data = sort { $a->{$sort_key} <=> $b->{$sort_key} } @$data;
+   $cmpfn = sub { $_[0] <=> $_[1] };
} else {
-   @$data = sort { $a->{$sort_key} cmp $b->{$sort_key} } @$data;
+   $cmpfn = sub { $_[0] cmp $_[1] };
}
+   @$data = sort {
+   PVE::Tools::safe_compare($a->{$sort_key}, $b->{$sort_key}, $cmpfn)
+   } @$data;
 }
 
 my $colopts = {};
diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
index f02c0ae..f6b18f1 100644
--- a/src/PVE/Tools.pm
+++ b/src/PVE/Tools.pm
@@ -1766,4 +1766,13 @@ sub mount($) {
 );
 }
 
+sub safe_compare {
+my ($left, $right, $cmp) = @_;
+
+return 0 if !defined($left) && !defined($right);
+return -1 if !defined($left);
+return 1 if !defined($right);
+return $cmp->($left, $right);
+}
+
 1;
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH v2 common] print_text_table: handle undefined values in comparision

2020-04-28 Thread Oğuz Bektaş 
Tested-by: Oguz Bektas 


> Fabian Ebner  hat am 28. April 2020 10:18 geschrieben:
> 
>  
> by introducing a safe_compare helper. Fixes warnings, e.g.
> pvesh get /nodes//network
> would print "use of uninitialized"-warnings if there are inactive
> network interfaces, because for those, 'active' is undef.
> 
> Signed-off-by: Fabian Ebner 
> ---
> 
> Changes from v1:
> * don't change sort_key depending on data
> * instead handle undefined values directly
>   in comparision
> 
>  src/PVE/CLIFormatter.pm | 11 +++
>  src/PVE/Tools.pm|  9 +
>  2 files changed, 16 insertions(+), 4 deletions(-)
> 
> diff --git a/src/PVE/CLIFormatter.pm b/src/PVE/CLIFormatter.pm
> index 0e9cbe6..4f18fa9 100644
> --- a/src/PVE/CLIFormatter.pm
> +++ b/src/PVE/CLIFormatter.pm
> @@ -150,8 +150,7 @@ sub data_to_text {
>  # $props_to_print - ordered list of properties to print
>  # $options
>  # - sort_key: can be used to sort after a specific column, if it isn't set 
> we sort
> -#   after the leftmost column (with no undef value in $data) this can be
> -#   turned off by passing 0 as sort_key
> +#   after the leftmost column. This can be turned off by passing 0 as 
> sort_key
>  # - noborder: print without asciiart border
>  # - noheader: print without table header
>  # - columns: limit output width (if > 0)
> @@ -174,11 +173,15 @@ sub print_text_table {
>  
>  if (defined($sort_key) && $sort_key ne 0) {
>   my $type = $returnprops->{$sort_key}->{type} // 'string';
> + my $cmpfn;
>   if ($type eq 'integer' || $type eq 'number') {
> - @$data = sort { $a->{$sort_key} <=> $b->{$sort_key} } @$data;
> + $cmpfn = sub { $_[0] <=> $_[1] };
>   } else {
> - @$data = sort { $a->{$sort_key} cmp $b->{$sort_key} } @$data;
> + $cmpfn = sub { $_[0] cmp $_[1] };
>   }
> + @$data = sort {
> + PVE::Tools::safe_compare($a->{$sort_key}, $b->{$sort_key}, $cmpfn)
> + } @$data;
>  }
>  
>  my $colopts = {};
> diff --git a/src/PVE/Tools.pm b/src/PVE/Tools.pm
> index f02c0ae..f6b18f1 100644
> --- a/src/PVE/Tools.pm
> +++ b/src/PVE/Tools.pm
> @@ -1766,4 +1766,13 @@ sub mount($) {
>  );
>  }
>  
> +sub safe_compare {
> +my ($left, $right, $cmp) = @_;
> +
> +return 0 if !defined($left) && !defined($right);
> +return -1 if !defined($left);
> +return 1 if !defined($right);
> +return $cmp->($left, $right);
> +}
> +
>  1;
> -- 
> 2.20.1
> 
> 
> ___
> pve-devel mailing list
> pve-devel@pve.proxmox.com
> https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH guest-common 2/2] fix 2682: make sure configuration file is up-to-date for lock_config-variants

2020-04-28 Thread Fabian Ebner 

This patch breaks container creation for some reason. It'll fail with:
unable to create CT : config file already exists
I'll investigate why this happens.

On 23.04.20 13:51, Fabian Ebner wrote:

See [0] for the details. The call tree for the variants is
lock_config -> lock_config_full -> lock_config_mode
so it is sufficient to adapt lock_config_mode.

[0]: https://bugzilla.proxmox.com/show_bug.cgi?id=2682

Suggested-by: Fabian Grünbichler 
Signed-off-by: Fabian Ebner 
---
  PVE/AbstractConfig.pm | 8 +++-
  1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/PVE/AbstractConfig.pm b/PVE/AbstractConfig.pm
index f1b395c..15368de 100644
--- a/PVE/AbstractConfig.pm
+++ b/PVE/AbstractConfig.pm
@@ -292,7 +292,13 @@ sub lock_config_mode {
  
  my $filename = $class->config_file_lock($vmid);
  
-my $res = lock_file_full($filename, $timeout, $shared, $code, @param);

+# make sure configuration file is up-to-date
+my $realcode = sub {
+   PVE::Cluster::cfs_update();
+   $code->(@param);
+};
+
+my $res = lock_file_full($filename, $timeout, $shared, $realcode, @param);
  
  die $@ if $@;
  



___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH guest-common 2/2] fix 2682: make sure configuration file is up-to-date for lock_config-variants

2020-04-28 Thread Fabian Ebner 

On 28.04.20 13:00, Fabian Ebner wrote:

This patch breaks container creation for some reason. It'll fail with:
unable to create CT : config file already exist

the real error message is:
unable to create CT : CT  already exists on node ''
I pasted the wrong one in a hurry


I'll investigate why this happens.


The reason is that in create_vm for containers, update_pct_config is 
called before create_and_lock_config and the former leads to a 
write_config call (via vmconfig_apply_pending).




On 23.04.20 13:51, Fabian Ebner wrote:

See [0] for the details. The call tree for the variants is
lock_config -> lock_config_full -> lock_config_mode
so it is sufficient to adapt lock_config_mode.

[0]: https://bugzilla.proxmox.com/show_bug.cgi?id=2682

Suggested-by: Fabian Grünbichler 
Signed-off-by: Fabian Ebner 
---
  PVE/AbstractConfig.pm | 8 +++-
  1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/PVE/AbstractConfig.pm b/PVE/AbstractConfig.pm
index f1b395c..15368de 100644
--- a/PVE/AbstractConfig.pm
+++ b/PVE/AbstractConfig.pm
@@ -292,7 +292,13 @@ sub lock_config_mode {
  my $filename = $class->config_file_lock($vmid);
-    my $res = lock_file_full($filename, $timeout, $shared, $code, 
@param);

+    # make sure configuration file is up-to-date
+    my $realcode = sub {
+    PVE::Cluster::cfs_update();
+    $code->(@param);
+    };
+
+    my $res = lock_file_full($filename, $timeout, $shared, $realcode, 
@param);

  die $@ if $@;



___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Re: [pve-devel] [PATCH container 1/2] LXC: drop unused imported locking functions

2020-04-28 Thread Fabian Ebner 
Played around for a bit with your patches applied on top of mine and 
found no obvious issue, except for LXC creation [0] which is exposed by 
my second patch.


So for Fabian G.'s patches:
Tested-By: Fabian Ebner 

[0]: https://pve.proxmox.com/pipermail/pve-devel/2020-April/043210.html

On 27.04.20 10:24, Fabian Grünbichler wrote:

Signed-off-by: Fabian Grünbichler 
---
  src/PVE/LXC.pm | 4 ++--
  1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/PVE/LXC.pm b/src/PVE/LXC.pm
index fbe736e..651ec08 100644
--- a/src/PVE/LXC.pm
+++ b/src/PVE/LXC.pm
@@ -20,8 +20,8 @@ use PVE::SafeSyslog;
  use PVE::INotify;
  use PVE::JSONSchema qw(get_standard_option);
  use PVE::Tools qw(
-dir_glob_foreach file_get_contents file_set_contents lock_file
-lock_file_full AT_FDCWD O_PATH $IPV4RE $IPV6RE
+dir_glob_foreach file_get_contents file_set_contents
+AT_FDCWD O_PATH $IPV4RE $IPV6RE
  );
  use PVE::CpuSet;
  use PVE::Network;



___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server] restore: use correct storage for format check for cloudinit drives

2020-04-28 Thread Dominik Csapak 
when a backup includes a cloudinit disk on a non-existent storage,
the restore fails with 'storage' does not exist

this happens because we want to get the format of the disk, by
checking the source storage

we fix this by using the target storage first and only the source as
fallback

this will still fail if neither storage exists
(which is ok, since we cannot restore then anyway)

Signed-off-by: Dominik Csapak 
---
 PVE/QemuServer.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 37c7320..e355810 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -5739,7 +5739,7 @@ my $parse_backup_hints = sub {
my $drive = parse_drive($virtdev, $2);
if (drive_is_cloudinit($drive)) {
my ($storeid, $volname) = 
PVE::Storage::parse_volume_id($drive->{file});
-   my $scfg = PVE::Storage::storage_config($storecfg, $storeid);
+   my $scfg = PVE::Storage::storage_config($storecfg, 
$options->{storage} // $storeid);
my $format = qemu_img_format($scfg, $volname); # has 'raw' 
fallback
 
$virtdev_hash->{$virtdev} = {
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 00/17] Fix: #2124 zstd

2020-04-28 Thread Alwin Antreich 
Zstandard (zstd) [0] is a data compression algorithm, in addition to
gzip, lzo for our backup/restore. It can utilize multiple core CPUs. But
by default it has one compression and one writer thread.


Here some quick tests I made on my workstation. The files where placed
on a ram disk. And with dd filled from /dev/urandom and /dev/zero.

__Compression__
file size: 1073741824 bytes
  = urandom = = zero =
  995ms 1073766414328ms 98192zstd -k
  732ms 1073766414295ms 98192zstd -k -T4
  906ms 1073791036562ms 4894779  lzop -k
31992ms 1073915558   5594ms 1042087  gzip -k
30832ms 1074069541   5776ms 1171491  pigz -k -p 1
 7814ms 1074069541   1567ms 1171491  pigz -k -p 4

__Decompression__
file size: 1073741824 bytes
= urandom =   = zero =
   712ms  869ms  zstd -d
   685ms  872ms  zstd -k -d -T4
   841ms 2462ms  lzop -d
  5417ms 4754ms  gzip -k -d
  1248ms 3118ms  pigz -k -d -p 1
  1236ms 2379ms  pigz -k -d -p 4


And I used the same ramdisk to move a VM onto it and run a quick
backup/restore.

__vzdump backup__
INFO: transferred 34359 MB in 69 seconds (497 MB/s) zstd -T1
INFO: transferred 34359 MB in 37 seconds (928 MB/s) zstd -T4
INFO: transferred 34359 MB in 51 seconds (673 MB/s) lzo
INFO: transferred 34359 MB in 1083 seconds (31 MB/s) gzip
INFO: transferred 34359 MB in 241 seconds (142 MB/s) pigz -n 4

__qmrestore__
progress 100% (read 34359738368 bytes, duration 36 sec)
total bytes read 34359738368, sparse bytes 8005484544 (23.3%) zstd -d -T4

progress 100% (read 34359738368 bytes, duration 38 sec)
total bytes read 34359738368, sparse bytes 8005484544 (23.3%) lzo

progress 100% (read 34359738368 bytes, duration 175 sec)
total bytes read 34359738368, sparse bytes 8005484544 (23.3%) pigz -n 4


v4 -> v5:
* fixup, use File::stat directly without overwriting CORE::stat,
  thanks Dietmar for pointing this out
  https://pve.proxmox.com/pipermail/pve-devel/2020-April/043134.html
* rebase to current master

v3 -> v4:
* fixed styling issues discovered by f.ebner (thanks)
* incorporated tests of d.jaeger into patches
* added fixes discovered by tests (f.ebner thanks)

v2 -> v3:
* split archive_info into decompressor_info and archive_info
* "compact" regex pattern is now a constant and used in
  multiple modules
* added tests for regex matching
* bug fix for ctime of backup files

v1 -> v2:
* factored out the decompressor info first, as Thomas suggested
* made the regex pattern of backup files more compact, easier to
  read (hopefully)
* less code changes for container restores

Thanks for any comment or suggestion in advance.

[0] https://facebook.github.io/zstd/

Alwin Antreich (17):
__pve-storage__
  storage: test: split archive format/compressor
  storage: replace build-in stat with File::stat
  test: parse_volname
  test: list_volumes
  Fix: backup: ctime was from stat not file name
  test: path_to_volume_id
  Fix: path_to_volume_id returned wrong content
  Fix: add missing snippets subdir
  backup: compact regex for backup file filter
  Fix: #2124 storage: add zstd support
  test: get_subdir
  test: filesystem_path

 test/Makefile  |   5 +-
 PVE/Diskmanage.pm  |   9 +-
 PVE/Storage.pm |  91 --
 PVE/Storage/Plugin.pm  |  47 ++-
 test/archive_info_test.pm  | 127 
 test/filesystem_path_test.pm   |  91 ++
 test/get_subdir_test.pm|  44 +++
 test/list_volumes_test.pm  | 537 +
 test/parse_volname_test.pm | 253 
 test/path_to_volume_id_test.pm | 274 +
 test/run_plugin_tests.pl   |  18 ++
 11 files changed, 1440 insertions(+), 56 deletions(-)
 create mode 100644 test/archive_info_test.pm
 create mode 100644 test/filesystem_path_test.pm
 create mode 100644 test/get_subdir_test.pm
 create mode 100644 test/list_volumes_test.pm
 create mode 100644 test/parse_volname_test.pm
 create mode 100644 test/path_to_volume_id_test.pm
 create mode 100755 test/run_plugin_tests.pl


__guest_common__
  Fix: #2124 add zstd support

 PVE/VZDump/Common.pm | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)


__qemu-server__
  restore: replace archive format/compression
  Fix #2124: Add support for zstd

 PVE/QemuServer.pm | 38 +++---
 1 file changed, 7 insertions(+), 31 deletions(-)


__pve-container__
  Fix: #2124 add zstd

 src/PVE/LXC/Create.pm | 1 +
 1 file changed, 1 insertion(+)


__pve-manager__
  Fix #2124: Add support for zstd

 PVE/VZDump.pm| 11 +--
 debian/control   |  1 +
 www/manager6/form/CompressionSelector.js |  3 ++-
 3 files changed, 12 insertions(+), 3 deletions(-)
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH guest-common v5] Fix: #2124 add zstd support

2020-04-28 Thread Alwin Antreich 
Signed-off-by: Alwin Antreich 
---
 PVE/VZDump/Common.pm | 9 -
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/PVE/VZDump/Common.pm b/PVE/VZDump/Common.pm
index 4789a50..909e3af 100644
--- a/PVE/VZDump/Common.pm
+++ b/PVE/VZDump/Common.pm
@@ -88,7 +88,7 @@ my $confdesc = {
type => 'string',
description => "Compress dump file.",
optional => 1,
-   enum => ['0', '1', 'gzip', 'lzo'],
+   enum => ['0', '1', 'gzip', 'lzo', 'zstd'],
default => '0',
 },
 pigz=> {
@@ -98,6 +98,13 @@ my $confdesc = {
optional => 1,
default => 0,
 },
+zstd => {
+   type => "integer",
+   description => "Zstd threads. N=0 uses half of the available cores,".
+   " N>0 uses N as thread count.",
+   optional => 1,
+   default => 1,
+},
 quiet => {
type => 'boolean',
description => "Be quiet.",
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH manager v5] Fix #2124: Add support for zstd

2020-04-28 Thread Alwin Antreich 
This patch adds the zstd to the compression selection for backup on the
GUI and add .zst to the backup file filter. Including zstd as package
install dependency.

Signed-off-by: Alwin Antreich 
---
 PVE/VZDump.pm| 11 +--
 debian/control   |  1 +
 www/manager6/form/CompressionSelector.js |  3 ++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/PVE/VZDump.pm b/PVE/VZDump.pm
index f3274196..80f4734c 100644
--- a/PVE/VZDump.pm
+++ b/PVE/VZDump.pm
@@ -609,6 +609,13 @@ sub compressor_info {
} else {
return ('gzip --rsyncable', 'gz');
}
+} elsif ($opt_compress eq 'zstd') {
+   my $zstd_threads = $opts->{zstd} // 1;
+   if ($zstd_threads == 0) {
+   my $cpuinfo = PVE::ProcFSTools::read_cpuinfo();
+   $zstd_threads = int(($cpuinfo->{cpus} + 1)/2);
+   }
+   return ("zstd --threads=${zstd_threads}", 'zst');
 } else {
die "internal error - unknown compression option '$opt_compress'";
 }
@@ -620,7 +627,7 @@ sub get_backup_file_list {
 my $bklist = [];
 foreach my $fn (<$dir/${bkname}-*>) {
next if $exclude_fn && $fn eq $exclude_fn;
-   if ($fn =~ 
m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?)))$!)
 {
+   if ($fn =~ 
m!/(${bkname}-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)))$!)
 {
$fn = "$dir/$1"; # untaint
my $t = timelocal ($7, $6, $5, $4, $3 - 1, $2);
push @$bklist, [$fn, $t];
@@ -928,7 +935,7 @@ sub exec_backup_task {
debugmsg ('info', "delete old backup '$d->[0]'", $logfd);
unlink $d->[0];
my $logfn = $d->[0];
-   $logfn =~ s/\.(tgz|((tar|vma)(\.(gz|lzo))?))$/\.log/;
+   $logfn =~ 
s/\.(tgz|((tar|vma)(\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?))$/\.log/;
unlink $logfn;
}
}
diff --git a/debian/control b/debian/control
index edb2833d..318b4f0e 100644
--- a/debian/control
+++ b/debian/control
@@ -60,6 +60,7 @@ Depends: apt-transport-https | apt (>= 1.5~),
  logrotate,
  lsb-base,
  lzop,
+ zstd,
  novnc-pve,
  pciutils,
  perl (>= 5.10.0-19),
diff --git a/www/manager6/form/CompressionSelector.js 
b/www/manager6/form/CompressionSelector.js
index 8938fc0e..842b7710 100644
--- a/www/manager6/form/CompressionSelector.js
+++ b/www/manager6/form/CompressionSelector.js
@@ -4,6 +4,7 @@ Ext.define('PVE.form.CompressionSelector', {
 comboItems: [
 ['0', Proxmox.Utils.noneText],
 ['lzo', 'LZO (' + gettext('fast') + ')'],
-['gzip', 'GZIP (' + gettext('good') + ')']
+['gzip', 'GZIP (' + gettext('good') + ')'],
+['zstd', 'ZSTD (' + gettext('better') + ')'],
 ]
 });
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 03/12] test: parse_volname

2020-04-28 Thread Alwin Antreich 
Test to reduce the potential for accidental breakage on regex changes.
And to make sure that all vtype_subdirs are parsed.

Signed-off-by: Alwin Antreich 
---
 PVE/Storage/Plugin.pm  |   4 +
 test/parse_volname_test.pm | 253 +
 test/run_plugin_tests.pl   |   2 +-
 3 files changed, 258 insertions(+), 1 deletion(-)
 create mode 100644 test/parse_volname_test.pm

diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index dba6eb9..0925910 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -457,6 +457,10 @@ my $vtype_subdirs = {
 snippets => 'snippets',
 };
 
+sub get_vtype_subdirs {
+return $vtype_subdirs;
+}
+
 sub get_subdir {
 my ($class, $scfg, $vtype) = @_;
 
diff --git a/test/parse_volname_test.pm b/test/parse_volname_test.pm
new file mode 100644
index 000..87c758c
--- /dev/null
+++ b/test/parse_volname_test.pm
@@ -0,0 +1,253 @@
+package PVE::Storage::TestParseVolname;
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use PVE::Storage;
+use Test::More;
+
+my $vmid = 1234;
+
+# an array of test cases, each test is comprised of the following keys:
+# description => to identify a single test
+# volname => the input for parse_volname
+# expected=> the array that parse_volname returns
+my $tests = [
+#
+# VM images
+#
+{
+   description => 'VM disk image, linked, qcow2, vm- as base-',
+   volname => 
"$vmid/vm-$vmid-disk-0.qcow2/$vmid/vm-$vmid-disk-0.qcow2",
+   expected=> [ 'images', "vm-$vmid-disk-0.qcow2", "$vmid", 
"vm-$vmid-disk-0.qcow2", "$vmid", undef, 'qcow2', ],
+},
+#
+# iso
+#
+{
+   description => 'ISO image, iso',
+   volname => 'iso/some-installation-disk.iso',
+   expected=> ['iso', 'some-installation-disk.iso'],
+},
+{
+   description => 'ISO image, img',
+   volname => 'iso/some-other-installation-disk.img',
+   expected=> ['iso', 'some-other-installation-disk.img'],
+},
+#
+# container templates
+#
+{
+   description => 'Container template tar.gz',
+   volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
+   expected=> ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.gz'],
+},
+{
+   description => 'Container template tar.xz',
+   volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.xz',
+   expected=> ['vztmpl', 'debian-10.0-standard_10.0-1_amd64.tar.xz'],
+},
+#
+# container rootdir
+#
+{
+   description => 'Container rootdir, sub directory',
+   volname => "rootdir/$vmid",
+   expected=> ['rootdir', "$vmid", "$vmid"],
+},
+{
+   description => 'Container rootdir, subvol',
+   volname => "$vmid/subvol-$vmid-disk-0.subvol",
+   expected=> [ 'images', "subvol-$vmid-disk-0.subvol", "$vmid", 
undef, undef, undef, 'subvol' ],
+},
+{
+   description => 'Backup archive, no virtualization type',
+   volname => "backup/vzdump-none-$vmid-2020_03_30-21_39_30.tar",
+   expected=> ['backup', "vzdump-none-$vmid-2020_03_30-21_39_30.tar"],
+},
+#
+# Snippets
+#
+{
+   description => 'Snippets, yaml',
+   volname => 'snippets/userconfig.yaml',
+   expected=> ['snippets', 'userconfig.yaml'],
+},
+{
+   description => 'Snippets, perl',
+   volname => 'snippets/hookscript.pl',
+   expected=> ['snippets', 'hookscript.pl'],
+},
+#
+# failed matches
+#
+{
+   description => "Failed match: VM disk image, base, raw",
+   volname => "/base-$vmid-disk-0.raw",
+   expected=> "unable to parse directory volume name 
'/base-$vmid-disk-0.raw'\n",
+},
+{
+   description => 'Failed match: ISO image, dvd',
+   volname => 'iso/yet-again-a-installation-disk.dvd',
+   expected=> "unable to parse directory volume name 
'iso/yet-again-a-installation-disk.dvd'\n",
+},
+{
+   description => 'Failed match: Container template, zip.gz',
+   volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.zip.gz',
+   expected=> "unable to parse directory volume name 
'vztmpl/debian-10.0-standard_10.0-1_amd64.zip.gz'\n",
+},
+{
+   description => 'Failed match: Container template, tar.bz2',
+   volname => 'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2',
+   expected=> "unable to parse directory volume name 
'vztmpl/debian-10.0-standard_10.0-1_amd64.tar.bz2'\n",
+},
+{
+   description => 'Failed match: Container rootdir, subvol',
+   volname => "rootdir/subvol-$vmid-disk-0",
+   expected=> "unable to parse directory volume name 
'rootdir/subvol-$vmid-disk-0'\n",
+},
+{
+   description => 'Failed match: VM disk image, linked, vhdx',
+   volname => 
"$vmid/base-$vmid-disk-0.vhdx/$vmid/vm-$vmid-disk-0.vhdx",
+   expected=> "unable to

[pve-devel] [PATCH storage v5 05/12] Fix: backup: ctime was from stat not file name

2020-04-28 Thread Alwin Antreich 
The vzdump file was passed with the full path to the regex. That regex
captures the time from the file name, to calculate the epoch.

As the regex didn't match, the ctime from stat was taken instead. This
resulted in the ctime shown when the file was changed, not when the
backup was made.

Signed-off-by: Alwin Antreich 
---
 PVE/Storage/Plugin.pm |  3 ++-
 test/list_volumes_test.pm | 16 
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 0925910..a6071eb 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -942,7 +942,8 @@ my $get_subdir_files = sub {
next if $fn !~ 
m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!;
 
my $format = $2;
-   $info = { volid => "$sid:backup/$1", format => $format };
+   $fn = $1;
+   $info = { volid => "$sid:backup/$fn", format => $format };
 
if ($fn =~ 
m!^vzdump\-(?:lxc|qemu)\-(?:[1-9][0-9]{2,8})\-(\d{4})_(\d{2})_(\d{2})\-(\d{2})_(\d{2})_(\d{2})\.${format}$!)
 {
my $epoch = timelocal($6, $5, $4, $3, $2-1, $1 - 1900);
diff --git a/test/list_volumes_test.pm b/test/list_volumes_test.pm
index a215617..941b903 100644
--- a/test/list_volumes_test.pm
+++ b/test/list_volumes_test.pm
@@ -129,7 +129,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1585595500,
'format'  => 'vma.gz',
'size'=> DEFAULT_SIZE,
'vmid'=> '16110',
@@ -137,7 +137,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1585595565,
'format'  => 'vma.lzo',
'size'=> DEFAULT_SIZE,
'vmid'=> '16110',
@@ -145,7 +145,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1585595635,
'format'  => 'vma',
'size'=> DEFAULT_SIZE,
'vmid'=> '16110',
@@ -189,7 +189,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1585597170,
'format'  => 'tar.lzo',
'size'=> DEFAULT_SIZE,
'vmid'=> '16112',
@@ -197,7 +197,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1585597770,
'format'  => 'tar.gz',
'size'=> DEFAULT_SIZE,
'vmid'=> '16112',
@@ -205,7 +205,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1585598370,
'format'  => 'tgz',
'size'=> DEFAULT_SIZE,
'vmid'=> '16112',
@@ -347,7 +347,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1580756263,
'format'  => 'tar.gz',
'size'=> DEFAULT_SIZE,
'vmid'=> '19253',
@@ -355,7 +355,7 @@ my @tests = (
},
{
'content' => 'backup',
-   'ctime'   => DEFAULT_CTIME,
+   'ctime'   => 1548095359,
'format'  => 'tar',
'size'=> DEFAULT_SIZE,
'vmid'=> '19254',
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 11/12] test: get_subdir

2020-04-28 Thread Alwin Antreich 
Co-Authored-by: Dominic Jaeger 
Signed-off-by: Alwin Antreich 
---
 test/get_subdir_test.pm  | 44 
 test/run_plugin_tests.pl |  1 +
 2 files changed, 45 insertions(+)
 create mode 100644 test/get_subdir_test.pm

diff --git a/test/get_subdir_test.pm b/test/get_subdir_test.pm
new file mode 100644
index 000..576c475
--- /dev/null
+++ b/test/get_subdir_test.pm
@@ -0,0 +1,44 @@
+package PVE::Storage::TestGetSubdir;
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use PVE::Storage::Plugin;
+use Test::More;
+
+my $scfg_with_path = { path => '/some/path' };
+my $vtype_subdirs = PVE::Storage::Plugin::get_vtype_subdirs();
+
+# each test is comprised of the following array keys:
+# [0] => storage config; positive with path key
+# [1] => storage type;  see $vtype_subdirs
+# [2] => expected return from get_subdir
+my $tests = [
+# failed matches
+[ $scfg_with_path, 'none', "unknown vtype 'none'\n" ],
+[ {}, 'iso', "storage definintion has no path\n" ],
+];
+
+# creates additional positive tests
+foreach my $type (keys %$vtype_subdirs) {
+my $path = "$scfg_with_path->{path}/$vtype_subdirs->{$type}";
+push @$tests, [ $scfg_with_path, $type, $path ];
+}
+
+plan tests => scalar @$tests;
+
+foreach my $tt (@$tests) {
+my ($scfg, $type, $expected) = @$tt;
+
+my $got;
+eval { $got = PVE::Storage::Plugin->get_subdir($scfg, $type) };
+$got = $@ if $@;
+
+is ($got, $expected, "get_subdir for $type") || diag(explain($got));
+}
+
+done_testing();
+
+1;
diff --git a/test/run_plugin_tests.pl b/test/run_plugin_tests.pl
index 770b407..9e427eb 100755
--- a/test/run_plugin_tests.pl
+++ b/test/run_plugin_tests.pl
@@ -11,6 +11,7 @@ my $res = $harness->runtests(
 "parse_volname_test.pm",
 "list_volumes_test.pm",
 "path_to_volume_id_test.pm",
+"get_subdir_test.pm",
 );
 
 exit -1 if !$res || $res->{failed} || $res->{parse_errors};
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 02/12] storage: replace build-in stat occurrences

2020-04-28 Thread Alwin Antreich 
with File::stat::stat to minimize variable declarations. And allow to
mock this method in tests instead of the perl build-in stat.

Signed-off-by: Alwin Antreich 
---
 PVE/Diskmanage.pm |  9 +
 PVE/Storage/Plugin.pm | 34 ++
 2 files changed, 15 insertions(+), 28 deletions(-)

diff --git a/PVE/Diskmanage.pm b/PVE/Diskmanage.pm
index 13e7cd8..cac944d 100644
--- a/PVE/Diskmanage.pm
+++ b/PVE/Diskmanage.pm
@@ -6,6 +6,7 @@ use PVE::ProcFSTools;
 use Data::Dumper;
 use Cwd qw(abs_path);
 use Fcntl ':mode';
+use File::stat;
 use JSON;
 
 use PVE::Tools qw(extract_param run_command file_get_contents 
file_read_firstline dir_glob_regex dir_glob_foreach trim);
@@ -673,11 +674,11 @@ sub get_disks {
 sub get_partnum {
 my ($part_path) = @_;
 
-my ($mode, $rdev) = (stat($part_path))[2,6];
+my $st = stat($part_path);
 
-next if !$mode || !S_ISBLK($mode) || !$rdev;
-my $major = PVE::Tools::dev_t_major($rdev);
-my $minor = PVE::Tools::dev_t_minor($rdev);
+next if !$st->mode || !S_ISBLK($st->mode) || !$st->rdev;
+my $major = PVE::Tools::dev_t_major($st->rdev);
+my $minor = PVE::Tools::dev_t_minor($st->rdev);
 my $partnum_path = "/sys/dev/block/$major:$minor/";
 
 my $partnum;
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 4489a77..dba6eb9 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -7,6 +7,7 @@ use Fcntl ':mode';
 use File::chdir;
 use File::Path;
 use File::Basename;
+use File::stat qw();
 use Time::Local qw(timelocal);
 
 use PVE::Tools qw(run_command);
@@ -718,12 +719,10 @@ sub free_image {
 sub file_size_info {
 my ($filename, $timeout) = @_;
 
-my @fs = stat($filename);
-my $mode = $fs[2];
-my $ctime = $fs[10];
+my $st = File::stat::stat($filename);
 
-if (S_ISDIR($mode)) {
-   return wantarray ? (0, 'subvol', 0, undef, $ctime) : 1;
+if (S_ISDIR($st->mode)) {
+   return wantarray ? (0, 'subvol', 0, undef, $st->ctime) : 1;
 }
 
 my $json = '';
@@ -741,7 +740,7 @@ sub file_size_info {
 
 my ($size, $format, $used, $parent) = $info->@{qw(virtual-size format 
actual-size backing-filename)};
 
-return wantarray ? ($size, $format, $used, $parent, $ctime) : $size;
+return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size;
 }
 
 sub volume_size_info {
@@ -918,22 +917,9 @@ my $get_subdir_files = sub {
 
 foreach my $fn (<$path/*>) {
 
-   my ($dev,
-   $ino,
-   $mode,
-   $nlink,
-   $uid,
-   $gid,
-   $rdev,
-   $size,
-   $atime,
-   $mtime,
-   $ctime,
-   $blksize,
-   $blocks
-   ) = stat($fn);
-
-   next if S_ISDIR($mode);
+   my $st = File::stat::stat($fn);
+
+   next if S_ISDIR($st->mode);
 
my $info;
 
@@ -972,8 +958,8 @@ my $get_subdir_files = sub {
};
}
 
-   $info->{size} = $size;
-   $info->{ctime} //= $ctime;
+   $info->{size} = $st->size;
+   $info->{ctime} //= $st->ctime;
 
push @$res, $info;
 }
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server v5 1/2] restore: replace archive format/compression

2020-04-28 Thread Alwin Antreich 
regex to reduce the code duplication, as archive_info and
decompressor_info provides the same information as well.

Signed-off-by: Alwin Antreich 
---
 PVE/QemuServer.pm | 36 ++--
 1 file changed, 6 insertions(+), 30 deletions(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 37c7320..265d4f8 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -5627,28 +5627,9 @@ sub tar_restore_cleanup {
 sub restore_file_archive {
 my ($archive, $vmid, $user, $opts) = @_;
 
-my $format = $opts->{format};
-my $comp;
-
-if ($archive =~ m/\.tgz$/ || $archive =~ m/\.tar\.gz$/) {
-   $format = 'tar' if !$format;
-   $comp = 'gzip';
-} elsif ($archive =~ m/\.tar$/) {
-   $format = 'tar' if !$format;
-} elsif ($archive =~ m/.tar.lzo$/) {
-   $format = 'tar' if !$format;
-   $comp = 'lzop';
-} elsif ($archive =~ m/\.vma$/) {
-   $format = 'vma' if !$format;
-} elsif ($archive =~ m/\.vma\.gz$/) {
-   $format = 'vma' if !$format;
-   $comp = 'gzip';
-} elsif ($archive =~ m/\.vma\.lzo$/) {
-   $format = 'vma' if !$format;
-   $comp = 'lzop';
-} else {
-   $format = 'vma' if !$format; # default
-}
+my $info = PVE::Storage::archive_info($archive);
+my $format = $opts->{format} // $info->{format};
+my $comp = $info->{compression};
 
 # try to detect archive format
 if ($format eq 'tar') {
@@ -6235,14 +6216,9 @@ sub restore_vma_archive {
 }
 
 if ($comp) {
-   my $cmd;
-   if ($comp eq 'gzip') {
-   $cmd = ['zcat', $readfrom];
-   } elsif ($comp eq 'lzop') {
-   $cmd = ['lzop', '-d', '-c', $readfrom];
-   } else {
-   die "unknown compression method '$comp'\n";
-   }
+   my $info = PVE::Storage::decompressor_info('vma', $comp);
+   my $cmd = $info->{decompressor};
+   push @$cmd, $readfrom;
$add_pipe->($cmd);
 }
 
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 07/12] Fix: path_to_volume_id returned wrong content

2020-04-28 Thread Alwin Antreich 
type for backup files. Patch includes changes of the test as well.

Signed-off-by: Alwin Antreich 
---
 PVE/Storage.pm | 2 +-
 test/path_to_volume_id_test.pm | 8 
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index bdd6ebc..1ef5ed2 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -536,7 +536,7 @@ sub path_to_volume_id {
return ('rootdir', "$sid:rootdir/$vmid");
} elsif ($path =~ 
m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) {
my $name = $1;
-   return ('iso', "$sid:backup/$name");
+   return ('backup', "$sid:backup/$name");
}
 }
 
diff --git a/test/path_to_volume_id_test.pm b/test/path_to_volume_id_test.pm
index 744c3ee..7d69869 100644
--- a/test/path_to_volume_id_test.pm
+++ b/test/path_to_volume_id_test.pm
@@ -72,7 +72,7 @@ my @tests = (
description => 'Backup, vma.gz',
volname => 
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
expected=> [
-   'iso',
+   'backup',
'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz',
],
 },
@@ -80,7 +80,7 @@ my @tests = (
description => 'Backup, vma.lzo',
volname => 
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
expected=> [
-   'iso',
+   'backup',
'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo',
],
 },
@@ -88,7 +88,7 @@ my @tests = (
description => 'Backup, vma',
volname => 
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
expected=> [
-   'iso',
+   'backup',
'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma',
],
 },
@@ -96,7 +96,7 @@ my @tests = (
description => 'Backup, tar.lzo',
volname => 
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
expected=> [
-   'iso',
+   'backup',
'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo',
],
 },
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 09/12] backup: compact regex for backup file filter

2020-04-28 Thread Alwin Antreich 
the more compact form of the regex should allow easier addition of new
file extensions.

Signed-off-by: Alwin Antreich 
---
 PVE/Storage.pm| 4 ++--
 PVE/Storage/Plugin.pm | 6 --
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 5df074d..0b2745e 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -535,7 +535,7 @@ sub path_to_volume_id {
} elsif ($path =~ m!^$privatedir/(\d+)$!) {
my $vmid = $1;
return ('rootdir', "$sid:rootdir/$vmid");
-   } elsif ($path =~ 
m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) {
+   } elsif ($path =~ 
m!^$backupdir/([^/]+\.(?:tgz|(?:(?:tar|vma)(?:\.(?:${\PVE::Storage::Plugin::COMPRESSOR_RE}))?)))$!)
 {
my $name = $1;
return ('backup', "$sid:backup/$name");
} elsif ($path =~ m!^$snippetsdir/([^/]+)$!) {
@@ -1392,7 +1392,7 @@ sub archive_info {
 my $info;
 
 my $volid = basename($archive);
-if ($volid =~ 
/vzdump-(lxc|openvz|qemu)-\d+-(?:\d{4})_(?:\d{2})_(?:\d{2})-(?:\d{2})_(?:\d{2})_(?:\d{2})\.(tgz$|tar|vma)(?:\.(gz|lzo))?$/)
 {
+if ($volid =~ 
/vzdump-(lxc|openvz|qemu)-\d+-(?:\d{4})_(?:\d{2})_(?:\d{2})-(?:\d{2})_(?:\d{2})_(?:\d{2})\.(tgz$|tar|vma)(?:\.(${\PVE::Storage::Plugin::COMPRESSOR_RE}))?$/)
 {
$info = decompressor_info($2, $3);
$info->{type} = $1;
 } else {
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index a6071eb..5f3e4c1 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -18,6 +18,8 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
+use constant COMPRESSOR_RE => 'gz|lzo';
+
 our @COMMON_TAR_FLAGS = qw(
 --one-file-system
 -p --sparse --numeric-owner --acls
@@ -435,7 +437,7 @@ sub parse_volname {
return ('vztmpl', $1);
 } elsif ($volname =~ m!^rootdir/(\d+)$!) {
return ('rootdir', $1, $1);
-} elsif ($volname =~ 
m!^backup/([^/]+(\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo)))$!) {
+} elsif ($volname =~ 
m!^backup/([^/]+(?:\.(?:tgz|(?:(?:tar|vma)(?:\.(?:${\COMPRESSOR_RE}))?$!) {
my $fn = $1;
if ($fn =~ m/^vzdump-(openvz|lxc|qemu)-(\d+)-.+/) {
return ('backup', $fn, $2);
@@ -939,7 +941,7 @@ my $get_subdir_files = sub {
 
} elsif ($tt eq 'backup') {
next if defined($vmid) && $fn !~  m/\S+-$vmid-\S+/;
-   next if $fn !~ 
m!/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!;
+   next if $fn !~ 
m!/([^/]+\.(tgz|(?:(?:tar|vma)(?:\.(${\COMPRESSOR_RE}))?)))$!;
 
my $format = $2;
$fn = $1;
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH qemu-server v5 2/2] Fix #2124: Add support for zstd

2020-04-28 Thread Alwin Antreich 
Signed-off-by: Alwin Antreich 
---
 PVE/QemuServer.pm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
index 265d4f8..fda1acb 100644
--- a/PVE/QemuServer.pm
+++ b/PVE/QemuServer.pm
@@ -7165,7 +7165,7 @@ sub complete_backup_archives {
 my $res = [];
 foreach my $id (keys %$data) {
foreach my $item (@{$data->{$id}}) {
-   next if $item->{format} !~ m/^vma\.(gz|lzo)$/;
+   next if $item->{format} !~ 
m/^vma\.(${\PVE::Storage::Plugin::COMPRESSOR_RE})$/;
push @$res, $item->{volid} if defined($item->{volid});
}
 }
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH container v5] Fix: #2124 add zstd

2020-04-28 Thread Alwin Antreich 
Signed-off-by: Alwin Antreich 
---
 src/PVE/LXC/Create.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/PVE/LXC/Create.pm b/src/PVE/LXC/Create.pm
index 52b0b48..39902a2 100644
--- a/src/PVE/LXC/Create.pm
+++ b/src/PVE/LXC/Create.pm
@@ -123,6 +123,7 @@ sub restore_tar_archive {
'.bz2' => '-j',
'.xz'  => '-J',
'.lzo'  => '--lzop',
+   '.zst'  => '--zstd',
);
if ($archive =~ /\.tar(\.[^.]+)?$/) {
if (defined($1)) {
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 08/12] Fix: add missing snippets subdir

2020-04-28 Thread Alwin Antreich 
since it is a valid content type and adapt the path_to_volume_id_test.
Also adds an extra check if all vtype_subdirs are returned.

Signed-off-by: Alwin Antreich 
---
 PVE/Storage.pm |  4 
 test/path_to_volume_id_test.pm | 26 +-
 2 files changed, 25 insertions(+), 5 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 1ef5ed2..5df074d 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -512,6 +512,7 @@ sub path_to_volume_id {
my $tmpldir = $plugin->get_subdir($scfg, 'vztmpl');
my $backupdir = $plugin->get_subdir($scfg, 'backup');
my $privatedir = $plugin->get_subdir($scfg, 'rootdir');
+   my $snippetsdir = $plugin->get_subdir($scfg, 'snippets');
 
if ($path =~ m!^$imagedir/(\d+)/([^/\s]+)$!) {
my $vmid = $1;
@@ -537,6 +538,9 @@ sub path_to_volume_id {
} elsif ($path =~ 
m!^$backupdir/([^/]+\.(tar|tar\.gz|tar\.lzo|tgz|vma|vma\.gz|vma\.lzo))$!) {
my $name = $1;
return ('backup', "$sid:backup/$name");
+   } elsif ($path =~ m!^$snippetsdir/([^/]+)$!) {
+   my $name = $1;
+   return ('snippets', "$sid:snippets/$name");
}
 }
 
diff --git a/test/path_to_volume_id_test.pm b/test/path_to_volume_id_test.pm
index 7d69869..e5e24c1 100644
--- a/test/path_to_volume_id_test.pm
+++ b/test/path_to_volume_id_test.pm
@@ -134,18 +134,24 @@ my @tests = (
'local:1234/subvol-1234-disk-0.subvol'
],
 },
-
-# no matches
 {
description => 'Snippets, yaml',
volname => "$storage_dir/snippets/userconfig.yaml",
-   expected => [''],
+   expected => [
+   'snippets',
+   'local:snippets/userconfig.yaml',
+   ],
 },
 {
description => 'Snippets, hookscript',
volname => "$storage_dir/snippets/hookscript.pl",
-   expected=> [''],
+   expected=> [
+   'snippets',
+   'local:snippets/hookscript.pl',
+   ],
 },
+
+# no matches
 {
description => 'CT template, tar.xz',
volname => 
"$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.xz",
@@ -210,7 +216,10 @@ my @tests = (
 },
 );
 
-plan tests => scalar @tests;
+plan tests => scalar @tests + 1;
+
+my $seen_vtype;
+my $vtype_subdirs = { map { $_ => 1 } keys %{ 
PVE::Storage::Plugin::get_vtype_subdirs() } };
 
 foreach my $tt (@tests) {
 my $file = $tt->{volname};
@@ -232,8 +241,15 @@ foreach my $tt (@tests) {
 $got = $@ if $@;
 
 is_deeply($got, $expected, $description) || diag(explain($got));
+
+$seen_vtype->{@$expected[0]} = 1
+   if ( @$expected[0] ne '' && scalar @$expected > 1);
 }
 
+# to check if all $vtype_subdirs are defined in path_to_volume_id
+# or have a test
+is_deeply($seen_vtype, $vtype_subdirs, "vtype_subdir check");
+
 #cleanup
 # File::Temp unlinks tempdir on exit
 
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 01/12] storage: test: split archive format/compressor

2020-04-28 Thread Alwin Antreich 
detection into separate functions so they are reusable and easier
modifiable. This patch also adds the test for archive_info.

Signed-off-by: Alwin Antreich 
---
 test/Makefile |   5 +-
 PVE/Storage.pm|  79 +---
 test/archive_info_test.pm | 125 ++
 test/run_plugin_tests.pl  |  12 
 4 files changed, 199 insertions(+), 22 deletions(-)
 create mode 100644 test/archive_info_test.pm
 create mode 100755 test/run_plugin_tests.pl

diff --git a/test/Makefile b/test/Makefile
index 833a597..c54b10f 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -1,6 +1,6 @@
 all: test
 
-test: test_zfspoolplugin test_disklist test_bwlimit
+test: test_zfspoolplugin test_disklist test_bwlimit test_plugin
 
 test_zfspoolplugin: run_test_zfspoolplugin.pl
./run_test_zfspoolplugin.pl
@@ -10,3 +10,6 @@ test_disklist: run_disk_tests.pl
 
 test_bwlimit: run_bwlimit_tests.pl
./run_bwlimit_tests.pl
+
+test_plugin: run_plugin_tests.pl
+   ./run_plugin_tests.pl
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 0848176..bdd6ebc 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1351,6 +1351,53 @@ sub foreach_volid {
 }
 }
 
+sub decompressor_info {
+my ($format, $comp) = @_;
+
+if ($format eq 'tgz' && !defined($comp)) {
+   ($format, $comp) = ('tar', 'gz');
+}
+
+my $decompressor = {
+   tar => {
+   gz => ['tar', '-z'],
+   lzo => ['tar', '--lzop'],
+   },
+   vma => {
+   gz => ['zcat'],
+   lzo => ['lzop', '-d', '-c'],
+   },
+};
+
+die "ERROR: archive format not defined\n"
+   if !defined($decompressor->{$format});
+
+my $decomp = $decompressor->{$format}->{$comp} if $comp;
+
+my $info = {
+   format => $format,
+   compression => $comp,
+   decompressor => $decomp,
+};
+
+return $info;
+}
+
+sub archive_info {
+my ($archive) = shift;
+my $info;
+
+my $volid = basename($archive);
+if ($volid =~ 
/vzdump-(lxc|openvz|qemu)-\d+-(?:\d{4})_(?:\d{2})_(?:\d{2})-(?:\d{2})_(?:\d{2})_(?:\d{2})\.(tgz$|tar|vma)(?:\.(gz|lzo))?$/)
 {
+   $info = decompressor_info($2, $3);
+   $info->{type} = $1;
+} else {
+   die "ERROR: couldn't determine format and compression type\n";
+}
+
+return $info;
+}
+
 sub extract_vzdump_config_tar {
 my ($archive, $conf_re) = @_;
 
@@ -1396,16 +1443,12 @@ sub extract_vzdump_config_vma {
 };
 
 
+my $info = archive_info($archive);
+$comp //= $info->{compression};
+my $decompressor = $info->{decompressor};
+
 if ($comp) {
-   my $uncomp;
-   if ($comp eq 'gz') {
-   $uncomp = ["zcat", $archive];
-   } elsif ($comp eq 'lzo') {
-   $uncomp = ["lzop", "-d", "-c", $archive];
-   } else {
-   die "unknown compression method '$comp'\n";
-   }
-   $cmd = [$uncomp, ["vma", "config", "-"]];
+   $cmd = [ [@$decompressor, $archive], ["vma", "config", "-"] ];
 
# in some cases, lzop/zcat exits with 1 when its stdout pipe is
# closed early by vma, detect this and ignore the exit code later
@@ -1455,20 +1498,14 @@ sub extract_vzdump_config {
 }
 
 my $archive = abs_filesystem_path($cfg, $volid);
+my $info = archive_info($archive);
+my $format = $info->{format};
+my $comp = $info->{compression};
+my $type = $info->{type};
 
-if ($volid =~ 
/vzdump-(lxc|openvz)-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|(tar(\.(gz|lzo))?))$/)
 {
+if ($type eq 'lxc' || $type eq 'openvz') {
return extract_vzdump_config_tar($archive, 
qr!^(\./etc/vzdump/(pct|vps)\.conf)$!);
-} elsif ($volid =~ 
/vzdump-qemu-\d+-(\d{4})_(\d{2})_(\d{2})-(\d{2})_(\d{2})_(\d{2})\.(tgz|((tar|vma)(\.(gz|lzo))?))$/)
 {
-   my $format;
-   my $comp;
-   if ($7 eq 'tgz') {
-   $format = 'tar';
-   $comp = 'gz';
-   } else {
-   $format = $9;
-   $comp = $11 if defined($11);
-   }
-
+} elsif ($type eq 'qemu') {
if ($format eq 'tar') {
return extract_vzdump_config_tar($archive, 
qr!\(\./qemu-server\.conf\)!);
} else {
diff --git a/test/archive_info_test.pm b/test/archive_info_test.pm
new file mode 100644
index 000..c9bb1b7
--- /dev/null
+++ b/test/archive_info_test.pm
@@ -0,0 +1,125 @@
+package PVE::Storage::TestArchiveInfo;
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use PVE::Storage;
+use Test::More;
+
+my $vmid = 16110;
+
+# an array of test cases, each test is comprised of the following keys:
+# description => to identify a single test
+# archive => the input filename for archive_info
+# expected=> the hash that archive_info returns
+#
+# most of them are created further below
+my $tests = [
+# backup archives
+{
+   description => 'Backup archive, lxc, tgz',
+   archive => "backup/vzdump-lxc-$vmid-2020_03_30-21_39_30.tgz",
+   expected=> {
+   't

[pve-devel] [PATCH storage v5 06/12] test: path_to_volume_id

2020-04-28 Thread Alwin Antreich 
Test to reduce the potential for accidental breakage on regex changes.

Signed-off-by: Alwin Antreich 
---
 test/path_to_volume_id_test.pm | 242 +
 test/run_plugin_tests.pl   |   2 +-
 2 files changed, 243 insertions(+), 1 deletion(-)
 create mode 100644 test/path_to_volume_id_test.pm

diff --git a/test/path_to_volume_id_test.pm b/test/path_to_volume_id_test.pm
new file mode 100644
index 000..744c3ee
--- /dev/null
+++ b/test/path_to_volume_id_test.pm
@@ -0,0 +1,242 @@
+package PVE::Storage::TestPathToVolumeId;
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use PVE::Storage;
+
+use Test::More;
+
+use Cwd;
+use File::Basename;
+use File::Path qw(make_path remove_tree);
+use File::Temp;
+
+my $storage_dir = File::Temp->newdir();
+my $scfg = {
+'digest' => 'd29306346b8b25b90a4a96165f1e8f52d1af1eda',
+'ids'=> {
+   'local' => {
+   'shared'   => 0,
+   'path' => "$storage_dir",
+   'type' => 'dir',
+   'maxfiles' => 0,
+   'content'  => {
+   'snippets' => 1,
+   'rootdir'  => 1,
+   'images'   => 1,
+   'iso'  => 1,
+   'backup'   => 1,
+   'vztmpl'   => 1,
+   },
+   },
+},
+'order' => {
+   'local' => 1,
+},
+};
+
+# the tests array consists of hashes with the following keys:
+# description => to identify the test case
+# volname => to create the test file
+# expected=> the result that path_to_volume_id should return
+my @tests = (
+{
+   description => 'Image, qcow2',
+   volname => "$storage_dir/images/16110/vm-16110-disk-0.qcow2",
+   expected=> [
+   'images',
+   'local:16110/vm-16110-disk-0.qcow2',
+   ],
+},
+{
+   description => 'Image, raw',
+   volname => "$storage_dir/images/16112/vm-16112-disk-0.raw",
+   expected=> [
+   'images',
+   'local:16112/vm-16112-disk-0.raw',
+   ],
+},
+{
+   description => 'Image template, qcow2',
+   volname => "$storage_dir/images/9004/base-9004-disk-0.qcow2",
+   expected=> [
+   'images',
+   'local:9004/base-9004-disk-0.qcow2',
+   ],
+},
+
+{
+   description => 'Backup, vma.gz',
+   volname => 
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
+   expected=> [
+   'iso',
+   'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz',
+   ],
+},
+{
+   description => 'Backup, vma.lzo',
+   volname => 
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
+   expected=> [
+   'iso',
+   'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo',
+   ],
+},
+{
+   description => 'Backup, vma',
+   volname => 
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
+   expected=> [
+   'iso',
+   'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma',
+   ],
+},
+{
+   description => 'Backup, tar.lzo',
+   volname => 
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
+   expected=> [
+   'iso',
+   'local:backup/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo',
+   ],
+},
+
+{
+   description => 'ISO file',
+   volname => 
"$storage_dir/template/iso/yet-again-a-installation-disk.iso",
+   expected=> [
+   'iso',
+   'local:iso/yet-again-a-installation-disk.iso',
+   ],
+},
+{
+   description => 'CT template, tar.gz',
+   volname => 
"$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.gz",
+   expected=> [
+   'vztmpl',
+   'local:vztmpl/debian-10.0-standard_10.0-1_amd64.tar.gz',
+   ],
+},
+
+{
+   description => 'Rootdir',
+   volname => "$storage_dir/private/1234/", # fileparse needs / at the 
end
+   expected=> [
+   'rootdir',
+   'local:rootdir/1234',
+   ],
+},
+{
+   description => 'Rootdir, folder subvol',
+   volname => "$storage_dir/images/1234/subvol-1234-disk-0.subvol/", # 
fileparse needs / at the end
+   expected=> [
+   'images',
+   'local:1234/subvol-1234-disk-0.subvol'
+   ],
+},
+
+# no matches
+{
+   description => 'Snippets, yaml',
+   volname => "$storage_dir/snippets/userconfig.yaml",
+   expected => [''],
+},
+{
+   description => 'Snippets, hookscript',
+   volname => "$storage_dir/snippets/hookscript.pl",
+   expected=> [''],
+},
+{
+   description => 'CT template, tar.xz',
+   volname => 
"$storage_dir/template/cache/debian-10.0-standard_10.0-1_amd64.tar.xz",
+   expected=> [''],
+},
+
+# no matches, path or files with failures
+{
+   description

[pve-devel] [PATCH storage v5 10/12] Fix: #2124 storage: add zstd support

2020-04-28 Thread Alwin Antreich 
Signed-off-by: Alwin Antreich 
---
 PVE/Storage.pm |  4 +++-
 PVE/Storage/Plugin.pm  |  2 +-
 test/archive_info_test.pm  |  4 +++-
 test/list_volumes_test.pm  | 18 ++
 test/parse_volname_test.pm |  6 +++---
 test/path_to_volume_id_test.pm | 16 
 6 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index 0b2745e..87550b1 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -1366,10 +1366,12 @@ sub decompressor_info {
tar => {
gz => ['tar', '-z'],
lzo => ['tar', '--lzop'],
+   zst => ['tar', '--zstd'],
},
vma => {
gz => ['zcat'],
lzo => ['lzop', '-d', '-c'],
+   zst => ['zstd', '-q', '-d', '-c'],
},
 };
 
@@ -1460,7 +1462,7 @@ sub extract_vzdump_config_vma {
my $errstring;
my $err = sub {
my $output = shift;
-   if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: 
stdout: Broken pipe/) {
+   if ($output =~ m/lzop: Broken pipe: / || $output =~ m/gzip: 
stdout: Broken pipe/ || $output =~ m/zstd: error 70 : Write error : Broken 
pipe/) {
$broken_pipe = 1;
} elsif (!defined ($errstring) && $output !~ m/^\s*$/) {
$errstring = "Failed to extract config from VMA archive: 
$output\n";
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 5f3e4c1..e9da403 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -18,7 +18,7 @@ use JSON;
 
 use base qw(PVE::SectionConfig);
 
-use constant COMPRESSOR_RE => 'gz|lzo';
+use constant COMPRESSOR_RE => 'gz|lzo|zst';
 
 our @COMMON_TAR_FLAGS = qw(
 --one-file-system
diff --git a/test/archive_info_test.pm b/test/archive_info_test.pm
index c9bb1b7..283fe47 100644
--- a/test/archive_info_test.pm
+++ b/test/archive_info_test.pm
@@ -45,10 +45,12 @@ my $decompressor = {
 tar => {
gz  => ['tar', '-z'],
lzo => ['tar', '--lzop'],
+   zst => ['tar', '--zstd'],
 },
 vma => {
gz  => ['zcat'],
lzo => ['lzop', '-d', '-c'],
+   zst => ['zstd', '-q', '-d', '-c'],
 },
 };
 
@@ -85,7 +87,7 @@ foreach my $virt (keys %$bkp_suffix) {
 my $non_bkp_suffix = {
 'openvz' => [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ],
 'lxc'=> [ 'zip', 'tgz.lzo', 'tar.bz2', 'zip.gz', '', ],
-'qemu'   => [ 'vma.xz', 'vms.gz', '', ],
+'qemu'   => [ 'vma.xz', 'vms.gz', 'vmx.zst', '', ],
 'none'   => [ 'tar.gz', ],
 };
 
diff --git a/test/list_volumes_test.pm b/test/list_volumes_test.pm
index 941b903..efcb547 100644
--- a/test/list_volumes_test.pm
+++ b/test/list_volumes_test.pm
@@ -93,6 +93,7 @@ my @tests = (
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
"$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
+   "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst",
"$storage_dir/snippets/userconfig.yaml",
"$storage_dir/snippets/hookscript.pl",
],
@@ -151,6 +152,14 @@ my @tests = (
'vmid'=> '16110',
'volid'   => 
'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma',
},
+   {
+   'content' => 'backup',
+   'ctime'   => 1585595635,
+   'format'  => 'vma.zst',
+   'size'=> DEFAULT_SIZE,
+   'vmid'=> '16110',
+   'volid'   => 
'local:backup/vzdump-qemu-16110-2020_03_30-21_13_55.vma.zst',
+   },
{
'content' => 'snippets',
'ctime'   => DEFAULT_CTIME,
@@ -174,6 +183,7 @@ my @tests = (
"$storage_dir/images/16112/vm-16112-disk-0.raw",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_39_30.tar.lzo",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz",
+   "$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst",
"$storage_dir/dump/vzdump-lxc-16112-2020_03_30-21_59_30.tgz",
],
expected => [
@@ -203,6 +213,14 @@ my @tests = (
'vmid'=> '16112',
'volid'   => 
'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.gz',
},
+   {
+   'content' => 'backup',
+   'ctime'   => 1585597770,
+   'format'  => 'tar.zst',
+   'size'=> DEFAULT_SIZE,
+   'vmid'=> '16112',
+   'volid'   => 
'local:backup/vzdump-lxc-16112-2020_03_30-21_49_30.tar.zst',
+   },
{
'content' => 'backup',
'ctime'   => 1585598370,
diff --git a/test/parse_volname_test.pm b/test/parse_volname_test.pm
index 87c758c..d6ac885 100644
--- a/test/parse_volname_test.pm
+++ b/test/parse_volname_test.pm
@@ -176,9 +176,9 @@ foreach my $s

[pve-devel] [PATCH storage v5 12/12] test: filesystem_path

2020-04-28 Thread Alwin Antreich 
Signed-off-by: Alwin Antreich 
---
 test/filesystem_path_test.pm | 91 
 test/run_plugin_tests.pl |  1 +
 2 files changed, 92 insertions(+)
 create mode 100644 test/filesystem_path_test.pm

diff --git a/test/filesystem_path_test.pm b/test/filesystem_path_test.pm
new file mode 100644
index 000..c1b6d90
--- /dev/null
+++ b/test/filesystem_path_test.pm
@@ -0,0 +1,91 @@
+package PVE::Storage::TestFilesystemPath;
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use PVE::Storage;
+use Test::More;
+
+my $path = '/some/path';
+
+# each array entry is a test that consists of the following keys:
+# volname  => image name that is passed to parse_volname
+# snapname => to test the die condition
+# expected => the array of return values; or the die message
+my $tests = [
+{
+   volname  => '1234/vm-1234-disk-0.raw',
+   snapname => undef,
+   expected => [
+   "$path/images/1234/vm-1234-disk-0.raw",
+   '1234',
+   'images'
+   ],
+},
+{
+   volname  => '1234/vm-1234-disk-0.raw',
+   snapname => 'my_snap',
+   expected => "can't snapshot this image format\n"
+},
+{
+   volname  => '1234/vm-1234-disk-0.qcow2',
+   snapname => undef,
+   expected => [
+   "$path/images/1234/vm-1234-disk-0.qcow2",
+   '1234',
+   'images'
+   ],
+},
+{
+   volname  => '1234/vm-1234-disk-0.qcow2',
+   snapname => 'my_snap',
+   expected => [
+   "$path/images/1234/vm-1234-disk-0.qcow2",
+   '1234',
+   'images'
+   ],
+},
+{
+   volname  => 'iso/my-awesome-proxmox.iso',
+   snapname => undef,
+   expected => [
+   "$path/template/iso/my-awesome-proxmox.iso",
+   undef,
+   'iso'
+   ],
+},
+{
+   volname  => "backup/vzdump-qemu-1234-2020_03_30-21_12_40.vma",
+   snapname => undef,
+   expected => [
+   "$path/dump/vzdump-qemu-1234-2020_03_30-21_12_40.vma",
+   1234,
+   'backup'
+   ],
+},
+];
+
+plan tests => scalar @$tests;
+
+foreach my $tt (@$tests) {
+my $volname = $tt->{volname};
+my $snapname = $tt->{snapname};
+my $expected = $tt->{expected};
+my $scfg = { path => $path };
+my $got;
+
+eval {
+   $got = [ PVE::Storage::Plugin->filesystem_path($scfg, $volname, 
$snapname) ];
+};
+$got = $@ if $@;
+
+is_deeply($got, $expected, "wantarray: filesystem_path for $volname")
+|| diag(explain($got));
+
+}
+
+done_testing();
+
+1;
diff --git a/test/run_plugin_tests.pl b/test/run_plugin_tests.pl
index 9e427eb..e29fc88 100755
--- a/test/run_plugin_tests.pl
+++ b/test/run_plugin_tests.pl
@@ -12,6 +12,7 @@ my $res = $harness->runtests(
 "list_volumes_test.pm",
 "path_to_volume_id_test.pm",
 "get_subdir_test.pm",
+"filesystem_path_test.pm",
 );
 
 exit -1 if !$res || $res->{failed} || $res->{parse_errors};
-- 
2.20.1


___
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

[pve-devel] [PATCH storage v5 04/12] test: list_volumes

2020-04-28 Thread Alwin Antreich 
Test to reduce the potential for accidental breakage on regex changes.

Co-Authored-by: Dominic Jaeger 
Signed-off-by: Alwin Antreich 
---
 test/list_volumes_test.pm | 519 ++
 test/run_plugin_tests.pl  |   6 +-
 2 files changed, 524 insertions(+), 1 deletion(-)
 create mode 100644 test/list_volumes_test.pm

diff --git a/test/list_volumes_test.pm b/test/list_volumes_test.pm
new file mode 100644
index 000..a215617
--- /dev/null
+++ b/test/list_volumes_test.pm
@@ -0,0 +1,519 @@
+package PVE::Storage::TestListVolumes;
+
+use strict;
+use warnings;
+
+use lib qw(..);
+
+use PVE::Storage;
+use PVE::Cluster;
+use PVE::Tools qw(run_command);
+
+use Test::More;
+use Test::MockModule;
+
+use Cwd;
+use File::Basename;
+use File::Path qw(make_path remove_tree);
+use File::stat qw();
+use File::Temp;
+use Storable qw(dclone);
+
+use constant DEFAULT_SIZE => 131072; # 128 kiB
+use constant DEFAULT_USED => 262144; # 256 kiB
+use constant DEFAULT_CTIME => 1234567890;
+
+# get_vmlist() return values
+my $mocked_vmlist = {
+'version' => 1,
+'ids' => {
+   '16110' => {
+   'node'=> 'x42',
+   'type'=> 'qemu',
+   'version' => 4,
+   },
+   '16112' => {
+   'node'=> 'x42',
+   'type'=> 'lxc',
+   'version' => 7,
+   },
+   '16114' => {
+   'node'=> 'x42',
+   'type'=> 'qemu',
+   'version' => 2,
+   },
+   '16113' => {
+   'node'=> 'x42',
+   'type'=> 'qemu',
+   'version' => 5,
+   },
+   '16115' => {
+   'node'=> 'x42',
+   'type'=> 'qemu',
+   'version' => 1,
+   },
+   '9004' => {
+   'node'=> 'x42',
+   'type'=> 'qemu',
+   'version' => 6,
+   }
+}
+};
+
+my $storage_dir = File::Temp->newdir();
+my $scfg = {
+'type' => 'dir',
+'maxfiles' => 0,
+'path' => $storage_dir,
+'shared'   => 0,
+'content'  => {
+   'iso'  => 1,
+   'rootdir'  => 1,
+   'vztmpl'   => 1,
+   'images'   => 1,
+   'snippets' => 1,
+   'backup'   => 1,
+},
+};
+
+# The test cases are comprised of an arry of hashes with the following keys:
+# description => displayed on error by Test::More
+# vmid=> used for image matches by list_volume
+# files   => array of files for qemu-img to create
+# expected=> returned result hash
+#(content, ctime, format, parent, size, used, vimd, volid)
+my @tests = (
+{
+   description => 'VMID: 16110, VM, qcow2, backup, snippets',
+   vmid => '16110',
+   files => [
+   "$storage_dir/images/16110/vm-16110-disk-0.qcow2",
+   "$storage_dir/images/16110/vm-16110-disk-1.raw",
+   "$storage_dir/images/16110/vm-16110-disk-2.vmdk",
+   "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz",
+   "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo",
+   "$storage_dir/dump/vzdump-qemu-16110-2020_03_30-21_13_55.vma",
+   "$storage_dir/snippets/userconfig.yaml",
+   "$storage_dir/snippets/hookscript.pl",
+   ],
+   expected => [
+   {
+   'content' => 'images',
+   'ctime'   => DEFAULT_CTIME,
+   'format'  => 'qcow2',
+   'parent'  => undef,
+   'size'=> DEFAULT_SIZE,
+   'used'=> DEFAULT_USED,
+   'vmid'=> '16110',
+   'volid'   => 'local:16110/vm-16110-disk-0.qcow2',
+   },
+   {
+   'content' => 'images',
+   'ctime'   => DEFAULT_CTIME,
+   'format'  => 'raw',
+   'parent'  => undef,
+   'size'=> DEFAULT_SIZE,
+   'used'=> DEFAULT_USED,
+   'vmid'=> '16110',
+   'volid'   => 'local:16110/vm-16110-disk-1.raw',
+   },
+   {
+   'content' => 'images',
+   'ctime'   => DEFAULT_CTIME,
+   'format'  => 'vmdk',
+   'parent'  => undef,
+   'size'=> DEFAULT_SIZE,
+   'used'=> DEFAULT_USED,
+   'vmid'=> '16110',
+   'volid'   => 'local:16110/vm-16110-disk-2.vmdk',
+   },
+   {
+   'content' => 'backup',
+   'ctime'   => DEFAULT_CTIME,
+   'format'  => 'vma.gz',
+   'size'=> DEFAULT_SIZE,
+   'vmid'=> '16110',
+   'volid'   => 
'local:backup/vzdump-qemu-16110-2020_03_30-21_11_40.vma.gz',
+   },
+   {
+   'content' => 'backup',
+   'ctime'   => DEFAULT_CTIME,
+   'format'  => 'vma.lzo',
+   'size'=> DEFAULT_SIZE,
+   'vmid'=> '16110',
+   'volid'   => 
'local:backup/vzdump-qemu-16110-2020_03_30-21_12_45.vma.lzo',