On 9/30/19 12:58 PM, Stefan Reiter wrote:
> The package will be used for custom CPU models as a SectionConfig, hence
> the name. For now we simply move some CPU related helper functions and
> declarations over from QemuServer to reduce clutter there.
Single thing I'm not too sure is how "qemu_machine_feature_enabled" fit's
into the CPUConfig here..
AFAICT, this rather looks like and attempt to avoid a cyclic module
dependency?
Maybe we could also do a Machine module which then includes stuff like:
machine_type_is_q35
get_basic_machine_info
But seems not really worth it... better ideas?
>
> Signed-off-by: Stefan Reiter <s.rei...@proxmox.com>
> ---
> PVE/QemuServer.pm | 242 +---------------------------------
> PVE/QemuServer/CPUConfig.pm | 251 ++++++++++++++++++++++++++++++++++++
> PVE/QemuServer/Makefile | 1 +
> 3 files changed, 256 insertions(+), 238 deletions(-)
> create mode 100644 PVE/QemuServer/CPUConfig.pm
>
> diff --git a/PVE/QemuServer.pm b/PVE/QemuServer.pm
> index 20c1061..a95006f 100644
> --- a/PVE/QemuServer.pm
> +++ b/PVE/QemuServer.pm
> @@ -33,6 +33,7 @@ use PVE::QemuConfig;
> use PVE::QMPClient;
> use PVE::RPCEnvironment;
> use PVE::GuestHelpers;
> +use PVE::QemuServer::CPUConfig qw(qemu_machine_feature_enabled
> print_cpu_device get_cpu_options);
> use PVE::QemuServer::PCI qw(print_pci_addr print_pcie_addr
> print_pcie_root_port);
> use PVE::QemuServer::Memory;
> use PVE::QemuServer::USB qw(parse_usb_device);
> @@ -116,105 +117,6 @@ mkdir $var_run_tmpdir;
> my $lock_dir = "/var/lock/qemu-server";
> mkdir $lock_dir;
>
> -my $cpu_vendor_list = {
> - # Intel CPUs
> - 486 => 'GenuineIntel',
> - pentium => 'GenuineIntel',
> - pentium2 => 'GenuineIntel',
> - pentium3 => 'GenuineIntel',
> - coreduo => 'GenuineIntel',
> - core2duo => 'GenuineIntel',
> - Conroe => 'GenuineIntel',
> - Penryn => 'GenuineIntel',
> - Nehalem => 'GenuineIntel',
> - 'Nehalem-IBRS' => 'GenuineIntel',
> - Westmere => 'GenuineIntel',
> - 'Westmere-IBRS' => 'GenuineIntel',
> - SandyBridge => 'GenuineIntel',
> - 'SandyBridge-IBRS' => 'GenuineIntel',
> - IvyBridge => 'GenuineIntel',
> - 'IvyBridge-IBRS' => 'GenuineIntel',
> - Haswell => 'GenuineIntel',
> - 'Haswell-IBRS' => 'GenuineIntel',
> - 'Haswell-noTSX' => 'GenuineIntel',
> - 'Haswell-noTSX-IBRS' => 'GenuineIntel',
> - Broadwell => 'GenuineIntel',
> - 'Broadwell-IBRS' => 'GenuineIntel',
> - 'Broadwell-noTSX' => 'GenuineIntel',
> - 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
> - 'Skylake-Client' => 'GenuineIntel',
> - 'Skylake-Client-IBRS' => 'GenuineIntel',
> - 'Skylake-Server' => 'GenuineIntel',
> - 'Skylake-Server-IBRS' => 'GenuineIntel',
> -
> - # AMD CPUs
> - athlon => 'AuthenticAMD',
> - phenom => 'AuthenticAMD',
> - Opteron_G1 => 'AuthenticAMD',
> - Opteron_G2 => 'AuthenticAMD',
> - Opteron_G3 => 'AuthenticAMD',
> - Opteron_G4 => 'AuthenticAMD',
> - Opteron_G5 => 'AuthenticAMD',
> - EPYC => 'AuthenticAMD',
> - 'EPYC-IBPB' => 'AuthenticAMD',
> -
> - # generic types, use vendor from host node
> - host => 'default',
> - kvm32 => 'default',
> - kvm64 => 'default',
> - qemu32 => 'default',
> - qemu64 => 'default',
> - max => 'default',
> -};
> -
> -my @supported_cpu_flags = (
> - 'pcid',
> - 'spec-ctrl',
> - 'ibpb',
> - 'ssbd',
> - 'virt-ssbd',
> - 'amd-ssbd',
> - 'amd-no-ssb',
> - 'pdpe1gb',
> - 'md-clear',
> - 'hv-tlbflush',
> - 'hv-evmcs',
> - 'aes'
> -);
> -my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
> -
> -my $cpu_fmt = {
> - cputype => {
> - description => "Emulated CPU type.",
> - type => 'string',
> - enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
> - default => 'kvm64',
> - default_key => 1,
> - },
> - hidden => {
> - description => "Do not identify as a KVM virtual machine.",
> - type => 'boolean',
> - optional => 1,
> - default => 0
> - },
> - 'hv-vendor-id' => {
> - type => 'string',
> - pattern => qr/[a-zA-Z0-9]{1,12}/,
> - format_description => 'vendor-id',
> - description => 'The Hyper-V vendor ID. Some drivers or programs inside
> Windows guests need a specific ID.',
> - optional => 1,
> - },
> - flags => {
> - description => "List of additional CPU flags separated by ';'."
> - . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
> - . " Currently supported flags: @{[join(', ',
> @supported_cpu_flags)]}.",
> - format_description => '+FLAG[;-FLAG...]',
> - type => 'string',
> - pattern => qr/$cpu_flag(;$cpu_flag)*/,
> - optional => 1,
> - },
> -};
> -
> my $watchdog_fmt = {
> model => {
> default_key => 1,
> @@ -603,7 +505,7 @@ EODESCR
> optional => 1,
> description => "Emulated CPU type.",
> type => 'string',
> - format => $cpu_fmt,
> + format => $PVE::QemuServer::CPUConfig::cpu_fmt,
> },
> parent => get_standard_option('pve-snapshot-name', {
> optional => 1,
> @@ -2133,26 +2035,6 @@ sub print_netdev_full {
> return $netdev;
> }
>
> -
> -sub print_cpu_device {
> - my ($conf, $id) = @_;
> -
> - my $kvm = $conf->{kvm} // 1;
> - my $cpu = $kvm ? "kvm64" : "qemu64";
> - if (my $cputype = $conf->{cpu}) {
> - my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
> - or die "Cannot parse cpu description: $cputype\n";
> - $cpu = $cpuconf->{cputype};
> - }
> -
> - my $cores = $conf->{cores} || 1;
> -
> - my $current_core = ($id - 1) % $cores;
> - my $current_socket = int(($id - 1 - $current_core)/$cores);
> -
> - return
> "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
> -}
> -
> my $vga_map = {
> 'cirrus' => 'cirrus-vga',
> 'std' => 'VGA',
> @@ -3643,62 +3525,6 @@ sub query_understood_cpu_flags {
> return $flags;
> }
>
> -sub get_cpu_options {
> - my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion,
> $gpu_passthrough) = @_;
> -
> - my $cpuFlags = [];
> - my $ostype = $conf->{ostype};
> -
> - my $cpu = $kvm ? "kvm64" : "qemu64";
> - if ($arch eq 'aarch64') {
> - $cpu = 'cortex-a57';
> - }
> - my $hv_vendor_id;
> - if (my $cputype = $conf->{cpu}) {
> - my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
> - or die "Cannot parse cpu description: $cputype\n";
> - $cpu = $cpuconf->{cputype};
> - $kvm_off = 1 if $cpuconf->{hidden};
> - $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
> -
> - if (defined(my $flags = $cpuconf->{flags})) {
> - push @$cpuFlags, split(";", $flags);
> - }
> - }
> -
> - push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
> -
> - push @$cpuFlags , '-x2apic'
> - if $conf->{ostype} && $conf->{ostype} eq 'solaris';
> -
> - push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
> -
> - push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
> -
> - if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch
> eq 'x86_64') {
> -
> - push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
> - push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
> - }
> -
> - add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type,
> $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
> -
> - push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq
> 'x86_64';
> -
> - push @$cpuFlags, 'kvm=off' if $kvm_off;
> -
> - if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
> - push @$cpuFlags, "vendor=${cpu_vendor}"
> - if $cpu_vendor ne 'default';
> - } elsif ($arch ne 'aarch64') {
> - die "internal error"; # should not happen
> - }
> -
> - $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
> -
> - return ('-cpu', $cpu);
> -}
> -
> sub config_to_command {
> my ($storecfg, $vmid, $conf, $defaults, $forcemachine) = @_;
>
> @@ -4084,7 +3910,8 @@ sub config_to_command {
> push @$rtcFlags, 'base=localtime';
> }
>
> - push @$cmd, get_cpu_options($conf, $arch, $kvm, $machine_type, $kvm_off,
> $kvmver, $winversion, $gpu_passthrough);
> + push @$cmd, get_cpu_options($conf, $arch, $kvm,
> + $machine_type, $kvm_off, $kvmver, $winversion, $gpu_passthrough);
>
> PVE::QemuServer::Memory::config($conf, $vmid, $sockets, $cores,
> $defaults, $hotplug_features, $cmd);
>
> @@ -7319,28 +7146,6 @@ sub get_running_qemu_version {
> return "$res->{qemu}->{major}.$res->{qemu}->{minor}";
> }
>
> -sub qemu_machine_feature_enabled {
> - my ($machine, $kvmver, $version_major, $version_minor) = @_;
> -
> - my $current_major;
> - my $current_minor;
> -
> - if ($machine && $machine =~
> m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
> -
> - $current_major = $3;
> - $current_minor = $4;
> -
> - } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
> -
> - $current_major = $1;
> - $current_minor = $2;
> - }
> -
> - return 1 if $current_major > $version_major ||
> - ($current_major == $version_major &&
> - $current_minor >= $version_minor);
> -}
> -
> sub qemu_machine_pxe {
> my ($vmid, $conf) = @_;
>
> @@ -7426,45 +7231,6 @@ sub scsihw_infos {
> return ($maxdev, $controller, $controller_prefix);
> }
>
> -sub add_hyperv_enlightenments {
> - my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios,
> $gpu_passthrough, $hv_vendor_id) = @_;
> -
> - return if $winversion < 6;
> - return if $bios && $bios eq 'ovmf' && $winversion < 8;
> -
> - if ($gpu_passthrough || defined($hv_vendor_id)) {
> - $hv_vendor_id //= 'proxmox';
> - push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
> - }
> -
> - if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
> - push @$cpuFlags , 'hv_spinlocks=0x1fff';
> - push @$cpuFlags , 'hv_vapic';
> - push @$cpuFlags , 'hv_time';
> - } else {
> - push @$cpuFlags , 'hv_spinlocks=0xffff';
> - }
> -
> - if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
> - push @$cpuFlags , 'hv_reset';
> - push @$cpuFlags , 'hv_vpindex';
> - push @$cpuFlags , 'hv_runtime';
> - }
> -
> - if ($winversion >= 7) {
> - push @$cpuFlags , 'hv_relaxed';
> -
> - if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
> - push @$cpuFlags , 'hv_synic';
> - push @$cpuFlags , 'hv_stimer';
> - }
> -
> - if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
> - push @$cpuFlags , 'hv_ipi';
> - }
> - }
> -}
> -
> sub windows_version {
> my ($ostype) = @_;
>
> diff --git a/PVE/QemuServer/CPUConfig.pm b/PVE/QemuServer/CPUConfig.pm
> new file mode 100644
> index 0000000..c994228
> --- /dev/null
> +++ b/PVE/QemuServer/CPUConfig.pm
> @@ -0,0 +1,251 @@
> +package PVE::QemuServer::CPUConfig;
> +
> +use strict;
> +use warnings;
please do a empty newline after above
> +use PVE::JSONSchema;
> +
> +use base 'Exporter';
> +our @EXPORT_OK = qw(
> +print_cpu_device
> +get_cpu_options
> +qemu_machine_feature_enabled
> +);
> +
> +my $cpu_vendor_list = {
> + # Intel CPUs
> + 486 => 'GenuineIntel',
> + pentium => 'GenuineIntel',
> + pentium2 => 'GenuineIntel',
> + pentium3 => 'GenuineIntel',
> + coreduo => 'GenuineIntel',
> + core2duo => 'GenuineIntel',
> + Conroe => 'GenuineIntel',
> + Penryn => 'GenuineIntel',
> + Nehalem => 'GenuineIntel',
> + 'Nehalem-IBRS' => 'GenuineIntel',
> + Westmere => 'GenuineIntel',
> + 'Westmere-IBRS' => 'GenuineIntel',
> + SandyBridge => 'GenuineIntel',
> + 'SandyBridge-IBRS' => 'GenuineIntel',
> + IvyBridge => 'GenuineIntel',
> + 'IvyBridge-IBRS' => 'GenuineIntel',
> + Haswell => 'GenuineIntel',
> + 'Haswell-IBRS' => 'GenuineIntel',
> + 'Haswell-noTSX' => 'GenuineIntel',
> + 'Haswell-noTSX-IBRS' => 'GenuineIntel',
> + Broadwell => 'GenuineIntel',
> + 'Broadwell-IBRS' => 'GenuineIntel',
> + 'Broadwell-noTSX' => 'GenuineIntel',
> + 'Broadwell-noTSX-IBRS' => 'GenuineIntel',
> + 'Skylake-Client' => 'GenuineIntel',
> + 'Skylake-Client-IBRS' => 'GenuineIntel',
> + 'Skylake-Server' => 'GenuineIntel',
> + 'Skylake-Server-IBRS' => 'GenuineIntel',
> +
> + # AMD CPUs
> + athlon => 'AuthenticAMD',
> + phenom => 'AuthenticAMD',
> + Opteron_G1 => 'AuthenticAMD',
> + Opteron_G2 => 'AuthenticAMD',
> + Opteron_G3 => 'AuthenticAMD',
> + Opteron_G4 => 'AuthenticAMD',
> + Opteron_G5 => 'AuthenticAMD',
> + EPYC => 'AuthenticAMD',
> + 'EPYC-IBPB' => 'AuthenticAMD',
> +
> + # generic types, use vendor from host node
> + host => 'default',
> + kvm32 => 'default',
> + kvm64 => 'default',
> + qemu32 => 'default',
> + qemu64 => 'default',
> + max => 'default',
> +};
> +
> +my @supported_cpu_flags = (
> + 'pcid',
> + 'spec-ctrl',
> + 'ibpb',
> + 'ssbd',
> + 'virt-ssbd',
> + 'amd-ssbd',
> + 'amd-no-ssb',
> + 'pdpe1gb',
> + 'md-clear',
> + 'hv-tlbflush',
> + 'hv-evmcs',
> + 'aes'
> +);
> +my $cpu_flag = qr/[+-](@{[join('|', @supported_cpu_flags)]})/;
> +
> +our $cpu_fmt = {
> + cputype => {
> + description => "Emulated CPU type.",
> + type => 'string',
> + enum => [ sort { "\L$a" cmp "\L$b" } keys %$cpu_vendor_list ],
> + default => 'kvm64',
> + default_key => 1,
> + },
> + hidden => {
> + description => "Do not identify as a KVM virtual machine.",
> + type => 'boolean',
> + optional => 1,
> + default => 0
> + },
> + 'hv-vendor-id' => {
> + type => 'string',
> + pattern => qr/[a-zA-Z0-9]{1,12}/,
> + format_description => 'vendor-id',
> + description => 'The Hyper-V vendor ID. Some drivers or programs inside
> Windows guests need a specific ID.',
> + optional => 1,
> + },
> + flags => {
> + description => "List of additional CPU flags separated by ';'."
> + . " Use '+FLAG' to enable, '-FLAG' to disable a flag."
> + . " Currently supported flags: @{[join(', ',
> @supported_cpu_flags)]}.",
> + format_description => '+FLAG[;-FLAG...]',
> + type => 'string',
> + pattern => qr/$cpu_flag(;$cpu_flag)*/,
> + optional => 1,
> + },
> +};
> +
> +# Print a QEMU device node for a given VM configuration for hotplugging CPUs
> +sub print_cpu_device {
> + my ($conf, $id) = @_;
> +
> + my $kvm = $conf->{kvm} // 1;
> + my $cpu = $kvm ? "kvm64" : "qemu64";
> + if (my $cputype = $conf->{cpu}) {
> + my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
> + or die "Cannot parse cpu description: $cputype\n";
> + $cpu = $cpuconf->{cputype};
> + }
> +
> + my $cores = $conf->{cores} || 1;
> +
> + my $current_core = ($id - 1) % $cores;
> + my $current_socket = int(($id - 1 - $current_core)/$cores);
> +
> + return
> "$cpu-x86_64-cpu,id=cpu$id,socket-id=$current_socket,core-id=$current_core,thread-id=0";
> +}
> +
> +# Calculate QEMU's '-cpu' argument from a given VM configuration
> +sub get_cpu_options {
> + my ($conf, $arch, $kvm, $machine_type, $kvm_off, $kvmver, $winversion,
> $gpu_passthrough) = @_;
> +
> + my $cpuFlags = [];
> + my $ostype = $conf->{ostype};
> +
> + my $cpu = $kvm ? "kvm64" : "qemu64";
> + if ($arch eq 'aarch64') {
> + $cpu = 'cortex-a57';
> + }
> + my $hv_vendor_id;
> + if (my $cputype = $conf->{cpu}) {
> + my $cpuconf = PVE::JSONSchema::parse_property_string($cpu_fmt, $cputype)
> + or die "Cannot parse cpu description: $cputype\n";
> + $cpu = $cpuconf->{cputype};
> + $kvm_off = 1 if $cpuconf->{hidden};
> + $hv_vendor_id = $cpuconf->{'hv-vendor-id'};
> +
> + if (defined(my $flags = $cpuconf->{flags})) {
> + push @$cpuFlags, split(";", $flags);
> + }
> + }
> +
> + push @$cpuFlags , '+lahf_lm' if $cpu eq 'kvm64' && $arch eq 'x86_64';
> +
> + push @$cpuFlags , '-x2apic'
> + if $conf->{ostype} && $conf->{ostype} eq 'solaris';
> +
> + push @$cpuFlags, '+sep' if $cpu eq 'kvm64' || $cpu eq 'kvm32';
> +
> + push @$cpuFlags, '-rdtscp' if $cpu =~ m/^Opteron/;
> +
> + if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3) && $arch
> eq 'x86_64') {
> +
> + push @$cpuFlags , '+kvm_pv_unhalt' if $kvm;
> + push @$cpuFlags , '+kvm_pv_eoi' if $kvm;
> + }
> +
> + add_hyperv_enlightenments($cpuFlags, $winversion, $machine_type,
> $kvmver, $conf->{bios}, $gpu_passthrough, $hv_vendor_id) if $kvm;
> +
> + push @$cpuFlags, 'enforce' if $cpu ne 'host' && $kvm && $arch eq
> 'x86_64';
> +
> + push @$cpuFlags, 'kvm=off' if $kvm_off;
> +
> + if (my $cpu_vendor = $cpu_vendor_list->{$cpu}) {
> + push @$cpuFlags, "vendor=${cpu_vendor}"
> + if $cpu_vendor ne 'default';
> + } elsif ($arch ne 'aarch64') {
> + die "internal error"; # should not happen
> + }
> +
> + $cpu .= "," . join(',', @$cpuFlags) if scalar(@$cpuFlags);
> +
> + return ('-cpu', $cpu);
> +}
> +
> +sub add_hyperv_enlightenments {
> + my ($cpuFlags, $winversion, $machine_type, $kvmver, $bios,
> $gpu_passthrough, $hv_vendor_id) = @_;
> +
> + return if $winversion < 6;
> + return if $bios && $bios eq 'ovmf' && $winversion < 8;
> +
> + if ($gpu_passthrough || defined($hv_vendor_id)) {
> + $hv_vendor_id //= 'proxmox';
> + push @$cpuFlags , "hv_vendor_id=$hv_vendor_id";
> + }
> +
> + if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 3)) {
> + push @$cpuFlags , 'hv_spinlocks=0x1fff';
> + push @$cpuFlags , 'hv_vapic';
> + push @$cpuFlags , 'hv_time';
> + } else {
> + push @$cpuFlags , 'hv_spinlocks=0xffff';
> + }
> +
> + if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 6)) {
> + push @$cpuFlags , 'hv_reset';
> + push @$cpuFlags , 'hv_vpindex';
> + push @$cpuFlags , 'hv_runtime';
> + }
> +
> + if ($winversion >= 7) {
> + push @$cpuFlags , 'hv_relaxed';
> +
> + if (qemu_machine_feature_enabled ($machine_type, $kvmver, 2, 12)) {
> + push @$cpuFlags , 'hv_synic';
> + push @$cpuFlags , 'hv_stimer';
> + }
> +
> + if (qemu_machine_feature_enabled ($machine_type, $kvmver, 3, 1)) {
> + push @$cpuFlags , 'hv_ipi';
> + }
> + }
> +}
> +
> +sub qemu_machine_feature_enabled {
> + my ($machine, $kvmver, $version_major, $version_minor) = @_;
> +
> + my $current_major;
> + my $current_minor;
> +
> + if ($machine && $machine =~
> m/^((?:pc(-i440fx|-q35)?|virt)-(\d+)\.(\d+))/) {
> +
> + $current_major = $3;
> + $current_minor = $4;
> +
> + } elsif ($kvmver =~ m/^(\d+)\.(\d+)/) {
> +
> + $current_major = $1;
> + $current_minor = $2;
> + }
> +
> + return 1 if $current_major > $version_major ||
> + ($current_major == $version_major &&
> + $current_minor >= $version_minor);
> +}
> +
> +1;
> diff --git a/PVE/QemuServer/Makefile b/PVE/QemuServer/Makefile
> index afc39a3..a3054f1 100644
> --- a/PVE/QemuServer/Makefile
> +++ b/PVE/QemuServer/Makefile
> @@ -5,6 +5,7 @@ SOURCES=PCI.pm \
> OVF.pm \
> Cloudinit.pm \
> Agent.pm \
> + CPUConfig.pm \
>
> .PHONY: install
> install: ${SOURCES}
>
_______________________________________________
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
