Signed-off-by: Dzmitry Kotsikau <dkotsi...@gmail.com> --- PVE/Storage/LunCmd/Makefile | 2 +- PVE/Storage/LunCmd/Scst.pm | 557 ++++++++++++++++++++++++++++++++++++++++++++ PVE/Storage/ZFSPlugin.pm | 15 +- 3 files changed, 570 insertions(+), 4 deletions(-) create mode 100644 PVE/Storage/LunCmd/Scst.pm
diff --git a/PVE/Storage/LunCmd/Makefile b/PVE/Storage/LunCmd/Makefile index b959255..5a44cbd 100644 --- a/PVE/Storage/LunCmd/Makefile +++ b/PVE/Storage/LunCmd/Makefile @@ -1,4 +1,4 @@ -SOURCES=Comstar.pm Istgt.pm Iet.pm +SOURCES=Comstar.pm Istgt.pm Iet.pm Scst.pm .PHONY: install install: diff --git a/PVE/Storage/LunCmd/Scst.pm b/PVE/Storage/LunCmd/Scst.pm new file mode 100644 index 0000000..4e0a67b --- /dev/null +++ b/PVE/Storage/LunCmd/Scst.pm @@ -0,0 +1,557 @@ +package PVE::Storage::LunCmd::Scst; + +#1) Install scst on proxmox node + +#apt-get install git gawk build-essential flex bison automake autoconf pve-headers dkms +#git clone https://github.com/bvanassche/scst.git +#cd scst +#make scst scst_local iscsi-scst scst srpt usr +#make scst_install scst_local_install iscsi_install scst_install srpt_install usr_install +#depmod -a +#cat > /etc/default/scst << 'EOF' +#ISCSID_OPTIONS="-a 192.168.254.10 -u0 -g0 -p3260" +#SCST_TARGET_MODULES="scst scst_vdisk iscsi-scst" +#EOF + +#2) Create portal + +#cat > /etc/scst.conf <'EOF' +#TARGET_DRIVER iscsi { +# DefaultTime2Wait 2 +# DefaultTime2Retain 90 +# enabled 1 +#} +#EOF + + + + +use strict; +use warnings; +use PVE::Tools qw(run_command file_read_firstline trim dir_glob_regex dir_glob_foreach); +use Data::Dumper; + +sub get_base; + +# A logical unit can max have 16383 LUNs +my $MAX_LUNS = 16383; +my $HANDLER = 'vdisk_fileio'; + +my $CONFIG_FILE = '/etc/scst.conf'; +my $CONFIG_FILE_TMP = '/tmp/pve-scst.conf'; + +my $DAEMON = '/usr/local/sbin/iscsi-scstd'; +my $SETTINGS = undef; +my $CONFIG; + +my @ssh_opts = ('-o', 'BatchMode=yes', '-o','PreferredAuthentications=publickey'); +my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts); +my @scp_cmd = ('/usr/bin/scp', @ssh_opts); +my $id_rsa_path = '/etc/pve/priv/zfs'; +my $scstadmin = '/usr/local/sbin/scstadmin'; + +my $execute_command = sub { + my ($scfg, $exec, $timeout, $method, @params) = @_; + + my $msg = ''; + my $err = undef; + my $target; + my $cmd; + my $res = (); + + $timeout = 10 if !$timeout; + + my $output = sub { + my $line = shift; + $msg .= "$line\n"; + }; + + my $errfunc = sub { + my $line = shift; + $err .= "$line"; + }; + + if ($exec eq 'scp') { + $target = 'root@[' . $scfg->{portal} . ']'; + $cmd = [@scp_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", '--', $method, "$target:$params[0]"]; + } else { + $target = 'root@' . $scfg->{portal}; + $cmd = [@ssh_cmd, '-i', "$id_rsa_path/$scfg->{portal}_id_rsa", $target, '--', $method, @params]; + } + + eval { + run_command($cmd, outfunc => $output, errfunc => $errfunc, timeout => $timeout); + }; + if ($@) { + $res = { + result => 0, + msg => $err, + } + } else { + $res = { + result => 1, + msg => $msg, + } + } + + return $res; +}; + +my $update_config = sub { + my ($scfg) = @_; + + my @params = ('-write_config ', $CONFIG_FILE); + my $res = $execute_command->($scfg, 'ssh', undef, $scstadmin, @params); + die $res->{msg} unless $res->{result}; +}; + +my $get_target_tid = sub { + my ($scfg) = @_; + + my $res = {msg => undef }; + + my @params = ("/sys/kernel/scst_tgt/targets/iscsi/$scfg->{target}/tid"); + $res = $execute_command->($scfg, 'ssh', 10, 'test', '-f', @params); + die "The target not found! ". $res->{msg} unless $res->{result}; + + my $tid = undef; + + $res = $execute_command->($scfg, 'ssh', undef, 'cat', @params); + die $res->{msg} unless $res->{result}; + my @lines = split "\n", $res->{msg}; + $tid = $lines[0]; + return $tid; +}; + + +sub parseLine { + my $line = shift; + my $hash = shift; + my $child = shift; + return if (! $line); + return if ($line =~ /^\s*$/); + + $line =~ s/^\s+//; $line =~ s/\s+$//; + + my @elements; + while ($line =~ m/"([^"\\]*(\\.[^"\\]*)*)"|([^\s]+)/g) { + push @elements, defined($1) ? $1:$3; + } + + my $attribute = $elements[0]; + my $value = $elements[1]; + my $value2 = $elements[2]; + + if (defined($attribute) && defined($value) && defined($value2)) { + $$hash{$attribute}->{$value}->{$value2} = $child; + } elsif (defined($attribute) && defined($value)) { + $$hash{$attribute}->{$value} = $child; + } elsif (defined($attribute)) { + $$hash{$attribute} = $child; + } +}; + + +sub parseStanza { + my $buffer = shift; + my $line; + my %hash; + my $attribute; + my $value; + my $value2; + my $quoted; + + while ($#{$buffer} > -1) { + my $char = shift @{$buffer}; + + if ($char eq '{') { + my $child = parseStanza($buffer); + if ($line) { + parseLine($line, \%hash, $child); + $line = undef; + } + next; + } + + if ($char eq '}') { + return \%hash + }; + + if ($char eq "\n") { + my %empty; + parseLine($line, \%hash, \%empty); + $line = undef; + } else { + $line .= $char; + } + } + return \%hash; +}; + +sub parseLuns { + my $scfg = shift; + my $luns = shift; + my $h_luns = (); + my $base = get_base; + + foreach my $lun (keys %{$$luns} ) { + foreach my $device (keys %{$$luns->{$lun}}) { + if (!defined($$CONFIG{'HANDLER'}) || + !defined($$CONFIG{'HANDLER'}->{$HANDLER}) || + !defined($$CONFIG{'HANDLER'}->{$HANDLER}->{'DEVICE'}->{$device})) { + die "Wrong configuration file $CONFIG_FILE!"; + } + my $dev = $$CONFIG{'HANDLER'}->{$HANDLER}->{'DEVICE'}->{$device}; + + my $conf = undef; + $conf->{Device} = (keys %{$$dev{'prod_id'}})[0]; + $conf->{Path} = (keys %{$$dev{'filename'}})[0]; + $conf->{blocksize} = (keys %{$$dev{'blocksize'}})[0]; + $conf->{size} = (keys %{$$dev{'size'}})[0]; + if ($conf->{Path} && $conf->{Path} =~ /^$base\/$scfg->{pool}\/([\w\-]+)$/) { + $conf->{include} = 1; + } else { + $conf->{include} = 0; + } + $conf->{lun} = $lun; + push @{$h_luns}, $conf; + } + } + return $h_luns; +} + +my $readConfigFile = sub { + my ($scfg) = @_; + my $buffer; + my @stanza; + my $level; + my $used = (); + + my @commands = ("touch '$CONFIG_FILE_TMP'", + "chmod 600 '$CONFIG_FILE_TMP'", + "$scstadmin -write_config '$CONFIG_FILE_TMP' -nonkey", + "cat '$CONFIG_FILE_TMP'", + "rm -f '$CONFIG_FILE_TMP'" + ); + my $res = $execute_command->($scfg, 'ssh', 20, join('&&', @commands)); + die $res->{msg} unless $res->{result}; + my @lines = split "\n", $res->{msg}; + foreach my $line (@lines){ + $line =~ s/^\#.*//; + $line =~ s/[^\\]\#.*//; + $line =~ s/\\(.)/$1/g; + $buffer .= $line."\n"; + } + my @buff_a; + @buff_a = split(//, $buffer); + $CONFIG = parseStanza(\@buff_a); + + if (!defined($$CONFIG{'TARGET_DRIVER'}) || + !(scalar keys %{$$CONFIG{'TARGET_DRIVER'}}) || + !defined($$CONFIG{'TARGET_DRIVER'}->{'iscsi'}) || + !defined($$CONFIG{'TARGET_DRIVER'}->{'iscsi'}->{'TARGET'}->{$scfg->{target}}) + ) + { + die Dumper($CONFIG) ."\nWrong configuration file $CONFIG_FILE!"; + + } + + $SETTINGS->{target} = $scfg->{target}; + my $tgt = $$CONFIG{'TARGET_DRIVER'}->{'iscsi'}->{'TARGET'}->{$scfg->{target}}; + if (defined($$tgt{'LUN'})) { + push @{$SETTINGS->{luns}}, @{parseLuns($scfg ,\$tgt->{'LUN'})}; + } + + if (defined($$tgt{'GROUP'})) { + foreach my $group (keys %{$$tgt{'GROUP'}}) { + if (defined($$tgt{'GROUP'}->{$group}->{'LUN'})) { + push @{$SETTINGS->{luns}}, @{parseLuns($scfg , \$tgt->{'GROUP'}->{$group}->{'LUN'} )}; + } + } + } + foreach my $lun (@{$SETTINGS->{luns}}) { + $used->{$lun->{lun}} = 1; + } + + $SETTINGS->{used} = $used; +# die Dumper($SETTINGS); + return 0; +}; + +my $get_lu_name = sub { + my $i; + + my $used = $SETTINGS->{used}; + for ($i = 0; $i < $MAX_LUNS; $i++) { + last unless $used->{$i}; + } + $SETTINGS->{used}->{$i} = 1; + + return $i; +}; + +my $init_lu_name = sub { + foreach my $lun (@{$SETTINGS->{luns}}) { + $SETTINGS->{used}->{$lun->{lun}} = 1; + } +}; + +my $free_lu_name = sub { + my ($lu_name) = @_; + my $new; + + foreach my $lun (@{$SETTINGS->{luns}}) { + if ($lun->{lun} != $lu_name) { + push @$new, $lun; + } + } + + $SETTINGS->{luns} = $new; + $SETTINGS->{used}->{$lu_name} = 0; +}; + +my $make_lun = sub { + my ($scfg, $path) = @_; + + die 'Maximum number of LUNs per target is 16383' if scalar @{$SETTINGS->{luns}} >= $MAX_LUNS; + + my $lun = $get_lu_name->(); + my $conf = { + lun => $lun, + Path => $path, + include => 1, + }; + push @{$SETTINGS->{luns}}, $conf; + $SETTINGS->{used}->{$lun} = 1; + # print Dumper ($SETTINGS); + return $conf; +}; + +my $list_view = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $lun = undef; + + my $object = $params[0]; + foreach my $lun (@{$SETTINGS->{luns}}) { + next unless $lun->{include} == 1; + if ($lun->{Path} =~ /^$object$/) { + return $lun->{lun} if (defined($lun->{lun})); + die "$lun->{Path}: Missing LUN"; + } + } + + return $lun; +}; + +my $list_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $name = undef; + + my $object = $params[0]; + foreach my $lun (@{$SETTINGS->{luns}}) { + next unless $lun->{include} == 1; + if ($lun->{Path} =~ /^$object$/) { + return $lun->{Path}; + } + } + return $name; +}; + +my $parse_size = sub { + my ($text) = @_; + + return 0 if !$text; + + if ($text =~ m/^(\d+(\.\d+)?)(k)?$/) { + my ($size, $reminder, $unit) = ($1, $2, $3); + return $size if !$unit; + if ($unit eq 'k') { + $size *= 1024; + } + if ($reminder) { + $size = ceil($size); + } + return $size; + } else { + return 0; + } +}; + +sub append_group_params { + my $scfg = shift; + my $params = shift; + if ($scfg->{comstar_tg}) { + push @{$params}, ('-group', $scfg->{comstar_tg} ); + } +} + +my $create_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + + if ($list_lun->($scfg, $timeout, $method, @params)) { + die "$params[0]: LUN exists"; + } + my $lun = $params[0]; + $lun = $make_lun->($scfg, $lun); + + my $tid = $get_target_tid->($scfg); + my $device = 'diskt'.$tid.'l'.$lun->{lun}; + print "Create $HANDLER device $device using filename=$lun->{Path}..."; + my $blocksize = $parse_size->($scfg->{blocksize}); + my $res = {msg => undef}; + + + my @commands = ( + "(i=1; while [ ! -e '$lun->{Path}' -a \$i -le 20 ]; do sleep 1; i=\$((i+1)); done)", + "[ -e '$lun->{Path}' ]", + "$scstadmin"); + @params = ('-open_dev', $device, '-handler', $HANDLER, '-attributes', "filename=$lun->{Path},nv_cache=1,blocksize=512,thin_provisioned=1,zero_copy=1"); + + $res = $execute_command->($scfg, 'ssh', 30, join ('&&', @commands), @params); +# $res = $execute_command->($scfg, 'ssh', 30, $scstadmin, @params); + do { + $free_lu_name->($lun->{lun}); + if ($res->{msg}) { + die $res->{msg} ; + } else { + die 'create_lun: timeout or unknown error! command: '. join ('&&', @commands) . ' ' . join ' ' , @params; + } + } unless $res->{result}; + + print "Done!\n"; + print "Create LUN $lun->{lun} using the device $device for target $scfg->{target}..."; + + @params = ('-add_lun', $lun->{lun}, '-driver', 'iscsi', '-device', $device, '-target', "$scfg->{target}"); + append_group_params ($scfg, \@params); + + $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params); + + do { + print "Failed!\nRemoving device..."; + @params = ('-close_dev', $device, '-handler', $HANDLER,'-noprompt'); + $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params); + print "Done!\n"; + $free_lu_name->($lun->{lun}); + $update_config->($scfg); + die $res->{msg}; + } unless $res->{result}; + print "Done!\n"; + $update_config->($scfg); + return $res->{msg}; +}; + +my $delete_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $res = {msg => undef}; + + my $path = $params[0]; + + + foreach my $lun (@{$SETTINGS->{luns}}) { + if ($lun->{Path} eq $path) { + + my $tid = $get_target_tid->($scfg); + my $device = 'diskt'.$tid.'l'.$lun->{lun}; + print "Delete $HANDLER device $device using filename=$lun->{Path}..."; + + @params = ('-rem_lun', $lun->{lun}, '-driver', 'iscsi', '-device', $device, '-target', "$scfg->{target}", '-noprompt'); + append_group_params ($scfg, \@params); + + $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params); + do { + $free_lu_name->($lun->{lun}); + die $res->{msg}; + } unless $res->{result}; + + print "Done!\n"; + print "Delete LUN $lun->{lun} using the device $device for target $scfg->{target}..."; + + @params = ('-close_dev', $device, '-handler', $HANDLER,'-noprompt'); + $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params); + + if ($res->{result}) { + $free_lu_name->($lun->{lun}); + print "Done!\n"; + last; + } else { + $update_config->($scfg); + die $res->{msg}; + } + } + } + $update_config->($scfg); + return $res->{msg}; +}; + +my $import_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + + print "Import LUN\n"; + return $create_lun->($scfg, $timeout, $method, @params); +}; + +my $modify_lun = sub { + my ($scfg, $timeout, $method, @params) = @_; + my $lun; + my $res = {msg => undef}; + + my $path = $params[1]; + + foreach my $cfg (@{$SETTINGS->{luns}}) { + if ($cfg->{Path} eq $path) { + $lun = $cfg; + last; + } + } + + my $tid = $get_target_tid->($scfg); + my $device = 'diskt'.$tid.'l'.$lun->{lun}; + + + @params = ('-resync_dev', $device, '-handler', $HANDLER,'-noprompt'); + $res = $execute_command->($scfg, 'ssh', $timeout, $scstadmin, @params); + die $res->{msg} unless $res->{result}; + + $update_config->($scfg); + return $res->{msg}; +}; + +my $add_view = sub { + my ($scfg, $timeout, $method, @params) = @_; + return ''; +}; + +my $get_lun_cmd_map = sub { + my ($method) = @_; + + my $cmdmap = { + create_lu => { cmd => $create_lun }, + delete_lu => { cmd => $delete_lun }, + import_lu => { cmd => $import_lun }, + modify_lu => { cmd => $modify_lun }, + add_view => { cmd => $add_view }, + list_view => { cmd => $list_view }, + list_lu => { cmd => $list_lun }, + }; + + die "unknown command '$method'" unless exists $cmdmap->{$method}; + + return $cmdmap->{$method}; +}; + +sub run_lun_command { + my ($scfg, $timeout, $method, @params) = @_; + + $readConfigFile->($scfg) unless $SETTINGS; + my $cmdmap = $get_lun_cmd_map->($method); + my $msg = $cmdmap->{cmd}->($scfg, $timeout, $method, @params); + + return $msg; +} + +sub get_base { + return '/dev/zvol'; +} + +1; + diff --git a/PVE/Storage/ZFSPlugin.pm b/PVE/Storage/ZFSPlugin.pm index f88fe94..6694d4e 100644 --- a/PVE/Storage/ZFSPlugin.pm +++ b/PVE/Storage/ZFSPlugin.pm @@ -12,9 +12,11 @@ use base qw(PVE::Storage::ZFSPoolPlugin); use PVE::Storage::LunCmd::Comstar; use PVE::Storage::LunCmd::Istgt; use PVE::Storage::LunCmd::Iet; +use PVE::Storage::LunCmd::Scst; +use Data::Dumper; -my @ssh_opts = ('-o', 'BatchMode=yes'); +my @ssh_opts = ('-o', 'BatchMode=yes','-o','PreferredAuthentications=publickey'); my @ssh_cmd = ('/usr/bin/ssh', @ssh_opts); my $id_rsa_path = '/etc/pve/priv/zfs'; @@ -31,7 +33,7 @@ my $lun_cmds = { my $zfs_unknown_scsi_provider = sub { my ($provider) = @_; - die "$provider: unknown iscsi provider. Available [comstar, istgt, iet]"; + die "$provider: unknown iscsi provider. Available [comstar, istgt, iet, scst]"; }; my $zfs_get_base = sub { @@ -43,6 +45,8 @@ my $zfs_get_base = sub { return PVE::Storage::LunCmd::Istgt::get_base; } elsif ($scfg->{iscsiprovider} eq 'iet') { return PVE::Storage::LunCmd::Iet::get_base; + } elsif ($scfg->{iscsiprovider} eq 'scst') { + return PVE::Storage::LunCmd::Scst::get_base; } else { $zfs_unknown_scsi_provider->($scfg->{iscsiprovider}); } @@ -63,6 +67,8 @@ sub zfs_request { $msg = PVE::Storage::LunCmd::Istgt::run_lun_command($scfg, $timeout, $method, @params); } elsif ($scfg->{iscsiprovider} eq 'iet') { $msg = PVE::Storage::LunCmd::Iet::run_lun_command($scfg, $timeout, $method, @params); + } elsif ($scfg->{iscsiprovider} eq 'scst') { + $msg = PVE::Storage::LunCmd::Scst::run_lun_command($scfg, $timeout, $method, @params); } else { $zfs_unknown_scsi_provider->($scfg->{iscsiprovider}); } @@ -98,11 +104,13 @@ sub zfs_get_lu_name { my $object = ($zvol =~ /^.+\/.+/) ? "$base/$zvol" : "$base/$scfg->{pool}/$zvol"; + print "$object\n"; + my $lu_name = $class->zfs_request($scfg, undef, 'list_lu', $object); return $lu_name if $lu_name; - die "Could not find lu_name for zvol $zvol"; + die "Could not find lu_name for zvol $zvol: $object"; } sub zfs_add_lun_mapping_entry { @@ -351,6 +359,7 @@ sub volume_has_feature { clone => { base => 1}, template => { current => 1}, copy => { base => 1, current => 1}, + sparseinit => { base => 1, current => 1}, }; my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) = -- 2.11.0 _______________________________________________ pve-devel mailing list pve-devel@pve.proxmox.com https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel
