Thanks. This makes sense and more what I was expecting.
iscsiadm is stuck on the scan completion. It might have sent some
inquiry and maybe report luns (can't tell from the below how far we
got). While this scan is going on a couple mutexes are held. This is why
if you did try a logout command later we would see the other hang you sent.
For why we are hanging in the scan, could you send enable iscsi logging
when you run the iscsiadm rescan command:
echo 1 > /sys/module/libiscsi/paramters/debug_libiscsi_conn
echo 1 > /sys/module/libiscsi/paramters/debug_libiscsi_session
echo 1 > /sys/module/libiscsi/paramters/debug_libiscsi_eh
echo 1 > /sys/module/libiscsi_tcp/paramters/debug_libiscsi_tcp
echo 1 > /sys/module/iscsi_tcp/paramters/debug_iscsi_tcp
Could you also enable scsi scan logging? If you do not know how to do
that, I attached the scsi_logging_level script from the s390 tools
(works on any arch). Just run
scsi_logging_level --scan 7 -s
On 4/17/14, 6:14 AM, Cale, Yonatan wrote:
Hi Mike,
Regarding the analysis of waiting-tasks.txt, I'd like to add some information
about the issue:
After running the "rescan" that gets stuck, any other iscsiadm command we try
running - gets stuck too. I'm guessing that this is because the first command locks some
mutex that all other commands wait on. (I didn't verify this, it's mainly a guess)
I'm guessing that the command you analyzed in the previous mail is not the original
"rescan" but something else that was run later, or in other words, I think it's
not the root cause.
Here are 2 example stack traces (from 2 different runs) that I believe belong
to the stuck iscsiadm process:
{
iscsiadm R running task 0 4597 1 0x00000004
ffff88009bab38b8 ffffffff815ec84c ffff88009bab37d8 ffff88009bab2010
0000000000004000 ffff88009ba1d5c0 ffff88009bab3fd8 ffff88009bab3fd8
0000000000000000 ffff88009ba1d5c0 ffff88013b088c40 ffff88009ba1d858
Call Trace:
[<ffffffff815ec84c>] ? __schedule+0x3fc/0x8d0
[<ffffffff8103c765>] ? wake_up_process+0x15/0x20
[<ffffffff810488b5>] ? raise_softirq_irqoff+0x95/0xb0
[<ffffffff81279536>] ? number+0x316/0x340
[<ffffffff8126df97>] ? cfq_set_request+0x257/0x470
[<ffffffffa0028002>] ? iscsi_queuecommand+0x102/0x4f0 [libiscsi]
[<ffffffff81051c4b>] ? lock_timer_base+0x3b/0x70
[<ffffffff810e1d3d>] free_percpu+0x9d/0x130
[<ffffffff810e22b9>] ? pcpu_alloc+0x1e9/0x900
[<ffffffff81273bda>] ? kobject_release+0x4a/0xa0
[<ffffffff8127400d>] ? kobject_set_name_vargs+0x6d/0x80
[<ffffffff8112e0b6>] ? bio_phys_segments+0x26/0x30
[<ffffffff8125ca20>] ? blk_rq_bio_prep+0x30/0xc0
[<ffffffff8125f795>] ? blk_put_queue+0x15/0x20
[<ffffffff81310a8f>] ? scsi_device_dev_release_usercontext+0x17f/0x1d0
[<ffffffff8125f61c>] ? blk_alloc_queue_node+0x6c/0x1b0
[<ffffffff8125fcb4>] ? blk_init_queue_node+0x34/0x80
[<ffffffff812739b7>] ? kobject_init+0x37/0xa0
[<ffffffff8130cd26>] ? scsi_execute_req+0xa6/0x140
[<ffffffff8130e0f9>] ? scsi_probe_and_add_lun+0x289/0xce0
[<ffffffff810cbb5d>] ? free_one_page+0x20d/0x410
[<ffffffff815f01ce>] ? apic_timer_interrupt+0xe/0x20
[<ffffffff81273bed>] ? kobject_release+0x5d/0xa0
[<ffffffff8130f168>] ? __scsi_scan_target+0x3d8/0x620
[<ffffffff81279d5b>] ? vsscanf+0x2ab/0x790
[<ffffffff8130fa28>] ? scsi_scan_target+0xc8/0xd0
[<ffffffffa00111b6>] ? iscsi_user_scan_session+0x156/0x1a0
[scsi_transport_iscsi]
[<ffffffffa0011060>] ? iscsi_host_for_each_session+0x20/0x20
[scsi_transport_iscsi]
[<ffffffff812dcc73>] ? device_for_each_child+0x53/0x80
[<ffffffffa001103d>] ? iscsi_user_scan+0x2d/0x30 [scsi_transport_iscsi]
[<ffffffff813105cc>] ? store_scan+0xec/0x120
[<ffffffff8110b568>] ? do_filp_open+0x48/0xa0
[<ffffffff812dbdc0>] ? dev_attr_store+0x20/0x30
[<ffffffff81162f31>] ? sysfs_write_file+0xd1/0x150
[<ffffffff810fcf3b>] ? vfs_write+0xcb/0x170
[<ffffffff810fd0e5>] ? sys_write+0x55/0x90
[<ffffffff815ef86b>] ? system_call_fastpath+0x16/0x1b
}
{
iscsiadm R running task 0 2845 1 0x00000004
ffff8800682078b8 ffffffff815ec84c ffff8800682077d8 ffff880068206010
0000000000004000 ffff8800680e8c40 ffff880068207fd8 ffff880068207fd8
0000000000000000 ffff8800680e8c40 ffff88013b088c40 ffff8800680e8ed8
Call Trace:
[<ffffffff815ec84c>] ? __schedule+0x3fc/0x8d0
[<ffffffff8103c765>] ? wake_up_process+0x15/0x20
[<ffffffff810488b5>] ? raise_softirq_irqoff+0x95/0xb0
[<ffffffff81279536>] ? number+0x316/0x340
[<ffffffff8126df97>] ? cfq_set_request+0x257/0x470
[<ffffffff8130c02d>] ? scsi_setup_blk_pc_cmnd+0x12d/0x160
[<ffffffff81304eb1>] ? scsi_dispatch_cmd+0xf1/0x210
[<ffffffff815ecdc5>] ? _cond_resched+0x35/0x50
[<ffffffff815ece1b>] ? wait_for_common+0x3b/0x180
[<ffffffff8128ae8a>] ? percpu_counter_destroy+0x1a/0x30
[<ffffffff812760c9>] ? prop_local_destroy_percpu+0x9/0x10
[<ffffffff810e22b9>] ? pcpu_alloc+0x1e9/0x900
[<ffffffff81273bda>] ? kobject_release+0x4a/0xa0
[<ffffffff81273fcd>] ? kobject_set_name_vargs+0x2d/0x80
[<ffffffff812dca69>] ? dev_set_name+0x69/0x70
[<ffffffff8125ca20>] ? blk_rq_bio_prep+0x30/0xc0
[<ffffffff8125f795>] ? blk_put_queue+0x15/0x20
[<ffffffff81310a8f>] ? scsi_device_dev_release_usercontext+0x17f/0x1d0
[<ffffffff8125f61c>] ? blk_alloc_queue_node+0x6c/0x1b0
[<ffffffff8125fcb4>] ? blk_init_queue_node+0x34/0x80
[<ffffffff8125fd13>] ? blk_init_queue+0x13/0x20
[<ffffffff812e3e55>] ? transport_setup_device+0x15/0x20
[<ffffffff8130cd26>] ? scsi_execute_req+0xa6/0x140
[<ffffffff8130e0f9>] ? scsi_probe_and_add_lun+0x289/0xce0
[<ffffffff810cbb5d>] ? free_one_page+0x20d/0x410
[<ffffffff815f01ce>] ? apic_timer_interrupt+0xe/0x20
[<ffffffff81273bed>] ? kobject_release+0x5d/0xa0
[<ffffffff8130f168>] ? __scsi_scan_target+0x3d8/0x620
[<ffffffff81279d5b>] ? vsscanf+0x2ab/0x790
[<ffffffff8130fa28>] ? scsi_scan_target+0xc8/0xd0
[<ffffffffa00111b6>] ? iscsi_user_scan_session+0x156/0x1a0
[scsi_transport_iscsi]
[<ffffffffa0011060>] ? iscsi_host_for_each_session+0x20/0x20
[scsi_transport_iscsi]
[<ffffffff812dcc73>] ? device_for_each_child+0x53/0x80
[<ffffffffa001103d>] ? iscsi_user_scan+0x2d/0x30 [scsi_transport_iscsi]
[<ffffffff813105cc>] ? store_scan+0xec/0x120
[<ffffffff8110b568>] ? do_filp_open+0x48/0xa0
[<ffffffff812dbdc0>] ? dev_attr_store+0x20/0x30
[<ffffffff81162f31>] ? sysfs_write_file+0xd1/0x150
[<ffffffff810fcf3b>] ? vfs_write+0xcb/0x170
[<ffffffff810fd0e5>] ? sys_write+0x55/0x90
[<ffffffff815ef86b>] ? system_call_fastpath+0x16/0x1b
}
They have a lot of similarities but also differences. I provided 2 traces
hoping that it will be more informative for you.
They were obtained by running:
echo t > /proc/sysrq-trigger
dmesg > dmesg.txt
I know that it's the stuck process because I ran "killall iscsid" first, and because
"dmesg.txt" only contains one "iscsiadm" process.
Does this make sense?
-----Original Message-----
From: Cale, Yonatan
Sent: Wednesday, April 16, 2014 7:22 PM
To: 'Mike Christie'
Cc: [email protected]; [email protected]
Subject: RE: Target reboot -> iscsiadm rescan Stuck
Hi,
Attached is a compressed tcpdump capture (iscsiadm_spreboot_3.gz, ~300kb), Some
detailes about it:
It was obtained by running " tcpdump icmp and src or dst 10.76.18.174 or src or dst
10.76.18.175 -i any -w iscsiadm_spreboot_3".
My local iscsi port IPs are 10.76.18.174 & 10.76.18.175.
The target iscsi IPs are 10.76.18.31 & 10.76.18.33.
Chronological order:
-Start capture
-A few seconds later, reboot SP A (one of the iscsi targets) -A long time (~10
minutes) later, the bug happens (I notice it because iscsiadm starts taking 85%
CPU) -A few seconds later, stop capture
I'm mentioning the order mainly to point out that the bug happens in the last
few seconds of the capture
And thank you for sitting on this bug with me.
:)
-----Original Message-----
From: Mike Christie [mailto:[email protected]]
Sent: Monday, April 14, 2014 8:07 PM
To: Cale, Yonatan
Cc: [email protected]; [email protected]
Subject: Re: Target reboot -> iscsiadm rescan Stuck
On 04/13/2014 07:29 AM, Cale, Yonatan wrote:
# cat waiting-tasks.txt
SysRq : Show Blocked State
task PC stack pid father
iscsid D 0000000000000000 0 2842 1 0x00000000
ffff880137f83918 0000000000000086 ffff88010e3be2d0 ffff880137f82010
0000000000004000 ffff88013b2b8c40 ffff880137f83fd8 ffff880137f83fd8
0000000000000000 ffff88013b2b8c40 ffffffff81a0b020 ffff88013b2b8ed0
Call Trace:
[<ffffffff81273a97>] ? kobject_put+0x27/0x60 [<ffffffff812dc447>] ?
put_device+0x17/0x20 [<ffffffff8103418f>] ? complete+0x4f/0x60
[<ffffffff815ed07f>] schedule+0x3f/0x60 [<ffffffff815eda02>]
__mutex_lock_slowpath+0x102/0x180 [<ffffffff815edf0b>]
mutex_lock+0x2b/0x50 [<ffffffffa0011a97>]
__iscsi_unbind_session+0x67/0x160 [scsi_transport_iscsi]
[<ffffffffa0011ca1>] iscsi_remove_session+0x111/0x1f0
[scsi_transport_iscsi] [<ffffffffa0011d96>]
iscsi_destroy_session+0x16/0x60 [scsi_transport_iscsi]
[<ffffffffa002573d>] iscsi_session_teardown+0x9d/0xd0 [libiscsi]
[<ffffffffa0032300>] iscsi_sw_tcp_session_destroy+0x50/0x70
[iscsi_tcp] [<ffffffffa0012c9d>] iscsi_if_rx+0x7dd/0xaa0
[scsi_transport_iscsi] [<ffffffff814f50ee>]
netlink_unicast+0x2ae/0x2c0 [<ffffffff814d11dc>] ?
memcpy_fromiovec+0x7c/0xa0 [<ffffffff814f5aae>]
netlink_sendmsg+0x33e/0x380 [<ffffffff814c55f8>]
sock_sendmsg+0xe8/0x120 [<ffffffff811078bf>] ? do_lookup+0xcf/0x360
[<ffffffff8111ad6f>] ? mntput+0x1f/0x40 [<ffffffff81107012>] ?
path_put+0x22/0x30 [<ffffffff814c512b>] ?
move_addr_to_kernel+0x6b/0x70 [<ffffffff814d13a1>] ?
verify_iovec+0x51/0x100 [<ffffffff814c683f>]
__sys_sendmsg+0x3df/0x3f0 [<ffffffff810f8419>] ?
kmem_cache_free+0xe9/0xf0 [<ffffffff81100dac>] ?
cp_new_stat+0xfc/0x120 [<ffffffff814c6a79>] sys_sendmsg+0x49/0x80
[<ffffffff815ef86b>] system_call_fastpath+0x16/0x1b
Ok, that was more interesting than I expected. If you are not running a
iscsiadm logout command, then it looks like the target returned a error code
indicating that target is not coming back. iscsid handled this by trying to
remove the target and we are stuck in there. This and the scan command should
be fast at this time because it looks like the replacement/recovery timeout has
expired:
session1: session recovery timed out after 120 secs
session2: session recovery timed out after 120 secs
At this time, all IO is just fast failed so we are not waiting for IO and any
time we need that mutex it should be quick.
1. Can you send me a tcpdump trace? I am guessing that you do not want the
session to be deleted during this time, so we need the trace to see what error
code is leading to this.
2. I am wondering if the hang is because the device is offlined when this
happened. I did not try that case. I will setup that kernel and try here.
--
You received this message because you are subscribed to the Google Groups
"open-iscsi" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
To post to this group, send email to [email protected].
Visit this group at http://groups.google.com/group/open-iscsi.
For more options, visit https://groups.google.com/d/optout.
#! /bin/bash
###############################################################################
# Conveniently create and set scsi logging level, show SCSI_LOG fields in human
# readable form.
#
# (C) Copyright IBM Corp. 2006
###############################################################################
REVISION="1.0"
SCRIPTNAME="scsi_logging_level"
declare -i LOG_ERROR=0
declare -i LOG_TIMEOUT=0
declare -i LOG_SCAN=0
declare -i LOG_MLQUEUE=0
declare -i LOG_MLCOMPLETE=0
declare -i LOG_LLQUEUE=0
declare -i LOG_LLCOMPLETE=0
declare -i LOG_HLQUEUE=0
declare -i LOG_HLCOMPLETE=0
declare -i LOG_IOCTL=0
declare -i LEVEL=0
SET=0
GET=0
CREATE=0
OPTS=`getopt -o hvcgsa:E:T:S:I:M:L:H: --long \
help,version,create,get,set,all:,error:,timeout:,scan:,ioctl:,\
midlevel:,mlqueue:,mlcomplete:,lowlevel:,llqueue:,llcomplete:,\
highlevel:,hlqueue:,hlcomplete: -n \'$SCRIPTNAME\' -- "$@"`
eval set -- "$OPTS"
# print version info
printversion()
{
cat <<EOF
%S390_TOOLS_VERSION% ($SCRIPTNAME $REVISION)
(C) Copyright IBM Corp. 2006
EOF
}
# print usage and help
printhelp()
{
cat <<EOF
Usage: $SCRIPTNAME [OPTIONS]
Create, get or set scsi logging level.
Options:
-h, --help print this help
-v, --version print version information
-s, --set create and set logging level as specified on
command line
-g, --get get current logging level and display it
-c, --create create logging level as specified on command line
-a, --all specify value for all SCSI_LOG fields
-E, --error specify SCSI_LOG_ERROR
-T, --timeout specify SCSI_LOG_TIMEOUT
-S, --scan specify SCSI_LOG_SCAN
-M, --midlevel specify SCSI_LOG_MLQUEUE and SCSI_LOG_MLCOMPLETE
--mlqueue specify SCSI_LOG_MLQUEUE
--mlcomplete specify SCSI_LOG_MLCOMPLETE
-L, --lowlevel specify SCSI_LOG_LLQUEUE and SCSI_LOG_LLCOMPLETE
--llqueue specify SCSI_LOG_LLQUEUE
--llcomplete specify SCSI_LOG_LLCOMPLETE
-H, --highlevel specify SCSI_LOG_HLQUEUE and SCSI_LOG_HLCOMPLETE
--hlqueue specify SCSI_LOG_HLQUEUE
--hlcomplete specify SCSI_LOG_HLCOMPLETE
-I, --ioctl specify SCSI_LOG_IOCTL
Exactly one of the options "-c", "-g" and "-s" has to be specified.
Valid values for SCSI_LOG fields are integers from 0 to 7.
Note: Several SCSI_LOG fields can be specified using several options.
When multiple options specify same SCSI_LOG field the most specific
option has precedence.
Example: "scsi_logging_level --hlqueue 3 --highlevel 2 --all 1 -s" sets
SCSI_LOG_HLQUEUE=3, SCSI_LOG_HLCOMPLETE=2 and assigns all other SCSI_LOG
fields the value 1.
EOF
}
check_level()
{
if [ `echo -n $1 | tr --complement [:digit:] 'a' | grep -s 'a'` ]
then
invalid_cmdline "log level '$1' out of range [0, 7]"
fi
if [ $1 -lt 0 -o $1 -gt 7 ]
then
invalid_cmdline "log level '$1' out of range [0, 7]"
fi
}
# check cmd line arguments
check_cmdline()
{
while true ; do
case "$1" in
-a|--all) _ALL=$2; check_level $2
shift 2;;
-c|--create) CREATE=1;
shift 1;;
-g|--get) GET=1
shift 1;;
-h|--help) printhelp
exit 0;;
-s|--set) SET=1
shift 1;;
-v|--version) printversion
exit 0;;
-E|--error) _ERROR=$2; check_level $2
shift 2;;
-T|--timeout) _TIMEOUT=$2; check_level $2
shift 2;;
-S|--scan) _SCAN=$2; check_level $2
shift 2;;
-M|--midlevel) _ML=$2; check_level $2
shift 2;;
--mlqueue) _MLQUEUE=$2; check_level $2
shift 2;;
--mlcomplete) _MLCOMPLETE=$2; check_level $2
shift 2;;
-L|--lowlevel) _LL=$2; check_level $2
shift 2;;
--llqueue) _LLQUEUE=$2; check_level $2
shift 2;;
--llcomplete) _LLCOMPLETE=$2; check_level $2
shift 2;;
-H|--highlevel) _HL=$2; check_level $2
shift 2;;
--hlqueue) _HLQUEUE=$2; check_level $2
shift 2;;
--hlcomplete) _HLCOMPLETE=$2; check_level $2
shift 2;;
-I|--ioctl) _IOCTL=$2; check_level $2
shift 2;;
--) shift; break;;
*) echo "Internal error!" ; exit 1;;
esac
done
if [ -n "$*" ]
then
invalid_cmdline invalid parameter $*
fi
if [ $GET = "1" -a $SET = "1" ]
then
invalid_cmdline options \'-c\', \'-g\' and \'-s\' are mutual exclusive
elif [ $GET = "1" -a $CREATE = "1" ]
then
invalid_cmdline options \'-c\', \'-g\' and \'-s\' are mutual exclusive
elif [ $SET = "1" -a $CREATE = "1" ]
then
invalid_cmdline options \'-c\', \'-g\' and \'-s\' are mutual exclusive
fi
LOG_ERROR=${_ERROR:-${_ALL:-0}}
LOG_TIMEOUT=${_TIMEOUT:-${_ALL:-0}}
LOG_SCAN=${_SCAN:-${_ALL:-0}}
LOG_MLQUEUE=${_MLQUEUE:-${_ML:-${_ALL:-0}}}
LOG_MLCOMPLETE=${_MLCOMPLETE:-${_ML:-${_ALL:-0}}}
LOG_LLQUEUE=${_LLQUEUE:-${_LL:-${_ALL:-0}}}
LOG_LLCOMPLETE=${_LLCOMPLETE:-${_LL:-${_ALL:-0}}}
LOG_HLQUEUE=${_HLQUEUE:-${_HL:-${_ALL:-0}}}
LOG_HLCOMPLETE=${_HLCOMPLETE:-${_HL:-${_ALL:-0}}}
LOG_IOCTL=${_IOCTL:-${_ALL:-0}}
}
invalid_cmdline()
{
echo "$SCRIPTNAME: $*"
echo "$SCRIPTNAME: Try '$SCRIPTNAME --help' for more information."
exit 1
}
get_logging_level()
{
echo "Current scsi logging level:"
LEVEL=`sysctl -n dev.scsi.logging_level`
if [ $? != 0 ]
then
echo "$SCRIPTNAME: could not read scsi logging level" \
"(kernel probably without SCSI_LOGGING support)"
exit 1
fi
}
show_logging_level()
{
echo "dev.scsi.logging_level = $LEVEL"
LOG_ERROR=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_TIMEOUT=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_SCAN=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_MLQUEUE=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_MLCOMPLETE=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_LLQUEUE=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_LLCOMPLETE=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3))
LOG_HLQUEUE=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3));
LOG_HLCOMPLETE=$(($LEVEL & 7)); LEVEL=$(($LEVEL>>3));
LOG_IOCTL=$(($LEVEL & 7))
echo "SCSI_LOG_ERROR=$LOG_ERROR"
echo "SCSI_LOG_TIMEOUT=$LOG_TIMEOUT"
echo "SCSI_LOG_SCAN=$LOG_SCAN"
echo "SCSI_LOG_MLQUEUE=$LOG_MLQUEUE"
echo "SCSI_LOG_MLCOMPLETE=$LOG_MLCOMPLETE"
echo "SCSI_LOG_LLQUEUE=$LOG_LLQUEUE"
echo "SCSI_LOG_LLCOMPLETE=$LOG_LLCOMPLETE"
echo "SCSI_LOG_HLQUEUE=$LOG_HLQUEUE"
echo "SCSI_LOG_HLCOMPLETE=$LOG_HLCOMPLETE"
echo "SCSI_LOG_IOCTL=$LOG_IOCTL"
}
set_logging_level()
{
echo "New scsi logging level:"
sysctl -q -w dev.scsi.logging_level=$LEVEL
if [ $? != 0 ]
then
echo "$SCRIPTNAME: could not write scsi logging level" \
"(kernel probably without SCSI_LOGGING support)"
exit 1
fi
}
create_logging_level()
{
LEVEL=$(($LOG_IOCTL & 7)); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_HLCOMPLETE & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_HLQUEUE & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_LLCOMPLETE & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_LLQUEUE & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_MLCOMPLETE & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_MLQUEUE & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_SCAN & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_TIMEOUT & 7))); LEVEL=$(($LEVEL<<3))
LEVEL=$(($LEVEL|($LOG_ERROR & 7)))
}
check_cmdline $*
if [ $SET = "1" ]
then
create_logging_level
set_logging_level
show_logging_level
elif [ $GET = "1" ]
then
get_logging_level
show_logging_level
elif [ $CREATE = "1" ]
then
create_logging_level
show_logging_level
else
invalid_cmdline missing option \'-g\', \'-s\' or \'-c\'
fi
