On Fri, Jul 28, 2006 at 06:24:56PM +0100, James Coleman wrote:
> Hello,
>
> I am regularily looking at memory use of processes on solaris these days.
> Much of my work recently has been using libumem to track exactly what is
> using memory and
> it has proved very useful. I have found a limitation of using libumem in
> this way is that
> large allocations get classed as "oversize" and do not get tracked in
> libumem's caches so they do
> not appear in the output of libumem commands (except ::findleaks for which
> there is some special
> handling of oversize).
>
> Could someone confirm that I am correct about this that oversize
> allocations are not tracked
> in libumem?
They are tracked, but in a different way; you can get all of the oversized
allocations by doing:
> *umem_oversize_arena::walk vmem_alloc | ::vmem_seg -v
Which dcmds in particular are you using that you'd like oversize support for?
> It makes sense really. I am thinking it might be possible to
> extend libumem to
> allow tracking of oversize allocations. Would that be a relatively
> straight-forward task?
> I have the source >;) eek!
Depending upon the dcmd, it is probably quite possible to extend it to take
oversize allocations into account.
Cheers,
- jonathan
>
> To further our ability to test and debug what is the VERY best memory
> profiler tool on solaris? :)
> I have the feeling I am missing something? Am I?
>
> Thanks all for getting this far and bearing with yet another libumem email!
>
> James.
>
>
> aside:
>
> Attached is mdebug.sh script extended so it takes corefile on command-line.
> And extended with a -u option that runs libumem commands.
> e.g. mdebug.sh -u app core or mdebug.sh app core
> I find the ability to specify files on command-line much better.
> Simple changes but makes script much more usable - depending on your
> context maybe.
> #!/usr/bin/bash
> #set -xv
> # Author: Gopinath Rao
> # Developer Technical Support
> # Sun Microsystems Inc
> #
> # 28/07/2006 - James Coleman - Commprove
> # added command-line handling and usage
> # added libumem (-u option)
>
> #
> # Description:
> # adv_analysis() called when the advanced analysis option is chosen
> # on startup
> # Function performs advanced crash dump analysis
> # It calls basic_analysis() to do a basic analysis first
>
> adv_analysis()
> {
>
> basic_analysis
>
> mdb -k $g <<EOA
>
> =nn"** /etc/system entries **"
> ="-------------------------"
> ::system
>
> =nn"** IPC identifiers **"
> ="----------------------"
> ::ipcs -l
>
> =nn"** Callout table **"
> ="----------------------"
> ::callout
>
> =nn"** Modinfo output **"
> ="----------------------"
> ::modinfo
>
> =nn"** Prtconf output **"
> ="----------------------"
> ::prtconf -v
>
> =nn"** Filesystem output **"
> ="----------------------"
> ::fsinfo
>
> =nn"** Kmastat output **"
> ="----------------------"
> ::kmastat
> EOA
>
> }
>
> #
> # Description:
> # special_cases() called when advanced analysis option is chosen
> # Function checks
> # a) Whether it was a forced crash dump and if so runs the "threadlist"
> # macro and the output is written to the file "threadlist.txt"
> # b) Whether the kmem_flags is set to greater than 0xf, and runs the
> # "findleaks" macro. Its output is written to the file
> # "memleaks.txt".
>
> special_cases()
> {
>
> kmem_flags=`echo "kmem_flags/D"| mdb -k unix.$g vmcore.$g| cut -f2 -d":"`
> if [ $kmem_flags -ge 15 ]
> then
> echo -n "Do you want to run the findleaks macro(y/n)(default:n)?"
> read s
> if [ "$s" != "n" ]
> then
> echo "Running findleaks Macro"
> mdb -k $g <<EOA 1>memleaks.txt
> =nn"Findleaks output"
> ="-----------------"
> ::findleaks
> EOA
> fi
> fi
>
> panicstring=`echo "*panicstr/s" | mdb -k unix.$g vmcore.$g | cut -f2 -d":"`
> if [ "$panicstring" = "zero" ]
> then
> mdb -k $g <<EOA 1>threadlist.txt
> =nn"Threadlist output"
> ="------------------"
> \$<threadlist
> EOA
> fi
>
> adv_analysis > outfile.`date |cut -f4 -d" "`
>
> }
>
> #
> # Description:
> # basic_analysis() called when the advanced analysis option is chosen
> # on startup
> # Function performs basic crash dump analysis
>
> basic_analysis()
> {
>
> echo "Working....."
>
> cat <<EOC
>
>
> ******************************************************************************
> System Crash Dump Analysis Output MDeBug Rev 1.0
> `date` Files: unix.$g vmcore.$g
>
> ******************************************************************************
>
> EOC
>
> mdb -k $g <<EOA
>
> =nn"Time of Boot"
> ="---------------"
> ::eval *boot_time=y
> =nn"Time of Crash"
> ="---------------"
> ::eval *time=y
>
> =nn"System Information"
> ="--------------------"
> \$<utsname
> =nn"Panic String"
> ="--------------"
> *panicstr/s
>
> =nn"Stack Backtrace"
> ="-----------------"
> \$c
>
> =nn"** Per CPU information **"
> ="---------------------------"
> ncpus/X
> ncpus_online/X
> =nn
>
> =nn"** Cpuinfo output **"
> ="------------------ -"
> ::cpuinfo -v
> =nn
> =nn"** CPU structures **"
> ="--------------------"
> \$<cpus
> =nn
> =nn"** Process table **"
> ="--------------------"
> ::ps -f
> =nn
> ="** Msgbuf **"
> ="------------"
> \$<msgbuf
> EOA
>
> }
>
> #
> # Description:
> # appcore_analysis() called when the application core analysis option is
> # chosen on startup and the operating system version is Solaris 8 OE or
> # version.
> # Function performs basic application core dump analysis
>
> appcore_analysis()
> {
>
> cat <<EOC
>
>
> ******************************************************************************
> Application core Dump Analysis Output MDeBug Rev 1.0
> `date` Files: $bin $cor
>
> ******************************************************************************
>
> EOC
>
> mdb $bin $cor <<EOA
>
> =nn"** Core file status **"
> ="------------------------"
> ::status
>
> =nn"** Thread stack(\$c) **"
> ="----------------------"
> \$c
>
> =nn"** Shared objects **"
> ="----------------------"
> ::objects
>
> EOA
>
> }
>
> ### Main body of the script
>
> # set up the PATH to the commands used inside this script
> # /usr/ucb set to get the echo behaviour without carriage return
>
> PATH=/usr/ucb:/usr/bin:$PATH
> export PATH
>
> outfile=outfile.appcore
>
> if [[ "$1" == "" ]] ; then
>
> echo "Start of the script"
> tput clear
>
> echo " Welcome to the MDeBug Session "
> echo " ****************************** "
> echo " "
> echo "Select one of the following:"
>
> echo " 1. Run MDeBug against a Kernel Crash dump"
> echo " 2. Run MDeBug against an Application core"
> echo " 3. Exit "
> echo -n "Enter your selection:"
>
> read sel
>
> else
>
> USAGE="usage: $0 [-u|-k] binary_file core\n
> -u: libumem analysis\n
> -k: kernel crash dump analysis";
>
> sel=2
>
> while [[ "$1" != "" ]] ; do
> if [[ "$1" == "-k" ]] ; then
> sel=1
> elif [[ "$1" == "-u" ]] ; then
> umem=1
> elif [[ "$binary_filename" == "" ]] ; then
> binary_filename="$1"
> elif [[ "$core_filename" == "" ]] ; then
> core_filename="$1"
> else
> echo "warning: unknown param: $1"
> fi
> shift
> done
>
> if [[ ! -f $binary_filename ]] ; then
> echo "ERROR: $binary_filename (binary file) not a file?"
> echo $USAGE
> exit
> fi
>
> if [[ ! -f $core_filename ]] ; then
> echo "ERROR: $core_filename (core file) not a file?"
> echo $USAGE
> exit
> fi
>
> bin=$binary_filename
> cor=$core_filename
> outfile=${core_filename}.appcore
>
> fi
>
> case $sel in
> 1)
> if [ `uname -r | cut -f2 -d"."` -lt 8 ]
> then
> echo "OS release needs to be atleast Solaris 5.8 to use this option,
> Aborting !"
> exit
> fi
>
> echo -n "Do you want basic or advanced crash dump analysis[b/a](default:b)?"
> read n
>
> echo -n "Please enter the suffix digit for your crash dump file:"
> read g
>
> if [ "$g" = "" ]
> then
> echo "You did not specify the crash dump file suffix, Aborting!"
> exit
> fi
>
> if [ "$n" = "b" ] || [ "$n" = "" ]
> then
> basic_analysis > outfile.`date |cut -f4 -d" "`
> echo ""
> echo "Done!"
> elif [ "$n" = "a" ]
> then
> special_cases
> echo ""
> echo "Done!"
> fi
> ;;
>
> 2)
>
> if [[ "$bin" == "" ]] ; then
> echo -n "Enter the binary name which generated the core:"
> read bin
> fi
>
> if [[ "$cor" == "" ]] ; then
> echo -n "Enter the core file name:"
> read cor
> fi
>
> if [ `uname -r | cut -f2 -d"."` -lt 9 ]
> then
> echo "OS release needs to be Solaris 5.9 to use this option, Aborting !"
> exit
> fi
>
> appcore_analysis > ${outfile}
>
> thr_model=`echo "::status" | mdb $bin $cor | grep thread | cut -f2 -d":"`
> if [ "$thr_model" = " multi-threaded" ]
> then
> mdb $bin $cor <<EOA 1>>${outfile}
> =nn"Thread stack for MT app"
> ="------------------------"
> ::walk thread | ::findstack
> EOA
> fi
>
> if [ $umem ] ; then
> mdb_dmods=`mdb $bin $cor <<EOA
> ::dmods
> EOA`
> mdb_libumem=`echo ${mdb_dmods//*libumem/libumem} |cut -d" " -f 1`
>
> if [[ ! $mdb_libumem == "" ]] ; then
>
> echo "::umem_status
> ::findleaks
> ::umem_verify
> ::umastat
> ::umausers
> \$q
> " > libumem-check.mdb
>
> mdb $bin $cor < libumem-check.mdb >> ${outfile}
>
> else
> echo "WARNING: no libumem info available" >> ${outfile}
> echo "WARNING: no libumem info available"
> fi
>
> fi
>
> echo "Done!output: ${outfile}"
> ;;
>
> 3)
> echo "Bye !"
> exit;;
>
> *)
> echo "Invalid selection - Aborting !"
> ;;
>
> esac
> # End of the script -- 02/21/2002 --grao
>
> _______________________________________________
> mdb-discuss mailing list
> mdb-discuss at opensolaris.org
--
Jonathan Adams, Solaris Kernel Development
