On Oct 26, 2009, at 4:00 PM, Cindy Swearingen wrote:
Hi Jeremy,
Can you use the command below and send me the output, please?
Thanks,
Cindy
# mdb -k
> ::stacks -m zfs
Ok, it did it again. I replaced the drive and it's currently
resilvering (13.66% done, 135h33m to go it says) and the output of
that command is this:
> ::stacks -m zfs
THREAD STATE SOBJ COUNT
ffffff02f0078a80 SLEEP MUTEX 31
swtch+0x147
turnstile_block+0x764
mutex_vector_enter+0x261
zfs_zget+0x47
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
lookupname+0x28
chroot+0x30
_sys_sysenter_post_swapgs+0x14b
ffffff02efa33500 SLEEP CV 18
swtch+0x147
cv_wait+0x61
dbuf_read+0x237
dmu_buf_hold+0x96
zap_lockdir+0x67
zap_lookup_norm+0x55
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
stat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02da1a91e0 SLEEP CV 9
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_lockdir+0x67
zap_lookup_norm+0x55
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
stat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02d8c46ac0 SLEEP CV 7
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_hold_impl+0x81
dbuf_findbp+0xcf
dbuf_hold_impl+0x81
dbuf_hold+0x2e
dnode_hold_impl+0xb5
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
lookupname+0x28
chroot+0x30
_sys_sysenter_post_swapgs+0x14b
ffffff02da1a2720 SLEEP CV 6
swtch+0x147
cv_wait+0x61
txg_wait_open+0x7a
dmu_tx_wait+0xb3
dmu_tx_assign+0x4b
zfs_inactive+0xa8
fop_inactive+0xaf
vn_rele+0x5f
closef+0x75
closeandsetf+0x44a
close+0x18
_sys_sysenter_post_swapgs+0x14b
ffffff000f61bc60 SLEEP CV 5
swtch+0x147
cv_wait+0x61
txg_thread_wait+0x5f
txg_quiesce_thread+0x94
thread_start+8
ffffff02d8514aa0 SLEEP CV 5
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_hold_impl+0x81
dbuf_findbp+0xcf
dbuf_hold_impl+0x81
dbuf_findbp+0xcf
dbuf_hold_impl+0x81
dbuf_hold+0x2e
dnode_hold_impl+0xb5
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
lookupname+0x28
chroot+0x30
_sys_sysenter_post_swapgs+0x14b
ffffff02d89b4c20 SLEEP CV 3
swtch+0x147
cv_wait+0x61
txg_wait_synced+0x7f
dmu_tx_wait+0xcd
zfs_create+0x44d
fop_create+0xfc
vn_createat+0x5e1
vn_openat+0x1fb
copen+0x418
open64+0x34
_sys_sysenter_post_swapgs+0x14b
ffffff02dc0a5400 SLEEP CV 3
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_get_leaf_byblk+0x56
zap_deref_leaf+0x78
fzap_lookup+0x78
zap_lookup_norm+0x116
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0xd2
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02efc771a0 SLEEP CV 3
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dmu_buf_hold_array_by_dnode+0x220
dmu_buf_hold_array+0x73
dmu_read_uio+0x4d
zfs_read+0x19a
fop_read+0x6b
rfs3_read+0x393
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff001076dc60 SLEEP CV 3
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dsl_pool_sync+0xe1
spa_sync+0x32a
txg_sync_thread+0x265
thread_start+8
ffffff02d8c4d900 SLEEP CV 2
swtch+0x147
cv_wait+0x61
txg_wait_open+0x7a
dmu_tx_wait+0xb3
zfs_write+0x3bd
fop_write+0x6b
write+0x2e2
write32+0x22
_sys_sysenter_post_swapgs+0x14b
ffffff02efa98e80 SLEEP CV 2
swtch+0x147
cv_wait+0x61
zil_commit+0x62
zfs_putpage+0x2c9
fop_putpage+0x74
rfs3_commit+0x180
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02ef89a3c0 SLEEP CV 2
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_hold_impl+0x81
dbuf_findbp+0xcf
dbuf_hold_impl+0x81
dbuf_hold+0x2e
dnode_hold_impl+0xb5
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
stat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02efb24b00 SLEEP CV 2
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_lockdir+0x67
zap_cursor_retrieve+0x74
zfs_readdir+0x29e
fop_readdir+0xab
getdents64+0xbc
_sys_sysenter_post_swapgs+0x14b
ffffff02efce3c40 SLEEP CV 2
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dnode_hold_impl+0xd9
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_vget+0x25e
fsop_vget+0x67
nfs3_fhtovp+0x47
rfs3_getattr+0x40
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02efc763a0 SLEEP MUTEX 2
swtch+0x147
turnstile_block+0x764
mutex_vector_enter+0x261
zfs_zget+0x47
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
stat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff000f531c60 SLEEP CV 1
swtch+0x147
cv_timedwait+0xba
arc_reclaim_thread+0x17b
thread_start+8
ffffff000f537c60 SLEEP CV 1
swtch+0x147
cv_timedwait+0xba
l2arc_feed_thread+0xa5
thread_start+8
ffffff000f621c60 SLEEP CV 1
swtch+0x147
cv_timedwait+0xba
txg_thread_wait+0x7b
txg_sync_thread+0x114
thread_start+8
ffffff02d8c4ee60 SLEEP CV 1
swtch+0x147
cv_wait+0x61
dmu_buf_hold_array_by_dnode+0x2b7
dmu_buf_hold_array+0x73
dmu_read_uio+0x4d
zfs_read+0x19a
fop_read+0x6b
rfs3_read+0x393
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02dc0a6c80 SLEEP CV 1
swtch+0x147
cv_wait+0x61
spa_config_enter+0x7d
spa_vdev_enter+0x2f
spa_vdev_setpath+0x32
zfs_ioc_vdev_setpath+0x48
zfsdev_ioctl+0x10b
cdev_ioctl+0x45
spec_ioctl+0x83
fop_ioctl+0x7b
ioctl+0x18e
_sys_sysenter_post_swapgs+0x14b
ffffff052a2551c0 SLEEP CV 1
swtch+0x147
cv_wait+0x61
txg_wait_open+0x7a
dmu_tx_wait+0xb3
zfs_setattr+0xca8
fop_setattr+0xad
vpsetattr+0x12a
fdsetattr+0x30
fchmod+0x3a
_sys_sysenter_post_swapgs+0x14b
ffffff02d89b7a80 SLEEP CV 1
swtch+0x147
cv_wait+0x61
txg_wait_open+0x7a
dmu_tx_wait+0xb3
zfs_write+0x3bd
fop_write+0x6b
rfs3_write+0x507
common_dispatch+0x4a7
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02dc0a5780 SLEEP CV 1
swtch+0x147
cv_wait+0x61
txg_wait_synced+0x7f
dmu_tx_wait+0xcd
zfs_rename+0x553
fop_rename+0xc5
vn_renameat+0x2ff
vn_rename+0x2f
rename+0x17
_sys_sysenter_post_swapgs+0x14b
ffffff02d8c34720 SLEEP CV 1
swtch+0x147
cv_wait+0x61
txg_wait_synced+0x7f
spa_vdev_state_exit+0x4e
vdev_fault+0xde
zfs_ioc_vdev_set_state+0xa6
zfsdev_ioctl+0x10b
cdev_ioctl+0x45
spec_ioctl+0x83
fop_ioctl+0x7b
ioctl+0x18e
_sys_sysenter_post_swapgs+0x14b
ffffff02efa3f540 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zil_commit+0x62
zfs_fsync+0xd3
fop_fsync+0x5a
rfs3_create+0x767
common_dispatch+0x4a7
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02efa3e3c0 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zil_commit+0x62
zfs_fsync+0xd3
fop_fsync+0x5a
rfs3_setattr+0x447
common_dispatch+0x4a7
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02efc77520 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_hold_impl+0x81
dbuf_findbp+0xcf
dbuf_hold_impl+0x81
dbuf_findbp+0xcf
dbuf_hold_impl+0x81
dbuf_hold+0x2e
dnode_hold_impl+0xb5
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
stat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02ef791c60 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_hold_impl+0x81
dbuf_hold+0x2e
dnode_hold_impl+0xb5
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
lookupname+0x28
chroot+0x30
_sys_sysenter_post_swapgs+0x14b
ffffff02d8c40b00 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_hold_impl+0x81
dbuf_hold+0x2e
dnode_hold_impl+0xb5
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_vget+0x25e
fsop_vget+0x67
nfs3_fhtovp+0x47
rfs3_getattr+0x40
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02efb398e0 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dbuf_findbp+0xe7
dbuf_prefetch+0x93
dmu_zfetch_fetch+0x62
dmu_zfetch_dofetch+0xb8
dmu_zfetch_find+0x394
dmu_zfetch+0xac
dbuf_read+0x170
dnode_hold_impl+0xd9
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_dirent_lock+0x3fc
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff0372bdac60 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_get_leaf_byblk+0x56
zap_deref_leaf+0x78
fzap_lookup+0x78
zap_lookup_norm+0x116
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02da1a4380 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_lockdir+0x67
zap_lookup_norm+0x55
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0x91
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02efce3540 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_lockdir+0x67
zap_lookup_norm+0x55
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
rfs3_lookup+0x395
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02d8c44e60 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dmu_buf_hold+0x96
zap_table_load+0x76
zap_idx_to_blk+0x56
zap_deref_leaf+0x60
fzap_lookup+0x78
zap_lookup_norm+0x116
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0xd2
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff02d8c45c60 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dnode_hold_impl+0xd9
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_dirent_lock+0x3fc
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
rfs3_lookup+0x395
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02da1a7c80 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dnode_hold_impl+0xd9
dnode_hold+0x2b
dmu_bonus_hold+0x36
zfs_zget+0x5a
zfs_root+0x57
fsop_root+0x2e
traverse+0x61
lookuppnvp+0x423
lookuppnat+0x12c
lookupnameat+0x91
lookupname+0x28
chroot+0x30
_sys_sysenter_post_swapgs+0x14b
ffffff02d8c50ac0 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dbuf_read+0x1e8
dnode_hold_impl+0xd9
dnode_hold+0x2b
dmu_object_info+0x30
bplist_open+0x36
dsl_dataset_get_ref+0x139
dsl_dataset_hold+0x100
dmu_objset_prefetch+0x24
findfunc+0x23
dmu_objset_find_spa+0x30d
dmu_objset_find+0x40
zfs_ioc_snapshot_list_next+0x59
zfsdev_ioctl+0x10b
cdev_ioctl+0x45
spec_ioctl+0x83
fop_ioctl+0x7b
ioctl+0x18e
_sys_sysenter_post_swapgs+0x14b
ffffff02efa3fc40 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dmu_tx_hold_free+0x19f
zfs_remove+0x2b9
fop_remove+0xa5
vn_removeat+0x2f0
vn_remove+0x2c
unlink+0x19
_sys_sysenter_post_swapgs+0x14b
ffffff001073dc60 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
dsl_pool_sync+0x1f6
spa_sync+0x3cd
txg_sync_thread+0x265
thread_start+8
ffffff02d8c40780 SLEEP CV 1
swtch+0x147
cv_wait+0x61
zio_wait+0x5d
zil_commit_writer+0x2ac
zil_commit+0x8c
zfs_putpage+0x2c9
fop_putpage+0x74
rfs3_commit+0x180
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff02d89bac80 SLEEP MUTEX 1
swtch+0x147
turnstile_block+0x764
mutex_vector_enter+0x261
zfs_zget+0x47
zfs_vget+0x25e
fsop_vget+0x67
nfs3_fhtovp+0x47
rfs3_access+0x4a
common_dispatch+0x3a0
rfs_dispatch+0x2d
svc_getreq+0x19c
svc_run+0x16b
svc_do_run+0x81
nfssys+0x765
_sys_sysenter_post_swapgs+0x14b
ffffff0010adfc60 SLEEP MUTEX 1
swtch+0x147
turnstile_block+0x764
mutex_vector_enter+0x261
zfs_zinactive+0x3c
zfs_inactive+0xee
fop_inactive+0xaf
vn_rele_dnlc+0x6c
do_dnlc_reduce_cache+0x16a
taskq_d_thread+0xb1
thread_start+8
ffffff02da197560 SLEEP CV 1
zfs_lookup+0xb1
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0xd2
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
ffffff000fc92c60 SLEEP CV 1
zio_destroy+0x5c
zio_done+0xc4
zio_execute+0xa0
zio_notify_parent+0xa6
zio_done+0x2c9
zio_execute+0xa0
taskq_thread+0x193
thread_start+8
ffffff001006ac60 SLEEP CV 1
zio_vdev_io_done+0x62
zio_execute+0xa0
taskq_thread+0x193
thread_start+8
ffffff052a254e40 ONPROC <NONE> 1
apic_intr_exit+0x32
hilevel_intr_epilog+0x123
do_interrupt+0xe9
_sys_rtt_ints_disabled+8
dmu_zfetch_colinear+0x87
dmu_zfetch+0xd3
dbuf_read+0x272
dnode_hold_impl+0x22b
dnode_hold+0x2b
dmu_buf_hold+0x75
zap_get_leaf_byblk+0x56
zap_deref_leaf+0x78
fzap_lookup+0x78
zap_lookup_norm+0x116
zap_lookup+0x2d
zfs_match_find+0xfd
zfs_dirent_lock+0x3d1
zfs_dirlook+0xd9
zfs_lookup+0x104
fop_lookup+0xed
lookuppnvp+0x3a3
lookuppnat+0x12c
lookupnameat+0xd2
cstatat_getvp+0x164
cstatat64_32+0x82
lstat64_32+0x31
_sys_sysenter_post_swapgs+0x14b
-Jeremy
_______________________________________________
zfs-discuss mailing list
zfs-discuss@opensolaris.org
http://mail.opensolaris.org/mailman/listinfo/zfs-discuss
