Stack traces from a second hi1.4xlarge running ami-8baa73e2: CPU0 rip: ffffffff8105711a try_to_wake_up+0xca flags: 00001202 i nz rsp: ffff880f22dfc740 rax: 0000000000000006 rcx: ffff880f22dfc870 rdx: 0000000000000082 rbx: ffff880b13d90000 rsi: 0000000000000000 rdi: 0000000000000001 rbp: ffff880f22dfc790 r8: ffffea0008b0b118 r9: 0000000000000000 r10: 57ffc73d4450b118 r11: 0000000000000202 r12: ffff880b13d905f8 r13: 0000000000000000 r14: 0000000000000000 r15: 0000000000000003 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f6132beb700 gs: 0000 @ ffff880f22df9000/0000000000000000 Code (instr addr ffffffff8105711a) 66 90 45 85 ff 74 0e 44 8b 7d cc e9 a8 00 00 00 0f 1f 00 f3 90 <8b> 7b 28 85 ff 75 f7 48 8b 13 31
Stack: ffffffff81007b02 0000000000012940 0000000000012940 0000000621c005e8 ffffffff8110b7f7 ffff880b13dbfc30 0000000000000000 ffff880f22fc47d0 ffff880f22dfc870 0000000000000000 ffff880f22dfc7a0 ffffffff81057262 ffff880f22dfc7c0 ffffffff810813c6 ffffffff810073ed ffff880f22fc47b8 Call Trace: [<ffffffff8105711a>] try_to_wake_up+0xca <-- [<ffffffff81007b02>] check_events+0x12 [<ffffffff8110b7f7>] mempool_free_slab+0x17 [<ffffffff81057262>] default_wake_function+0x12 [<ffffffff810813c6>] autoremove_wake_function+0x16 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff8108142b>] wake_bit_function+0x3b [<ffffffff81044be8>] __wake_up_common+0x58 [<ffffffff8110b7f7>] mempool_free_slab+0x17 [<ffffffff810466d8>] __wake_up+0x48 [<ffffffff81081061>] __wake_up_bit+0x31 [<ffffffff81108bda>] unlock_page+0x2a [<ffffffff8119f726>] mpage_end_io+0x46 [<ffffffff8119996d>] bio_endio+0x1d [<ffffffff814b24d7>] dec_pending+0x87 [<ffffffff814b2833>] clone_endio+0xa3 [<ffffffff8119996d>] bio_endio+0x1d [<ffffffff812cd413>] req_bio_endio.isra.45+0xa3 [<ffffffff812cdcf5>] blk_update_request+0xf5 [<ffffffff812ce041>] blk_update_bidi_request+0x31 [<ffffffff812cf257>] __blk_end_request_all+0x37 [<ffffffff813e3677>] blkif_interrupt+0x167 [<ffffffff810ccb65>] handle_irq_event_percpu+0x55 [<ffffffff812ec66b>] radix_tree_lookup+0xb [<ffffffff810ccd7b>] handle_irq_event+0x4b [<ffffffff810cf53c>] handle_edge_irq+0x7c [<ffffffff81384a69>] __xen_evtchn_do_upcall+0x199 [<ffffffff813869ef>] xen_evtchn_do_upcall+0x2f [<ffffffff8160f62e>] xen_do_hypervisor_callback+0x1e [<ffffffff810013aa>] hypercall_page+0x3aa [<ffffffff810013aa>] hypercall_page+0x3aa [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e23>] xen_spin_lock_flags+0x63 [<ffffffff8160619e>] _raw_spin_lock_irqsave+0x2e [<ffffffff8104e662>] update_shares+0x92 [<ffffffff81544540>] tcp_init_xmit_timers+0x30 [<ffffffff81056a68>] rebalance_domains+0x48 [<ffffffff810076ac>] xen_timer_interrupt+0x2c [<ffffffff81056d08>] run_rebalance_domains+0x48 [<ffffffff81065bf8>] __do_softirq+0xa8 [<ffffffff81384ad7>] __xen_evtchn_do_upcall+0x207 [<ffffffff8160f5dc>] call_softirq+0x1c [<ffffffff8100c265>] do_softirq+0x65 [<ffffffff81065fde>] irq_exit+0x8e [<ffffffff813869f5>] xen_evtchn_do_upcall+0x35 [<ffffffff8160f62e>] xen_do_hypervisor_callback+0x1e CPU1 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff88001001b7a0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88001001b7b8 rdi: 0000000000000003 rbp: ffff88001001b7e8 r8: ffff880ec85a3000 r9: ffff880f21c000d0 r10: 000000006de60240 r11: 0000000000000206 r12: 0000000000000017 r13: 0000000000000001 r14: ffff880ec5acd801 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fa13a3ac700 gs: 0000 @ ffff880f22e14000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000001 00000000fffffffa ffffffff8138512e ffff88001001b7d4 ffffffff00000001 0000000000000000 0000000a81384b3e ffff88001001b7e8 ffffffff81efce20 ffff88001001b7f8 ffffffff81386b80 ffff88001001b848 ffffffff815ecdcd 0000000000001000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff814b4154>] _dm_request.isra.21+0x104 [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff8119aac3>] bio_add_page+0x53 [<ffffffff8119fd90>] do_mpage_readpage+0x470 [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff811a0010>] mpage_readpages+0x120 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff81114068>] read_pages+0x48 [<ffffffff81114283>] __do_page_cache_readahead+0x163 [<ffffffff811145f1>] ra_submit+0x21 [<ffffffff815f2a11>] do_sync_mmap_readahead.isra.26+0x94 [<ffffffff8110b53e>] filemap_fault+0x33e [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff81032b15>] pvclock_clocksource_read+0x55 [<ffffffff810076e0>] xen_clocksource_read+0x20 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff8108bc57>] getnstimeofday+0x57 [<ffffffff81606715>] page_fault+0x25 CPU2 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880f22e32c68 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880f22e32c80 rdi: 0000000000000003 rbp: ffff880f22e32cb0 r8: ffff880ec85a3600 r9: ffff880f21c00100 r10: 0000000000000001 r11: 0000000000000202 r12: 000000000000001d r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e92900 cs: e033 ss: e02b ds: 002b es: 002b fs: 0000 @ 00007f8883dfd700 gs: 0000 @ ffff880f22e2f000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880f22e32c9c ffffffff00000001 0000000000000000 0000001081384b3e ffff880f22e32cb0 ffff880f22e0b940 ffff880f22e32cc0 ffffffff81386b80 ffff880f22e32d10 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff81056ab9>] rebalance_domains+0x99 [<ffffffff81056c5f>] nohz_idle_balance+0xbf [<ffffffff810076ac>] xen_timer_interrupt+0x2c [<ffffffff81056d12>] run_rebalance_domains+0x52 [<ffffffff81065bf8>] __do_softirq+0xa8 [<ffffffff81384ad7>] __xen_evtchn_do_upcall+0x207 [<ffffffff8160f5dc>] call_softirq+0x1c [<ffffffff8100c265>] do_softirq+0x65 [<ffffffff81065fde>] irq_exit+0x8e [<ffffffff813869f5>] xen_evtchn_do_upcall+0x35 [<ffffffff8160f62e>] xen_do_hypervisor_callback+0x1e CPU3 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff88000cddd850 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88000cddd868 rdi: 0000000000000003 rbp: ffff88000cddd898 r8: ffff880ec85a3c00 r9: ffff880f21c00130 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000023 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e5c900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f6131fdf700 gs: 0000 @ ffff880f22e4a000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff88000cddd884 ffffffff00000001 0000000000000000 0000001681384b3e ffff88000cddd898 ffff880f22e0b940 ffff88000cddd8a8 ffffffff81386b80 ffff88000cddd8f8 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff816040af>] schedule+0x3f [<ffffffff816046f5>] schedule_timeout+0x2a5 [<ffffffff81007aef>] xen_restore_fl_direct_reloc+0x4 [<ffffffff814df0b1>] sk_wait_data+0xd1 [<ffffffff810813b0>] add_wait_queue+0x60 [<ffffffff81535cf5>] tcp_recvmsg+0x4c5 [<ffffffff81559c1b>] inet_recvmsg+0x6b [<ffffffff81559c94>] inet_sendmsg+0x64 [<ffffffff814d9c9d>] sock_recvmsg+0x11d [<ffffffff814d9e2e>] sock_sendmsg+0x10e [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff81007b02>] check_events+0x12 [<ffffffff814dd18e>] sys_recvfrom+0xee [<ffffffff810076e0>] xen_clocksource_read+0x20 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff8108bc57>] getnstimeofday+0x57 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU4 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880b8186d6d0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880b8186d6e8 rdi: 0000000000000003 rbp: ffff880b8186d718 r8: ffff880ec86e0200 r9: ffff880f21c00160 r10: ffff880c18589bc0 r11: 0000000000000202 r12: 0000000000000029 r13: 0000000000000001 r14: ffff880c18589b01 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fb347fff700 gs: 0000 @ ffff880f22e65000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000200 00000000fffffffa ffffffff8138512e ffff880b8186d704 ffffffff00000001 0000000000000000 0000001c81384b3e ffff880b8186d718 ffffffff81efce20 ffff880b8186d728 ffffffff81386b80 ffff880b8186d778 ffffffff815ecdcd 0000000000001000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8115f0b7>] mem_cgroup_lru_add_after_commit+0x37 [<ffffffff8119fd66>] do_mpage_readpage+0x446 [<ffffffff8110996d>] add_to_page_cache_locked+0x7d [<ffffffff8119ffde>] mpage_readpages+0xee [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81220d58>] __ext4_journal_stop+0x78 [<ffffffff81202ead>] ext4_readpages+0x1d [<ffffffff81114068>] read_pages+0x48 [<ffffffff81114283>] __do_page_cache_readahead+0x163 [<ffffffff811145f1>] ra_submit+0x21 [<ffffffff815f2a11>] do_sync_mmap_readahead.isra.26+0x94 [<ffffffff8110b53e>] filemap_fault+0x33e [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff81167042>] do_sync_write+0xd2 [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff81167850>] vfs_write+0x110 [<ffffffff81606715>] page_fault+0x25 CPU5 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff88001010bab0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88001010bac8 rdi: 0000000000000003 rbp: ffff88001010baf8 r8: ffff880ec86e0800 r9: ffff880f21c00190 r10: 0000000000000001 r11: 0000000000000206 r12: 000000000000002f r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22e92900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f95c028a700 gs: 0000 @ ffff880f22e80000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff88001010bae4 ffffffff00000001 0000000000000000 0000002281384b3e ffff88001010baf8 ffff880f22e0b940 ffff88001010bb08 ffffffff81386b80 ffff88001010bb58 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81094989>] futex_wait_queue_me+0xc9 [<ffffffff81095940>] futex_wait+0x100 [<ffffffff81057470>] wake_up_state+0x10 [<ffffffff8109489b>] wake_futex+0x3b [<ffffffff81095bb3>] futex_wake+0x113 [<ffffffff8109768c>] do_futex+0xbc [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff810978a5>] sys_futex+0x105 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU6 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880b13dbf910 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880b13dbf928 rdi: 0000000000000003 rbp: ffff880b13dbf958 r8: ffff880ec86e0e00 r9: ffff880f21c001c0 r10: 0000000000000001 r11: 0000000000000206 r12: 0000000000000035 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22ead900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f8888890700 gs: 0000 @ ffff880f22e9b000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880b13dbf944 ffffffff00000001 0000000000000000 0000002881384b3e ffff880b13dbf958 ffff880f22e0b940 ffff880b13dbf968 ffffffff81386b80 ffff880b13dbf9b8 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff81108b70>] sleep_on_page+0x20 [<ffffffff816040af>] schedule+0x3f [<ffffffff8160415f>] io_schedule+0x8f [<ffffffff81108b7e>] sleep_on_page_killable+0xe [<ffffffff8160497f>] __wait_on_bit+0x5f [<ffffffff8110b120>] wait_on_page_bit_killable+0x80 [<ffffffff810813f0>] autoremove_wake_function+0x40 [<ffffffff8110b1d6>] __lock_page_or_retry+0xa6 [<ffffffff8110b503>] filemap_fault+0x303 [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff81097678>] do_futex+0xa8 [<ffffffff810978a5>] sys_futex+0x105 [<ffffffff81003f26>] xen_clts+0x66 [<ffffffff81606715>] page_fault+0x25 CPU7 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff88091e9d96d0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88091e9d96e8 rdi: 0000000000000003 rbp: ffff88091e9d9718 r8: ffff880ec86e1400 r9: ffff880f21c001f0 r10: ffff88000c51ae40 r11: 0000000000000206 r12: 000000000000003b r13: 0000000000000001 r14: ffff88000c51ae01 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f6138dae700 gs: 0000 @ ffff880f22eb6000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000200 00000000fffffffa ffffffff8138512e ffff88091e9d9704 ffffffff00000001 0000000000000000 0000002e81384b3e ffff88091e9d9718 ffffffff81efce20 ffff88091e9d9728 ffffffff81386b80 ffff88091e9d9778 ffffffff815ecdcd 0000000000002000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8115f0b7>] mem_cgroup_lru_add_after_commit+0x37 [<ffffffff8119fd66>] do_mpage_readpage+0x446 [<ffffffff8110996d>] add_to_page_cache_locked+0x7d [<ffffffff8119ffde>] mpage_readpages+0xee [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81220d58>] __ext4_journal_stop+0x78 [<ffffffff81202ead>] ext4_readpages+0x1d [<ffffffff81114068>] read_pages+0x48 [<ffffffff81114283>] __do_page_cache_readahead+0x163 [<ffffffff811145f1>] ra_submit+0x21 [<ffffffff815f2a11>] do_sync_mmap_readahead.isra.26+0x94 [<ffffffff8110b53e>] filemap_fault+0x33e [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff81167042>] do_sync_write+0xd2 [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff81167850>] vfs_write+0x110 [<ffffffff81606715>] page_fault+0x25 CPU8 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880a240856d0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880a240856e8 rdi: 0000000000000003 rbp: ffff880a24085718 r8: ffff880ec86e1a00 r9: ffff880f21c00490 r10: ffff88000ca56240 r11: 0000000000000202 r12: 0000000000000041 r13: 0000000000000001 r14: ffff88000ca56201 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f9a688ca700 gs: 0000 @ ffff880f22ed1000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000001 00000000fffffffa ffffffff8138512e ffff880a24085704 ffffffff00000001 0000000000000000 0000003481384b3e ffff880a24085718 ffffffff81efce20 ffff880a24085728 ffffffff81386b80 ffff880a24085778 ffffffff815ecdcd 000000000000b000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8115f0b7>] mem_cgroup_lru_add_after_commit+0x37 [<ffffffff8119fdae>] do_mpage_readpage+0x48e [<ffffffff8110996d>] add_to_page_cache_locked+0x7d [<ffffffff8119ffde>] mpage_readpages+0xee [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81202ead>] ext4_readpages+0x1d [<ffffffff81114068>] read_pages+0x48 [<ffffffff81114283>] __do_page_cache_readahead+0x163 [<ffffffff811145f1>] ra_submit+0x21 [<ffffffff815f2a11>] do_sync_mmap_readahead.isra.26+0x94 [<ffffffff8110b53e>] filemap_fault+0x33e [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff8112b994>] tlb_finish_mmu+0x14 [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff8112a78d>] madvise_vma+0xfd [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff8108bc57>] getnstimeofday+0x57 [<ffffffff8112a9f5>] sys_madvise+0x225 [<ffffffff81606715>] page_fault+0x25 CPU9 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001206 i nz p rsp: ffff880ec5051bf8 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880ec5051c10 rdi: 0000000000000003 rbp: ffff880ec5051c40 r8: ffff880ec8780000 r9: ffff880f21c004c0 r10: 0000000000000001 r11: 0000000000000206 r12: 0000000000000047 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22efe900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f95c29b1700 gs: 0000 @ ffff880f22eec000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880ec5051c2c ffffffff00000001 0000000000000000 0000003a81384b3e ffff880ec5051c40 ffff880f22e0b940 ffff880ec5051c50 ffffffff81386b80 ffff880ec5051ca0 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff8107bf0e>] worker_thread+0x23e [<ffffffff8107bcd0>] manage_workers.isra.30+0x130 [<ffffffff8108090c>] kthread+0x8c [<ffffffff8160f4e4>] kernel_thread_helper+0x4 [<ffffffff8160e5e3>] int_ret_from_sys_call+0x7 [<ffffffff816064a1>] retint_restore_args+0x5 [<ffffffff8160f4e0>] gs_change+0x13 CPU10 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff8804860116d0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff8804860116e8 rdi: 0000000000000003 rbp: ffff880486011718 r8: ffff880ec8780600 r9: ffff880f21c004f0 r10: ffff880ec7d21500 r11: 0000000000000202 r12: 000000000000004d r13: 0000000000000001 r14: ffff880ec7d21501 r15: 0000000000000000 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007fa13a0a9700 gs: 0000 @ ffff880f22f07000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000200 00000000fffffffa ffffffff8138512e ffff880486011704 ffffffff00000001 0000000000000000 0000004081384b3e ffff880486011718 ffffffff81efce20 ffff880486011728 ffffffff81386b80 ffff880486011778 ffffffff815ecdcd 0000000000001000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff816061f5>] _raw_spin_lock_irq+0x15 [<ffffffff812d036d>] __make_request+0xbd [<ffffffff812cd72a>] generic_make_request.part.51+0x24a [<ffffffff812cda35>] generic_make_request+0x45 [<ffffffff812cdad7>] submit_bio+0x87 [<ffffffff8115f0b7>] mem_cgroup_lru_add_after_commit+0x37 [<ffffffff8119fd66>] do_mpage_readpage+0x446 [<ffffffff8110996d>] add_to_page_cache_locked+0x7d [<ffffffff8119ffde>] mpage_readpages+0xee [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81112129>] __alloc_pages_nodemask+0x109 [<ffffffff81207d50>] noalloc_get_block_write+0x30 [<ffffffff81220d58>] __ext4_journal_stop+0x78 [<ffffffff81202ead>] ext4_readpages+0x1d [<ffffffff81114068>] read_pages+0x48 [<ffffffff81114283>] __do_page_cache_readahead+0x163 [<ffffffff811145f1>] ra_submit+0x21 [<ffffffff815f2a11>] do_sync_mmap_readahead.isra.26+0x94 [<ffffffff8110b53e>] filemap_fault+0x33e [<ffffffff8112b354>] __do_fault+0x54 [<ffffffff8112ea4a>] handle_pte_fault+0xfa [<ffffffff81005cce>] xen_pmd_val+0xe [<ffffffff81004759>] __raw_callee_save_xen_pmd_val+0x11 [<ffffffff8112ef18>] handle_mm_fault+0x1f8 [<ffffffff81167042>] do_sync_write+0xd2 [<ffffffff81609a0e>] do_page_fault+0x14e [<ffffffff81007b02>] check_events+0x12 [<ffffffff81167850>] vfs_write+0x110 [<ffffffff81606715>] page_fault+0x25 CPU11 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff88000d859ab0 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88000d859ac8 rdi: 0000000000000003 rbp: ffff88000d859af8 r8: ffff880ec8780c00 r9: ffff880f21c00520 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000053 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22f34900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f8882ded700 gs: 0000 @ ffff880f22f22000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff88000d859ae4 ffffffff00000001 0000000000000000 0000004681384b3e ffff88000d859af8 ffff880f22e0b940 ffff88000d859b08 ffffffff81386b80 ffff88000d859b58 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff816040af>] schedule+0x3f [<ffffffff81094989>] futex_wait_queue_me+0xc9 [<ffffffff81095940>] futex_wait+0x100 [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff81095bb3>] futex_wake+0x113 [<ffffffff8109768c>] do_futex+0xbc [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff810978a5>] sys_futex+0x105 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU12 rip: ffffffff8105711d try_to_wake_up+0xcd flags: 00001202 i nz rsp: ffff88001017ddd8 rax: 0000000000000005 rcx: ffff880010183dc8 rdx: 0000000000000001 rbx: ffff88000d861700 rsi: ffff8800101002c8 rdi: 0000000000000001 rbp: ffff88001017de28 r8: 000000002297fc12 r9: 0000000000000000 r10: 0000000000000001 r11: 0000000000000000 r12: ffff88000d861cf8 r13: 0000000000000000 r14: ffff8800101002c8 r15: 0000000000000003 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f95b24e4700 gs: 0000 @ ffff880f22f3d000/0000000000000000 Code (instr addr ffffffff8105711d) 85 ff 74 0e 44 8b 7d cc e9 a8 00 00 00 0f 1f 00 f3 90 8b 7b 28 <85> ff 75 f7 48 8b 13 31 c0 83 e2 Stack: 0000000000000000 ffffffff81e58400 ffff88001017de58 00000005810955ee 0000000000000fc0 ffff88000d861700 ffff88001010bdb0 00000000ffffffff 0000000000000001 ffff880010183db0 ffff88001017de38 ffffffff81057470 ffff88001017de58 ffffffff8109489b 0000000000000001 ffffffff81e58400 Call Trace: [<ffffffff8105711d>] try_to_wake_up+0xcd <-- [<ffffffff81057470>] wake_up_state+0x10 [<ffffffff8109489b>] wake_futex+0x3b [<ffffffff81095b89>] futex_wake+0xe9 [<ffffffff81097678>] do_futex+0xa8 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff810978a5>] sys_futex+0x105 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU13 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880460741d80 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880460741d98 rdi: 0000000000000003 rbp: ffff880460741dc8 r8: ffff880ec8781800 r9: ffff880f21c00580 r10: 00007fa13b65714c r11: 0000000000000202 r12: 000000000000005f r13: 0000000000000201 r14: 0000000000000001 r15: 00007f95a001bc00 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f95abefe700 gs: 0000 @ ffff880f22f58000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000000 00000000fffffffa ffffffff8138512e ffff880460741db4 ffffffff00000001 0000000000000000 0000005281384b3e ffff880460741dc8 ffffffff81e58400 ffff880460741dd8 ffffffff81386b80 ffff880460741e28 ffffffff815ecdcd ffff880460741e58 0000000000000000 0000000000000fc0 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff81095b0d>] futex_wake+0x6d [<ffffffff81097678>] do_futex+0xa8 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff810978a5>] sys_futex+0x105 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU14 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff880b14225850 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff880b14225868 rdi: 0000000000000003 rbp: ffff880b14225898 r8: ffff880ec8781e00 r9: ffff880f21c005b0 r10: 0000000000000001 r11: 0000000000000202 r12: 0000000000000065 r13: 0000000000000001 r14: ffff880f22e0b901 r15: ffff880f22f85900 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f612f0b0700 gs: 0000 @ ffff880f22f73000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000049 00000000fffffffa ffffffff8138512e ffff880b14225884 ffffffff00000001 0000000000000000 0000005881384b3e ffff880b14225898 ffff880f22e0b940 ffff880b142258a8 ffffffff81386b80 ffff880b142258f8 ffffffff815ecdcd 0000000000000000 0000000000000000 0000000000000000 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff810495bc>] double_rq_lock+0x2c [<ffffffff810567aa>] load_balance+0xfa [<ffffffff8160611e>] _raw_spin_unlock_irqrestore+0x1e [<ffffffff815ef880>] idle_balance+0xab [<ffffffff81603f89>] __schedule+0x6a9 [<ffffffff810073ed>] xen_force_evtchn_callback+0xd [<ffffffff816040af>] schedule+0x3f [<ffffffff816046f5>] schedule_timeout+0x2a5 [<ffffffff81007aef>] xen_restore_fl_direct_reloc+0x4 [<ffffffff814df0b1>] sk_wait_data+0xd1 [<ffffffff810813b0>] add_wait_queue+0x60 [<ffffffff81535cf5>] tcp_recvmsg+0x4c5 [<ffffffff81559c1b>] inet_recvmsg+0x6b [<ffffffff81559c94>] inet_sendmsg+0x64 [<ffffffff814d9c9d>] sock_recvmsg+0x11d [<ffffffff814d9e2e>] sock_sendmsg+0x10e [<ffffffff81095986>] futex_wait+0x146 [<ffffffff810955ee>] get_futex_key+0x14e [<ffffffff814dd18e>] sys_recvfrom+0xee [<ffffffff810076e0>] xen_clocksource_read+0x20 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff8108bc57>] getnstimeofday+0x57 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 CPU15 rip: ffffffff810013aa hypercall_page+0x3aa flags: 00001202 i nz rsp: ffff88000d805d80 rax: 0000000000000000 rcx: ffffffff810013aa rdx: 0000000000000000 rbx: 0000000000000000 rsi: ffff88000d805d98 rdi: 0000000000000003 rbp: ffff88000d805dc8 r8: ffff880ec8002400 r9: ffff880f21c005e0 r10: 00007fa13b6573f0 r11: 0000000000000202 r12: 000000000000006b r13: 0000000000000201 r14: 0000000000000001 r15: 00007fa4410bff00 cs: e033 ss: e02b ds: 0000 es: 0000 fs: 0000 @ 00007f95b11d1700 gs: 0000 @ ffff880f22f8e000/0000000000000000 Code (instr addr ffffffff810013aa) cc cc cc cc cc cc cc cc cc cc cc 51 41 53 b8 1d 00 00 00 0f 05 <41> 5b 59 c3 cc cc cc cc cc cc cc Stack: 0000000000000000 00000000fffffffa ffffffff8138512e ffff88000d805db4 ffffffff00000001 0000000000000000 0000005e81384b3e ffff88000d805dc8 ffffffff81e58400 ffff88000d805dd8 ffffffff81386b80 ffff88000d805e28 ffffffff815ecdcd ffff88000d805e58 0000000000000000 0000000000000fc0 Call Trace: [<ffffffff810013aa>] hypercall_page+0x3aa <-- [<ffffffff8138512e>] xen_poll_irq_timeout+0x3e [<ffffffff81386b80>] xen_poll_irq+0x10 [<ffffffff815ecdcd>] xen_spin_lock_slow+0x98 [<ffffffff81008e8a>] xen_spin_lock+0x4a [<ffffffff81605f0e>] _raw_spin_lock+0xe [<ffffffff81095b0d>] futex_wake+0x6d [<ffffffff81097678>] do_futex+0xa8 [<ffffffff810076f9>] xen_clocksource_get_cycles+0x9 [<ffffffff810978a5>] sys_futex+0x105 [<ffffffff8108bd4a>] do_gettimeofday+0x1a [<ffffffff8160e3c2>] system_call_fastpath+0x16 -- You received this bug notification because you are a member of Ubuntu Bugs, which is subscribed to Ubuntu. https://bugs.launchpad.net/bugs/1011792 Title: Scheduler deadlock running 3.0.0 on multiple EC2 instance types To manage notifications about this bug go to: https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1011792/+subscriptions -- ubuntu-bugs mailing list ubuntu-bugs@lists.ubuntu.com https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
