Using ceph-objectstore-info on PG 25.0 (which indeed, was the one I remember having the error) shows this:
struct_v 10 { "pgid": "25.0", "last_update": "7592'106", "last_complete": "7592'106", "log_tail": "0'0", "last_user_version": 106, "last_backfill": "MIN", "last_backfill_bitwise": 1, "purged_snaps": [], "history": { "epoch_created": 5191, "epoch_pool_created": 5191, "last_epoch_started": 7489, "last_interval_started": 7488, "last_epoch_clean": 7489, "last_interval_clean": 7488, "last_epoch_split": 0, "last_epoch_marked_full": 0, "same_up_since": 7593, "same_interval_since": 7593, "same_primary_since": 7593, "last_scrub": "7592'106", "last_scrub_stamp": "2019-04-25 21:34:52.079721", "last_deep_scrub": "7485'70", "last_deep_scrub_stamp": "2019-04-22 10:15:40.532014", "last_clean_scrub_stamp": "2019-04-19 14:52:44.047548" }, "stats": { "version": "7592'105", "reported_seq": "2621", "reported_epoch": "7592", "state": "active+clean+inconsistent", "last_fresh": "2019-04-25 20:02:55.620028", "last_change": "2019-04-24 19:52:45.072473", "last_active": "2019-04-25 20:02:55.620028", "last_peered": "2019-04-25 20:02:55.620028", "last_clean": "2019-04-25 20:02:55.620028", "last_became_active": "2019-04-22 17:55:37.578239", "last_became_peered": "2019-04-22 17:55:37.578239", "last_unstale": "2019-04-25 20:02:55.620028", "last_undegraded": "2019-04-25 20:02:55.620028", "last_fullsized": "2019-04-25 20:02:55.620028", "mapping_epoch": 7593, "log_start": "0'0", "ondisk_log_start": "0'0", "created": 5191, "last_epoch_clean": 7489, "parent": "0.0", "parent_split_bits": 0, "last_scrub": "7592'88", "last_scrub_stamp": "2019-04-24 19:52:45.072367", "last_deep_scrub": "7485'70", "last_deep_scrub_stamp": "2019-04-22 10:15:40.532014", "last_clean_scrub_stamp": "2019-04-19 14:52:44.047548", "log_size": 105, "ondisk_log_size": 105, "stats_invalid": false, "dirty_stats_invalid": false, "omap_stats_invalid": false, "hitset_stats_invalid": false, "hitset_bytes_stats_invalid": false, "pin_stats_invalid": false, "manifest_stats_invalid": false, "snaptrimq_len": 0, "stat_sum": { "num_bytes": 0, "num_objects": 9, "num_object_clones": 0, "num_object_copies": 27, "num_objects_missing_on_primary": 0, "num_objects_missing": 0, "num_objects_degraded": 0, "num_objects_misplaced": 0, "num_objects_unfound": 0, "num_objects_dirty": 9, "num_whiteouts": 0, "num_read": 87, "num_read_kb": 87, "num_write": 98, "num_write_kb": 98, "num_scrub_errors": 0, "num_shallow_scrub_errors": 0, "num_deep_scrub_errors": 0, "num_objects_recovered": 0, "num_bytes_recovered": 0, "num_keys_recovered": 0, "num_objects_omap": 9, "num_objects_hit_set_archive": 0, "num_bytes_hit_set_archive": 0, "num_flush": 0, "num_flush_kb": 0, "num_evict": 0, "num_evict_kb": 0, "num_promote": 0, "num_flush_mode_high": 0, "num_flush_mode_low": 0, "num_evict_mode_some": 0, "num_evict_mode_full": 0, "num_objects_pinned": 0, "num_legacy_snapsets": 0, "num_large_omap_objects": 0, "num_objects_manifest": 0, "num_omap_bytes": 0, "num_omap_keys": 0 }, "up": [], "acting": [], "blocked_by": [], "up_primary": -1, "acting_primary": -1, "purged_snaps": [] }, "empty": 0, "dne": 0, "incomplete": 1, "last_epoch_started": 7489, "hit_set_history": { "current_last_update": "0'0", "history": [] } } On Fri, Apr 26, 2019 at 7:35 PM Elise Burke <elise.n...@gmail.com> wrote: > Thanks for the suggestions. I've uploaded the surprisingly large (1.5G!) > log file: ceph-post-file: 2d8d22f4-580b-4b57-a13a-f49dade34ba7 > > Looks like these are the relevant lines: > > -52> 2019-04-26 19:23:05.190 7fb2657dc700 20 osd.2 op_wq(2) _process > empty q, waiting > -51> 2019-04-26 19:23:05.190 7fb2667de700 20 osd.2 op_wq(0) _process > OpQueueItem(25.0 PGDelete(25.0 e7648) prio 5 cost 1048576 e7648) queued > -50> 2019-04-26 19:23:05.190 7fb2717f4700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _kv_finalize_thread sleep > -49> 2019-04-26 19:23:05.190 7fb2667de700 20 osd.2 op_wq(0) _process > 25.0 to_process <OpQueueItem(25.0 PGDelete(25.0 e7648) prio 5 cost 1048576 > e7648)> waiting <> waiting_peering {} > -48> 2019-04-26 19:23:05.190 7fb2667de700 20 osd.2 op_wq(0) _process > OpQueueItem(25.0 PGDelete(25.0 e7648) prio 5 cost 1048576 e7648) pg > 0x7c20a00000 > -47> 2019-04-26 19:23:05.190 7fb2667de700 10 osd.2 pg_epoch: 7648 > pg[25.0( v 7592'106 (0'0,7592'106] lb MIN (bitwise) local-lis/les=7488/7489 > n=9 ec=5191/5191 lis/c 7488/7488 les/c/f 7489/7489/0 7593/7593/7593) [] > r=-1 lpr=7648 D > ELETING pi=[7488,7593)/1 crt=7592'106 lcod 0'0 unknown NOTIFY mbc={}] > do_peering_event: epoch_sent: 7648 epoch_requested: 7648 DeleteSome > -46> 2019-04-26 19:23:05.190 7fb2667de700 10 osd.2 pg_epoch: 7648 > pg[25.0( v 7592'106 (0'0,7592'106] lb MIN (bitwise) local-lis/les=7488/7489 > n=9 ec=5191/5191 lis/c 7488/7488 les/c/f 7489/7489/0 7593/7593/7593) [] > r=-1 lpr=7648 D > ELETING pi=[7488,7593)/1 crt=7592'106 lcod 0'0 unknown NOTIFY mbc={}] > _delete_some > -45> 2019-04-26 19:23:05.190 7fb2667de700 15 > bluestore(/var/lib/ceph/osd/ceph-2) collection_list 25.0_head start GHMIN > end GHMAX max 30 > -44> 2019-04-26 19:23:05.190 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list range > 0x7f7fffffffffffffe500000000 to 0x7f7fffffffffffffe5ffffffff and > 0x7f800000000000001900000000 to 0x7f8000000000000019ffffffff > start GHMIN > -43> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list pend > 0x7f7fffffffffffffe5ffffffff > -42> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list key > 0x7f7fffffffffffffff100020'P!osdmap.7114!='0x0000000000000000ffffffffffffffff'o' > >= GHMAX > -41> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list oid > #25:00000000::::head# end GHMAX > -40> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list oid > #25:b08b92bd::::head# end GHMAX > -39> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list key > 0x80800000000000000c1c00000021213dfffffffffffffffeffffffffffffffff'o' >= > GHMAX > -38> 2019-04-26 19:23:05.191 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) collection_list 25.0_head start GHMAX > end GHMAX max 30 = 0, ls.size() = 2, next = GHMAX > -37> 2019-04-26 19:23:05.191 7fb2667de700 20 osd.2 pg_epoch: 7648 > pg[25.0( v 7592'106 (0'0,7592'106] lb MIN (bitwise) local-lis/les=7488/7489 > n=9 ec=5191/5191 lis/c 7488/7488 les/c/f 7489/7489/0 7593/7593/7593) [] > r=-1 lpr=7648 D > ELETING pi=[7488,7593)/1 crt=7592'106 lcod 0'0 unknown NOTIFY mbc={}] > _delete_some [#25:00000000::::head#,#25:b08b92bd::::head#] > -36> 2019-04-26 19:23:05.191 7fb2667de700 20 osd.2 pg_epoch: 7648 > pg[25.0( v 7592'106 (0'0,7592'106] lb MIN (bitwise) local-lis/les=7488/7489 > n=9 ec=5191/5191 lis/c 7488/7488 les/c/f 7489/7489/0 7593/7593/7593) [] > r=-1 lpr=7648 D > ELETING pi=[7488,7593)/1 crt=7592'106 lcod 0'0 unknown NOTIFY mbc={}] > _delete_some finished > -35> 2019-04-26 19:23:05.191 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) queue_transactions ch 0x7c27d94b40 > 25.0_head > -34> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _txc_create osr 0x7c274f6c60 = > 0x7c2ef34c00 seq 3 > -33> 2019-04-26 19:23:05.191 7fb2667de700 15 > bluestore(/var/lib/ceph/osd/ceph-2) _remove 25.0_head #25:00000000::::head# > onode 0x7c20aaf200 txc 0x7c2ef34c00 > -32> 2019-04-26 19:23:05.191 7fb2667de700 15 > bluestore(/var/lib/ceph/osd/ceph-2) _do_truncate 25.0_head > #25:00000000::::head# 0x0 > -31> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore.onode(0x7c20aaf200).flush flush done > -30> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _do_omap_clear remove range start: > 0x0000000000b0295a'-' end: 0x0000000000b0295a'~' > -29> 2019-04-26 19:23:05.191 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) _remove 25.0_head #25:00000000::::head# > = 0 > -28> 2019-04-26 19:23:05.191 7fb2667de700 15 > bluestore(/var/lib/ceph/osd/ceph-2) _remove_collection 25.0_head > -27> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore.OnodeSpace(0x7c27d94c98 in 0x7c1f7e10a0) map_any > -26> 2019-04-26 19:23:05.191 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list range > 0x7f7fffffffffffffe500000000 to 0x7f7fffffffffffffe5ffffffff and > 0x7f800000000000001900000000 to 0x7f8000000000000019ffffffff > start GHMIN > -25> 2019-04-26 19:23:05.192 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list pend > 0x7f7fffffffffffffe5ffffffff > -24> 2019-04-26 19:23:05.192 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list key > 0x7f7fffffffffffffff100020'P!osdmap.7114!='0x0000000000000000ffffffffffffffff'o' > >= GHMAX > -23> 2019-04-26 19:23:05.192 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list oid > #25:00000000::::head# end GHMAX > -22> 2019-04-26 19:23:05.192 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list oid > #25:b08b92bd::::head# end GHMAX > -21> 2019-04-26 19:23:05.192 7fb2667de700 20 > bluestore(/var/lib/ceph/osd/ceph-2) _collection_list key > 0x80800000000000000c1c00000021213dfffffffffffffffeffffffffffffffff'o' >= > GHMAX > -20> 2019-04-26 19:23:05.192 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) _remove_collection oid > #25:00000000::::head# > -19> 2019-04-26 19:23:05.192 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) _remove_collection oid > #25:b08b92bd::::head# > -18> 2019-04-26 19:23:05.192 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) _remove_collection > #25:b08b92bd::::head# exists in db, not present in ram > -17> 2019-04-26 19:23:05.192 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) _remove_collection 25.0_head is > non-empty > -16> 2019-04-26 19:23:05.192 7fb2667de700 10 > bluestore(/var/lib/ceph/osd/ceph-2) _remove_collection 25.0_head = -39 > -15> 2019-04-26 19:23:05.192 7fb2667de700 -1 > bluestore(/var/lib/ceph/osd/ceph-2) _txc_add_transaction error (39) > Directory not empty not handled on operation 21 (op 1, counting from 0) > -14> 2019-04-26 19:23:05.192 7fb2667de700 0 > bluestore(/var/lib/ceph/osd/ceph-2) _dump_transaction transaction dump: > { > "ops": [ > { > "op_num": 0, > "op_name": "remove", > "collection": "25.0_head", > "oid": "#25:00000000::::head#" > }, > { > "op_num": 1, > "op_name": "rmcoll", > "collection": "25.0_head" > } > > -13> 2019-04-26 19:23:05.199 7fb2667de700 -1 > /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.0/rpm/el7/BUILD/ceph-14.2.0/src/os/bluestore/BlueStore.cc: > In function 'void BlueStore::_txc_add_transaction(BlueStore::TransContext*, > ObjectStore::Transaction*)' thread 7fb2667de700 time 2019-04-26 > 19:23:05.193826 > /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.0/rpm/el7/BUILD/ceph-14.2.0/src/os/bluestore/BlueStore.cc: > 11069: abort() > > ceph version 14.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus > (stable) > 1: (ceph::__ceph_abort(char const*, int, char const*, std::string > const&)+0xd8) [0x7c1454ee40] > 2: (BlueStore::_txc_add_transaction(BlueStore::TransContext*, > ObjectStore::Transaction*)+0x2a85) [0x7c14b2d5f5] > 3: > (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, > std::vector<ObjectStore::Transaction, > std::allocator<ObjectStore::Transaction> >&, > boost::intrusive_ptr<TrackedOp>, ThreadPool::TPHandle*)+0x526) > [0x7c14b2e366] > 4: > (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, > ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, > ThreadPool::TPHandle*)+0x7f) [0x7c1470a81f] > 5: (PG::_delete_some(ObjectStore::Transaction*)+0x83d) [0x7c1476d70d] > 6: (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x38) > [0x7c1476e528] > 7: (boost::statechart::simple_state<PG::RecoveryState::Deleting, > PG::RecoveryState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, > mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, > mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na, > mpl_::na, mpl_::na, mpl_::na>, > (boost::statechart::history_mode)0>::react_impl(boost::statechart::event_base > const&, void const*)+0x16a) [0x7c147acc8a] > 8: (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, > PG::RecoveryState::Initial, std::allocator<void>, > boost::statechart::null_exception_translator>::process_event(boost::statechart::event_base > const&)+0x5a) [0x7c1478a91a] > 9: (PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, > PG::RecoveryCtx*)+0x119) [0x7c14779c99] > 10: (OSD::dequeue_peering_evt(OSDShard*, PG*, > std::shared_ptr<PGPeeringEvent>, ThreadPool::TPHandle&)+0x1b4) > [0x7c146b4494] > 11: (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, > ThreadPool::TPHandle&)+0x234) [0x7c146b48d4] > 12: (OSD::ShardedOpWQ::_process(unsigned int, > ceph::heartbeat_handle_d*)+0x9f4) [0x7c146a8c14] > 13: (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x433) > [0x7c14ca0f43] > 14: (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0x7c14ca3fe0] > 15: (()+0x7dd5) [0x7fb284e3bdd5] > 16: (clone()+0x6d) [0x7fb283d01ead] > > -12> 2019-04-26 19:23:05.199 7fb280b24700 1 -- [v2: > 192.168.1.152:6800/1887364,v1:192.168.1.152:6801/1887364] <== > client.244681 v1:192.168.1.50:0/337348516 6 ==== > osd_op(client.244681.0:2571786 12.5es0 12.43dfa65e (undecoded) > ondisk+retry+write+known_if_redirected e7648) v8 ==== 229+0+4194304 > (unknown 2519985655 0 813231621) 0x7c2f3ee840 con 0x7c2f2fac00 > -11> 2019-04-26 19:23:05.199 7fb280b24700 15 osd.2 7648 enqueue_op > 0x7c300f8420 prio 63 cost 4194304 latency 0.035786 epoch 7648 > osd_op(client.244681.0:2571786 12.5es0 12.43dfa65e (undecoded) > ondisk+retry+write+known_if_redirected e7648) v8 > -10> 2019-04-26 19:23:05.199 7fb280b24700 20 osd.2 op_wq(4) _enqueue > OpQueueItem(12.5es0 PGOpItem(op=osd_op(client.244681.0:2571786 12.5es0 > 12.43dfa65e (undecoded) ondisk+retry+write+known_if_redirected e7648) v8) > prio 63 cost 4194304 e7648) > -9> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 op_wq(4) _process > 12.5es0 to_process <> waiting <> waiting_peering {} > -8> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 op_wq(4) _process > OpQueueItem(12.5es0 PGOpItem(op=osd_op(client.244681.0:2571786 12.5es0 > 12.43dfa65e (undecoded) ondisk+retry+write+known_if_redirected e7648) v8) > prio 63 cost 4194304 e7648) queued > -7> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 op_wq(4) _process > 12.5es0 to_process <OpQueueItem(12.5es0 > PGOpItem(op=osd_op(client.244681.0:2571786 12.5es0 12.43dfa65e (undecoded) > ondisk+retry+write+known_if_redirected e7648) v8) prio 63 cost 4194304 > e7648)> waiting <> waiting_peering {} > -6> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 op_wq(4) _process > OpQueueItem(12.5es0 PGOpItem(op=osd_op(client.244681.0:2571786 12.5es0 > 12.43dfa65e (undecoded) ondisk+retry+write+known_if_redirected e7648) v8) > prio 63 cost 4194304 e7648) pg 0x7c271c6000 > -5> 2019-04-26 19:23:05.200 7fb2647da700 10 osd.2 7648 dequeue_op > 0x7c300f8420 prio 63 cost 4194304 latency 0.036428 > osd_op(client.244681.0:2571786 12.5es0 12.43dfa65e (undecoded) > ondisk+retry+write+known_if_redirected e7648) v8 pg pg[12.5es0( v > 7592'250137 (5872'250020,7592'250137] local-lis/les=7488/7489 n=22756 > ec=5088/733 lis/c 7488/7488 les/c/f 7489/7489/5043 7648/7648/7648) > [2,2147483647,0,7,4]p2(0) r=0 lpr=7648 pi=[7488,7648)/1 crt=7592'250135 > lcod 0'0 mlcod 0'0 peering mbc={}] > -4> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 7648 share_map > client.244681 v1:192.168.1.50:0/337348516 7648 > -3> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 7648 > should_share_map client.244681 v1:192.168.1.50:0/337348516 7648 > -2> 2019-04-26 19:23:05.200 7fb2647da700 10 osd.2 7648 dequeue_op > 0x7c300f8420 finish > -1> 2019-04-26 19:23:05.200 7fb2647da700 20 osd.2 op_wq(4) _process > empty q, waiting > 0> 2019-04-26 19:23:05.206 7fb2667de700 -1 *** Caught signal > (Aborted) ** > in thread 7fb2667de700 thread_name:tp_osd_tp > > > On Fri, Apr 26, 2019 at 3:57 PM Gregory Farnum <gfar...@redhat.com> wrote: > >> You'll probably want to generate a log with "debug osd = 20" and >> "debug bluestore = 20", then share that or upload it with >> ceph-post-file, to get more useful info about which PGs are breaking >> (is it actually the ones that were supposed to delete?). >> >> If there's a particular set of PGs you need to rescue, you can also >> look at using the ceph-objectstore-tool to export them off the busted >> OSD stores and import them into OSDs that still work. >> >> >> On Fri, Apr 26, 2019 at 12:01 PM Elise Burke <elise.n...@gmail.com> >> wrote: >> > >> > Hi, >> > >> > I upgraded to Nautilus a week or two ago and things had been mostly >> fine. I was interested in trying the device health stats feature and >> enabled it. In doing so it created a pool, device_health_metrics, which >> contained zero bytes. >> > >> > Unfortunately this pool developed a PG that could not be repaired with >> `ceph pg repair`. That's okay, I thought, this pool is empty (zero bytes), >> so I'll just remove it and discard the PG entirely. >> > >> > So I did: `ceph osd pool rm device_health_metrics device_health_metrics >> --yes-i-really-really-mean-it` >> > >> > Within a few seconds three OSDs had gone missing (this pool was size=3) >> and now crashloop at startup. >> > >> > Any assistance in getting these OSDs up (such as by discarding the >> errant PG) would be appreciated. I'm most concerned about the other pools >> in the system, as losing three OSDs at once has not been ideal. >> > >> > This is made more difficult as these are in the Bluestore configuration >> and were set up with ceph-deploy to bare metal (using LVM mode). >> > >> > Here's the traceback as noted in journalctl: >> > >> > Apr 26 11:01:43 databox ceph-osd[1878533]: -5381> 2019-04-26 >> 11:01:08.902 7f8a00866d80 -1 Falling back to public interface >> > Apr 26 11:01:43 databox ceph-osd[1878533]: -4241> 2019-04-26 >> 11:01:41.835 7f8a00866d80 -1 osd.2 7630 log_to_monitors {default=true} >> > Apr 26 11:01:43 databox ceph-osd[1878533]: -3> 2019-04-26 11:01:43.203 >> 7f89dee53700 -1 bluestore(/var/lib/ceph/osd/ceph-2) _txc_add_transaction >> error (39) Directory not empty not handled on operation 21 (op 1, counting >> from 0) >> > Apr 26 11:01:43 databox ceph-osd[1878533]: -1> 2019-04-26 11:01:43.209 >> 7f89dee53700 -1 >> /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14 >> > Apr 26 11:01:43 databox ceph-osd[1878533]: >> /home/jenkins-build/build/workspace/ceph-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos7/DIST/centos7/MACHINE_SIZE/huge/release/14.2.0/rpm/el7/BUILD/ceph-14.2.0/src/os/bluest >> > Apr 26 11:01:43 databox ceph-osd[1878533]: ceph version 14.2.0 >> (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable) >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 1: (ceph::__ceph_abort(char >> const*, int, char const*, std::string const&)+0xd8) [0xfc63afe40] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 2: >> (BlueStore::_txc_add_transaction(BlueStore::TransContext*, >> ObjectStore::Transaction*)+0x2a85) [0xfc698e5f5] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 3: >> (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, >> std::vector<ObjectStore::Transaction, >> std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 4: >> (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, >> ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, >> ThreadPool::TPHandle*)+0x7f) [0xfc656b81f >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 5: >> (PG::_delete_some(ObjectStore::Transaction*)+0x83d) [0xfc65ce70d] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 6: >> (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x38) >> [0xfc65cf528] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 7: >> (boost::statechart::simple_state<PG::RecoveryState::Deleting, >> PG::RecoveryState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, >> mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::na >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 8: >> (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, >> PG::RecoveryState::Initial, std::allocator<void>, >> boost::statechart::null_exception_translator>::process_event(boost >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 9: >> (PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, >> PG::RecoveryCtx*)+0x119) [0xfc65dac99] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 10: >> (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, >> ThreadPool::TPHandle&)+0x1b4) [0xfc6515494] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 11: >> (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, >> ThreadPool::TPHandle&)+0x234) [0xfc65158d4] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 12: >> (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x9f4) >> [0xfc6509c14] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 13: >> (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x433) >> [0xfc6b01f43] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 14: >> (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xfc6b04fe0] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 15: (()+0x7dd5) >> [0x7f89fd4b0dd5] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 16: (clone()+0x6d) >> [0x7f89fc376ead] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 0> 2019-04-26 11:01:43.217 >> 7f89dee53700 -1 *** Caught signal (Aborted) ** >> > Apr 26 11:01:43 databox ceph-osd[1878533]: in thread 7f89dee53700 >> thread_name:tp_osd_tp >> > Apr 26 11:01:43 databox ceph-osd[1878533]: ceph version 14.2.0 >> (3a54b2b6d167d4a2a19e003a705696d4fe619afc) nautilus (stable) >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 1: (()+0xf5d0) >> [0x7f89fd4b85d0] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 2: (gsignal()+0x37) >> [0x7f89fc2af207] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 3: (abort()+0x148) >> [0x7f89fc2b08f8] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 4: (ceph::__ceph_abort(char >> const*, int, char const*, std::string const&)+0x19c) [0xfc63aff04] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 5: >> (BlueStore::_txc_add_transaction(BlueStore::TransContext*, >> ObjectStore::Transaction*)+0x2a85) [0xfc698e5f5] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 6: >> (BlueStore::queue_transactions(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, >> std::vector<ObjectStore::Transaction, >> std::allocator<ObjectStore::Transaction> >&, boost::intrusive_ptr >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 7: >> (ObjectStore::queue_transaction(boost::intrusive_ptr<ObjectStore::CollectionImpl>&, >> ObjectStore::Transaction&&, boost::intrusive_ptr<TrackedOp>, >> ThreadPool::TPHandle*)+0x7f) [0xfc656b81f >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 8: >> (PG::_delete_some(ObjectStore::Transaction*)+0x83d) [0xfc65ce70d] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 9: >> (PG::RecoveryState::Deleting::react(PG::DeleteSome const&)+0x38) >> [0xfc65cf528] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 10: >> (boost::statechart::simple_state<PG::RecoveryState::Deleting, >> PG::RecoveryState::ToDelete, boost::mpl::list<mpl_::na, mpl_::na, mpl_::na, >> mpl_::na, mpl_::na, mpl_::na, mpl_::na, mpl_::n >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 11: >> (boost::statechart::state_machine<PG::RecoveryState::RecoveryMachine, >> PG::RecoveryState::Initial, std::allocator<void>, >> boost::statechart::null_exception_translator>::process_event(boos >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 12: >> (PG::do_peering_event(std::shared_ptr<PGPeeringEvent>, >> PG::RecoveryCtx*)+0x119) [0xfc65dac99] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 13: >> (OSD::dequeue_peering_evt(OSDShard*, PG*, std::shared_ptr<PGPeeringEvent>, >> ThreadPool::TPHandle&)+0x1b4) [0xfc6515494] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 14: >> (OSD::dequeue_delete(OSDShard*, PG*, unsigned int, >> ThreadPool::TPHandle&)+0x234) [0xfc65158d4] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 15: >> (OSD::ShardedOpWQ::_process(unsigned int, ceph::heartbeat_handle_d*)+0x9f4) >> [0xfc6509c14] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 16: >> (ShardedThreadPool::shardedthreadpool_worker(unsigned int)+0x433) >> [0xfc6b01f43] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 17: >> (ShardedThreadPool::WorkThreadSharded::entry()+0x10) [0xfc6b04fe0] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 18: (()+0x7dd5) >> [0x7f89fd4b0dd5] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: 19: (clone()+0x6d) >> [0x7f89fc376ead] >> > Apr 26 11:01:43 databox ceph-osd[1878533]: NOTE: a copy of the >> executable, or `objdump -rdS <executable>` is needed to interpret this. >> > >> > Thanks! >> > >> > -Elise >> > _______________________________________________ >> > ceph-users mailing list >> > ceph-users@lists.ceph.com >> > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com >> >
_______________________________________________ ceph-users mailing list ceph-users@lists.ceph.com http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com