Assuming the one who lost its filesystem is totally gone, mark it
lost. That will tell the OSDs to give up on whatever data it might
have had and you should be good to go (modulo whatever data you might
have lost from only having it on the dead OSD during the reboot).
-Greg
Software Engineer #42 @ http://inktank.com | http://ceph.com
On Tue, Jan 7, 2014 at 1:32 PM, Bryan Morris <br...@internalfx.com> wrote:
>
> I have 4 PGs stuck inactive.....
>
> I have tried everything I can find online, and could use some help.
>
> I had 3 OSDs... was in process of rebooting one and another totally crashed
> and corrupted its filesystem (BAD).
>
> So now I have 4 incomplete PGs.
>
> Things I've tried:
>
> Rebooting all OSDs.
> Running 'ceph pg {pg-id} mark_unfound_lost revert'
> Running 'ceph osd lost {OSD ID}'
>
>
> Here is a PG query from one of the 4 PGs
>
>
>
> [[{ "state": "down+incomplete",
> "epoch": 7994,
> "up": [
> 1,
> 2],
> "acting": [
> 1,
> 2],
> "info": { "pgid": "4.1ca",
> "last_update": "0'0",
> "last_complete": "0'0",
> "log_tail": "0'0",
> "last_backfill": "MAX",
> "purged_snaps": "[]",
> "history": { "epoch_created": 75,
> "last_epoch_started": 6719,
> "last_epoch_clean": 6710,
> "last_epoch_split": 0,
> "same_up_since": 7973,
> "same_interval_since": 7973,
> "same_primary_since": 7969,
> "last_scrub": "6239'25314",
> "last_scrub_stamp": "2014-01-05 18:48:01.122717",
> "last_deep_scrub": "6235'25313",
> "last_deep_scrub_stamp": "2014-01-03 18:47:48.390112",
> "last_clean_scrub_stamp": "2014-01-05 18:48:01.122717"},
> "stats": { "version": "0'0",
> "reported_seq": "1259",
> "reported_epoch": "7994",
> "state": "down+incomplete",
> "last_fresh": "2014-01-07 14:04:36.269587",
> "last_change": "2014-01-07 14:04:36.269587",
> "last_active": "0.000000",
> "last_clean": "0.000000",
> "last_became_active": "0.000000",
> "last_unstale": "2014-01-07 14:04:36.269587",
> "mapping_epoch": 7971,
> "log_start": "0'0",
> "ondisk_log_start": "0'0",
> "created": 75,
> "last_epoch_clean": 6710,
> "parent": "0.0",
> "parent_split_bits": 0,
> "last_scrub": "6239'25314",
> "last_scrub_stamp": "2014-01-05 18:48:01.122717",
> "last_deep_scrub": "6235'25313",
> "last_deep_scrub_stamp": "2014-01-03 18:47:48.390112",
> "last_clean_scrub_stamp": "2014-01-05 18:48:01.122717",
> "log_size": 0,
> "ondisk_log_size": 0,
> "stats_invalid": "0",
> "stat_sum": { "num_bytes": 0,
> "num_objects": 0,
> "num_object_clones": 0,
> "num_object_copies": 0,
> "num_objects_missing_on_primary": 0,
> "num_objects_degraded": 0,
> "num_objects_unfound": 0,
> "num_read": 0,
> "num_read_kb": 0,
> "num_write": 0,
> "num_write_kb": 0,
> "num_scrub_errors": 0,
> "num_shallow_scrub_errors": 0,
> "num_deep_scrub_errors": 0,
> "num_objects_recovered": 0,
> "num_bytes_recovered": 0,
> "num_keys_recovered": 0},
> "stat_cat_sum": {},
> "up": [
> 1,
> 2],
> "acting": [
> 1,
> 2]},
> "empty": 1,
> "dne": 0,
> "incomplete": 0,
> "last_epoch_started": 0},
> "recovery_state": [
> { "name": "Started\/Primary\/Peering",
> "enter_time": "2014-01-07 14:04:36.269509",
> "past_intervals": [
> { "first": 5221,
> "last": 5226,
> "maybe_went_rw": 1,
> "up": [
> 0,
> 1],
> "acting": [
> 0,
> 1]},
> { "first": 5227,
> "last": 5439,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 5440,
> "last": 5441,
> "maybe_went_rw": 0,
> "up": [
> 1],
> "acting": [
> 1]},
> { "first": 5442,
> "last": 6388,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 6389,
> "last": 6395,
> "maybe_went_rw": 1,
> "up": [
> 0],
> "acting": [
> 0]},
> { "first": 6396,
> "last": 6708,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 6709,
> "last": 6717,
> "maybe_went_rw": 1,
> "up": [
> 1],
> "acting": [
> 1]},
> { "first": 6718,
> "last": 6721,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 2],
> "acting": [
> 1,
> 2]},
> { "first": 6722,
> "last": 6724,
> "maybe_went_rw": 1,
> "up": [
> 2],
> "acting": [
> 2]},
> { "first": 6725,
> "last": 6737,
> "maybe_went_rw": 1,
> "up": [
> 0,
> 2],
> "acting": [
> 0,
> 2]},
> { "first": 6738,
> "last": 7946,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 7947,
> "last": 7948,
> "maybe_went_rw": 1,
> "up": [
> 1],
> "acting": [
> 1]},
> { "first": 7949,
> "last": 7950,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 7951,
> "last": 7952,
> "maybe_went_rw": 1,
> "up": [
> 0],
> "acting": [
> 0]},
> { "first": 7953,
> "last": 7954,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 7955,
> "last": 7956,
> "maybe_went_rw": 1,
> "up": [
> 1],
> "acting": [
> 1]},
> { "first": 7957,
> "last": 7958,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 2],
> "acting": [
> 1,
> 2]},
> { "first": 7959,
> "last": 7966,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 7967,
> "last": 7968,
> "maybe_went_rw": 1,
> "up": [
> 0],
> "acting": [
> 0]},
> { "first": 7969,
> "last": 7970,
> "maybe_went_rw": 1,
> "up": [
> 1,
> 0],
> "acting": [
> 1,
> 0]},
> { "first": 7971,
> "last": 7972,
> "maybe_went_rw": 1,
> "up": [
> 1],
> "acting": [
> 1]}],
> "probing_osds": [
> 1,
> 2],
> "down_osds_we_would_probe": [
> 0],
> "peering_blocked_by": []},
> { "name": "Started",
> "enter_time": "2014-01-07 14:04:36.269462"}]}
>
>
>
> CEPH OSD TREE
>
> ceph osd tree
> # id weight type name up/down reweight
> -1 4.5 pool default
> -4 1.8 host ceph2
> 1 1.8 osd.1 up 1
> -6 2.7 host ceph3
> 2 2.7 osd.2 up 1
>
>
>
> _______________________________________________
> ceph-users mailing list
> ceph-users@lists.ceph.com
> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>
_______________________________________________
ceph-users mailing list
ceph-users@lists.ceph.com
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
