4 PGs stuck inactive

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 




I have 4 PGs stuck inactive.....

I have tried everything I can find online, and could use some help.

I had 3 OSDs... was in process of rebooting one and another totally crashed and corrupted its filesystem (BAD).

So now I have 4 incomplete PGs.

Things I've tried:
Rebooting all OSDs.
Running 'ceph pg {pg-id} mark_unfound_lost revert'
Running 'ceph osd lost {OSD ID}'

Here is a PG query from one of the 4 PGs



[[{ "state": "down+incomplete",
  "epoch": 7994,
  "up": [
        1,
        2],
  "acting": [
        1,
        2],
  "info": { "pgid": "4.1ca",
      "last_update": "0'0",
      "last_complete": "0'0",
      "log_tail": "0'0",
      "last_backfill": "MAX",
      "purged_snaps": "[]",
      "history": { "epoch_created": 75,
          "last_epoch_started": 6719,
          "last_epoch_clean": 6710,
          "last_epoch_split": 0,
          "same_up_since": 7973,
          "same_interval_since": 7973,
          "same_primary_since": 7969,
          "last_scrub": "6239'25314",
          "last_scrub_stamp": "2014-01-05 18:48:01.122717",
          "last_deep_scrub": "6235'25313",
          "last_deep_scrub_stamp": "2014-01-03 18:47:48.390112",
          "last_clean_scrub_stamp": "2014-01-05 18:48:01.122717"},
      "stats": { "version": "0'0",
          "reported_seq": "1259",
          "reported_epoch": "7994",
          "state": "down+incomplete",
          "last_fresh": "2014-01-07 14:04:36.269587",
          "last_change": "2014-01-07 14:04:36.269587",
          "last_active": "0.000000",
          "last_clean": "0.000000",
          "last_became_active": "0.000000",
          "last_unstale": "2014-01-07 14:04:36.269587",
          "mapping_epoch": 7971,
          "log_start": "0'0",
          "ondisk_log_start": "0'0",
          "created": 75,
          "last_epoch_clean": 6710,
          "parent": "0.0",
          "parent_split_bits": 0,
          "last_scrub": "6239'25314",
          "last_scrub_stamp": "2014-01-05 18:48:01.122717",
          "last_deep_scrub": "6235'25313",
          "last_deep_scrub_stamp": "2014-01-03 18:47:48.390112",
          "last_clean_scrub_stamp": "2014-01-05 18:48:01.122717",
          "log_size": 0,
          "ondisk_log_size": 0,
          "stats_invalid": "0",
          "stat_sum": { "num_bytes": 0,
              "num_objects": 0,
              "num_object_clones": 0,
              "num_object_copies": 0,
              "num_objects_missing_on_primary": 0,
              "num_objects_degraded": 0,
              "num_objects_unfound": 0,
              "num_read": 0,
              "num_read_kb": 0,
              "num_write": 0,
              "num_write_kb": 0,
              "num_scrub_errors": 0,
              "num_shallow_scrub_errors": 0,
              "num_deep_scrub_errors": 0,
              "num_objects_recovered": 0,
              "num_bytes_recovered": 0,
              "num_keys_recovered": 0},
          "stat_cat_sum": {},
          "up": [
                1,
                2],
          "acting": [
                1,
                2]},
      "empty": 1,
      "dne": 0,
      "incomplete": 0,
      "last_epoch_started": 0},
  "recovery_state": [
        { "name": "Started\/Primary\/Peering",
          "enter_time": "2014-01-07 14:04:36.269509",
          "past_intervals": [
                { "first": 5221,
                  "last": 5226,
                  "maybe_went_rw": 1,
                  "up": [
                        0,
                        1],
                  "acting": [
                        0,
                        1]},
                { "first": 5227,
                  "last": 5439,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 5440,
                  "last": 5441,
                  "maybe_went_rw": 0,
                  "up": [
                        1],
                  "acting": [
                        1]},
                { "first": 5442,
                  "last": 6388,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 6389,
                  "last": 6395,
                  "maybe_went_rw": 1,
                  "up": [
                        0],
                  "acting": [
                        0]},
                { "first": 6396,
                  "last": 6708,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 6709,
                  "last": 6717,
                  "maybe_went_rw": 1,
                  "up": [
                        1],
                  "acting": [
                        1]},
                { "first": 6718,
                  "last": 6721,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        2],
                  "acting": [
                        1,
                        2]},
                { "first": 6722,
                  "last": 6724,
                  "maybe_went_rw": 1,
                  "up": [
                        2],
                  "acting": [
                        2]},
                { "first": 6725,
                  "last": 6737,
                  "maybe_went_rw": 1,
                  "up": [
                        0,
                        2],
                  "acting": [
                        0,
                        2]},
                { "first": 6738,
                  "last": 7946,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 7947,
                  "last": 7948,
                  "maybe_went_rw": 1,
                  "up": [
                        1],
                  "acting": [
                        1]},
                { "first": 7949,
                  "last": 7950,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 7951,
                  "last": 7952,
                  "maybe_went_rw": 1,
                  "up": [
                        0],
                  "acting": [
                        0]},
                { "first": 7953,
                  "last": 7954,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 7955,
                  "last": 7956,
                  "maybe_went_rw": 1,
                  "up": [
                        1],
                  "acting": [
                        1]},
                { "first": 7957,
                  "last": 7958,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        2],
                  "acting": [
                        1,
                        2]},
                { "first": 7959,
                  "last": 7966,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 7967,
                  "last": 7968,
                  "maybe_went_rw": 1,
                  "up": [
                        0],
                  "acting": [
                        0]},
                { "first": 7969,
                  "last": 7970,
                  "maybe_went_rw": 1,
                  "up": [
                        1,
                        0],
                  "acting": [
                        1,
                        0]},
                { "first": 7971,
                  "last": 7972,
                  "maybe_went_rw": 1,
                  "up": [
                        1],
                  "acting": [
                        1]}],
          "probing_osds": [
                1,
                2],
          "down_osds_we_would_probe": [
                0],
          "peering_blocked_by": []},
        { "name": "Started",
          "enter_time": "2014-01-07 14:04:36.269462"}]}



CEPH OSD TREE

ceph osd tree
# id    weight  type name       up/down reweight
-1      4.5     pool default
-4      1.8             host ceph2
1       1.8                     osd.1   up      1
-6      2.7             host ceph3
2       2.7                     osd.2   up      1


_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com

[Index of Archives]     [Information on CEPH]     [Linux Filesystem Development]     [Ceph Development]     [Ceph Large]     [Ceph Dev]     [Linux USB Development]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]     [xfs]


  Powered by Linux