> That doesn't look like an RBD object -- any idea who is
> "client.34720596.1:212637720"?
So I think these might be proxy ops from the cache tier, as there are also
block ops on one of the cache tier OSD's, but this time it actually lists
the object name. Block op on cache tier.
"description": "osd_op(client.34720596.1:212637720 17.ae78c1cf
17:f3831e75:::rbd_data.15a5e20238e1f29.00000000000388ad:head [set-alloc-hint
object_size 4194304 write_size 4194304,write 2584576~16384] snapc 0=[]
RETRY=2 ondisk+retry+write+known_if_redirected e104841)",
"initiated_at": "2017-12-12 16:25:32.435718",
"age": 13996.681147,
"duration": 13996.681203,
"type_data": {
"flag_point": "reached pg",
"client_info": {
"client": "client.34720596",
"client_addr": "10.3.31.41:0/2600619462",
"tid": 212637720
I'm a bit baffled at the moment what's going. The pg query (attached) is not
showing in the main status that it has been blocked from peering or that
there are any missing objects. I've tried restarting all OSD's I can see
relating to the PG in case they needed a bit of a nudge.
>
> On Tue, Dec 12, 2017 at 12:36 PM, Nick Fisk <nick@xxxxxxxxxx> wrote:
> > Does anyone know what this object (0.ae78c1cf) might be, it's not your
> > normal run of the mill RBD object and I can't seem to find it in the
> > pool using rados --all ls . It seems to be leaving the 0.1cf PG stuck
> > in an
> > activating+remapped state and blocking IO. Pool 0 is just a pure RBD
> > activating+pool
> > with a cache tier above it. There is no current mention of unfound
> > objects or any other obvious issues.
> >
> > There is some backfilling going on, on another OSD which was upgraded
> > to bluestore, which was when the issue started. But I can't see any
> > link in the PG dump with upgraded OSD. My only thought so far is to
> > wait for this backfilling to finish and then deep-scrub this PG and
> > see if that reveals anything?
> >
> > Thanks,
> > Nick
> >
> > "description": "osd_op(client.34720596.1:212637720 0.1cf 0.ae78c1cf
> > (undecoded)
> > ondisk+retry+write+ignore_cache+ignore_overlay+known_if_redirected
> > e105014)",
> > "initiated_at": "2017-12-12 17:10:50.030660",
> > "age": 335.948290,
> > "duration": 335.948383,
> > "type_data": {
> > "flag_point": "delayed",
> > "events": [
> > {
> > "time": "2017-12-12 17:10:50.030660",
> > "event": "initiated"
> > },
> > {
> > "time": "2017-12-12 17:10:50.030692",
> > "event": "queued_for_pg"
> > },
> > {
> > "time": "2017-12-12 17:10:50.030719",
> > "event": "reached_pg"
> > },
> > {
> > "time": "2017-12-12 17:10:50.030727",
> > "event": "waiting for peered"
> > },
> > {
> > "time": "2017-12-12 17:10:50.197353",
> > "event": "reached_pg"
> > },
> > {
> > "time": "2017-12-12 17:10:50.197355",
> > "event": "waiting for peered"
> >
> > _______________________________________________
> > ceph-users mailing list
> > ceph-users@xxxxxxxxxxxxxx
> > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>
>
>
> --
> Jason
> _______________________________________________
> ceph-users mailing list
> ceph-users@xxxxxxxxxxxxxx
> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
{
"state": "activating+remapped",
"snap_trimq": "[]",
"epoch": 105385,
"up": [
34,
68,
8
],
"acting": [
34,
8,
53
],
"backfill_targets": [
"68"
],
"actingbackfill": [
"8",
"34",
"53",
"68"
],
"info": {
"pgid": "0.1cf",
"last_update": "104744'7509486",
"last_complete": "104744'7509486",
"log_tail": "104532'7507938",
"last_user_version": 132304619,
"last_backfill": "MAX",
"last_backfill_bitwise": 0,
"purged_snaps": [
{
"start": "1",
"length": "d"
}
],
"history": {
"epoch_created": 1,
"epoch_pool_created": 1,
"last_epoch_started": 104744,
"last_interval_started": 104743,
"last_epoch_clean": 104744,
"last_interval_clean": 104743,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 105371,
"same_interval_since": 105371,
"same_primary_since": 105014,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927"
},
"stats": {
"version": "104744'7509486",
"reported_seq": "6171655",
"reported_epoch": "105385",
"state": "activating+remapped",
"last_fresh": "2017-12-12 20:21:58.171078",
"last_change": "2017-12-12 20:16:18.598517",
"last_active": "2017-12-12 16:08:08.229538",
"last_peered": "2017-12-12 15:59:38.592595",
"last_clean": "2017-12-12 15:59:38.592595",
"last_became_active": "2017-12-12 15:59:38.072033",
"last_became_peered": "2017-12-12 15:59:38.072033",
"last_unstale": "2017-12-12 20:21:58.171078",
"last_undegraded": "2017-12-12 20:21:58.171078",
"last_fullsized": "2017-12-12 20:21:58.171078",
"mapping_epoch": 105371,
"log_start": "104532'7507938",
"ondisk_log_start": "104532'7507938",
"created": 1,
"last_epoch_clean": 104744,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927",
"log_size": 1548,
"ondisk_log_size": 1548,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": true,
"stat_sum": {
"num_bytes": 16467992576,
"num_objects": 3946,
"num_object_clones": 0,
"num_object_copies": 11838,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 3946,
"num_whiteouts": 0,
"num_read": 644272,
"num_read_kb": 451588661,
"num_write": 7184317,
"num_write_kb": 692140524,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 42584,
"num_bytes_recovered": 178448003072,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
34,
68,
8
],
"acting": [
34,
8,
53
],
"blocked_by": [],
"up_primary": 34,
"acting_primary": 34
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 105374,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
"peer_info": [
{
"peer": "8",
"pgid": "0.1cf",
"last_update": "104744'7509486",
"last_complete": "104744'7509486",
"log_tail": "104532'7507938",
"last_user_version": 132304619,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "d"
}
],
"history": {
"epoch_created": 1,
"epoch_pool_created": 1,
"last_epoch_started": 104744,
"last_interval_started": 104743,
"last_epoch_clean": 104744,
"last_interval_clean": 104743,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 105371,
"same_interval_since": 105371,
"same_primary_since": 105014,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927"
},
"stats": {
"version": "104744'7509486",
"reported_seq": "6171159",
"reported_epoch": "105013",
"state": "remapped+peering",
"last_fresh": "2017-12-12 17:10:49.084788",
"last_change": "2017-12-12 17:10:49.056360",
"last_active": "2017-12-12 16:08:08.229538",
"last_peered": "2017-12-12 15:59:38.592595",
"last_clean": "2017-12-12 15:59:38.592595",
"last_became_active": "2017-12-12 15:59:38.072033",
"last_became_peered": "2017-12-12 15:59:38.072033",
"last_unstale": "2017-12-12 17:10:49.084788",
"last_undegraded": "2017-12-12 17:10:49.084788",
"last_fullsized": "2017-12-12 17:10:49.084788",
"mapping_epoch": 105371,
"log_start": "104532'7507938",
"ondisk_log_start": "104532'7507938",
"created": 1,
"last_epoch_clean": 104744,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927",
"log_size": 1548,
"ondisk_log_size": 1548,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": true,
"stat_sum": {
"num_bytes": 16467992576,
"num_objects": 3946,
"num_object_clones": 0,
"num_object_copies": 11838,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 3946,
"num_whiteouts": 0,
"num_read": 644272,
"num_read_kb": 451588661,
"num_write": 7184317,
"num_write_kb": 692140524,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 42584,
"num_bytes_recovered": 178448003072,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
34,
68,
8
],
"acting": [
34,
8,
53
],
"blocked_by": [],
"up_primary": 34,
"acting_primary": 34
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 105348,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "38",
"pgid": "0.1cf",
"last_update": "104744'7509486",
"last_complete": "104744'7509486",
"log_tail": "104532'7507986",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "d"
}
],
"history": {
"epoch_created": 1,
"epoch_pool_created": 1,
"last_epoch_started": 104744,
"last_interval_started": 104743,
"last_epoch_clean": 104744,
"last_interval_clean": 104743,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 105371,
"same_interval_since": 105371,
"same_primary_since": 105014,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 105371,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
34,
68,
8
],
"acting": [
34,
8,
53
],
"blocked_by": [],
"up_primary": 34,
"acting_primary": 34
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 105348,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "53",
"pgid": "0.1cf",
"last_update": "104744'7509486",
"last_complete": "104744'7509486",
"log_tail": "104532'7507938",
"last_user_version": 132304619,
"last_backfill": "MAX",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "d"
}
],
"history": {
"epoch_created": 1,
"epoch_pool_created": 1,
"last_epoch_started": 104744,
"last_interval_started": 104743,
"last_epoch_clean": 104744,
"last_interval_clean": 104743,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 105371,
"same_interval_since": 105371,
"same_primary_since": 105014,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927"
},
"stats": {
"version": "104744'7509485",
"reported_seq": "6171063",
"reported_epoch": "104744",
"state": "active+clean+remapped",
"last_fresh": "2017-12-12 15:59:38.592595",
"last_change": "2017-12-12 15:59:38.081819",
"last_active": "2017-12-12 15:59:38.592595",
"last_peered": "2017-12-12 15:59:38.592595",
"last_clean": "2017-12-12 15:59:38.592595",
"last_became_active": "2017-12-12 15:59:38.072033",
"last_became_peered": "2017-12-12 15:59:38.072033",
"last_unstale": "2017-12-12 15:59:38.592595",
"last_undegraded": "2017-12-12 15:59:38.592595",
"last_fullsized": "2017-12-12 15:59:38.592595",
"mapping_epoch": 105371,
"log_start": "104532'7507938",
"ondisk_log_start": "104532'7507938",
"created": 1,
"last_epoch_clean": 104744,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927",
"log_size": 1547,
"ondisk_log_size": 1547,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": true,
"stat_sum": {
"num_bytes": 16467992576,
"num_objects": 3946,
"num_object_clones": 0,
"num_object_copies": 11838,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 3946,
"num_whiteouts": 0,
"num_read": 644272,
"num_read_kb": 451588661,
"num_write": 7184317,
"num_write_kb": 692140524,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 42584,
"num_bytes_recovered": 178448003072,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [
34,
68,
8
],
"acting": [
34,
8,
53
],
"blocked_by": [],
"up_primary": 34,
"acting_primary": 34
},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 105348,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
},
{
"peer": "68",
"pgid": "0.1cf",
"last_update": "104744'7509486",
"last_complete": "104744'7509486",
"log_tail": "104532'7507986",
"last_user_version": 0,
"last_backfill": "MIN",
"last_backfill_bitwise": 1,
"purged_snaps": [
{
"start": "1",
"length": "d"
}
],
"history": {
"epoch_created": 1,
"epoch_pool_created": 1,
"last_epoch_started": 104744,
"last_interval_started": 104743,
"last_epoch_clean": 104744,
"last_interval_clean": 104743,
"last_epoch_split": 0,
"last_epoch_marked_full": 0,
"same_up_since": 105371,
"same_interval_since": 105371,
"same_primary_since": 105014,
"last_scrub": "104525'7503532",
"last_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_deep_scrub": "104525'7503532",
"last_deep_scrub_stamp": "2017-12-11 13:07:39.418927",
"last_clean_scrub_stamp": "2017-12-11 13:07:39.418927"
},
"stats": {
"version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "unknown",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_peered": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_became_peered": "0.000000",
"last_unstale": "0.000000",
"last_undegraded": "0.000000",
"last_fullsized": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": false,
"dirty_stats_invalid": false,
"omap_stats_invalid": false,
"hitset_stats_invalid": false,
"hitset_bytes_stats_invalid": false,
"pin_stats_invalid": false,
"stat_sum": {
"num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_missing": 0,
"num_objects_degraded": 0,
"num_objects_misplaced": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0,
"num_bytes_hit_set_archive": 0,
"num_flush": 0,
"num_flush_kb": 0,
"num_evict": 0,
"num_evict_kb": 0,
"num_promote": 0,
"num_flush_mode_high": 0,
"num_flush_mode_low": 0,
"num_evict_mode_some": 0,
"num_evict_mode_full": 0,
"num_objects_pinned": 0,
"num_legacy_snapsets": 0
},
"up": [],
"acting": [],
"blocked_by": [],
"up_primary": -1,
"acting_primary": -1
},
"empty": 0,
"dne": 0,
"incomplete": 1,
"last_epoch_started": 105374,
"hit_set_history": {
"current_last_update": "0'0",
"history": []
}
}
],
"recovery_state": [
{
"name": "Started/Primary/Active",
"enter_time": "2017-12-12 20:16:18.597553",
"might_have_unfound": [],
"recovery_progress": {
"backfill_targets": [
"68"
],
"waiting_on_backfill": [],
"last_backfill_started": "MIN",
"backfill_info": {
"begin": "MIN",
"end": "MIN",
"objects": []
},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": {
"pull_from_peer": [],
"pushing": []
}
},
"scrub": {
"scrubber.epoch_start": "0",
"scrubber.active": false,
"scrubber.state": "INACTIVE",
"scrubber.start": "MIN",
"scrubber.end": "MIN",
"scrubber.subset_last_update": "0'0",
"scrubber.deep": false,
"scrubber.seed": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []
}
},
{
"name": "Started",
"enter_time": "2017-12-12 20:16:15.939167"
}
],
"agent_state": {}
}
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
