>> Yesterday I removed two OSD's, to replace them with new disks. Ceph was
>> not able to completely reach all active+clean state, but some degraded
>> objects remain. However, the amount of degraded objects is negative
>> (-82), see below:
>>
>
> So why didn't it reach that state?
Well, I dunno, I was hoping this list would know why? I simply sat there
waiting for the process to complete and it didn't.
> Could you query those PGs and see why they are remapped?
>
> $ ceph pg <pg id> query
I queried one of the PG's, see below for the output. Can you tell why
they are remapped?
# ceph pg 5.af query
{ "state": "active+remapped",
"epoch": 1105,
"up": [
14,
11],
"acting": [
14,
11,
12],
"actingbackfill": [
"11",
"12",
"14"],
"info": { "pgid": "5.af",
"last_update": "533'1976",
"last_complete": "533'1976",
"log_tail": "0'0",
"last_user_version": 1976,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 197,
"last_epoch_started": 772,
"last_epoch_clean": 772,
"last_epoch_split": 0,
"same_up_since": 721,
"same_interval_since": 771,
"same_primary_since": 721,
"last_scrub": "533'1976",
"last_scrub_stamp": "2014-10-29 00:09:56.095703",
"last_deep_scrub": "533'1976",
"last_deep_scrub_stamp": "2014-10-27 00:09:48.770622",
"last_clean_scrub_stamp": "2014-10-29 00:09:56.095703"},
"stats": { "version": "533'1976",
"reported_seq": "2846",
"reported_epoch": "1105",
"state": "active+remapped",
"last_fresh": "2014-10-30 01:55:27.177249",
"last_change": "2014-10-29 23:07:40.579020",
"last_active": "2014-10-30 01:55:27.177249",
"last_clean": "2014-10-26 21:49:13.064622",
"last_became_active": "0.000000",
"last_unstale": "2014-10-30 01:55:27.177249",
"mapping_epoch": 766,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 197,
"last_epoch_clean": 772,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "533'1976",
"last_scrub_stamp": "2014-10-29 00:09:56.095703",
"last_deep_scrub": "533'1976",
"last_deep_scrub_stamp": "2014-10-27 00:09:48.770622",
"last_clean_scrub_stamp": "2014-10-29 00:09:56.095703",
"log_size": 1976,
"ondisk_log_size": 1976,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 4194304,
"num_objects": 13,
"num_object_clones": 0,
"num_object_copies": 39,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 13,
"num_whiteouts": 0,
"num_read": 36,
"num_read_kb": 40,
"num_write": 2047,
"num_write_kb": 17259,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 26,
"num_bytes_recovered": 8388608,
"num_keys_recovered": 222,
"num_objects_omap": 12,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
14,
11],
"acting": [
14,
11,
12],
"up_primary": 14,
"acting_primary": 14},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 772,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "11",
"pgid": "5.af",
"last_update": "533'1976",
"last_complete": "533'1976",
"log_tail": "0'0",
"last_user_version": 1976,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 197,
"last_epoch_started": 772,
"last_epoch_clean": 772,
"last_epoch_split": 0,
"same_up_since": 721,
"same_interval_since": 771,
"same_primary_since": 721,
"last_scrub": "533'1976",
"last_scrub_stamp": "2014-10-29 00:09:56.095703",
"last_deep_scrub": "533'1976",
"last_deep_scrub_stamp": "2014-10-27 00:09:48.770622",
"last_clean_scrub_stamp": "2014-10-29 00:09:56.095703"},
"stats": { "version": "533'1976",
"reported_seq": "2430",
"reported_epoch": "723",
"state": "remapped+peering",
"last_fresh": "2014-10-29 23:03:18.847590",
"last_change": "2014-10-29 23:03:17.673820",
"last_active": "2014-10-29 22:41:29.551558",
"last_clean": "2014-10-26 21:49:13.064622",
"last_became_active": "0.000000",
"last_unstale": "2014-10-29 23:03:18.847590",
"mapping_epoch": 766,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 197,
"last_epoch_clean": 719,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "533'1976",
"last_scrub_stamp": "2014-10-29 00:09:56.095703",
"last_deep_scrub": "533'1976",
"last_deep_scrub_stamp": "2014-10-27 00:09:48.770622",
"last_clean_scrub_stamp": "2014-10-29 00:09:56.095703",
"log_size": 1976,
"ondisk_log_size": 1976,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 4194304,
"num_objects": 13,
"num_object_clones": 0,
"num_object_copies": 39,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 13,
"num_whiteouts": 0,
"num_read": 36,
"num_read_kb": 40,
"num_write": 2047,
"num_write_kb": 17259,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 13,
"num_bytes_recovered": 4194304,
"num_keys_recovered": 111,
"num_objects_omap": 12,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
14,
11],
"acting": [
14,
11,
12],
"up_primary": 14,
"acting_primary": 14},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 772,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "12",
"pgid": "5.af",
"last_update": "533'1976",
"last_complete": "533'1976",
"log_tail": "0'0",
"last_user_version": 1976,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 197,
"last_epoch_started": 772,
"last_epoch_clean": 772,
"last_epoch_split": 0,
"same_up_since": 721,
"same_interval_since": 771,
"same_primary_since": 721,
"last_scrub": "533'1976",
"last_scrub_stamp": "2014-10-29 00:09:56.095703",
"last_deep_scrub": "533'1976",
"last_deep_scrub_stamp": "2014-10-27 00:09:48.770622",
"last_clean_scrub_stamp": "2014-10-29 00:09:56.095703"},
"stats": { "version": "533'1976",
"reported_seq": "2600",
"reported_epoch": "720",
"state": "active+clean",
"last_fresh": "2014-10-29 23:01:23.000209",
"last_change": "2014-10-29 22:53:33.121314",
"last_active": "2014-10-29 23:01:23.000209",
"last_clean": "2014-10-29 23:01:23.000209",
"last_became_active": "0.000000",
"last_unstale": "2014-10-29 23:01:23.000209",
"mapping_epoch": 766,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 197,
"last_epoch_clean": 719,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "533'1976",
"last_scrub_stamp": "2014-10-29 00:09:56.095703",
"last_deep_scrub": "533'1976",
"last_deep_scrub_stamp": "2014-10-27 00:09:48.770622",
"last_clean_scrub_stamp": "2014-10-29 00:09:56.095703",
"log_size": 1976,
"ondisk_log_size": 1976,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 4194304,
"num_objects": 13,
"num_object_clones": 0,
"num_object_copies": 39,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 13,
"num_whiteouts": 0,
"num_read": 42,
"num_read_kb": 54,
"num_write": 2047,
"num_write_kb": 17259,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 12,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
14,
11],
"acting": [
14,
11,
12],
"up_primary": 14,
"acting_primary": 14},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 772,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}}],
"recovery_state": [
{ "name": "Started\/Primary\/Active",
"enter_time": "2014-10-29 23:07:39.343666",
"might_have_unfound": [],
"recovery_progress": { "backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": { "pull_from_peer": [],
"pushing": []}},
"scrub": { "scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.block_writes": 0,
"scrubber.finalizing": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []}},
{ "name": "Started",
"enter_time": "2014-10-29 23:07:38.240102"}],
"agent_state": {}}
Thanks,
Erik.
>
> Wido
>
>> The cache should be empty, because I isseud "rados
>> cache-flush-evict-all"-command, and "rados -p cephfs-data-cache ls"
>> indeed shows zero objects in this pool.
>>
>> "rados df" however does show 192 objects for this pool, with just 35KB
>> used and -82 degraded:
>>
>> pool name category KB objects clones
>> degraded unfound rd rd KB wr wr KB
>> cephfs-data-cache - 35 192 0
>> -82 0 1119 348800 1198371 1703673493
>>
>> Please advice...
>>
>> Thanks,
>>
>> Erik.
>> _______________________________________________
>> ceph-users mailing list
>> ceph-users@xxxxxxxxxxxxxx
>> http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
>>
>
>
_______________________________________________
ceph-users mailing list
ceph-users@xxxxxxxxxxxxxx
http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
