W dniu 14.08.2014 17:20, Sage Weil pisze:
> On Thu, 14 Aug 2014, Pawe? Sadowski wrote:
>> Hello,
>>
>> I've a cluster of 35 OSD (30 HDD, 5 SSD) with cache tiering configured.
>> During tests it looks like ceph is not respecting target_max_bytes
>> settings. Steps to reproduce:
>> - configure cache tiering
>> - set target_max_bytes to 32G (on hot pool)
>> - write more than 32G of data
>> - nothing happens
> Can you 'ceph pg dump pools -f json-pretty' at this point? And pick a
> random PG in the cache pool and capture the output of 'ceph pg <pgid>
> query'.
[root at host227 ~] ceph df detail
GLOBAL:
SIZE AVAIL RAW USED %RAW USED OBJECTS
54833G 53657G 1176G 2.14 91468
POOLS:
NAME ID CATEGORY USED %USED OBJECTS
DIRTY READ WRITE
data 0 - 0 0 0
0 0 0
metadata 1 - 0 0 0
0 0 0
rbd 2 - 57700M 0.10 14425
14425 0 14425
cache 3 - 118G 0.22 30247
12742 0 86180
volumes 4 - 0 0 0
0 0 0
images 5 - 0 0 0
0 0 0
backups 6 - 0 0 0
0 0 0
vms 7 - 182G 0.33 46796
46796 2 46799
*** set target_max_bytes to 128G
[root at host227 ~] ceph osd pool set cache target_max_bytes $[128 * 1024 *
1024 * 1024]
set pool 3 target_max_bytes to 137438953472
[root at host227 ~] ceph -s
cluster 9fd17ded-7fb2-4993-b56c-bfa9ba3d0c1e
health HEALTH_OK
monmap e1: 1 mons at {host227=a.b.c.18:6789/0}, election epoch 2,
quorum 0 host227
osdmap e83: 35 osds: 35 up, 35 in
pgmap v2494: 1472 pgs, 8 pools, 338 GB data, 86713 objects
1176 GB used, 53657 GB / 54833 GB avail
1472 active+clean
*** data writing ...
[root at host227 ~] ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
54833G 53429G 1403G 2.56
POOLS:
NAME ID USED %USED OBJECTS
data 0 0 0 0
metadata 1 0 0 0
rbd 2 57700M 0.10 14425
cache 3 140G 0.26 36054
volumes 4 0 0 0
images 5 0 0 0
backups 6 0 0 0
vms 7 182G 0.33 46796
*** ceph didn't move any objects from hot pool
[root at host227 ~] ceph pg dump pools -f json-pretty
dumped pools in format json-pretty
[
{ "poolid": 0,
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 0,
"ondisk_log_size": 0},
{ "poolid": 1,
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 0,
"ondisk_log_size": 0},
{ "poolid": 2,
"stat_sum": { "num_bytes": 60502835200,
"num_objects": 14425,
"num_object_clones": 0,
"num_object_copies": 43275,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 14425,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 14425,
"num_write_kb": 59084800,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 14425,
"ondisk_log_size": 14425},
{ "poolid": 3,
"stat_sum": { "num_bytes": 151447928844,
"num_objects": 36109,
"num_object_clones": 0,
"num_object_copies": 108327,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 22597,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 101554,
"num_write_kb": 281886739,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 216606,
"ondisk_log_size": 216606},
{ "poolid": 4,
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 0,
"ondisk_log_size": 0},
{ "poolid": 5,
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 0,
"ondisk_log_size": 0},
{ "poolid": 6,
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 0,
"ondisk_log_size": 0},
{ "poolid": 7,
"stat_sum": { "num_bytes": 196272455692,
"num_objects": 46796,
"num_object_clones": 0,
"num_object_copies": 140388,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 46796,
"num_whiteouts": 0,
"num_read": 2,
"num_read_kb": 0,
"num_write": 46799,
"num_write_kb": 191672324,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"log_size": 46799,
"ondisk_log_size": 46799}]
[root at host227 ~] ceph pg 3.f4 query
{ "state": "active+clean",
"epoch": 83,
"up": [
13,
12,
11],
"acting": [
13,
12,
11],
"actingbackfill": [
"11",
"12",
"13"],
"info": { "pgid": "3.f4",
"last_update": "83'958",
"last_complete": "83'958",
"log_tail": "0'0",
"last_user_version": 958,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 4,
"last_epoch_started": 32,
"last_epoch_clean": 32,
"last_epoch_split": 0,
"same_up_since": 31,
"same_interval_since": 31,
"same_primary_since": 31,
"last_scrub": "0'0",
"last_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "83'958",
"reported_seq": "713",
"reported_epoch": "83",
"state": "active+clean",
"last_fresh": "2014-08-14 21:21:32.175427",
"last_change": "2014-08-14 12:39:19.584560",
"last_active": "2014-08-14 21:21:32.175427",
"last_clean": "2014-08-14 21:21:32.175427",
"last_became_active": "0.000000",
"last_unstale": "2014-08-14 21:21:32.175427",
"mapping_epoch": 31,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 4,
"last_epoch_clean": 32,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_clean_scrub_stamp": "0.000000",
"log_size": 958,
"ondisk_log_size": 958,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 595591168,
"num_objects": 142,
"num_object_clones": 0,
"num_object_copies": 426,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 88,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 452,
"num_write_kb": 1216512,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
12,
11],
"acting": [
13,
12,
11],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 32,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
"peer_info": [
{ "peer": "11",
"pgid": "3.f4",
"last_update": "83'958",
"last_complete": "83'958",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 4,
"last_epoch_started": 32,
"last_epoch_clean": 32,
"last_epoch_split": 0,
"same_up_since": 0,
"same_interval_since": 0,
"same_primary_since": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "0",
"reported_epoch": "0",
"state": "inactive",
"last_fresh": "0.000000",
"last_change": "0.000000",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "0.000000",
"mapping_epoch": 0,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 0,
"last_epoch_clean": 0,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "0.000000",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "0.000000",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [],
"acting": [],
"up_primary": -1,
"acting_primary": -1},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 32,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}},
{ "peer": "12",
"pgid": "3.f4",
"last_update": "83'958",
"last_complete": "83'958",
"log_tail": "0'0",
"last_user_version": 0,
"last_backfill": "MAX",
"purged_snaps": "[]",
"history": { "epoch_created": 4,
"last_epoch_started": 32,
"last_epoch_clean": 32,
"last_epoch_split": 0,
"same_up_since": 31,
"same_interval_since": 31,
"same_primary_since": 31,
"last_scrub": "0'0",
"last_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2014-08-14 12:39:17.339462",
"last_clean_scrub_stamp": "0.000000"},
"stats": { "version": "0'0",
"reported_seq": "1",
"reported_epoch": "30",
"state": "peering",
"last_fresh": "2014-08-14 12:39:17.350848",
"last_change": "2014-08-14 12:39:17.350848",
"last_active": "0.000000",
"last_clean": "0.000000",
"last_became_active": "0.000000",
"last_unstale": "2014-08-14 12:39:17.350848",
"mapping_epoch": 30,
"log_start": "0'0",
"ondisk_log_start": "0'0",
"created": 4,
"last_epoch_clean": 4,
"parent": "0.0",
"parent_split_bits": 0,
"last_scrub": "0'0",
"last_scrub_stamp": "2014-08-14 12:39:13.908560",
"last_deep_scrub": "0'0",
"last_deep_scrub_stamp": "2014-08-14 12:39:13.908560",
"last_clean_scrub_stamp": "0.000000",
"log_size": 0,
"ondisk_log_size": 0,
"stats_invalid": "0",
"stat_sum": { "num_bytes": 0,
"num_objects": 0,
"num_object_clones": 0,
"num_object_copies": 0,
"num_objects_missing_on_primary": 0,
"num_objects_degraded": 0,
"num_objects_unfound": 0,
"num_objects_dirty": 0,
"num_whiteouts": 0,
"num_read": 0,
"num_read_kb": 0,
"num_write": 0,
"num_write_kb": 0,
"num_scrub_errors": 0,
"num_shallow_scrub_errors": 0,
"num_deep_scrub_errors": 0,
"num_objects_recovered": 0,
"num_bytes_recovered": 0,
"num_keys_recovered": 0,
"num_objects_omap": 0,
"num_objects_hit_set_archive": 0},
"stat_cat_sum": {},
"up": [
13,
12,
11],
"acting": [
13,
12,
11],
"up_primary": 13,
"acting_primary": 13},
"empty": 0,
"dne": 0,
"incomplete": 0,
"last_epoch_started": 32,
"hit_set_history": { "current_last_update": "0'0",
"current_last_stamp": "0.000000",
"current_info": { "begin": "0.000000",
"end": "0.000000",
"version": "0'0"},
"history": []}}],
"recovery_state": [
{ "name": "Started\/Primary\/Active",
"enter_time": "2014-08-14 12:39:19.557945",
"might_have_unfound": [],
"recovery_progress": { "backfill_targets": [],
"waiting_on_backfill": [],
"last_backfill_started": "0\/\/0\/\/-1",
"backfill_info": { "begin": "0\/\/0\/\/-1",
"end": "0\/\/0\/\/-1",
"objects": []},
"peer_backfill_info": [],
"backfills_in_flight": [],
"recovering": [],
"pg_backend": { "pull_from_peer": [],
"pushing": []}},
"scrub": { "scrubber.epoch_start": "0",
"scrubber.active": 0,
"scrubber.block_writes": 0,
"scrubber.finalizing": 0,
"scrubber.waiting_on": 0,
"scrubber.waiting_on_whom": []}},
{ "name": "Started",
"enter_time": "2014-08-14 12:39:18.464397"}],
"agent_state": { "flush_mode": "idle",
"evict_mode": "idle",
"evict_effort": 0,
"position":
"92404ef4\/benchmark_data_host227_27916_object1087\/head\/\/3",
"atime_hist": { "histogram": [],
"upper_bound": 1},
"temp_hist": { "histogram": [],
"upper_bound": 1}}}
*** set target_max_bytes to the same value (as before writing)
[root at host227 ~] ceph osd pool set cache target_max_bytes $[128 * 1024 *
1024 * 1024]
set pool 3 target_max_bytes to 137438953472
*** ceph is now moving data
[root at host227 ~] ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
54833G 53555G 1277G 2.33
POOLS:
NAME ID USED %USED OBJECTS
data 0 0 0 0
metadata 1 0 0 0
rbd 2 57700M 0.10 14425
cache 3 137G 0.25 35299
volumes 4 0 0 0
images 5 0 0 0
backups 6 0 0 0
vms 7 184G 0.34 47221
*** data moving stopped (102G ~ 128G * 0.8)
[root at host227 ~] ceph df
GLOBAL:
SIZE AVAIL RAW USED %RAW USED
54833G 53440G 1392G 2.54
POOLS:
NAME ID USED %USED OBJECTS
data 0 0 0 0
metadata 1 0 0 0
rbd 2 57700M 0.10 14425
cache 3 102G 0.19 26286
volumes 4 0 0 0
images 5 0 0 0
backups 6 0 0 0
vms 7 224G 0.41 57506
> Then 'ceph tell osd.* injectargs '--debug-ms 1 --debug-osd 20'.
>
>> If I set target_max_bytes again (to the same value or any other option,
>> for example cache_min_evict_age) ceph will start to move data from hot
>> to base pool.
> Once it starts going, capture an OSD log (/var/log/ceph/ceph-osd.NNN.log)
> for an OSD that is now moving data.
Will send log file directly to you (log is 2MB compressed).
> Thanks!
> sage
>
>> I'm using ceph in version 0.80.4 (with cherry-picked patch from bug
>> http://tracker.ceph.com/issues/8982.
>>
>> Is there away to make it work as expected?
Thankyou,
PS
