I am using Ceph to back Openstack Nova ephemeral, Cinder volumes, Glance images, and Openstack Manila File Share storage. Originally, I was using ceph-fuse with Manila, but performance and resource usage was poor, so I changed to using the CephFs kernel driver. Now however, I am getting messages from my MDS that all of my Manila file share clients are "failing to respond to cache pressure". Can anyone take a look and advise me? I have earlier increased mds_cache_size from 100000 to 500000. Am I missing some configuration on the clients to correctly enable them to respond to cache instruction?
thanks!
shorton
arcuser@arccloud01:~$ sudo ceph daemon mds.arccloud01 perf dump mds
{
"mds": {
"request": 16147252,
"reply": 16147249,
"reply_latency": {
"avgcount": 16147249,
"sum": 19885.911791960
},
"forward": 0,
"dir_fetch": 535805,
"dir_commit": 35493,
"dir_split": 0,
"inode_max": 500000,
"inodes": 499912,
"inodes_top": 129548,
"inodes_bottom": 365853,
"inodes_pin_tail": 4511,
"inodes_pinned": 314789,
"inodes_expired": 3214675,
"inodes_with_caps": 314579,
"caps": 5338001,
"subtrees": 2,
"traverse": 19339004,
"traverse_hit": 16500851,
"traverse_forward": 0,
"traverse_discover": 0,
"traverse_dir_fetch": 2738658,
"traverse_remote_ino": 0,
"traverse_lock": 2149,
"load_cent": 1614725200,
"q": 0,
"exported": 0,
"exported_inodes": 0,
"imported": 0,
"imported_inodes": 0
}
}
arcuser@arccloud01:~$ cat /etc/ceph/ceph.conf
[global]
fsid = 6e647506-631a-457e-a52a-f21a3866a023
mon_initial_members = arccloud01, arccloud02, arccloud03
mon_host = 10.155.92.128,10.155.92.129,10.155.92.130
mon_pg_warn_max_per_osd = 400
mon_lease = 50
mon_lease_renew_interval = 30
mon_lease_ack_timeout = 100
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
public_network = 10.155.92.0/22
cluster_network = 192.168.92.0/22
[client.glanceimages]
keyring = /etc/ceph/ceph.client.glanceimages.keyring
[client.novapool]
keyring = /etc/ceph/ceph.client.novapool.keyring
[client.cindervolumes]
keyring = /etc/ceph/ceph.client.cindervolumes.keyring
[client.manila]
client_mount_uid = 0
client_mount_gid = 0
log_file = /opt/stack/logs/ceph-client.manila.log
admin_socket = /opt/stack/status/stack/ceph-$name.$pid.asok
keyring = /etc/ceph/ceph.client.manila.keyring
[mon.arccloud01]
host = arccloud01
mon addr = 10.155.92.128:6789
[mon.arccloud02]
host = arccloud02
mon addr = 10.155.92.129:6789
[mon.arccloud03]
host = arccloud03
mon addr = 10.155.92.130:6789
[osd.2]
host = arccloud01
public addr = 10.155.92.128
cluster addr = 192.168.92.128
[osd.1]
host = arccloud02
public addr = 10.155.92.129
cluster addr = 192.168.92.129
[osd.0]
host = arccloud03
public addr = 10.155.92.130
cluster addr = 192.168.92.130
[mds]
mds cache size = 500000
arcuser@arccloud01:/usr/local/bin$ sudo ceph -s
cluster 6e647506-631a-457e-a52a-f21a3866a023
health HEALTH_WARN
mds0: Client ROSA-LIN-DESKTOP failing to respond to cache pressure
mds0: Client QI-DAI-DESKTOP failing to respond to cache pressure
monmap e1: 3 mons at {arccloud01=10.155.92.128:6789/0,arccloud02=10.155.92.129:6789/0,arccloud03=10.155.92.130:6789/0}
election epoch 5152, quorum 0,1,2 arccloud01,arccloud02,arccloud03
fsmap e1774: 1/1/1 up {0=arccloud01=up:active}
osdmap e1528: 3 osds: 3 up, 3 in
flags sortbitwise
pgmap v1637495: 384 pgs, 6 pools, 458 GB data, 1992 kobjects
1752 GB used, 40431 GB / 42184 GB avail
384 active+clean
client io 36944 B/s wr, 0 op/s rd, 7 op/s wr
arcuser@arccloud01:/usr/local/bin$ sudo ceph auth list
installed auth entries:
mds.arccloud01
key: AQCqk6tXSuIhBxAAhBWtOpaezVMooYlWJyRXCQ==
caps: [mds] allow
caps: [mon] allow profile mds
caps: [osd] allow rwx
osd.0
key: AQDBnZ9XMV9MBhAAj9xjqDh97KguIXk0FnV1ww==
caps: [mon] allow profile osd
caps: [osd] allow *
osd.1
key: AQDUnZ9X4QnrJxAACUqQHUzYIBGNLflqHMjmrQ==
caps: [mon] allow profile osd
caps: [osd] allow *
osd.2
key: AQDunZ9XNw7BMhAAof3w6p0HBt6brsAg+SDPXA==
caps: [mon] allow profile osd
caps: [osd] allow *
client.admin
key: AQC6lp9XAfd4KBAAWc+SURHa4WVPO7tAb847iw==
caps: [mds] allow *
caps: [mon] allow *
caps: [osd] allow *
client.bootstrap-mds
key: AQC7lp9XkoMJCBAAuG70hJueP4a3lI7+Bm7YBA==
caps: [mon] allow profile bootstrap-mds
client.bootstrap-osd
key: AQC6lp9Xu3eiMRAAIrCEAy4APS5xRwnjqAAiIw==
caps: [mon] allow profile bootstrap-osd
client.bootstrap-rgw
key: AQC6lp9XRp6BOhAAlsL8we1uuvEg2PRQVZTE8w==
caps: [mon] allow profile bootstrap-rgw
client.cephadmin
key: AQDbochXcbglHhAAh07P01qJt5N/AqQp2W1WPA==
caps: [mds] allow rw path=/volumes/_nogroup/1cd9bc07-e883-4e90-8e88-6ee46b4f04de
caps: [mon] allow r
caps: [osd] allow rw pool=cephfs_data namespace=fsvolumens_1cd9bc07-e883-4e90-8e88-6ee46b4f04de
client.cindervolumes
key: AQDS96BXtfcADRAAOnw6+//qaJFpFbmhE/I9Tg==
caps: [mon] allow r
caps: [osd] allow class-read object_prefix rbd_children, allow rwx pool=cindervolumes, allow rwx pool=glanceimages, allow rwx pool=novapool, allow rwx pool=manila
client.glanceimages
key: AQB3rp9XXyi0OhAA3EJ1HG04YsMq+FKHbSazHw==
caps: [mon] allow r
caps: [osd] allow class-read object_prefix rbd_children, allow rwx pool=glanceimages
client.manila
key: AQBGoqtXFT0hLBAA9M4c+8cx1xU46bHZ9b+Lmg==
caps: [mds] allow *
caps: [mon] allow r,
allow command "auth del",
allow command "auth caps",
allow command "auth get",
allow command "auth get-or-create"
caps: [osd] allow rw
client.novapool
key: AQDGvJ9Xl1G3HxAAw0yy6/cOc3VZhlP0rfs7sw==
caps: [mon] allow r
caps: [osd] allow class-read object_prefix rbd_children, allow rwx pool=cindervolumes, allow rwx pool=glanceimages, allow rwx pool=novapool, allow rwx pool=manila
arcuser@arccloud01:/usr/local/bin$ sudo ceph daemon mds.arccloud01 session ls
[
{
"id": 1234461,
"num_leases": 0,
"num_caps": 304447,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234461 10.155.93.64:0\/795649277",
"client_metadata": {
"entity_id": "admin",
"hostname": "fort",
"kernel_version": "4.4.0-36-generic"
}
},
{
"id": 1234305,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234305 10.155.93.78:0\/816616354",
"client_metadata": {
"entity_id": "admin",
"hostname": "QI-DESKTOP",
"kernel_version": "4.4.0-38-generic"
}
},
{
"id": 1234248,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234248 10.155.93.70:0\/3285723029",
"client_metadata": {
"entity_id": "admin",
"hostname": "BIN-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234257,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234257 10.155.93.72:0\/512772364",
"client_metadata": {
"entity_id": "admin",
"hostname": "S-DESKTOP",
"kernel_version": "4.4.0-38-generic"
}
},
{
"id": 1234296,
"num_leases": 1,
"num_caps": 314078,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234296 10.155.93.73:0\/387190076",
"client_metadata": {
"entity_id": "admin",
"hostname": "RO-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234233,
"num_leases": 1,
"num_caps": 312807,
"state": "open",
"replay_requests": 0,
"completed_requests": 40,
"reconnecting": false,
"inst": "client.1234233 10.155.93.81:0\/60200371",
"client_metadata": {
"entity_id": "admin",
"hostname": "JI-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234224,
"num_leases": 0,
"num_caps": 313542,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234224 10.155.93.80:0\/2255163546",
"client_metadata": {
"entity_id": "admin",
"hostname": "CI-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234218,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234218 10.155.93.84:0\/2138779034",
"client_metadata": {
"entity_id": "admin",
"hostname": "JI-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234212,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234212 10.155.93.86:0\/245537925",
"client_metadata": {
"entity_id": "admin",
"hostname": "XI-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234230,
"num_leases": 1,
"num_caps": 313534,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234230 10.155.93.87:0\/3586471149",
"client_metadata": {
"entity_id": "admin",
"hostname": "ST-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234209,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234209 10.155.93.89:0\/721248196",
"client_metadata": {
"entity_id": "admin",
"hostname": "YA-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234221,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234221 10.155.93.83:0\/3593778124",
"client_metadata": {
"entity_id": "admin",
"hostname": "YO-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234206,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234206 10.155.93.82:0\/2319251827",
"client_metadata": {
"entity_id": "admin",
"hostname": "J-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234239,
"num_leases": 0,
"num_caps": 313508,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234239 10.155.93.75:0\/3783671045",
"client_metadata": {
"entity_id": "admin",
"hostname": "P-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234203,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234203 10.155.93.88:0\/63817652",
"client_metadata": {
"entity_id": "admin",
"hostname": "Y-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234200,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234200 10.155.93.74:0\/2010146939",
"client_metadata": {
"entity_id": "admin",
"hostname": "E-DESKTOP",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 1234185,
"num_leases": 0,
"num_caps": 1,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234185 10.155.92.130:0\/2356495099",
"client_metadata": {
"ceph_sha1": "45107e21c568dd033c2f0a3107dec8f0b0e58374",
"ceph_version": "ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374)",
"entity_id": "manila",
"hostname": "arccloud03",
"root": "\/"
}
},
{
"id": 1234245,
"num_leases": 0,
"num_caps": 313486,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.1234245 10.155.93.71:0\/350860345",
"client_metadata": {
"entity_id": "admin",
"hostname": "sxxxxxx",
"kernel_version": "4.4.0-34-generic"
}
},
{
"id": 575124,
"num_leases": 0,
"num_caps": 16385,
"state": "open",
"replay_requests": 0,
"completed_requests": 0,
"reconnecting": false,
"inst": "client.575124 10.155.93.50:0\/566906781",
"client_metadata": {
"ceph_sha1": "45107e21c568dd033c2f0a3107dec8f0b0e58374",
"ceph_version": "ceph version 10.2.2 (45107e21c568dd033c2f0a3107dec8f0b0e58374)",
"entity_id": "admin",
"hostname": "arccloudVM",
"mount_point": "\/home\/ubuntu\/arc_share",
"root": "\/"
}
}
]
