Performance really drops from 700MB/s to 10MB/s

[ganders@xxxxxxxxxxxx (German Anders)]

Hi to all,

      I'm having a particular behavior on a new Ceph cluster. I've map 
a RBD to a client and issue some performance tests with fio, at this 
point everything goes just fine (also the results :) ), but then I try 
to run another new test on a new RBD on the same client, and suddenly  
the performance goes below 10MB/s and it took almost 10 minutes to 
complete a 10G file test, if I issue a ceph -w I don't see anything 
suspicious, any idea what can be happening here?

      The cluster is made of:

3 x MON Servers
4 x OSD Servers (3TB SAS 6G disks for OSD daemons & tmpfs for Journal 
-> there's one tmpfs of 36GB that is share by 9 OSD daemons, on each 
server)
2 x Network SW (Cluster and Public)
10GbE speed on both networks

      The ceph.conf file is the following:

[global]
fsid = 56e56e4c-ea59-4157-8b98-acae109bebe1
mon_initial_members = cephmon01, cephmon02, cephmon03
mon_host = 10.97.10.1,10.97.10.2,10.97.10.3
auth_client_required = cephx
auth_cluster_required = cephx
auth_service_required = cephx
filestore_xattr_use_omap = true
public_network = 10.97.0.0/16
cluster_network = 192.168.10.0/24
osd_pool_default_size = 2
glance_api_version = 2

[mon]
debug_optracker = 0

[mon.cephmon01]
host = cephmon01
mon_addr = 10.97.10.1:6789

[mon.cephmon02]
host = cephmon02
mon_addr = 10.97.10.2:6789

[mon.cephmon03]
host = cephmon03
mon_addr = 10.97.10.3:6789

[osd]
journal_dio = false
osd_journal_size = 4096
fstype = btrfs
debug_optracker = 0

[osd.0]
host = cephosd01
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.1]
host = cephosd01
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.2]
host = cephosd01
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.3]
host = cephosd01
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.4]
host = cephosd01
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.5]
host = cephosd01
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.6]
host = cephosd01
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.7]
host = cephosd01
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.8]
host = cephosd01
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.9]
host = cephosd02
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.10]
host = cephosd02
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.11]
host = cephosd02
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.12]
host = cephosd02
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.13]
host = cephosd02
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.14]
host = cephosd02
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.15]
host = cephosd02
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.16]
host = cephosd02
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.17]
host = cephosd02
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.18]
host = cephosd03
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.19]
host = cephosd03
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.20]
host = cephosd03
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.21]
host = cephosd03
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.22]
host = cephosd03
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.23]
host = cephosd03
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.24]
host = cephosd03
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.25]
host = cephosd03
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.26]
host = cephosd03
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.27]
host = cephosd04
devs = /dev/sdc1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.28]
host = cephosd04
devs = /dev/sdd1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.29]
host = cephosd04
devs = /dev/sdf1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.30]
host = cephosd04
devs = /dev/sdg1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.31]
host = cephosd04
devs = /dev/sdi1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.32]
host = cephosd04
devs = /dev/sdj1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.33]
host = cephosd04
devs = /dev/sdl1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.34]
host = cephosd04
devs = /dev/sdm1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[osd.35]
host = cephosd04
devs = /dev/sdn1
osd_journal = /mnt/ramdisk/$cluster-$id-journal

[client.volumes]
keyring = /etc/ceph/ceph.client.volumes.keyring


Thanks in advance,

Best regards,


German Anders

















-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.ceph.com/pipermail/ceph-users-ceph.com/attachments/20140813/b138a57f/attachment.htm>