On the RBD performance issue, you may want to look at: http://tracker.ceph.com/issues/9192 Eric On Tue, Oct 27, 2015 at 8:59 PM, FaHui Lin <fahui.lin@xxxxxxxxxx> wrote: > Dear Ceph experts, > > I found something strange about the performance of my Ceph cluster: Read-out > much slower than write-in. > > I have 3 machines running OSDs, each has 8 OSDs running on 8 raid0s (each > made up of 2 HDDs) respectively. The OSD journal and data the is on the same > device. All machines in my clusters have 10Gb network. > > I used both Ceph RBD and CephFS, the client on another machine outside > cluster or on one of the running OSD (to rule out possible network issue), > an so on. All of these end up in a similar results: write-in can almost > reach the network limit, say 1200 MB/s, while read-out is only 350~450 MB/s. > > Trying to figure out, I did an extra test using CephFS: > > Version and Config: > [root@dl-disk1 ~]# ceph --version > ceph version 0.94.3 (95cefea9fd9ab740263bf8bb4796fd864d9afe2b) > [root@dl-disk1 ~]# cat /etc/ceph/ceph.conf > [global] > fsid = (hidden) > mon_initial_members = dl-disk1, dl-disk2, dl-disk3 > mon_host = (hidden) > auth_cluster_required = cephx > auth_service_required = cephx > auth_client_required = cephx > filestore_xattr_use_omap = true > > OSD tree: > # ceph osd tree > ID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY > -1 258.88000 root default > -2 87.28000 host dl-disk1 > 0 10.90999 osd.0 up 1.00000 1.00000 > 1 10.90999 osd.1 up 1.00000 1.00000 > 2 10.90999 osd.2 up 1.00000 1.00000 > 3 10.90999 osd.3 up 1.00000 1.00000 > 4 10.90999 osd.4 up 1.00000 1.00000 > 5 10.90999 osd.5 up 1.00000 1.00000 > 6 10.90999 osd.6 up 1.00000 1.00000 > 7 10.90999 osd.7 up 1.00000 1.00000 > -3 87.28000 host dl-disk2 > 8 10.90999 osd.8 up 1.00000 1.00000 > 9 10.90999 osd.9 up 1.00000 1.00000 > 10 10.90999 osd.10 up 1.00000 1.00000 > 11 10.90999 osd.11 up 1.00000 1.00000 > 12 10.90999 osd.12 up 1.00000 1.00000 > 13 10.90999 osd.13 up 1.00000 1.00000 > 14 10.90999 osd.14 up 1.00000 1.00000 > 15 10.90999 osd.15 up 1.00000 1.00000 > -4 84.31999 host dl-disk3 > 16 10.53999 osd.16 up 1.00000 1.00000 > 17 10.53999 osd.17 up 1.00000 1.00000 > 18 10.53999 osd.18 up 1.00000 1.00000 > 19 10.53999 osd.19 up 1.00000 1.00000 > 20 10.53999 osd.20 up 1.00000 1.00000 > 21 10.53999 osd.21 up 1.00000 1.00000 > 22 10.53999 osd.22 up 1.00000 1.00000 > 23 10.53999 osd.23 up 1.00000 1.00000 > > Pools and PG (each pool has 128 PGs): > # ceph osd lspools > 0 rbd,2 fs_meta,3 fs_data0,4 fs_data1, > # ceph pg dump pools > dumped pools in format plain > pg_stat objects mip degr misp unf bytes log disklog > pool 0 0 0 0 0 0 0 0 0 > pool 2 20 0 0 0 0 356958 264 264 > pool 3 3264 0 0 0 0 16106127360 14657 > 14657 > pool 4 0 0 0 0 0 0 0 0 > > To simplify the problem, I made a new crush rule that the CephFS data pool > use OSDs on only one machine (dl-disk1 here), and size = 1. > # ceph osd crush rule dump osd_in_dl-disk1__ruleset > { > "rule_id": 1, > "rule_name": "osd_in_dl-disk1__ruleset", > "ruleset": 1, > "type": 1, > "min_size": 1, > "max_size": 10, > "steps": [ > { > "op": "take", > "item": -2, > "item_name": "dl-disk1" > }, > { > "op": "chooseleaf_firstn", > "num": 0, > "type": "osd" > }, > { > "op": "emit" > } > ] > } > # ceph osd pool get fs_data0 crush_ruleset > crush_ruleset: 1 > # ceph osd pool get fs_data0 size > size: 1 > > Here starts the test. > On an client machine, I used dd to write a 4GB-file to CephFS, and checked > dstat on the OSD node dl-disk1: > [root@client ~]# dd of=/mnt/cephfs/4Gfile if=/dev/zero bs=4096k count=1024 > 1024+0 records in > 1024+0 records out > 4294967296 bytes (4.3 GB) copied, 3.69993 s, 1.2 GB/s > > [root@dl-disk1 ~]# dstat ... > ---total-cpu-usage---- ------memory-usage----- -net/total- > --dsk/sdb-----dsk/sdc-----dsk/sdd-----dsk/sde-----dsk/sdf-----dsk/sdg-----dsk/sdh-----dsk/sdi-- > usr sys idl wai hiq siq| used buff cach free| recv send| read writ: > read writ: read writ: read writ: read writ: read writ: read writ: read > writ > > 0 0 100 0 0 0|3461M 67.2M 15.1G 44.3G| 19k 20k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3461M 67.2M 15.1G 44.3G| 32k 32k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 8 18 74 0 0 0|3364M 67.2M 11.1G 48.4G| 391k 391k| 0 2712k: 0 > 1096k: 0 556k: 0 1084k: 0 1200k: 0 1196k: 0 688k: 0 > 1252k > 0 0 100 0 0 0|3364M 67.2M 11.1G 48.4G| 82k 127k| 0 0 : 0 > 0 : 0 0 : 0 928k: 0 540k: 0 0 : 0 0 : 0 0 > 8 16 72 3 0 1|3375M 67.2M 11.8G 47.7G| 718M 2068k| 0 120M: 0 > 172M: 0 76M: 0 220M: 0 188M: 16k 289M: 0 53M: 0 36M > 6 13 77 4 0 1|3391M 67.2M 12.3G 47.1G| 553M 1517k| 0 160M: 0 > 176M: 0 88M: 0 208M: 0 225M: 0 213M: 0 8208k: 0 49M > 6 13 77 3 0 1|3408M 67.2M 12.9G 46.6G| 544M 1272k| 0 212M: 0 > 8212k: 0 36M: 0 0 : 0 37M: 0 3852k: 0 497M: 0 > 337M > 0 0 99 0 0 0|3407M 67.3M 12.9G 46.6G| 53k 114k| 0 36M: 0 > 37M: 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3407M 67.3M 12.9G 46.6G| 68k 110k| 0 0 : 0 > 0 : 0 0 : 0 36M: 0 0 : 0 0 : 0 0 : 0 0 > 0 0 99 0 0 0|3407M 67.3M 12.9G 46.6G| 38k 328k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 36M: 0 0 > 0 1 99 0 0 0|3406M 67.3M 12.9G 46.6G| 11M 132k| 0 0 : 0 > 0 : 0 8224k: 0 0 : 0 0 : 0 32M: 0 0 : 0 36M > 14 24 52 8 0 2|3436M 67.3M 13.8G 45.6G|1026M 2897k| 0 100M: 0 > 409M: 0 164M: 0 313M: 0 253M: 0 321M: 0 84M: 0 76M > 14 24 34 27 0 1|3461M 67.3M 14.7G 44.7G| 990M 2565k| 0 354M: 0 > 72M: 0 0 : 0 164M: 0 313M: 0 188M: 0 308M: 0 333M > 4 9 70 16 0 0|3474M 67.3M 15.1G 44.3G| 269M 646k| 0 324M: 0 > 0 : 0 0 : 0 36M: 0 0 : 0 0 : 0 349M: 0 172M > 0 0 99 0 0 0|3474M 67.3M 15.1G 44.3G| 24k 315k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 37M: 0 0 > 0 0 99 0 0 0|3474M 67.4M 15.1G 44.3G| 38k 102k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 36M: 0 0 : 0 36M > 0 0 99 0 0 0|3473M 67.4M 15.1G 44.3G| 22k 23k| 0 0 : 0 > 0 : 0 36M: 0 0 : 0 36M: 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3473M 67.4M 15.1G 44.3G| 39k 40k| 0 304k: 0 > 16k: 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3472M 67.4M 15.1G 44.3G| 28k 64k| 0 64M: 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3471M 67.4M 15.1G 44.3G| 31k 94k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3472M 67.4M 15.1G 44.3G| 38k 39k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > > The throughput is 1.2 GB/s, able to reach the network limit 10Gb. > > Then, on the client machine, I used dd to read that file back from CephFS, > redirecting the file to /dev/zero (or /dev/null) to rule out local HDD's IO: > [root@client ~]# dd if=/mnt/cephfs/4Gfile of=/dev/zero bs=4096k count=1024 > 1024+0 records in > 1024+0 records out > 4294967296 bytes (4.3 GB) copied, 8.85246 s, 485 MB/s > > [root@dl-disk1 ~]# dstat ... > 0 0 100 0 0 0|3462M 67.4M 15.1G 44.3G| 36k 36k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3462M 67.4M 15.1G 44.3G| 22k 22k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3463M 67.4M 15.1G 44.3G| 49k 49k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 1 99 0 0 0|3464M 67.4M 15.1G 44.3G| 282k 111M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 5 93 0 0 0|3466M 67.4M 15.1G 44.3G|1171k 535M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 5 93 0 0 0|3467M 67.4M 15.1G 44.3G|1124k 535M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3467M 67.4M 15.1G 44.3G|1124k 535M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3467M 67.4M 15.1G 44.3G|1109k 527M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 93 0 0 0|3471M 67.4M 15.1G 44.3G|1044k 504M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3470M 67.4M 15.1G 44.3G|1031k 504M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 5 93 0 0 0|3470M 67.4M 15.1G 44.3G|1103k 527M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 93 0 0 0|3471M 67.5M 15.1G 44.3G|1084k 504M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3470M 67.5M 15.1G 44.3G| 25k 24k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > ----total-cpu-usage---- ------memory-usage----- -net/total- > --dsk/sdb-----dsk/sdc-----dsk/sdd-----dsk/sde-----dsk/sdf-----dsk/sdg-----dsk/sdh-----dsk/sdi-- > usr sys idl wai hiq siq| used buff cach free| recv send| read writ: > read writ: read writ: read writ: read writ: read writ: read writ: read > writ > 0 0 100 0 0 0|3470M 67.5M 15.1G 44.3G| 43k 44k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3470M 67.5M 15.1G 44.3G| 22k 23k| 0 48k: 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 35k 38k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 23k 85k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 44k 44k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 24k 25k| 0 12k: 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 45k 43k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3468M 67.5M 15.1G 44.3G| 17k 18k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > > > The throughput here was only 400~500 MB/s here. > I noticed that there was NO disk I/O during the read-out, that means all the > objects of the file were already cached in memory on the OSD node. > Thus, HDDs does NOT seem to cause the lower throughput. > > I also tried read-out using cat (in case dd may not use read-ahead in file > system. ), ended up getting similar result: > > [root@client ~]# time cat /mnt/cephfs/4Gfile > /dev/zero > > real 0m9.352s > user 0m0.002s > sys 0m4.147s > > > [root@dl-disk1 ~]# dstat ... > 0 0 100 0 0 0|3465M 67.5M 15.1G 44.3G| 23k 22k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3465M 67.5M 15.1G 44.3G| 17k 18k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3465M 67.5M 15.1G 44.3G| 37k 37k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 1 2 97 0 0 0|3466M 67.5M 15.1G 44.3G| 633k 280M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3467M 67.5M 15.1G 44.3G|1057k 498M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3470M 67.5M 15.1G 44.3G|1078k 498M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3470M 67.5M 15.1G 44.3G| 996k 486M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3469M 67.5M 15.1G 44.3G| 988k 489M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3469M 67.5M 15.1G 44.3G|1012k 489M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3470M 67.5M 15.1G 44.3G|1017k 497M| 0 0 : 0 > 8192B: 0 28k: 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 2 4 94 0 0 0|3469M 67.5M 15.1G 44.3G|1032k 498M| 0 0 : 0 > 0 : 0 0 : 0 8192B: 0 104k: 0 0 : 0 0 : 0 0 > ----total-cpu-usage---- ------memory-usage----- -net/total- > --dsk/sdb-----dsk/sdc-----dsk/sdd-----dsk/sde-----dsk/sdf-----dsk/sdg-----dsk/sdh-----dsk/sdi-- > usr sys idl wai hiq siq| used buff cach free| recv send| read writ: > read writ: read writ: read writ: read writ: read writ: read writ: read > writ > 2 4 94 0 0 0|3469M 67.5M 15.1G 44.3G|1025k 496M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 40k: 0 80k: 0 0 > 0 1 99 0 0 0|3469M 67.5M 15.1G 44.3G| 127k 52M| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 120k > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 21k 21k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 66k 66k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > 0 0 100 0 0 0|3469M 67.5M 15.1G 44.3G| 35k 38k| 0 0 : 0 > 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 : 0 0 > > > The average throughput is 4GB / 9.35s = 438 MB/s. Still, unlikely to be > HDD's issue. > > I'm sure that the network can reach 10Gb in both ways via iperf or other > test, and there's no other user process occupying bandwidth. > > Could you please help me some to find out the main reason for this issue? > Thank you. > > Best Regards, > FaHui > > > > > > > _______________________________________________ > ceph-users mailing list > ceph-users@xxxxxxxxxxxxxx > http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com > _______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
 |