|
Morning all,
We starting to apply load to our test cephfs system and are noticing some odd latency numbers. We are using erasure coding for the cold data pools and replication for our our cache tiers (not on ssd yet) . We noticed the following high latency on one node and it seams to be slowing down writes and reads on the cluster.
Our next step is break out mds, mgr, and mons to different machines but we wanted to start the discussion here.
Here is a bunch of information you may find useful.
ceph.conf [global]
fsid = XXXXX
mon_initial_members = ceph-mon1, ceph-mon2, ceph-mon3
mon_host = 10.141.167.238,10.141.160.251,10.141.161.249
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx
cluster network = 10.85.8.0/22
public network = 10.141.0.0/16
# we tested this with bluestore and xfs and have the same results
[osd]
enable_experimental_unrecoverable_data_corrupting_features = bluestore
Status cluster 8f6ba9d6-314d-4725-bcfa-340e500697f0
health HEALTH_OK
monmap e2: 3 mons at {ceph-mon1=10.141.167.238:6789/0,ceph-mon2=10.141.160.251:6789/0,ceph-mon3=10.141.161.249:6789/0}
election epoch 12, quorum 0,1,2 ceph-mon2,ceph-mon3,ceph-mon1
fsmap e30: 1/1/1 up {0=ceph-mon3=up:active}, 2 up:standby
mgr active: ceph-mon3 standbys: ceph-mon2, ceph-mon1
osdmap e100: 12 osds: 12 up, 12 in
flags sortbitwise,require_jewel_osds,require_kraken_osds
pgmap v119525: 124 pgs, 6 pools, 471 GB data, 1141 kobjects
970 GB used, 2231 GB / 3202 GB avail
124 active+clean
client io 11962 B/s rd, 11 op/s rd, 0 op/s wr
Pool space usageGLOBAL:
SIZE AVAIL RAW USED %RAW USED
3202G 2231G 970G 30.31
POOLS:
NAME ID USED %USED MAX AVAIL OBJECTS
rbd 0 0 0 580G 0
cephfs-hot 1 76137M 11.35 580G 466451
cephfs-cold 2 397G 25.48 1161G 650158
cephfs_metadata 3 47237k 0 580G 52275
one-hot 4 0 0 580G 0
one 5 0 0 1161G 0
OSD Performance and Latencyosd commit_latency(ms) apply_latency(ms) 9 1 1 8 1 1 0 13 13 11 1 1 1 38 38 10 2 2 2 21 21 3 2 2 4 20 20 5 1 1 6 1 1 7 1 1 OSD Tree and StatusID WEIGHT TYPE NAME UP/DOWN REWEIGHT PRIMARY-AFFINITY -1 3.12685 root default -2 1.08875 host ceph-mon1 0 0.27219 osd.0 up 1.00000 1.00000 1 0.27219 osd.1 up 1.00000 1.00000 2 0.27219 osd.2 up 1.00000 1.00000 4 0.27219 osd.4 up 1.00000 1.00000 -3 0.94936 host ceph-mon2 3 0.27219 osd.3 up 1.00000 1.00000 5 0.27219 osd.5 up 1.00000 1.00000 7 0.27219 osd.7 up 1.00000 1.00000 9 0.13280 osd.9 up 1.00000 1.00000 -4 1.08875 host ceph-mon3 6 0.27219 osd.6 up 1.00000 1.00000 8 0.27219 osd.8 up 1.00000 1.00000 10 0.27219 osd.10 up 1.00000 1.00000 11 0.27219 osd.11 up 1.00000 1.00000 [root@ceph-mon1 ~]# ceph pg dump
dumped all in format plain
version 119550
stamp 2017-03-15 12:46:24.428486
last_osdmap_epoch 100
last_pg_scan 100
full_ratio 0.95
nearfull_ratio 0.85
PG_STAT OBJECTS MISSING_ON_PRIMARY DEGRADED MISPLACED UNFOUND BYTES LOG DISK_LOG STATE STATE_STAMP VERSION REPORTED UP UP_PRIMARY ACTING ACTING_PRIMARY LAST_SCRUB SCRUB_STAMP LAST_DEEP_SCRUB DEEP_SCRUB_STAMP
0.2d 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.119959 0'0 99:57 [9,11,1] 9 [9,11,1] 9 0'0 2017-03-15 08:40:09.815059 0'0 2017-03-14
08:12:37.425834
0.2c 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:27:47.959479 0'0 100:29 [8,0,3] 8 [8,0,3] 8 0'0 2017-03-15 11:27:47.959398 0'0 2017-03-12
22:50:44.393457
0.2b 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.798794 0'0 94:27 [9,6,0] 9 [9,6,0] 9 0'0 2017-03-14 05:48:38.301001 0'0 2017-03-12
22:50:44.393456
0.2a 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.783652 0'0 93:26 [11,7,0] 11 [11,7,0] 11 0'0 2017-03-15 08:28:01.524045 0'0 2017-03-14
01:44:10.916908
0.29 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086293 0'0 97:28 [8,4,9] 8 [8,4,9] 8 0'0 2017-03-14 05:40:30.224198 0'0 2017-03-12
22:50:44.393454
0.28 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:05.045972 0'0 99:52 [2,5,6] 2 [2,5,6] 2 0'0 2017-03-15 11:00:05.045921 0'0 2017-03-14
04:13:27.042516
0.27 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.085837 0'0 97:52 [4,5,10] 4 [4,5,10] 4 0'0 2017-03-15 07:42:35.345807 0'0 2017-03-14
06:58:32.545361
0.26 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:52:48.382356 0'0 100:43 [1,7,6] 1 [1,7,6] 1 0'0 2017-03-15 11:52:48.382303 0'0 2017-03-14
09:43:14.727045
0.25 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.121464 0'0 99:66 [3,1,11] 3 [3,1,11] 3 0'0 2017-03-15 10:38:54.423533 0'0 2017-03-13
23:40:26.077607
0.24 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.783790 0'0 93:24 [11,0,3] 11 [11,0,3] 11 0'0 2017-03-14 09:27:51.202384 0'0 2017-03-14
09:27:51.202384
0.23 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.799383 0'0 94:51 [3,8,0] 3 [3,8,0] 3 0'0 2017-03-15 04:59:02.254164 0'0 2017-03-12
22:50:44.393446
0.22 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.195410 0'0 99:39 [8,9,2] 8 [8,9,2] 8 0'0 2017-03-15 09:44:03.948601 0'0 2017-03-12
22:50:44.393445
0.21 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:36.528918 0'0 99:72 [3,10,1] 3 [3,10,1] 3 0'0 2017-03-15 11:00:36.528870 0'0 2017-03-12
22:50:44.393444
0.20 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.799609 0'0 94:38 [7,0,8] 7 [7,0,8] 7 0'0 2017-03-15 10:13:21.551419 0'0 2017-03-12
22:50:44.393442
0.1f 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:27:27.285609 0'0 100:54 [6,1,7] 6 [6,1,7] 6 0'0 2017-03-15 11:27:27.285530 0'0 2017-03-12
22:50:44.393441
0.1e 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:37.897933 0'0 99:39 [7,1,6] 7 [7,1,6] 7 0'0 2017-03-15 11:00:37.897892 0'0 2017-03-12
22:50:44.393440
0.1d 0 0 0 0 0 0 0 0 active+clean 2017-03-15 12:14:43.246527 0'0 100:29 [8,0,5] 8 [8,0,5] 8 0'0 2017-03-15 12:14:43.246470 0'0 2017-03-15
12:14:43.246470
0.1c 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.799019 0'0 94:63 [3,10,0] 3 [3,10,0] 3 0'0 2017-03-15 01:25:50.956186 0'0 2017-03-12
22:50:44.393437
0.1b 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.121571 0'0 99:39 [7,6,1] 7 [7,6,1] 7 0'0 2017-03-15 08:18:54.805154 0'0 2017-03-12
22:50:44.393436
0.1a 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.119244 0'0 99:48 [5,1,8] 5 [5,1,8] 5 0'0 2017-03-15 08:00:51.862520 0'0 2017-03-15
08:00:51.862520
0.19 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.784074 0'0 93:62 [0,11,7] 0 [0,11,7] 0 0'0 2017-03-15 07:57:13.935810 0'0 2017-03-12
22:50:44.393434
0.18 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:38:15.847932 0'0 100:30 [10,4,7] 10 [10,4,7] 10 0'0 2017-03-15 11:38:15.847796 0'0 2017-03-15
11:38:15.847796
0.17 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.784295 0'0 93:42 [0,5,8] 0 [0,5,8] 0 0'0 2017-03-14 05:05:08.239405 0'0 2017-03-12
22:50:44.393431
0.16 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.118421 0'0 99:30 [8,1,3] 8 [8,1,3] 8 0'0 2017-03-15 01:58:21.000099 0'0 2017-03-15
01:58:21.000099
3.a 3289 0 0 0 0 0 3028 3028 active+clean 2017-03-15 10:59:52.122364 100'67947 100:20697 [6,1,3] 6 [6,1,3] 6 85'60019 2017-03-15 06:37:03.261930 85'60019 2017-03-15
06:37:03.261930
0.9 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:59:38.409276 0'0 100:52 [1,7,8] 1 [1,7,8] 1 0'0 2017-03-15 11:59:38.409200 0'0 2017-03-12
22:50:44.393401
1.8 26893 0 0 0 0 4917872433 3062 3062 active+clean 2017-03-15 10:59:54.109028 100'264150 100:206321 [6,4,3] 6 [6,4,3] 6 85'201591 2017-03-14 09:54:17.628516 0'0 2017-03-12
22:58:51.551958
2.b 39708 0 0 0 0 26703758232 3043 3043 active+clean 2017-03-15 10:59:52.195669 100'53392 100:32320 [8,2,5] 8 [8,2,5] 8 85'44426 2017-03-15 10:02:33.753337 0'0 2017-03-12
22:58:53.643259
3.b 3227 0 0 0 0 0 3050 3050 active+clean 2017-03-15 11:28:27.671176 100'70617 100:24248 [10,4,7] 10 [10,4,7] 10 100'70610 2017-03-15 11:28:27.671107 85'57865 2017-03-14
08:57:23.371141
0.8 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.196853 0'0 99:58 [3,10,2] 3 [3,10,2] 3 0'0 2017-03-15 02:59:29.527039 0'0 2017-03-14
02:45:37.740896
1.9 26478 0 0 0 0 4906646660 3009 3009 active+clean 2017-03-15 10:59:57.382487 100'266677 100:222230 [8,3,2] 8 [8,3,2] 8 85'234486 2017-03-15 03:37:28.182922 0'0 2017-03-12
22:58:51.551964
2.a 39321 0 0 0 0 26438889950 3055 3055 active+clean 2017-03-15 10:59:52.194575 100'54185 100:34972 [11,7,2] 11 [11,7,2] 11 85'40485 2017-03-14 09:26:04.085556 0'0 2017-03-12
22:58:53.643255
5.2 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.788431 0'0 94:22 [0,3,8] 0 [0,3,8] 0 0'0 2017-03-15 08:50:51.367854 0'0 2017-03-12
22:59:15.463362
4.3 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.755072 0'0 93:19 [0,11,7] 0 [0,11,7] 0 0'0 2017-03-14 06:58:30.050414 0'0 2017-03-12
22:59:14.407022
0.7 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.113827 0'0 99:52 [1,11,7] 1 [1,11,7] 1 0'0 2017-03-15 02:36:48.848706 0'0 2017-03-12
22:50:44.393399
1.6 61802 0 0 0 0 9816799529 3024 3024 active+clean 2017-03-15 10:59:51.658259 100'576546 100:411873 [6,9,4] 6 [6,9,4] 6 85'500558 2017-03-15 08:11:10.883131 85'400651 2017-03-14
03:22:13.677665
2.5 82560 0 0 0 0 53753456206 3056 3056 active+clean 2017-03-15 11:28:09.997386 100'106323 100:57552 [8,1,5] 8 [8,1,5] 8 100'106261 2017-03-15 11:28:09.997315 85'78184 2017-03-14
06:22:40.250909
3.4 6621 0 0 0 0 8389230 3035 3035 active+clean 2017-03-15 10:59:58.242009 100'146755 100:61467 [7,8,2] 7 [7,8,2] 7 86'135686 2017-03-15 10:46:03.168639 0'0 2017-03-12
22:59:12.616738
5.3 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.120289 0'0 99:31 [9,1,11] 9 [9,1,11] 9 0'0 2017-03-15 06:57:38.870430 0'0 2017-03-14
01:55:43.658927
4.2 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.606935 0'0 99:18 [1,8,9] 1 [1,8,9] 1 0'0 2017-03-14 08:07:55.935833 0'0 2017-03-14
08:07:55.935833
0.6 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086175 0'0 97:59 [4,11,5] 4 [4,11,5] 4 0'0 2017-03-14 08:07:53.069761 0'0 2017-03-12
22:50:44.393397
1.7 61031 0 0 0 0 10116071611 3004 3004 active+clean 2017-03-15 11:25:12.479773 100'550772 100:396565 [1,9,6] 1 [1,9,6] 1 100'547295 2017-03-15 11:25:12.479725 100'547295 2017-03-15
11:25:12.479725
2.4 80387 0 0 0 0 53220149134 3041 3041 active+clean 2017-03-15 11:02:23.117512 100'107407 100:64688 [5,10,4] 5 [5,10,4] 5 99'101538 2017-03-15 11:02:23.117442 0'0 2017-03-12
22:58:53.643225
3.5 6551 0 0 0 0 4194350 3017 3017 active+clean 2017-03-15 10:59:50.088771 100'138904 100:44971 [10,7,4] 10 [10,7,4] 10 85'125387 2017-03-15 03:41:18.534394 85'125387 2017-03-15
03:41:18.534394
5.0 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:10.046656 0'0 99:22 [2,10,5] 2 [2,10,5] 2 0'0 2017-03-15 11:00:10.046567 0'0 2017-03-15
11:00:10.046567
4.1 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:28:24.292278 0'0 100:28 [6,9,1] 6 [6,9,1] 6 0'0 2017-03-15 11:28:24.292221 0'0 2017-03-12
22:59:14.407011
0.5 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086680 0'0 97:30 [10,4,5] 10 [10,4,5] 10 0'0 2017-03-15 08:34:52.484299 0'0 2017-03-12
22:50:44.393396
1.4 56928 0 0 0 0 9839309040 3049 3049 active+clean 2017-03-15 11:00:04.470324 100'531624 100:362602 [1,11,7] 1 [1,11,7] 1 85'414369 2017-03-14 10:49:35.949776 0'0 2017-03-12
22:58:51.551937
2.7 82322 0 0 0 0 53834805828 3047 3047 active+clean 2017-03-15 10:59:52.195096 100'104536 100:63955 [6,2,3] 6 [6,2,3] 6 85'78411 2017-03-14 09:38:09.990513 0'0 2017-03-12
22:58:53.643240
3.6 6532 0 0 0 0 2232513 3082 3082 active+clean 2017-03-15 10:59:52.120341 100'138906 100:54610 [5,6,1] 5 [5,6,1] 5 85'100301 2017-03-14 10:25:36.346634 0'0 2017-03-12
22:59:12.616750
5.1 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.087655 0'0 97:31 [11,7,4] 11 [11,7,4] 11 0'0 2017-03-15 09:41:14.016921 0'0 2017-03-12
22:59:15.463340
4.0 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:27:31.740731 0'0 100:27 [10,7,0] 10 [10,7,0] 10 0'0 2017-03-15 11:27:31.740654 0'0 2017-03-12
22:59:14.407004
0.4 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:01:58.543877 0'0 99:54 [3,6,4] 3 [3,6,4] 3 0'0 2017-03-15 11:01:58.543795 0'0 2017-03-12
22:50:44.393395
1.5 58177 0 0 0 0 10155300518 3012 3012 active+clean 2017-03-15 11:00:47.901456 100'524940 100:401847 [5,8,0] 5 [5,8,0] 5 99'502306 2017-03-15 11:00:47.901401 0'0 2017-03-12
22:58:51.551943
2.6 82467 0 0 0 0 52326091763 3015 3015 active+clean 2017-03-15 11:29:53.392379 100'102832 100:53079 [8,3,1] 8 [8,3,1] 8 100'102781 2017-03-15 11:29:53.392312 0'0 2017-03-12
22:58:53.643235
3.7 6550 0 0 0 0 16777770 3051 3051 active+clean 2017-03-15 11:27:32.528729 100'119971 100:45042 [8,9,4] 8 [8,9,4] 8 100'117836 2017-03-15 11:27:32.528657 0'0 2017-03-12
22:59:12.616753
5.6 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.196965 0'0 99:29 [7,11,2] 7 [7,11,2] 7 0'0 2017-03-14 07:18:11.475577 0'0 2017-03-12
22:59:15.463390
4.7 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.539380 0'0 99:18 [1,7,8] 1 [1,7,8] 1 0'0 2017-03-14 09:59:57.856166 0'0 2017-03-12
22:59:14.407040
0.3 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:35:42.366704 0'0 100:48 [5,4,10] 5 [5,4,10] 5 0'0 2017-03-15 11:35:42.366625 0'0 2017-03-12
22:50:44.393393
1.2 29771 0 0 0 0 4990621053 3036 3036 active+clean 2017-03-15 10:59:58.240268 100'263520 100:256811 [7,6,1] 7 [7,6,1] 7 85'228631 2017-03-15 02:16:52.733869 0'0 2017-03-12
22:58:51.551925
2.1 41092 0 0 0 0 27097439153 3038 3038 active+clean 2017-03-15 10:59:52.265330 100'53447 100:30297 [2,3,8] 2 [2,3,8] 2 85'44229 2017-03-15 07:26:32.606767 85'37031 2017-03-14
00:54:50.771691
3.0 3094 0 0 0 0 0 3095 3095 active+clean 2017-03-15 10:59:50.094902 100'59096 100:20746 [5,6,4] 5 [5,6,4] 5 85'43054 2017-03-14 05:12:42.118592 0'0 2017-03-12
22:59:12.616697
5.7 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:53.151396 0'0 99:22 [4,8,7] 4 [4,8,7] 4 0'0 2017-03-15 10:59:53.151304 0'0 2017-03-14
03:38:12.984983
4.6 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.796843 0'0 94:20 [0,3,10] 0 [0,3,10] 0 0'0 2017-03-15 08:50:44.367407 0'0 2017-03-12
22:59:14.407036
0.2 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.087220 0'0 97:39 [3,4,6] 3 [3,4,6] 3 0'0 2017-03-15 06:00:51.617930 0'0 2017-03-12
22:50:44.393392
1.3 30426 0 0 0 0 4975465340 3095 3095 active+clean 2017-03-15 11:00:03.517811 100'266637 100:166850 [3,10,1] 3 [3,10,1] 3 85'231830 2017-03-15 09:58:03.740174 0'0 2017-03-12
22:58:51.551930
2.0 40803 0 0 0 0 26221404597 3066 3066 active+clean 2017-03-15 10:59:52.193019 100'52830 100:31031 [2,10,7] 2 [2,10,7] 2 85'40136 2017-03-14 09:40:36.647549 0'0 2017-03-12
22:58:53.643202
3.1 3275 0 0 0 0 0 3088 3088 active+clean 2017-03-15 11:00:04.472750 100'73396 100:27940 [3,2,11] 3 [3,2,11] 3 85'61551 2017-03-15 03:50:33.520479 85'50888 2017-03-14
00:55:47.692695
5.4 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.123323 0'0 99:31 [10,5,1] 10 [10,5,1] 10 0'0 2017-03-15 09:21:52.870470 0'0 2017-03-14
02:44:15.992242
4.5 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086549 0'0 97:26 [6,4,7] 6 [6,4,7] 6 0'0 2017-03-14 08:45:26.927014 0'0 2017-03-12
22:59:14.407030
0.1 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:34.527441 0'0 99:39 [3,6,1] 3 [3,6,1] 3 0'0 2017-03-15 11:00:34.527350 0'0 2017-03-12
22:50:44.393390
1.0 29838 0 0 0 0 5068704725 3014 3014 active+clean 2017-03-15 11:00:00.926181 100'282196 100:190123 [7,2,10] 7 [7,2,10] 7 85'247679 2017-03-15 03:38:02.649817 85'193471 2017-03-14
00:16:35.785265
2.3 40414 0 0 0 0 26633712199 3082 3082 active+clean 2017-03-15 11:27:36.722862 100'51993 100:27760 [11,5,1] 11 [11,5,1] 11 100'51963 2017-03-15 11:27:36.722787 0'0 2017-03-12
22:58:53.643218
3.2 3161 0 0 0 0 4194304 3070 3070 active+clean 2017-03-15 10:59:50.087049 100'68811 100:42765 [4,6,5] 4 [4,6,5] 4 85'52803 2017-03-15 09:37:00.943205 0'0 2017-03-12
22:59:12.616728
5.5 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.799044 0'0 94:28 [7,0,10] 7 [7,0,10] 7 0'0 2017-03-14 07:44:07.690644 0'0 2017-03-12
22:59:15.463384
4.4 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.784938 0'0 93:24 [5,10,0] 5 [5,10,0] 5 0'0 2017-03-15 08:44:42.159768 0'0 2017-03-14
04:58:49.481122
0.0 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086403 0'0 97:70 [4,11,5] 4 [4,11,5] 4 0'0 2017-03-15 03:51:30.434269 0'0 2017-03-12
22:50:44.393379
1.1 29353 0 0 0 0 5040607354 3087 3087 active+clean 2017-03-15 11:00:00.474951 100'282930 100:159187 [10,7,1] 10 [10,7,1] 10 85'248719 2017-03-15 10:02:03.173127 85'248719 2017-03-15
10:02:03.173127
2.2 40836 0 0 0 0 27221671010 3027 3027 active+clean 2017-03-15 10:59:44.783666 100'52586 100:30004 [0,5,11] 0 [0,5,11] 0 85'40391 2017-03-14 11:08:04.337946 85'40391 2017-03-14
11:08:04.337946
3.3 3311 0 0 0 0 34 3041 3041 active+clean 2017-03-15 11:28:23.607721 100'65924 100:27358 [10,4,5] 10 [10,4,5] 10 100'65917 2017-03-15 11:28:23.607663 0'0 2017-03-12
22:59:12.616732
3.9 3309 0 0 0 0 8388608 3043 3043 active+clean 2017-03-15 10:59:50.085242 100'59533 100:34102 [4,5,10] 4 [4,5,10] 4 85'48687 2017-03-14 09:43:13.590688 0'0 2017-03-12
22:59:12.616762
0.a 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.194122 0'0 99:54 [6,7,2] 6 [6,7,2] 6 0'0 2017-03-15 09:32:30.428371 0'0 2017-03-15
09:32:30.428371
1.b 28192 0 0 0 0 4985785537 3052 3052 active+clean 2017-03-15 11:32:11.000959 100'262882 100:195337 [1,3,10] 1 [1,3,10] 1 100'262310 2017-03-15 11:32:11.000879 0'0 2017-03-12
22:58:51.551973
2.8 40224 0 0 0 0 26569935756 3082 3082 active+clean 2017-03-15 10:59:45.806257 100'54254 100:32405 [0,10,7] 0 [0,10,7] 0 85'39981 2017-03-14 04:59:12.951119 0'0 2017-03-12
22:58:53.643246
3.8 3355 0 0 0 0 4194304 3038 3038 active+clean 2017-03-15 10:59:50.087874 100'68609 100:28224 [11,7,4] 11 [11,7,4] 11 85'57531 2017-03-15 02:05:09.595286 0'0 2017-03-12
22:59:12.616757
0.b 0 0 0 0 0 0 0 0 active+clean 2017-03-15 12:43:34.813725 0'0 100:61 [4,9,8] 4 [4,9,8] 4 0'0 2017-03-15 12:43:34.813647 0'0 2017-03-12
22:50:44.393404
1.a 27562 0 0 0 0 5022517954 3096 3096 active+clean 2017-03-15 11:00:14.456373 100'262024 100:164259 [1,7,6] 1 [1,7,6] 1 99'251716 2017-03-15 11:00:14.456299 0'0 2017-03-12
22:58:51.551968
2.9 40024 0 0 0 0 26283858136 3071 3071 active+clean 2017-03-15 11:10:54.706535 100'54152 100:30993 [5,10,2] 5 [5,10,2] 5 100'51820 2017-03-15 11:10:54.706476 100'51820 2017-03-15
11:10:54.706476
5.9 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.540224 0'0 99:22 [1,3,11] 1 [1,3,11] 1 0'0 2017-03-15 02:59:29.046947 0'0 2017-03-12
22:59:15.463403
4.8 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.119633 0'0 99:20 [1,6,7] 1 [1,6,7] 1 0'0 2017-03-15 05:10:13.051843 0'0 2017-03-12
22:59:14.407045
0.c 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.120332 0'0 99:28 [5,6,1] 5 [5,6,1] 5 0'0 2017-03-14 04:52:28.427628 0'0 2017-03-12
22:50:44.393405
5.8 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:27:47.303985 0'0 100:31 [6,1,3] 6 [6,1,3] 6 0'0 2017-03-15 11:27:47.303913 0'0 2017-03-15
11:27:47.303913
4.9 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.540382 0'0 99:18 [1,7,6] 1 [1,7,6] 1 0'0 2017-03-14 10:23:02.048252 0'0 2017-03-12
22:59:14.407050
0.d 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.088091 0'0 97:39 [10,9,4] 10 [10,9,4] 10 0'0 2017-03-15 06:12:31.356825 0'0 2017-03-12
22:50:44.393406
5.b 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.193670 0'0 99:22 [2,3,8] 2 [2,3,8] 2 0'0 2017-03-15 08:17:40.715399 0'0 2017-03-12
22:59:15.463411
4.a 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:44.763882 0'0 93:24 [11,7,0] 11 [11,7,0] 11 0'0 2017-03-15 10:02:30.159822 0'0 2017-03-12
22:59:14.407055
0.e 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:54.146801 0'0 99:43 [4,5,8] 4 [4,5,8] 4 0'0 2017-03-15 10:59:54.146724 0'0 2017-03-14
04:21:15.290464
5.a 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.088314 0'0 97:29 [3,6,4] 3 [3,6,4] 3 0'0 2017-03-14 10:38:02.413221 0'0 2017-03-12
22:59:15.463407
4.b 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:49.486180 0'0 96:21 [0,11,9] 0 [0,11,9] 0 0'0 2017-03-15 10:59:49.486069 0'0 2017-03-13
23:44:23.802829
0.f 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.193296 0'0 99:41 [2,10,7] 2 [2,10,7] 2 0'0 2017-03-14 05:05:12.405058 0'0 2017-03-12
22:50:44.393409
0.10 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.085901 0'0 97:52 [4,8,3] 4 [4,8,3] 4 0'0 2017-03-15 04:55:45.934842 0'0 2017-03-12
22:50:44.393410
0.11 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:27:03.732936 0'0 100:61 [3,8,1] 3 [3,8,1] 3 0'0 2017-03-15 11:27:03.732878 0'0 2017-03-12
22:50:44.393411
0.12 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:33.956360 0'0 99:48 [9,6,1] 9 [9,6,1] 9 0'0 2017-03-15 11:00:33.956281 0'0 2017-03-14
00:37:12.089111
0.13 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086980 0'0 97:39 [3,10,4] 3 [3,10,4] 3 0'0 2017-03-15 07:02:29.992516 0'0 2017-03-15
07:02:29.992516
0.14 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:55.011464 0'0 99:52 [1,11,5] 1 [1,11,5] 1 0'0 2017-03-15 10:59:55.011430 0'0 2017-03-12
22:50:44.393415
0.15 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.072892 0'0 97:50 [4,6,5] 4 [4,6,5] 4 0'0 2017-03-14 09:43:14.862695 0'0 2017-03-12
22:50:44.393417
0.2e 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:57.044595 0'0 99:52 [2,7,8] 2 [2,7,8] 2 0'0 2017-03-15 10:59:57.044557 0'0 2017-03-12
22:50:44.393472
0.2f 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.195618 0'0 99:28 [11,7,2] 11 [11,7,2] 11 0'0 2017-03-14 09:04:49.071489 0'0 2017-03-12
22:50:44.393473
0.30 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.122884 0'0 99:42 [8,1,9] 8 [8,1,9] 8 0'0 2017-03-15 08:23:54.467250 0'0 2017-03-12
22:50:44.393474
0.31 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.540619 0'0 99:50 [1,11,3] 1 [1,11,3] 1 0'0 2017-03-14 09:40:26.722867 0'0 2017-03-12
22:50:44.393476
0.32 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.194257 0'0 99:61 [2,11,7] 2 [2,11,7] 2 0'0 2017-03-15 07:35:23.370607 0'0 2017-03-12
22:50:44.393477
0.33 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.196550 0'0 99:37 [3,10,2] 3 [3,10,2] 3 0'0 2017-03-14 10:22:36.232867 0'0 2017-03-12
22:50:44.393478
0.34 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:00:36.960876 0'0 99:48 [9,11,2] 9 [9,11,2] 9 0'0 2017-03-15 11:00:36.960827 0'0 2017-03-12
22:50:44.393479
0.35 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.119940 0'0 99:46 [5,6,1] 5 [5,6,1] 5 0'0 2017-03-14 05:38:36.747488 0'0 2017-03-12
22:50:44.393481
0.36 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.087236 0'0 97:48 [9,6,4] 9 [9,6,4] 9 0'0 2017-03-15 09:17:58.121500 0'0 2017-03-12
22:50:44.393482
0.37 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.194785 0'0 99:39 [9,2,8] 9 [9,2,8] 9 0'0 2017-03-15 04:47:09.503797 0'0 2017-03-12
22:50:44.393483
0.38 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:45.798599 0'0 94:56 [9,0,11] 9 [9,0,11] 9 0'0 2017-03-15 04:44:51.485759 0'0 2017-03-12
22:50:44.393484
0.39 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:52:52.392420 0'0 100:48 [9,1,6] 9 [9,1,6] 9 0'0 2017-03-15 11:52:52.392352 0'0 2017-03-12
22:50:44.393486
0.3a 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:50.086627 0'0 97:59 [4,8,7] 4 [4,8,7] 4 0'0 2017-03-14 09:43:11.863773 0'0 2017-03-14
09:43:11.863773
0.3b 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.171056 0'0 99:43 [2,8,5] 2 [2,8,5] 2 0'0 2017-03-15 07:41:47.424549 0'0 2017-03-12
22:50:44.393488
0.3c 0 0 0 0 0 0 0 0 active+clean 2017-03-15 11:02:00.981332 0'0 99:39 [9,2,8] 9 [9,2,8] 9 0'0 2017-03-15 11:02:00.981264 0'0 2017-03-12
22:50:44.393490
0.3d 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.540831 0'0 99:61 [1,10,7] 1 [1,10,7] 1 0'0 2017-03-15 07:55:33.313612 0'0 2017-03-12
22:50:44.393491
0.3e 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.195787 0'0 99:30 [8,2,3] 8 [8,2,3] 8 0'0 2017-03-15 02:28:17.185544 0'0 2017-03-12
22:50:44.393492
0.3f 0 0 0 0 0 0 0 0 active+clean 2017-03-15 10:59:52.265530 0'0 99:43 [2,11,5] 2 [2,11,5] 2 0'0 2017-03-15 00:30:38.822799 0'0 2017-03-12
22:50:44.393493
5 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0
3 52275 0 0 0 0 48371113 36638 36638
2 650158 0 0 0 0 426305171964 36623 36623
1 466451 0 0 0 0 79835701754 36540 36540
0 0 0 0 0 0 0 0 0
sum 1168884 0 0 0 0 506189244831 109801 109801
OSD_STAT USED AVAIL TOTAL HB_PEERS PG_SUM
11 56284M 223G 278G [0,1,2,3,4,5,7,8,9,10] 28
10 97062M 183G 278G [0,1,2,4,5,6,7,8,9,11] 30
0 41564M 238G 278G [1,3,4,5,6,7,8,9,10,11] 22
1 123G 154G 278G [0,2,3,5,6,7,8,9,10,11] 42
2 112G 166G 278G [1,3,4,5,6,7,8,9,10,11] 28
4 47643M 232G 278G [0,1,3,5,6,7,8,9,10,11] 32
3 97557M 183G 278G [0,1,2,4,6,7,8,9,10,11] 32
5 127G 151G 278G [0,1,2,4,6,7,8,9,10,11] 31
6 71151M 209G 278G [1,2,3,4,5,7,8,9,10,11] 32
7 79459M 201G 278G [0,1,2,4,5,6,8,9,10,11] 40
9 23961M 112G 136G [0,1,2,4,5,6,7,8,10,11] 21
8 104G 174G 278G [0,1,2,3,4,5,7,9,10,11] 34
sum 970G 2231G 3202G
Rhian Resnick Assistant Director Middleware and HPC Office of Information Technology
Florida Atlantic University 777 Glades Road, CM22, Rm 173B Boca Raton, FL 33431 Phone 561.297.2647 Fax 561.297.0222
|
_______________________________________________ ceph-users mailing list ceph-users@xxxxxxxxxxxxxx http://lists.ceph.com/listinfo.cgi/ceph-users-ceph.com
