I had 2 nodes that I loaded almost 51TB to, then I expanded the
cluster to 6 nodes, now I need to spread data out over them. Below I
have the df -h from all 6 nodes, and you see that the original two
nodes (clustr-04 clustr-05) have many mounts that are at 100% -- and
even doing an ls -Rl of some of these directories are error-ing out,
which is disconcerting after loading this much data. How can I level
things out? I'm currently running the scale-n-defrag.sh on clustr-06
that has the client mounted on /mnt/glusterfs, but running this the
other day from 04 didn't seem to help at all. Also, the run on
clustr-06 is spitting out tons of "find: `setfattr': No such file or
directory" What can I do to solve this?
clustr-01
=======
Filesystem Size Used Avail Use% Mounted on
/dev/sdy3 133G 1.6G 124G 2% /
tmpfs 4.0G 0 4.0G 0% /lib/init/rw
udev 10M 336K 9.7M 4% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
/dev/sdy1 7.4G 149M 6.9G 3% /var/log
/dev/sdo1 1.4T 608G 754G 45% /mnt/data01
/dev/sdw1 1.4T 613G 749G 46% /mnt/data02
/dev/sdq1 1.4T 509G 853G 38% /mnt/data03
/dev/sdf1 1.4T 518G 845G 38% /mnt/data04
/dev/sdj1 1.4T 535G 827G 40% /mnt/data05
/dev/sdp1 1.4T 543G 819G 40% /mnt/data06
/dev/sds1 1.4T 536G 827G 40% /mnt/data07
/dev/sdk1 1.4T 531G 831G 39% /mnt/data08
/dev/sdg1 1.4T 511G 851G 38% /mnt/data09
/dev/sde1 1.4T 506G 856G 38% /mnt/data10
/dev/sdd1 1.4T 652G 710G 48% /mnt/data11
/dev/sdl1 1.4T 632G 730G 47% /mnt/data12
/dev/sdr1 1.4T 515G 848G 38% /mnt/data13
/dev/sdm1 1.4T 539G 824G 40% /mnt/data14
/dev/sdh1 1.4T 530G 832G 39% /mnt/data15
/dev/sda1 1.4T 525G 837G 39% /mnt/data16
/dev/sdi1 1.4T 515G 847G 38% /mnt/data17
/dev/sdu1 1.4T 517G 845G 38% /mnt/data18
/dev/sdc1 1.4T 518G 845G 38% /mnt/data19
/dev/sdv1 1.4T 517G 846G 38% /mnt/data20
/dev/sdt1 1.4T 548G 815G 41% /mnt/data21
/dev/sdb1 1.4T 559G 804G 42% /mnt/data22
/dev/sdn1 1.4T 615G 748G 46% /mnt/data23
/dev/sdx1 1.4T 618G 744G 46% /mnt/data24
clustr-02
=======
Filesystem Size Used Avail Use% Mounted on
/dev/sdy3 133G 1.2G 125G 1% /
tmpfs 4.0G 0 4.0G 0% /lib/init/rw
udev 10M 336K 9.7M 4% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
/dev/sdy1 7.4G 151M 6.9G 3% /var/log
/dev/sdh1 1.4T 551G 812G 41% /mnt/data01
/dev/sds1 1.4T 566G 797G 42% /mnt/data02
/dev/sdc1 1.4T 533G 829G 40% /mnt/data03
/dev/sdx1 1.4T 549G 813G 41% /mnt/data04
/dev/sdu1 1.4T 519G 844G 39% /mnt/data05
/dev/sdf1 1.4T 539G 823G 40% /mnt/data06
/dev/sdt1 1.4T 559G 803G 42% /mnt/data07
/dev/sdj1 1.4T 530G 832G 39% /mnt/data08
/dev/sdk1 1.4T 549G 814G 41% /mnt/data09
/dev/sdd1 1.4T 544G 818G 40% /mnt/data10
/dev/sdr1 1.4T 559G 803G 42% /mnt/data11
/dev/sdi1 1.4T 529G 833G 39% /mnt/data12
/dev/sdb1 1.4T 525G 837G 39% /mnt/data13
/dev/sdq1 1.4T 522G 840G 39% /mnt/data14
/dev/sde1 1.4T 557G 805G 41% /mnt/data15
/dev/sdn1 1.4T 576G 787G 43% /mnt/data16
/dev/sdw1 1.4T 514G 849G 38% /mnt/data17
/dev/sdl1 1.4T 519G 843G 39% /mnt/data18
/dev/sdp1 1.4T 541G 821G 40% /mnt/data19
/dev/sdg1 1.4T 552G 810G 41% /mnt/data20
/dev/sdo1 1.4T 509G 853G 38% /mnt/data21
/dev/sda1 1.4T 510G 853G 38% /mnt/data22
/dev/sdm1 1.4T 503G 860G 37% /mnt/data23
/dev/sdv1 1.4T 515G 847G 38% /mnt/data24
clustr-03
=======
Filesystem Size Used Avail Use% Mounted on
/dev/sdy3 133G 1.5G 124G 2% /
tmpfs 4.0G 0 4.0G 0% /lib/init/rw
udev 10M 352K 9.7M 4% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
/dev/sdy1 7.4G 1.1G 6.0G 16% /var/log
/dev/sdn1 1.4T 558G 804G 41% /mnt/data01
/dev/sdi1 1.4T 556G 806G 41% /mnt/data02
/dev/sdm1 1.4T 511G 851G 38% /mnt/data03
/dev/sdj1 1.4T 487G 875G 36% /mnt/data04
/dev/sdl1 1.4T 523G 839G 39% /mnt/data05
/dev/sdr1 1.4T 542G 820G 40% /mnt/data06
/dev/sdu1 1.4T 472G 891G 35% /mnt/data07
/dev/sdq1 1.4T 464G 898G 35% /mnt/data08
/dev/sdh1 1.4T 501G 861G 37% /mnt/data09
/dev/sdg1 1.4T 489G 874G 36% /mnt/data10
/dev/sde1 1.4T 452G 910G 34% /mnt/data11
/dev/sdv1 1.4T 462G 900G 34% /mnt/data12
/dev/sdc1 1.4T 460G 903G 34% /mnt/data13
/dev/sdf1 1.4T 467G 896G 35% /mnt/data14
/dev/sdt1 1.4T 495G 867G 37% /mnt/data15
/dev/sdk1 1.4T 491G 871G 37% /mnt/data16
/dev/sds1 1.4T 378G 985G 28% /mnt/data18
/dev/sda1 1.4T 492G 870G 37% /mnt/data19
/dev/sdx1 1.4T 498G 864G 37% /mnt/data20
/dev/sdw1 1.4T 455G 908G 34% /mnt/data21
/dev/sdo1 1.4T 447G 915G 33% /mnt/data22
/dev/sdb1 1.4T 472G 891G 35% /mnt/data23
/dev/sdd1 1.4T 482G 880G 36% /mnt/data24
/dev/sdp1 1.4T 424G 939G 32% /mnt/data17
clustr-04
=======
Filesystem Size Used Avail Use% Mounted on
/dev/sdy4 141G 8.4G 126G 7% /
tmpfs 4.0G 0 4.0G 0% /lib/init/rw
udev 10M 344K 9.7M 4% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
/dev/sdy1 228M 49M 168M 23% /boot
/dev/sdy2 3.7G 82M 3.4G 3% /var/log
/dev/sdw1 1.4T 1.4T 0 100% /mnt/data01
/dev/sdp1 1.4T 1.4T 0 100% /mnt/data02
/dev/sdm1 1.4T 1.4T 5.3G 100% /mnt/data03
/dev/sde1 1.4T 1.4T 5.3G 100% /mnt/data04
/dev/sdh1 1.4T 1.4T 2.1G 100% /mnt/data05
/dev/sdt1 1.4T 1.4T 2.1G 100% /mnt/data06
/dev/sdg1 1.4T 1.4T 0 100% /mnt/data07
/dev/sdk1 1.4T 1.4T 0 100% /mnt/data08
/dev/sda1 1.4T 1.4T 0 100% /mnt/data09
/dev/sdn1 1.4T 1.4T 0 100% /mnt/data10
/dev/sdc1 1.4T 1.4T 4.3G 100% /mnt/data11
/dev/sdd1 1.4T 1.4T 4.3G 100% /mnt/data12
/dev/sdf1 1.4T 1.4T 9.3G 100% /mnt/data13
/dev/sdx1 1.4T 1.4T 9.3G 100% /mnt/data14
/dev/sdi1 1.4T 1.4T 2.6G 100% /mnt/data15
/dev/sdj1 1.4T 1.4T 2.6G 100% /mnt/data16
/dev/sdl1 1.4T 1.4T 0 100% /mnt/data17
/dev/sdq1 1.4T 1.4T 0 100% /mnt/data18
/dev/sdv1 1.4T 1.4T 6.2G 100% /mnt/data19
/dev/sdo1 1.4T 1.4T 6.2G 100% /mnt/data20
/dev/sds1 1.4T 1.4T 7.9G 100% /mnt/data21
/dev/sdb1 1.4T 1.4T 7.9G 100% /mnt/data22
/dev/sdr1 1.4T 1.4T 0 100% /mnt/data23
/dev/sdu1 1.4T 1.4T 0 100% /mnt/data24
/etc/glusterfs/glusterfs.vol
97T 51T 46T 53% /mnt/glusterfs
clustr-05
=======
Filesystem Size Used Avail Use% Mounted on
/dev/sdy4 141G 1.5G 133G 2% /
tmpfs 4.0G 0 4.0G 0% /lib/init/rw
udev 10M 356K 9.7M 4% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
/dev/sdy1 236M 39M 185M 18% /boot
/dev/sdy3 3.7G 80M 3.5G 3% /var/log
/dev/sdl1 1.4T 1.4T 6.1G 100% /mnt/data01
/dev/sdq1 1.4T 1.4T 6.1G 100% /mnt/data02
/dev/sdp1 1.4T 1.4T 0 100% /mnt/data03
/dev/sdt1 1.4T 1.4T 0 100% /mnt/data04
/dev/sdi1 1.4T 1.4T 0 100% /mnt/data05
/dev/sdf1 1.4T 1.4T 0 100% /mnt/data06
/dev/sdc1 1.4T 1.4T 0 100% /mnt/data07
/dev/sdv1 1.4T 1.4T 0 100% /mnt/data08
/dev/sdm1 1.4T 1.4T 0 100% /mnt/data09
/dev/sdr1 1.4T 1.4T 0 100% /mnt/data10
/dev/sdg1 1.4T 1.4T 15G 99% /mnt/data11
/dev/sdu1 1.4T 1.4T 15G 99% /mnt/data12
/dev/sdh1 1.4T 1.4T 0 100% /mnt/data13
/dev/sda1 1.4T 1.4T 0 100% /mnt/data14
/dev/sdb1 1.4T 1.4T 0 100% /mnt/data15
/dev/sdd1 1.4T 1.4T 0 100% /mnt/data16
/dev/sdn1 1.4T 1.4T 0 100% /mnt/data17
/dev/sdk1 1.4T 1.4T 0 100% /mnt/data18
/dev/sdj1 1.4T 1.4T 824M 100% /mnt/data19
/dev/sde1 1.4T 1.4T 824M 100% /mnt/data20
/dev/sdw1 1.4T 1.4T 0 100% /mnt/data21
/dev/sdx1 1.4T 1.4T 0 100% /mnt/data22
/dev/sds1 1.4T 1.4T 0 100% /mnt/data23
/dev/sdo1 1.4T 1.4T 0 100% /mnt/data24
clustr06
=======
Filesystem Size Used Avail Use% Mounted on
/dev/sdy3 138G 1.2G 130G 1% /
tmpfs 4.0G 0 4.0G 0% /lib/init/rw
udev 10M 344K 9.7M 4% /dev
tmpfs 4.0G 0 4.0G 0% /dev/shm
/dev/sdy1 7.4G 149M 6.9G 3% /var/log
/dev/sdb1 1.4T 483M 1.4T 1% /mnt/data01
/dev/sdi1 1.4T 483M 1.4T 1% /mnt/data02
/dev/sdj1 1.4T 286M 1.4T 1% /mnt/data03
/dev/sdp1 1.4T 286M 1.4T 1% /mnt/data04
/dev/sdr1 1.4T 286M 1.4T 1% /mnt/data05
/dev/sdx1 1.4T 286M 1.4T 1% /mnt/data06
/dev/sdv1 1.4T 286M 1.4T 1% /mnt/data07
/dev/sdt1 1.4T 286M 1.4T 1% /mnt/data08
/dev/sdl1 1.4T 286M 1.4T 1% /mnt/data09
/dev/sdg1 1.4T 286M 1.4T 1% /mnt/data10
/dev/sde1 1.4T 286M 1.4T 1% /mnt/data11
/dev/sdn1 1.4T 286M 1.4T 1% /mnt/data12
/dev/sda1 1.4T 286M 1.4T 1% /mnt/data13
/dev/sdc1 1.4T 286M 1.4T 1% /mnt/data14
/dev/sdk1 1.4T 285M 1.4T 1% /mnt/data15
/dev/sdw1 1.4T 286M 1.4T 1% /mnt/data16
/dev/sdq1 1.4T 286M 1.4T 1% /mnt/data17
/dev/sdu1 1.4T 286M 1.4T 1% /mnt/data18
/dev/sdd1 1.4T 286M 1.4T 1% /mnt/data19
/dev/sdh1 1.4T 286M 1.4T 1% /mnt/data20
/dev/sdm1 1.4T 285M 1.4T 1% /mnt/data21
/dev/sdo1 1.4T 285M 1.4T 1% /mnt/data22
/dev/sds1 1.4T 286M 1.4T 1% /mnt/data23
/dev/sdf1 1.4T 286M 1.4T 1% /mnt/data24
/etc/glusterfs/glusterfs.vol
97T 51T 46T 53% /mnt/glusterfs
--
http://philcryer.com
