Hi, thanks for quick reply root@ctplmon1:~# ceph pg dump pgs_brief | grep undersized dumped pgs_brief 9.5 active+undersized+degraded [72,85,54,120,2147483647] 72 [72,85,54,120,2147483647] 72 9.6 active+undersized+degraded [101,47,113,74,2147483647] 101 [101,47,113,74,2147483647] 101 9.2 active+undersized+degraded [86,118,74,2147483647,49] 86 [86,118,74,2147483647,49] 86 9.d active+undersized+degraded [49,136,83,90,2147483647] 49 [49,136,83,90,2147483647] 49 9.f active+undersized+degraded [55,103,81,128,2147483647] 55 [55,103,81,128,2147483647] 55 9.18 active+undersized+degraded [115,50,61,89,2147483647] 115 [115,50,61,89,2147483647] 115 9.1d active+undersized+degraded [61,90,31,2147483647,125] 61 [61,90,31,2147483647,125] 61 9.10 active+undersized+degraded [46,2147483647,71,86,122] 46 [46,2147483647,71,86,122] 46 9.17 active+undersized+degraded [60,95,114,2147483647,48] 60 [60,95,114,2147483647,48] 60 9.15 active+undersized+degraded [121,76,30,101,2147483647] 121 [121,76,30,101,2147483647] 121 root@ctplmon1:~# ceph osd tree ID CLASS WEIGHT TYPE NAME STATUS REWEIGHT PRI-AFF -1 764.11981 root default -3 152.82378 host ctplosd1 0 hdd 5.45798 osd.0 down 0 1.00000 1 hdd 5.45799 osd.1 down 0 1.00000 2 hdd 5.45799 osd.2 down 0 1.00000 3 hdd 5.45799 osd.3 down 0 1.00000 4 hdd 5.45799 osd.4 down 0 1.00000 5 hdd 5.45799 osd.5 down 0 1.00000 6 hdd 5.45799 osd.6 down 0 1.00000 7 hdd 5.45799 osd.7 down 0 1.00000 8 hdd 5.45799 osd.8 down 0 1.00000 9 hdd 5.45799 osd.9 down 0 1.00000 10 hdd 5.45799 osd.10 down 0 1.00000 11 hdd 5.45799 osd.11 down 0 1.00000 12 hdd 5.45799 osd.12 down 0 1.00000 13 hdd 5.45799 osd.13 down 0 1.00000 14 hdd 5.45799 osd.14 down 0 1.00000 15 hdd 5.45799 osd.15 down 0 1.00000 16 hdd 5.45799 osd.16 down 0 1.00000 17 hdd 5.45799 osd.17 down 0 1.00000 18 hdd 5.45799 osd.18 down 0 1.00000 19 hdd 5.45799 osd.19 down 0 1.00000 20 hdd 5.45799 osd.20 down 0 1.00000 21 hdd 5.45799 osd.21 down 0 1.00000 22 hdd 5.45799 osd.22 down 0 1.00000 23 hdd 5.45799 osd.23 down 0 1.00000 24 hdd 5.45799 osd.24 down 0 1.00000 25 hdd 5.45799 osd.25 down 0 1.00000 26 hdd 5.45799 osd.26 down 0 1.00000 27 hdd 5.45799 osd.27 down 0 1.00000 -11 152.82401 host ctplosd5 112 hdd 5.45799 osd.112 up 1.00000 1.00000 113 hdd 5.45799 osd.113 up 1.00000 1.00000 114 hdd 5.45799 osd.114 up 1.00000 1.00000 115 hdd 5.45799 osd.115 up 1.00000 1.00000 116 hdd 5.45799 osd.116 up 1.00000 1.00000 117 hdd 5.45799 osd.117 up 1.00000 1.00000 118 hdd 5.45799 osd.118 up 1.00000 1.00000 119 hdd 5.45799 osd.119 up 1.00000 1.00000 120 hdd 5.45799 osd.120 up 1.00000 1.00000 121 hdd 5.45799 osd.121 up 1.00000 1.00000 122 hdd 5.45799 osd.122 up 1.00000 1.00000 123 hdd 5.45799 osd.123 up 1.00000 1.00000 124 hdd 5.45799 osd.124 up 1.00000 1.00000 125 hdd 5.45799 osd.125 up 1.00000 1.00000 126 hdd 5.45799 osd.126 up 1.00000 1.00000 127 hdd 5.45799 osd.127 up 1.00000 1.00000 128 hdd 5.45799 osd.128 up 1.00000 1.00000 129 hdd 5.45799 osd.129 up 1.00000 1.00000 130 hdd 5.45799 osd.130 up 1.00000 1.00000 131 hdd 5.45799 osd.131 up 1.00000 1.00000 132 hdd 5.45799 osd.132 up 1.00000 1.00000 133 hdd 5.45799 osd.133 up 1.00000 1.00000 134 hdd 5.45799 osd.134 up 1.00000 1.00000 135 hdd 5.45799 osd.135 up 1.00000 1.00000 136 hdd 5.45799 osd.136 up 1.00000 1.00000 137 hdd 5.45799 osd.137 up 1.00000 1.00000 138 hdd 5.45799 osd.138 up 1.00000 1.00000 139 hdd 5.45799 osd.139 up 1.00000 1.00000 -7 152.82401 host ctplosd6 57 hdd 5.45799 osd.57 up 1.00000 1.00000 58 hdd 5.45799 osd.58 up 1.00000 1.00000 59 hdd 5.45799 osd.59 up 1.00000 1.00000 60 hdd 5.45799 osd.60 up 1.00000 1.00000 61 hdd 5.45799 osd.61 up 1.00000 1.00000 62 hdd 5.45799 osd.62 up 1.00000 1.00000 63 hdd 5.45799 osd.63 up 1.00000 1.00000 64 hdd 5.45799 osd.64 up 1.00000 1.00000 65 hdd 5.45799 osd.65 up 1.00000 1.00000 66 hdd 5.45799 osd.66 up 1.00000 1.00000 67 hdd 5.45799 osd.67 up 1.00000 1.00000 68 hdd 5.45799 osd.68 up 1.00000 1.00000 69 hdd 5.45799 osd.69 up 1.00000 1.00000 70 hdd 5.45799 osd.70 up 1.00000 1.00000 71 hdd 5.45799 osd.71 up 1.00000 1.00000 72 hdd 5.45799 osd.72 up 1.00000 1.00000 73 hdd 5.45799 osd.73 up 1.00000 1.00000 74 hdd 5.45799 osd.74 up 1.00000 1.00000 75 hdd 5.45799 osd.75 up 1.00000 1.00000 76 hdd 5.45799 osd.76 up 1.00000 1.00000 77 hdd 5.45799 osd.77 up 1.00000 1.00000 78 hdd 5.45799 osd.78 up 1.00000 1.00000 79 hdd 5.45799 osd.79 up 1.00000 1.00000 80 hdd 5.45799 osd.80 up 1.00000 1.00000 81 hdd 5.45799 osd.81 up 1.00000 1.00000 82 hdd 5.45799 osd.82 up 1.00000 1.00000 83 hdd 5.45799 osd.83 up 1.00000 1.00000 84 hdd 5.45799 osd.84 up 1.00000 1.00000 -5 152.82401 host ctplosd7 28 hdd 5.45799 osd.28 up 1.00000 1.00000 29 hdd 5.45799 osd.29 up 1.00000 1.00000 30 hdd 5.45799 osd.30 up 1.00000 1.00000 31 hdd 5.45799 osd.31 up 1.00000 1.00000 32 hdd 5.45799 osd.32 up 1.00000 1.00000 33 hdd 5.45799 osd.33 up 1.00000 1.00000 34 hdd 5.45799 osd.34 up 1.00000 1.00000 35 hdd 5.45799 osd.35 up 1.00000 1.00000 36 hdd 5.45799 osd.36 up 1.00000 1.00000 37 hdd 5.45799 osd.37 up 1.00000 1.00000 38 hdd 5.45799 osd.38 up 1.00000 1.00000 39 hdd 5.45799 osd.39 up 1.00000 1.00000 40 hdd 5.45799 osd.40 up 1.00000 1.00000 41 hdd 5.45799 osd.41 up 1.00000 1.00000 42 hdd 5.45799 osd.42 up 1.00000 1.00000 43 hdd 5.45799 osd.43 up 1.00000 1.00000 44 hdd 5.45799 osd.44 up 1.00000 1.00000 45 hdd 5.45799 osd.45 up 1.00000 1.00000 46 hdd 5.45799 osd.46 up 1.00000 1.00000 47 hdd 5.45799 osd.47 up 1.00000 1.00000 48 hdd 5.45799 osd.48 up 1.00000 1.00000 49 hdd 5.45799 osd.49 up 1.00000 1.00000 50 hdd 5.45799 osd.50 up 1.00000 1.00000 51 hdd 5.45799 osd.51 up 1.00000 1.00000 52 hdd 5.45799 osd.52 up 1.00000 1.00000 53 hdd 5.45799 osd.53 up 1.00000 1.00000 54 hdd 5.45799 osd.54 up 1.00000 1.00000 55 hdd 5.45799 osd.55 up 1.00000 1.00000 -9 152.82401 host ctplosd8 56 hdd 5.45799 osd.56 up 1.00000 1.00000 85 hdd 5.45799 osd.85 up 1.00000 1.00000 86 hdd 5.45799 osd.86 up 1.00000 1.00000 87 hdd 5.45799 osd.87 up 1.00000 1.00000 88 hdd 5.45799 osd.88 up 1.00000 1.00000 89 hdd 5.45799 osd.89 up 1.00000 1.00000 90 hdd 5.45799 osd.90 up 1.00000 1.00000 91 hdd 5.45799 osd.91 up 1.00000 1.00000 92 hdd 5.45799 osd.92 up 1.00000 1.00000 93 hdd 5.45799 osd.93 up 1.00000 1.00000 94 hdd 5.45799 osd.94 up 1.00000 1.00000 95 hdd 5.45799 osd.95 up 1.00000 1.00000 96 hdd 5.45799 osd.96 up 1.00000 1.00000 97 hdd 5.45799 osd.97 up 1.00000 1.00000 98 hdd 5.45799 osd.98 up 1.00000 1.00000 99 hdd 5.45799 osd.99 up 1.00000 1.00000 100 hdd 5.45799 osd.100 up 1.00000 1.00000 101 hdd 5.45799 osd.101 up 1.00000 1.00000 102 hdd 5.45799 osd.102 up 1.00000 1.00000 103 hdd 5.45799 osd.103 up 1.00000 1.00000 104 hdd 5.45799 osd.104 up 1.00000 1.00000 105 hdd 5.45799 osd.105 up 1.00000 1.00000 106 hdd 5.45799 osd.106 up 1.00000 1.00000 107 hdd 5.45799 osd.107 up 1.00000 1.00000 108 hdd 5.45799 osd.108 up 1.00000 1.00000 109 hdd 5.45799 osd.109 up 1.00000 1.00000 110 hdd 5.45799 osd.110 up 1.00000 1.00000 111 hdd 5.45799 osd.111 up 1.00000 1.00000 root@ctplmon1:~# ceph osd pool ls detail pool 9 'default.rgw.buckets.data' erasure profile ec-32-profile size 5 min_size 4 crush_rule 1 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode on last_change 128267 lfor 0/127784/127779 flags hashpspool,ec_overwrites stripe_width 12288 application rgw ---- The affected pool is pool number 9 and host is ctplosd1. This is the host I removed at the first place (to reinstall OS) and now I added this host back to the cluster, but osds on this host cannot be brought back to up state for some reason, even though osd processes are running on the host. Kind regards, Rok On Thu, May 27, 2021 at 3:32 PM Eugen Block <eblock@xxxxxx> wrote: > Hi, > > this sounds like your crush rule(s) for one or more pools can't place > the PGs because the host is missing. Please share > > ceph pg dump pgs_brief | grep undersized > ceph osd tree > ceph osd pool ls detail > > and the crush rule(s) for the affected pool(s). > > > Zitat von Rok Jaklič <rjaklic@xxxxxxxxx>: > > > Hi, > > > > I have removed one node, but now ceph seems to stuck in: > > Degraded data redundancy: 67/2393 objects degraded (2.800%), 12 pgs > > degraded, 12 pgs undersized > > > > How to "force" rebalancing? Or should I just wait a little bit more? > > > > Kind regards, > > rok > > _______________________________________________ > > ceph-users mailing list -- ceph-users@xxxxxxx > > To unsubscribe send an email to ceph-users-leave@xxxxxxx > > > _______________________________________________ > ceph-users mailing list -- ceph-users@xxxxxxx > To unsubscribe send an email to ceph-users-leave@xxxxxxx > _______________________________________________ ceph-users mailing list -- ceph-users@xxxxxxx To unsubscribe send an email to ceph-users-leave@xxxxxxx
 |