How can I tell whether a 3.6.34 way replicated volume is making progress
with self healing?
We did a yum update on one of the brick servers a while back, but due to
a kernel issue the server didn't reboot, so it was down for several days
and quite a few updates were done to the volumes before it came back up.
If I look at /var/log/glusterfs/glfsheal-callrec.log there are many
lines like this one:
[2016-03-02 11:34:06.257961] W [dict.c:499:dict_ref] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x186)[0x7f3b374544c6] (--> /lib64/libglusterfs.so.0(dict_ref+0x5e)[0x7f3b3744da7e] (--> /lib64/libglusterfs.so.0(syncop_getxattr_cbk+0x1b)[0x7f3b3748e0ab] (--> /usr/lib64/glusterfs/3.6.3/xlator/cluster/replicate.so(afr_get_heal_info+0x14a)[0x7f3b2bba791a] (--> /usr/lib64/glusterfs/3.6.3/xlator/cluster/replicate.so(afr_getxattr+0x442)[0x7f3b2bb7b6b2] ))))) 0-dict: dict is NULL
but I don't see much else in the log that would indicate that progress
is being made, and in fact these lines make up most of the log. On the
brick that was updated:
[root@gluster2a-1 glusterfs]# grep -v "0-dict: dict is NULL" glfsheal-callrec.log | wc
764 10329 134557
[root@gluster2a-1 glusterfs]# grep "0-dict: dict is NULL" glfsheal-callrec.log | wc
2435 46265 1171235
on one of the other bricks:
[root@gluster2b-1 glusterfs]# grep -v "0-dict: dict is NULL" glfsheal-callrec.log | wc
3250 44203 579110
[root@gluster2b-1 glusterfs]# grep "0-dict: dict is NULL" glfsheal-callrec.log | wc
20117 382223 9676277
If I run "gluster volume heal callrec info", it starts like below, but
last time I ran this command it was still going after 2 days:
# gluster volume heal callrec info
Brick gluster1a-1.dns99.co.uk:/data/brick/callrec/
<gfid:d6e91662-a395-42de-8af3-ba6164eb0f90>
<gfid:09872041-bc68-4eee-9c4d-ef11e4137f66>
<gfid:4885e073-75fd-461a-a70e-1e9578ef99cd>
<gfid:1881cc27-09ab-453e-b89c-afcce7aab5ea>
<gfid:3a50792e-0ce9-49a2-99d5-d59d1fd7e1a9>
<gfid:efbfc625-c42c-4d5e-a22a-173f84362f24>
<gfid:f48beb09-975f-4ff6-843e-d8906d3b21b3>
<gfid:21f767b1-b5f3-4441-8f0e-ac852d2cdd25>
<gfid:f7d3ef0f-3c01-412e-93fa-704171267d9e>
<gfid:75c6e98b-7700-46f0-8be4-e897f969a5df>
<gfid:8d059d2b-9e98-482b-aeac-e5decd126fe7>
<gfid:b23a64dd-4522-408c-8465-4096c2977da4>
<gfid:554ce007-a0bd-437b-8004-d70d110b5acc>
<gfid:fa7bbd79-3394-49b1-96c1-0bf7d3b3138d>
<gfid:db53921d-ca03-4c97-95cd-9fa4b514d585>
<gfid:27632fb9-ab59-45cb-bb7a-5c78697619ea>
<gfid:f6c76172-c73b-4cf0-ad8a-00a6f9c6d7d2>
<gfid:0912f760-51ce-48bd-93b1-fffbebd965a1>
<gfid:66a4001b-9456-44e2-b547-5036383500d8>
<gfid:8bcece86-49af-4d88-83f8-8672dbdc5ed1>
<gfid:7ad41386-9d57-4a93-9fc2-3354e40d9927>
<gfid:880cd05b-46e5-4cf4-b042-bdb729acdd56>
<gfid:8df8af4d-38cf-4cbd-b68e-4a6f2d41b308>
<gfid:8781eb74-7409-4b1b-aa69-e30bf9c2387d>
<gfid:322c7748-459c-437e-bfba-6f9096a938c5>
<gfid:5cf36470-8d7e-4e26-904c-39a15d773b43>
<gfid:5cb576bb-8389-4f34-b812-5a770e4c0013>
<gfid:7a1b72f5-ca99-4ede-aacd-ba1b867959c0>
<gfid:6539988f-2fe6-4500-aa7c-a7cd7cfcc5df>
<gfid:3dcce45c-8c06-45f6-8101-a2e9737441c5>
<gfid:de268a4d-06fd-479d-8efc-954002ff7222>
<gfid:1badcc6d-52e5-42ca-a42a-3877415cf21e>
<gfid:459313b5-7ac8-43c6-a3b8-3a8e80afcebd>
<gfid:21c1bcc5-4c49-4010-9b4f-e920634764be>
<gfid:e4ca593f-73f5-4d9f-b84f-16168b4f84f1>
<gfid:9d80ae3c-833d-4a7c-81fc-b4ba9e645d5f>
<gfid:c78213fd-a60f-41fe-8471-6c8a92fdd873>
<gfid:7ce8062a-37fd-4388-aa71-42e3f1125d20>
<gfid:85ed9df0-64e1-4d93-8572-a5506c2c2d01>
<gfid:3bc95362-be4f-4561-b95f-4cd1260c1781>
<gfid:81e9ab2f-3f36-4a1f-ba2a-86577501c5db>
<gfid:e97cb502-924d-4ea8-9730-f84eda0b69fd>
<gfid:e721ce14-9201-4ec4-a027-bbeef43ab401>
<gfid:a194006c-b0c6-49e9-a2ec-3613403f869f>
<gfid:c610a4ae-1bbb-464d-8ef3-4c50a64f1110>
<gfid:4197cd59-1cec-452e-8c3c-c212ea9f17f6>
<gfid:1e2b6e9e-8d19-48d8-9488-4d3f78e38213>
<gfid:3f9fcead-ffbe-4dd0-9dcf-8e815dfdc5b4>
<gfid:a6024ac9-fdde-4517-87e8-ede5845f3bb3>
<gfid:664f979e-5acc-42f1-93be-9a46d783d430>
<gfid:0b95c327-d880-4e93-9432-3fd205b46ca4>
<gfid:ae447d7b-04b9-414e-ada6-d1880a5c6555>
<gfid:fa685010-3cd1-4b18-bc37-7db6c2e543ee>
<gfid:403c0233-432f-42d8-8ed0-eeca68c4b3f1>
<gfid:ac75a8e5-0906-40ea-a2ab-145f3cfcce2e>
<gfid:700e2b0a-e6c0-4888-a5ea-f52cac55e770>
<gfid:eaa43674-b1a3-4833-a946-de7b7121bb88>
<gfid:793a81de-73e8-4d84-8ac1-be03c5ac2c47>
<gfid:c603b955-305b-421e-9056-6a8256c25f88>
<gfid:b1db87aa-c1fa-409b-aa10-429860c20dbe>
<gfid:c2370b73-7c1c-4f94-afcf-74755b9a68a0>
<gfid:bc3c227c-a68e-4b9e-a81f-ce14aa30a504>
<gfid:a29c32b6-e9aa-4e51-8b07-311bb7512d89>
<gfid:9e6c8760-48e1-4b84-9ad4-e6a33c881b6c>
<gfid:eb87548b-a90a-458d-b215-939ad59f5ec0>
<gfid:a1a4c18d-8807-4a43-b9d2-99cf33a02c03>
<gfid:247283c9-f427-43ce-869a-895e31a9e891>
<gfid:0d4431f1-10b1-4339-96a9-d8370bbd77a7>
<gfid:a4745c5f-a3ad-4bed-a504-5ae5b31200a3>
<gfid:16fd1032-3bd5-474a-b30e-85971d69aaa9>
<gfid:9cf45591-c304-4348-b6ce-9dc9f8f335b1>
<gfid:331058cb-2fcf-4198-910f-5c7b78100807>
<gfid:ef583ff4-72b3-408d-bf34-88ca5534c71e>
<gfid:d6c6e6b4-d476-4b9b-990d-b79e66c490bf>
<gfid:0d1671b4-e31f-4c81-8600-fe63ffc84272>
<gfid:9a2c64de-a948-4a45-9466-58a0fb556ba4>
<gfid:b1138fe2-34cf-4d0d-af99-3b9d9aec0317>
<gfid:93dcfe76-318c-43c3-908c-0332201387f4>
<gfid:18e2f17d-801f-428d-98c1-c35f7ac8a68d>
<gfid:0b07c592-f784-4d08-a3a2-3a0de0df666e>
<gfid:f34d746e-1555-4b54-9ebd-2bfe7c96b0ef>
<gfid:2feaebdb-6eba-4e01-80ae-0812e5c770c6>
<gfid:c3419ad6-3e32-4fc5-93da-2b04a6090cfa>
<gfid:522b0117-7d22-4e94-b767-d62a5c914a62>
<gfid:94b5924f-65b9-45d6-8a03-b7084f8c4bdb>
<gfid:4d6be1c5-2282-441f-97e1-71c8c1e42aa1>
<gfid:70eab476-d53d-4f2b-b56f-13cf12821a24>
<gfid:1c623eb2-0594-48bf-accb-0c7b0a3a530a>
<gfid:84d3674c-6ec4-43b7-b590-8a125cbfbe43>
<gfid:e1d92bdc-0fe9-4531-9fc0-0c9e227721c0>
<gfid:25af9c71-aab5-4f39-bd4a-bb4f0dab9342>
<gfid:45aed765-2ae2-4fc8-9982-8e1b5d6c19d4>
<gfid:26bef74c-9675-4c74-96d0-327310cb0983>
<gfid:9c6254df-e8dd-40a3-a7d1-760f6c19027a>
<gfid:4a3ed044-845f-438f-8d8a-cac1c0d853f0>
<gfid:2bbb8529-4874-440d-959b-6a0745fdfda9>
<gfid:d3f58efb-d1ac-42ab-957c-d7b684bf0972>
<gfid:8f3a7de0-173a-4507-9b46-bc9db0a6bc41>
<gfid:ca5b027b-dc2f-468f-8cb0-131f1a9099d8>
<gfid:19291239-e44c-4025-9f74-af7431aac6b9>
<gfid:2807b990-2d3c-4f82-b89b-36637cc7c181>
<gfid:3df80623-6bf4-47e3-a379-4e5605d0eda6>
<gfid:d14c8834-a624-4288-8200-5b42fa165043>
<gfid:5375e1e8-287f-4fc5-8b01-ec2f856eebf7>
<gfid:fe814468-5a2c-469c-89de-9a96b9aacbc1>
<gfid:50787eb7-90ad-41fb-bd5e-3ba058b69c32>
<gfid:3449d319-e2d9-4444-8823-8f7351c1d45f>
<gfid:9ea01327-d5ed-48bd-8049-cc552555b774>
<gfid:c4a200b9-bb31-46cb-92c6-551fd6ad9ec3>
<gfid:a5be4523-2d2c-4894-b332-a7e3370658d3>
<gfid:b0f4e951-63b3-4152-b0b0-7aad8e6ed729>
<gfid:9fce8690-a9f7-4cbb-b13a-ba149af652b4>
<gfid:f12b4d8e-b5e6-4f71-a06c-0c0d067a7eb9>
<gfid:455c2911-7a9d-401f-9ded-4380c6cec405>
<gfid:efbd6028-b9f8-45de-a653-b27f75570b81>
<gfid:f70b50fb-3061-4eb9-94f0-85fb2d789d27>
<gfid:338baad1-6191-47f3-9737-dca2daf79fd8>
<gfid:a9e1cad9-cd31-48f0-b9f8-322e1f602401>
<gfid:7d522a0b-75fa-4b82-a0a2-4a81c98eea03>
<gfid:70b7db21-b70f-4ae0-8660-921c3194f209>
<gfid:61bf5407-c250-4448-a68f-d3bd82821260>
<gfid:1ba8d90a-5f0c-4387-a79a-592427f3d1c5>
<gfid:23e75764-9794-4efd-b319-6eac717e6f28>
<gfid:5ee303e4-44fe-44a1-83b5-5cd6a91bc76a>
<gfid:4e86088b-f975-4228-ad5a-8d08fbd456fe>
<gfid:68127ab7-807d-4a99-a609-6569d366d3aa>
<gfid:e356342a-3dab-4049-b4bc-ea0de4a8ee87>
<gfid:a6cb8d18-f2b3-48d4-ac37-c022603b8f8e>
<gfid:dad27361-d38f-46b3-a77c-b6ac5df054d9>
<gfid:047f6d58-db66-41ab-953b-fe19717edb1e>
<gfid:8a485bc1-ce8e-4587-8343-3c702d5272de>/1464964354.5545818_3.mp3 - Possibly undergoing heal
<gfid:7a2c09f2-98a1-43cb-a4fd-ad025bef22f2> - Possibly undergoing heal
<gfid:d7660014-815b-483c-a81b-4fca26178a58> - Possibly undergoing heal
<gfid:76609c7b-2007-439a-be98-39790af2d300>
<gfid:476e4d64-7446-4a78-ac45-ce04eff30512>
<gfid:a91db3b5-fc0b-49c1-897b-6b89d79b1404>
<gfid:d47f1dfd-f88a-4d96-b578-0531c1252778>
<gfid:c984ba6d-9534-4ba5-8665-1f8a6b249c95>
<gfid:8dfc56b0-211c-4e4b-8138-2cd889eb7f55>
<gfid:26428557-7556-4585-be1c-cd0ef31c7da3>
<gfid:637c1b30-95a9-4c3b-b7c6-83700b10205d>
<gfid:307d4cbc-95f2-4748-8e70-624766d515f9>
--
Cheers,
Kingsley.
_______________________________________________
Gluster-users mailing list
Gluster-users@xxxxxxxxxxx
http://www.gluster.org/mailman/listinfo/gluster-users
