Hi All, The regression run has caused the core to generate for below patch. https://build.gluster.org/job/rackspace-regression-2GB-triggered/18859/console >From the initial analysis, it's a tiered setup where ec sub-volume is the cold tier and afr is the hot tier. The crash has happened during lookup, the lookup is wound to cold-tier, since it is not present there, dht issued discover onto hot-tier and while serializing dictionary, it found the 'data' is freed for the key 'trusted.ec.size'. (gdb) bt #0 0x00007fe059df9772 in memcpy () from ./lib64/libc.so.6 #1 0x00007fe05b209902 in dict_serialize_lk (this=0x7fe04809f7dc, buf=0x7fe0480a2b7c "") at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/dict.c:2533 #2 0x00007fe05b20a182 in dict_allocate_and_serialize (this=0x7fe04809f7dc, buf=0x7fe04ef6bb08, length=0x7fe04ef6bb00) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/dict.c:2780 #3 0x00007fe04e3492de in client3_3_lookup (frame=0x7fe0480a22dc, this=0x7fe048008c00, data=0x7fe04ef6bbe0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/protocol/client/src/client-rpc-fops.c:3368 #4 0x00007fe04e32c8c8 in client_lookup (frame=0x7fe0480a22dc, this=0x7fe048008c00, loc=0x7fe0480a4354, xdata=0x7fe04809f7dc) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/protocol/client/src/client.c:417 #5 0x00007fe04dbdaf5f in afr_lookup_do (frame=0x7fe04809f6dc, this=0x7fe048029e00, err=0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/afr/src/afr-common.c:2422 #6 0x00007fe04dbdb4bb in afr_lookup (frame=0x7fe04809f6dc, this=0x7fe048029e00, loc=0x7fe03c0082f4, xattr_req=0x7fe03c00810c) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/afr/src/afr-common.c:2532 #7 0x00007fe04de3c2b8 in dht_lookup (frame=0x7fe0480a0a3c, this=0x7fe04802c580, loc=0x7fe03c0082f4, xattr_req=0x7fe03c00810c) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/dht/src/dht-common.c:2429 #8 0x00007fe04d91f07e in dht_lookup_everywhere (frame=0x7fe03c0081ec, this=0x7fe04802d450, loc=0x7fe03c0082f4) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/dht/src/dht-common.c:1803 #9 0x00007fe04d920953 in dht_lookup_cbk (frame=0x7fe03c0081ec, cookie=0x7fe03c00902c, this=0x7fe04802d450, op_ret=-1, op_errno=2, inode=0x0, stbuf=0x0, xattr=0x0, postparent=0x0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/dht/src/dht-common.c:2056 #10 0x00007fe04de35b94 in dht_lookup_everywhere_done (frame=0x7fe03c00902c, this=0x7fe0480288a0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/dht/src/dht-common.c:1338 #11 0x00007fe04de38281 in dht_lookup_everywhere_cbk (frame=0x7fe03c00902c, cookie=0x7fe04809ed2c, this=0x7fe0480288a0, op_ret=-1, op_errno=2, inode=0x0, buf=0x0, xattr=0x0, postparent=0x0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/dht/src/dht-common.c:1768 #12 0x00007fe05b299997 in default_lookup_cbk (frame=0x7fe04809ed2c, cookie=0x7fe048099ddc, this=0x7fe048027590, op_ret=-1, op_errno=2, inode=0x0, buf=0x0, xdata=0x0, postparent=0x0) at defaults.c:1188 #13 0x00007fe04e0a4861 in ec_manager_lookup (fop=0x7fe048099ddc, state=-5) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/ec/src/ec-generic.c:864 #14 0x00007fe04e0a0b3a in __ec_manager (fop=0x7fe048099ddc, error=2) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/ec/src/ec-common.c:2098 #15 0x00007fe04e09c912 in ec_resume (fop=0x7fe048099ddc, error=0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/ec/src/ec-common.c:289 #16 0x00007fe04e09caf8 in ec_complete (fop=0x7fe048099ddc) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/ec/src/ec-common.c:362 #17 0x00007fe04e0a41a8 in ec_lookup_cbk (frame=0x7fe04800107c, cookie=0x5, this=0x7fe048027590, op_ret=-1, op_errno=2, inode=0x7fe03c00152c, buf=0x7fe04ef6c860, xdata=0x0, postparent=0x7fe04ef6c7f0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/cluster/ec/src/ec-generic.c:758 #18 0x00007fe04e348239 in client3_3_lookup_cbk (req=0x7fe04809dd4c, iov=0x7fe04809dd8c, count=1, myframe=0x7fe04809964c) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/xlators/protocol/client/src/client-rpc-fops.c:3028 #19 0x00007fe05afd83e6 in rpc_clnt_handle_reply (clnt=0x7fe048066350, pollin=0x7fe0480018f0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/rpc/rpc-lib/src/rpc-clnt.c:759 #20 0x00007fe05afd8884 in rpc_clnt_notify (trans=0x7fe0480667f0, mydata=0x7fe048066380, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7fe0480018f0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/rpc/rpc-lib/src/rpc-clnt.c:900 #21 0x00007fe05afd4b7a in rpc_transport_notify (this=0x7fe0480667f0, event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7fe0480018f0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/rpc/rpc-lib/src/rpc-transport.c:541 #22 0x00007fe050594e0f in socket_event_poll_in (this=0x7fe0480667f0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/rpc/rpc-transport/socket/src/socket.c:2233 #23 0x00007fe050595365 in socket_event_handler (fd=25, idx=14, data=0x7fe0480667f0, poll_in=1, poll_out=0, poll_err=0) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/rpc/rpc-transport/socket/src/socket.c:2346 #24 0x00007fe05b2712fc in event_dispatch_epoll_handler (event_pool=0x11ecce0, event=0x7fe04ef6ce70) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/event-epoll.c:571 #25 0x00007fe05b2716ea in event_dispatch_epoll_worker (data=0x122b470) at /home/jenkins/root/workspace/rackspace-regression-2GB-triggered/libglusterfs/src/event-epoll.c:674 #26 0x00007fe05a4efaa1 in start_thread () from ./lib64/libpthread.so.0 #27 0x00007fe059e5893d in clone () from ./lib64/libc.so.6 (gdb) p *pair->value $11 = {is_static = 0 '\000', is_const = 0 '\000', len = 186, data = 0x6500000000 <Address 0x6500000000 out of bounds>, refcount = 0, lock = 39} Thanks and Regards, Kotresh H R _______________________________________________ Gluster-devel mailing list Gluster-devel@xxxxxxxxxxx http://www.gluster.org/mailman/listinfo/gluster-devel
