> I'm trying to understand the poor performance, because network latency > should be eliminated by the cache. To get some more insight, I've run the whole small file read test while monitoring GlusterFS with Valgrind. The following are the caller trees with the biggest sysTime consumers, sorted in descending order. As expected, there's a lot of server/transport overhead going on: > -------------------------------------------------------------------------------- > Ir sysCount sysTime file:function > -------------------------------------------------------------------------------- > > 23,293,148 1,207 832 < /root/src/glusterfs-3.0.2/libglusterfs/src/xlator.c:xlator_notify'2 (17x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 77,802,615,945 7,781,126 7,302,677 < /root/src/glusterfs-3.0.2/libglusterfs/src/xlator.c:xlator_notify (1676710x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 19,966 13 16 < ???:glusterfs_graph_parent_up (2x) [/root/src/glusterfs-3.0.2/DIST/sbin/glusterfsd] > 5,808,764 294 200 < /root/src/glusterfs-3.0.2/libglusterfs/src/xlator.c:xlator_tree_init (1x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 86,712,113 129 575 * ???:notify [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > > 33,099,077,824 5,864,408 6,806,776 < ???:start_thread (4x) [/lib64/libpthread-2.11.so] > 38,686,342 107 403 * /root/src/glusterfs-3.0.2/libglusterfs/src/transport.c:transport_peerproc [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 18,512,246,936 3,866,216 4,557,625 < ???:notify (538806x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 19,397,016 47 101 * ???:protocol_server_pollin [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 18,371,926,768 3,865,089 4,555,135 < ???:protocol_server_pollin (538806x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 40,349,922 95 239 * ???:protocol_server_interpret [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 14,050,061,433 3,791,013 4,485,388 < ???:resolve_and_resume (518740x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 16,599,680 43 189 * ???:server_resolve_all [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 14,014,795,523 3,790,939 4,485,147 < ???:server_resolve_all (518740x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 17,707,728 29 79 * ???:server_resolve [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 12,136,817,206 3,787,022 4,471,507 < ???:server_resolve_all'2 (518740x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 22,305,820 43 92 * ???:server_resolve'2 [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 12,058,504,226 3,786,862 4,471,155 < ???:server_resolve_all'2 (518740x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 13,487,240 26 42 * ???:server_resolve_done [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 6 1 786 < ???:0x00000000000138d0 (1x) [/lib64/ld-2.11.so] > 18,643,217 2,583,454 3,658,345 < /root/src/glusterfs-3.0.2/libglusterfs/src/syscall.c:sys_lgetxattr (2632517x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 18,643,223 2,583,455 3,659,131 * ???:lgetxattr [/lib64/libc-2.11.so] > > 13,637,759,680 3,129,517 3,628,112 < ???:protocol_server_interpret (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 28,479,240 39 57 * ???:server_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 9,586,687,393 3,122,235 3,605,185 < ???:server_resolve_done (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 24,737,657 36 96 * ???:server_lookup_resume [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 9,444,739,583 3,121,989 3,603,913 < ???:server_lookup_resume (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 32,988,592 54 87 * ???:pl_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/features/locks.so.0.0.0] > > 9,669,192,769 3,122,299 3,601,785 < ???:server_resolve_entry (237320x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 2,454,535,276 655,611 866,729 < ???:server_resolve_inode (279216x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 12,095,845,126 3,786,939 4,471,305 < ???:server_resolve'2 (518740x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 51,467,541 9,177 3,151 < ???:server_resolve_fd (2204x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 38,386,760 74 190 * ???:server_resolve_all'2 [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 8,806,875,695 3,120,814 3,600,306 < ???:pl_lookup (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/features/locks.so.0.0.0] > 57,195,880 160 1,140 * ???:posix_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > > 17,175,939,421 2,558,068 3,163,293 < /root/src/glusterfs-3.0.2/libglusterfs/src/event.c:event_dispatch (1x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 26,101,023 58 125 * /root/src/glusterfs-3.0.2/libglusterfs/src/event.c:event_dispatch_epoll [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 41,821,621 712,604 1,551,850 < ???:posix_lstat_with_gen (733894x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > 151,612,747 2,572,884 2,850,906 < ???:_posix_xattr_get_set (1898624x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > 39,487,770 72 321 * /root/src/glusterfs-3.0.2/libglusterfs/src/syscall.c:sys_lgetxattr [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 67,099,225 1,176,978 1,749,609 < ???:__socket_rwv (1198199x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 32,301,007 562,795 1,127,809 < ???:fuse_thread_proc (576797x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 58,575,324 1,739,616 2,877,101 * ???:readv [/lib64/libc-2.11.so] > > 18,512,109,204 3,696,836 2,764,916 < ???:start_thread (1x) [/lib64/libpthread-2.11.so] > 40,375,907 101 156 * ???:fuse_thread_proc [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > > 418,706,306 1,767,980 2,437,643 < ???:__socket_proto_state_machine (1198197x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 26,360,334 52 372 * ???:__socket_readv [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > > 33,549,527 580,637 1,192,326 < ???:__socket_rwv (599098x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 32,287,721 560,432 1,119,602 < ???:send_fuse_iov (576565x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 38,796,999 1,140,967 2,311,666 * ???:writev [/lib64/libc-2.11.so] > > 17,116,612,591 2,192,357 2,257,638 < /root/src/glusterfs-3.0.2/libglusterfs/src/event.c:event_dispatch_epoll (599111x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 63,505,766 59 148 * /root/src/glusterfs-3.0.2/libglusterfs/src/event.c:event_dispatch_epoll_handler [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 16,956,598,718 2,191,999 2,255,979 < /root/src/glusterfs-3.0.2/libglusterfs/src/event.c:event_dispatch_epoll_handler (599105x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 29,356,163 106 297 * ???:socket_event_handler [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > > 35,901 10 23 < ???:0x00000000000138d0'2 (1x) [/lib64/ld-2.11.so] > 66,975 . . < /root/src/glusterfs-3.0.2/libglusterfs/src/authenticate.c:gf_authenticate (12x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 5,398 . . < ???:init (1x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 50,399 . . < /root/src/glusterfs-3.0.2/libglusterfs/src/dict.c:dict_copy_with_ref (6x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 2,204,267,686 1,874,203 2,119,199 < ???:posix_lookup_xattr_fill (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > 22,547,215 28 321 * /root/src/glusterfs-3.0.2/libglusterfs/src/dict.c:dict_foreach [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 2,173,181,418 1,874,161 2,118,845 < /root/src/glusterfs-3.0.2/libglusterfs/src/dict.c:dict_foreach (1423968x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 72,859,696 147 549 * ???:_posix_xattr_get_set [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > > 814,472,497 1,178,741 1,759,690 < ???:socket_proto_state_machine (599099x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 156,963,745 280 843 * ???:__socket_proto_state_machine [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > > 122,201,951 580,824 1,192,913 < ???:__socket_writev (599098x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 209,064,307 1,177,320 1,750,642 < ???:__socket_readv (1198197x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 165,950,323 379 1,210 * ???:__socket_rwv [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > > 200,062 121 8,241 < ???:protocol_client_xfer'2 (157x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 1,239,478,023 801,228 1,505,679 < ???:protocol_client_xfer (1137735x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 206,749,397 452,552 702,999 < ???:protocol_server_reply (538806x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 84,072,210 160 452 * /root/src/glusterfs-3.0.2/libglusterfs/src/transport.c:transport_submit [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 965,327,643 750,763 1,436,035 < /root/src/glusterfs-3.0.2/libglusterfs/src/transport.c:transport_submit (599098x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 41,337,762 80 198 * ???:socket_submit [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > > 29,520,948,862 1,957,199 1,371,390 < ???:notify (1137892x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 58,032,492 137 487 * ???:protocol_client_pollin [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > > 29,036,692,703 783,784 1,335,984 < ???:protocol_client_pollin (1137892x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 84,083,036 185 421 * ???:protocol_client_interpret [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > > 11,690,495,090 2,109,391 1,315,890 < ???:ioc_lookup (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/performance/io-cache.so.0.0.0] > 23,732,800 22 62 * /root/src/glusterfs-3.0.2/libglusterfs/src/defaults.c:default_lookup [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 12,949,103,069 2,112,772 1,315,828 < ???:fuse_thread_proc (237320x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 35,653,545 51 98 * ???:fuse_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > > 11,551,040,183 2,109,087 1,315,240 < /root/src/glusterfs-3.0.2/libglusterfs/src/defaults.c:default_lookup (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 140,383,058 172 334 * ???:afr_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/cluster/afr.so.0.0.0] > > 344,750 68 670 < ???:fuse_getattr (7x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 11,828,588,958 2,109,626 1,308,211 < ???:fuse_lookup (237320x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 65,723 20 7,673 < ???:fuse_first_lookup (1x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 23,732,800 21 25 * ???:ioc_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/performance/io-cache.so.0.0.0] > > 5,855,134,430 2,098,527 1,291,028 < ???:afr_lookup (711984x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/cluster/afr.so.0.0.0] > 138,124,080 203 404 * ???:client_lookup [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > > 469,145,673 522,813 10,818 < ???:client_stat (259246x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 360,067 438 135 < ???:client_start_ping (137x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 24,112 24 282 < ???:protocol_client_handshake (6x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 1,977,154,281 2,090,739 1,272,402 < ???:client_lookup (711984x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 123,461,119 133,304 88,998 < ???:client_readdirp (39978x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 17,669,697 19,848 5,401 < ???:client_readdir (6612x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 159,527,575 178,049 73,771 < ???:client_opendir (59886x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 156,744,855 195,324 107,332 < ???:client_fdctx_destroy (59886x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 202,517,721 502 1,607 * ???:protocol_client_xfer [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > > 837,945,437 1,410,016 1,214,844 < ???:posix_lookup (474648x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > 305,113,568 511,356 567,902 < ???:posix_stat (259246x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > 187,142,970 1,187 3,108 * ???:posix_lstat_with_gen [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > > 415,395,365 581,993 1,195,490 < ???:socket_submit (599098x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > 18,572,038 36 130 * ???:__socket_ioq_churn_entry [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/transport/socket.so.0.0.0] > > 92,371,848 540,736 1,100,723 < ???:send_fuse_data (556514x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 3,047,494 19,783 19,308 < ???:send_fuse_err (20051x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 42,385,096 50 379 * ???:send_fuse_iov [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > > 2,111,515,391 621,089 784,942 < ???:server_resolve_done (259246x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 24,887,616 36 745 * ???:server_stat_resume [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > > 2,237,749,344 526,686 19,399 < ???:fuse_getattr (259246x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 1,973,641,007 620,749 782,444 < ???:server_stat_resume (259246x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/server.so.0.0.0] > 50,812,216 60 152 * /root/src/glusterfs-3.0.2/libglusterfs/src/defaults.c:default_stat [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 1,835,268,867 620,471 781,849 < /root/src/glusterfs-3.0.2/libglusterfs/src/defaults.c:default_stat (259246x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 37,331,424 85 171 * ???:posix_stat [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/storage/posix.so.0.0.0] > > 24,954,637,314 280,616 739,635 < ???:protocol_client_interpret (711984x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > 165,446,928 590 1,310 * ???:client_lookup_cbk [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/protocol/client.so.0.0.0] > > 202,933,577 11,307 12,257 < ???:afr_fresh_lookup_cbk (11110x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/cluster/afr.so.0.0.0] > 3,533,933,487 227,406 629,001 < ???:afr_revalidate_lookup_cbk (226218x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/cluster/afr.so.0.0.0] > 36,912,078 43 67 * ???:afr_lookup_done [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/cluster/afr.so.0.0.0] > > 1,464,094,573 232,267 626,137 < ???:afr_lookup_done (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/cluster/afr.so.0.0.0] > 16,375,632 18 41 * /root/src/glusterfs-3.0.2/libglusterfs/src/defaults.c:default_lookup_cbk [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > > 1,385,788,238 232,110 625,859 < /root/src/glusterfs-3.0.2/libglusterfs/src/defaults.c:default_lookup_cbk (237328x) [/root/src/glusterfs-3.0.2/DIST/lib/libglusterfs.so.0.0.0] > 18,986,240 27 33 * ???:ioc_lookup_cbk [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/performance/io-cache.so.0.0.0] > > 58,579,546 253,893 367,128 < ???:fuse_attr_cbk (259253x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 9,033,947 39,152 82,195 < ???:fuse_readdir_cbk (39978x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 4,510,905 19,471 32,795 < ???:fuse_fd_cbk (19962x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 53,628,509 228,269 618,676 < ???:fuse_entry_cbk (237320x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 226 1 6 < ???:fuse_init (1x) [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] > 13,356,336 24 19 * ???:send_fuse_data [/root/src/glusterfs-3.0.2/DIST/lib/glusterfs/3.0.2/xlator/mount/fuse.so.0.0.0] >
