|
Hi Nithya, I've completed this, and attached the log. [root@gluster07 ~]# getfattr --absolute-names -m . -d -e hex /mnt/raid6-storage/storage/ # file: /mnt/raid6-storage/storage/ security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a756e6c6162656c65645f743a733000 trusted.gfid=0x00000000000000000000000000000001 trusted.glusterfs.6f95525a-94d7-4174-bac4-e1a18fe010a2.xtime=0x5d31d426000cee76 trusted.glusterfs.quota.dirty=0x3000 trusted.glusterfs.quota.size.2=0x00001b73f79d92000000000000764c7a000000000005ce79 trusted.glusterfs.volume-id=0x6f95525a94d74174bac4e1a18fe010a2 [root@gluster07 ~]# gluster volume get storage client-log-level Option Value ------ ----- diagnostics.client-log-level INFO [root@gluster07 ~]# gluster volume set storage client-log-level DEBUG volume set: success [root@gluster07 ~]# umount /storage/ umount: /storage/: not mounted [root@gluster07 ~]# mount /storage/ [root@gluster07 ~]# stat /storage/ File: ‘/storage/’ Size: 4096 Blocks: 8 IO Block: 131072 directory Device: 2bh/43d Inode: 1 Links: 6 Access: (0755/drwxr-xr-x) Uid: ( 0/ root) Gid: ( 0/ root) Context: system_u:object_r:fusefs_t:s0 Access: 2019-07-19 08:01:17.458527914 -0700 Modify: 2018-07-04 15:18:05.947569905 -0700 Change: 2019-07-19 08:01:10.350465738 -0700 Birth: - [root@gluster07 ~]# gluster volume set storage client-log-level INFO volume set: success [root@gluster07 ~]# getfattr --absolute-names -m . -d -e hex /mnt/raid6-storage/storage/ # file: /mnt/raid6-storage/storage/ security.selinux=0x756e636f6e66696e65645f753a6f626a6563745f723a756e6c6162656c65645f743a733000 trusted.gfid=0x00000000000000000000000000000001 trusted.glusterfs.6f95525a-94d7-4174-bac4-e1a18fe010a2.xtime=0x5d31dafd000a5aa3 trusted.glusterfs.quota.dirty=0x3000 trusted.glusterfs.quota.size.2=0x00001b73f79da0000000000000764c7a000000000005ce79 trusted.glusterfs.volume-id=0x6f95525a94d74174bac4e1a18fe010a2 Thanks, -Matthew
On 7/18/19 9:00 PM, Nithya Balachandran
wrote:
|
[2019-07-19 15:01:17.493010] D [MSGID: 0] [io-threads.c:842:__iot_workers_scale] 0-storage-io-threads: scaled threads to 1 (queue_size=0/1) [2019-07-19 15:01:17.493212] D [MSGID: 0] [quick-read.c:1183:check_cache_size_ok] 0-storage-quick-read: Max cache size is 67271393280 [2019-07-19 15:01:17.493298] D [MSGID: 0] [io-cache.c:1595:check_cache_size_ok] 0-storage-io-cache: Max cache size is 67271393280 [2019-07-19 15:01:17.493323] D [MSGID: 0] [options.c:1227:xlator_option_init_size_uint64] 0-storage-readdir-ahead: option rda-request-size using set value 131072 [2019-07-19 15:01:17.493339] D [MSGID: 0] [options.c:1227:xlator_option_init_size_uint64] 0-storage-readdir-ahead: option rda-cache-limit using set value 10MB [2019-07-19 15:01:17.493348] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-readdir-ahead: option parallel-readdir using set value off [2019-07-19 15:01:17.493441] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-dht: option lock-migration using set value off [2019-07-19 15:01:17.493457] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-dht: option force-migration using set value off [2019-07-19 15:01:17.493509] D [MSGID: 0] [dht-shared.c:356:dht_init_regex] 0-storage-dht: using regex rsync-hash-regex = ^\.(.+)\.[^.]+$ [2019-07-19 15:01:17.493599] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-6: option ping-timeout using set value 42 [2019-07-19 15:01:17.493599] I [MSGID: 101190] [event-epoll.c:622:event_dispatch_epoll_worker] 0-epoll: Started thread with index 2 [2019-07-19 15:01:17.493628] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-6: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.493643] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-6: option send-gids using set value true [2019-07-19 15:01:17.493668] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-6: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.493678] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-6: setting ping-timeout to 42 [2019-07-19 15:01:17.493688] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.493793] D [socket.c:4464:socket_init] 0-storage-client-6: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.493806] D [socket.c:4482:socket_init] 0-storage-client-6: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.493813] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-6: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.493819] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-6: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.493828] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-6: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.493837] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-6: client init successful [2019-07-19 15:01:17.493863] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-5: option ping-timeout using set value 42 [2019-07-19 15:01:17.493882] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-5: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.493895] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-5: option send-gids using set value true [2019-07-19 15:01:17.493904] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-5: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.493911] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-5: setting ping-timeout to 42 [2019-07-19 15:01:17.493922] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.494015] D [socket.c:4464:socket_init] 0-storage-client-5: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.494025] D [socket.c:4482:socket_init] 0-storage-client-5: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.494032] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-5: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.494037] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-5: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.494044] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-5: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.494051] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-5: client init successful [2019-07-19 15:01:17.494077] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-4: option ping-timeout using set value 42 [2019-07-19 15:01:17.494087] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-4: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.494103] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-4: option send-gids using set value true [2019-07-19 15:01:17.494112] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-4: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.494118] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-4: setting ping-timeout to 42 [2019-07-19 15:01:17.494126] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.494214] D [socket.c:4464:socket_init] 0-storage-client-4: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.494222] D [socket.c:4482:socket_init] 0-storage-client-4: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.494228] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-4: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.494233] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-4: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.494239] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-4: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.494249] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-4: client init successful [2019-07-19 15:01:17.494274] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-3: option ping-timeout using set value 42 [2019-07-19 15:01:17.494283] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-3: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.494297] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-3: option send-gids using set value true [2019-07-19 15:01:17.494312] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-3: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.494318] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-3: setting ping-timeout to 42 [2019-07-19 15:01:17.494327] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.494444] D [socket.c:4464:socket_init] 0-storage-client-3: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.494454] D [socket.c:4482:socket_init] 0-storage-client-3: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.494460] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-3: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.494466] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-3: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.494473] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-3: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.494479] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-3: client init successful [2019-07-19 15:01:17.494520] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-2: option ping-timeout using set value 42 [2019-07-19 15:01:17.494532] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-2: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.494545] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-2: option send-gids using set value true [2019-07-19 15:01:17.494556] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-2: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.494562] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-2: setting ping-timeout to 42 [2019-07-19 15:01:17.494585] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.494683] D [socket.c:4464:socket_init] 0-storage-client-2: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.494692] D [socket.c:4482:socket_init] 0-storage-client-2: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.494699] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-2: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.494709] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-2: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.494717] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-2: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.494725] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-2: client init successful [2019-07-19 15:01:17.494757] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-1: option ping-timeout using set value 42 [2019-07-19 15:01:17.494770] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-1: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.494784] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-1: option send-gids using set value true [2019-07-19 15:01:17.494792] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-1: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.494800] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-1: setting ping-timeout to 42 [2019-07-19 15:01:17.494811] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.494904] D [socket.c:4464:socket_init] 0-storage-client-1: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.494912] D [socket.c:4482:socket_init] 0-storage-client-1: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.494918] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-1: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.494924] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-1: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.494930] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-1: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.494945] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-1: client init successful [2019-07-19 15:01:17.494997] D [MSGID: 0] [options.c:1225:xlator_option_init_int32] 0-storage-client-0: option ping-timeout using set value 42 [2019-07-19 15:01:17.495012] D [MSGID: 0] [options.c:1232:xlator_option_init_path] 0-storage-client-0: option remote-subvolume using set value /mnt/raid6-storage/storage [2019-07-19 15:01:17.495028] D [MSGID: 0] [options.c:1230:xlator_option_init_bool] 0-storage-client-0: option send-gids using set value true [2019-07-19 15:01:17.495042] D [rpc-clnt.c:1002:rpc_clnt_connection_init] 0-storage-client-0: defaulting frame-timeout to 30mins [2019-07-19 15:01:17.495050] D [rpc-clnt.c:1010:rpc_clnt_connection_init] 0-storage-client-0: setting ping-timeout to 42 [2019-07-19 15:01:17.495058] D [rpc-transport.c:269:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/5.3/rpc-transport/socket.so [2019-07-19 15:01:17.495164] D [socket.c:4464:socket_init] 0-storage-client-0: Configued transport.tcp-user-timeout=0 [2019-07-19 15:01:17.495173] D [socket.c:4482:socket_init] 0-storage-client-0: Reconfigued transport.keepalivecnt=9 [2019-07-19 15:01:17.495179] D [socket.c:4167:ssl_setup_connection_params] 0-storage-client-0: SSL support on the I/O path is NOT enabled [2019-07-19 15:01:17.495185] D [socket.c:4170:ssl_setup_connection_params] 0-storage-client-0: SSL support for glusterd is NOT enabled [2019-07-19 15:01:17.495192] D [rpc-clnt.c:1579:rpcclnt_cbk_program_register] 0-storage-client-0: New program registered: GlusterFS Callback, Num: 52743234, Ver: 1 [2019-07-19 15:01:17.495198] D [MSGID: 0] [client.c:2543:client_init_rpc] 0-storage-client-0: client init successful [2019-07-19 15:01:17.495235] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-6: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495247] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-6: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495261] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-6: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495272] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-6: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495282] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-6: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495298] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-5: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495311] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-5: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495321] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-5: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495336] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-5: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495347] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-5: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495362] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-4: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495372] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-4: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495385] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-4: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495395] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-4: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495404] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-4: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495419] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-3: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495437] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-3: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495449] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-3: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495459] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-3: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495472] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-3: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495489] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-2: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495499] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-2: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495508] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-2: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495519] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-2: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495529] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-2: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495548] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-1: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495558] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-1: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495568] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-1: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495578] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-1: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495588] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-1: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495603] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-0: option 'transport.address-family' is not recognized [2019-07-19 15:01:17.495613] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-0: option 'transport.tcp-user-timeout' is not recognized [2019-07-19 15:01:17.495624] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-0: option 'transport.socket.keepalive-time' is not recognized [2019-07-19 15:01:17.495642] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-0: option 'transport.socket.keepalive-interval' is not recognized [2019-07-19 15:01:17.495658] D [MSGID: 101174] [graph.c:397:_log_if_unknown_option] 0-storage-client-0: option 'transport.socket.keepalive-count' is not recognized [2019-07-19 15:01:17.495673] D [fuse-bridge.c:5332:notify] 0-fuse: got event 12 on graph 0 [2019-07-19 15:01:17.495691] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-0: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.498183] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.50 (port-24007) for hostname: 10.0.231.50 and port: 24007 [2019-07-19 15:01:17.498203] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-0: disabling SSL for portmapper connection [2019-07-19 15:01:17.498375] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-1: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.498496] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-0: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.498905] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.50:24007: ping timer event already removed [2019-07-19 15:01:17.498985] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-0: detected portmapper on server [2019-07-19 15:01:17.499085] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-0: Ping latency is 0ms [2019-07-19 15:01:17.499161] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-0: changing port to 49154 (from 0) [2019-07-19 15:01:17.499192] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:13) (non-SSL) [2019-07-19 15:01:17.499207] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-0: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.499221] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-0: disconnected (skipped notify) [2019-07-19 15:01:17.500802] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.51 (port-24007) for hostname: 10.0.231.51 and port: 24007 [2019-07-19 15:01:17.500818] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-1: disabling SSL for portmapper connection [2019-07-19 15:01:17.500930] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-2: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.503047] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.50 (port-24007) for hostname: 10.0.231.50 and port: 24007 [2019-07-19 15:01:17.503434] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.503486] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-1: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.503694] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.51:24007: ping timer event already removed [2019-07-19 15:01:17.503733] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-0: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.503888] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.50:49154: ping timer event already removed [2019-07-19 15:01:17.503936] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-1: detected portmapper on server [2019-07-19 15:01:17.504040] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-1: Ping latency is 0ms [2019-07-19 15:01:17.504065] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-0: Ping latency is 0ms [2019-07-19 15:01:17.504081] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-1: changing port to 49155 (from 0) [2019-07-19 15:01:17.504100] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:15) (non-SSL) [2019-07-19 15:01:17.504111] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-1: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.504120] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-1: disconnected (skipped notify) [2019-07-19 15:01:17.505440] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.52 (port-24007) for hostname: 10.0.231.52 and port: 24007 [2019-07-19 15:01:17.505456] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-2: disabling SSL for portmapper connection [2019-07-19 15:01:17.505564] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-3: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.507649] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.51 (port-24007) for hostname: 10.0.231.51 and port: 24007 [2019-07-19 15:01:17.507787] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.507839] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-0: Connected to storage-client-0, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.507855] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-0: No fds to open - notifying all parents child up [2019-07-19 15:01:17.507925] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-2: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.508103] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.52:24007: ping timer event already removed [2019-07-19 15:01:17.508141] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-1: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.508277] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.51:49155: ping timer event already removed [2019-07-19 15:01:17.508319] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-2: detected portmapper on server [2019-07-19 15:01:17.508370] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-0': avail_percent is: 24.00 and avail_space is: 9879221805056 and avail_inodes is: 99.00 [2019-07-19 15:01:17.508430] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-2: Ping latency is 0ms [2019-07-19 15:01:17.508452] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-1: Ping latency is 0ms [2019-07-19 15:01:17.508467] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-2: changing port to 49155 (from 0) [2019-07-19 15:01:17.508486] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:14) (non-SSL) [2019-07-19 15:01:17.508497] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-2: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.508505] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-2: disconnected (skipped notify) [2019-07-19 15:01:17.510077] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.53 (port-24007) for hostname: 10.0.231.53 and port: 24007 [2019-07-19 15:01:17.510095] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-3: disabling SSL for portmapper connection [2019-07-19 15:01:17.510202] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-4: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.512424] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.52 (port-24007) for hostname: 10.0.231.52 and port: 24007 [2019-07-19 15:01:17.512574] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.512621] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-1: Connected to storage-client-1, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.512640] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-1: No fds to open - notifying all parents child up [2019-07-19 15:01:17.512697] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-3: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.512895] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.53:24007: ping timer event already removed [2019-07-19 15:01:17.512934] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-2: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.513066] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.52:49155: ping timer event already removed [2019-07-19 15:01:17.513115] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-1': avail_percent is: 23.00 and avail_space is: 9383555014656 and avail_inodes is: 99.00 [2019-07-19 15:01:17.513141] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-3: detected portmapper on server [2019-07-19 15:01:17.513204] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-3: Ping latency is 0ms [2019-07-19 15:01:17.513226] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-2: Ping latency is 0ms [2019-07-19 15:01:17.513253] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-3: changing port to 49153 (from 0) [2019-07-19 15:01:17.513273] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:15) (non-SSL) [2019-07-19 15:01:17.513283] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-3: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.513292] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-3: disconnected (skipped notify) [2019-07-19 15:01:17.515002] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.54 (port-24007) for hostname: 10.0.231.54 and port: 24007 [2019-07-19 15:01:17.515021] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-4: disabling SSL for portmapper connection [2019-07-19 15:01:17.515159] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-5: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.517280] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.53 (port-24007) for hostname: 10.0.231.53 and port: 24007 [2019-07-19 15:01:17.517434] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.517481] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-2: Connected to storage-client-2, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.517493] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-2: No fds to open - notifying all parents child up [2019-07-19 15:01:17.517540] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-4: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.517730] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.54:24007: ping timer event already removed [2019-07-19 15:01:17.517780] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-3: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.517906] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.53:49153: ping timer event already removed [2019-07-19 15:01:17.517948] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-4: detected portmapper on server [2019-07-19 15:01:17.517994] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-2': avail_percent is: 23.00 and avail_space is: 9500197941248 and avail_inodes is: 99.00 [2019-07-19 15:01:17.518053] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-4: Ping latency is 0ms [2019-07-19 15:01:17.518075] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-3: Ping latency is 0ms [2019-07-19 15:01:17.518090] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-4: changing port to 49153 (from 0) [2019-07-19 15:01:17.518109] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:14) (non-SSL) [2019-07-19 15:01:17.518120] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-4: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.518128] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-4: disconnected (skipped notify) [2019-07-19 15:01:17.519815] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.55 (port-24007) for hostname: 10.0.231.55 and port: 24007 [2019-07-19 15:01:17.519833] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-5: disabling SSL for portmapper connection [2019-07-19 15:01:17.519952] I [MSGID: 114020] [client.c:2354:notify] 0-storage-client-6: parent translators are ready, attempting connect on transport [2019-07-19 15:01:17.522022] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.54 (port-24007) for hostname: 10.0.231.54 and port: 24007 [2019-07-19 15:01:17.522159] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.522200] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-3: Connected to storage-client-3, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.522210] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-3: No fds to open - notifying all parents child up [2019-07-19 15:01:17.522285] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-5: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.522463] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.55:24007: ping timer event already removed [2019-07-19 15:01:17.522500] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-4: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.522621] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.54:49153: ping timer event already removed [2019-07-19 15:01:17.522667] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-5: detected portmapper on server [2019-07-19 15:01:17.522706] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-3': avail_percent is: 23.00 and avail_space is: 9281959972864 and avail_inodes is: 99.00 [2019-07-19 15:01:17.522770] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-5: Ping latency is 0ms [2019-07-19 15:01:17.522792] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-4: Ping latency is 0ms [2019-07-19 15:01:17.522806] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-5: changing port to 49155 (from 0) [2019-07-19 15:01:17.522825] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:15) (non-SSL) [2019-07-19 15:01:17.522836] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-5: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.522845] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-5: disconnected (skipped notify) [2019-07-19 15:01:17.524583] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.56 (port-24007) for hostname: 10.0.231.56 and port: 24007 [2019-07-19 15:01:17.524601] D [socket.c:3221:socket_fix_ssl_opts] 0-storage-client-6: disabling SSL for portmapper connection Final graph: +------------------------------------------------------------------------------+ 1: volume storage-client-0 2: type protocol/client 3: option opversion 50000 4: option clnt-lk-version 1 5: option volfile-checksum 0 6: option volfile-key /storage 7: option client-version 5.3 8: option process-name fuse 9: option process-uuid CTX_ID:d0875391-ca96-40d7-9350-63f487ef1210-GRAPH_ID:0-PID:23336-HOST:gluster07.pcic.uvic.ca-PC_NAME:storage-client-0-RECON_NO:-0 10: option fops-version 1298437 11: option ping-timeout 42 12: option remote-host 10.0.231.50 13: option remote-subvolume /mnt/raid6-storage/storage 14: option transport-type socket 15: option transport.address-family inet 16: option username affc2c53-f158-48fa-8191-2ae39ced889e 17: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 18: option transport.tcp-user-timeout 0 19: option transport.socket.keepalive-time 20 20: option transport.socket.keepalive-interval 2 21: option transport.socket.keepalive-count 9 22: option send-gids true 23: end-volume 24: 25: volume storage-client-1 26: type protocol/client 27: option opversion 50000 28: option clnt-lk-version 1 29: option volfile-checksum 0 30: option volfile-key /storage 31: option client-version 5.3 32: option process-name fuse 33: option process-uuid CTX_ID:d0875391-ca96-40d7-9350-63f487ef1210-GRAPH_ID:0-PID:23336-HOST:gluster07.pcic.uvic.ca-PC_NAME:storage-client-1-RECON_NO:-0 34: option fops-version 1298437 35: option ping-timeout 42 36: option remote-host 10.0.231.51 37: option remote-subvolume /mnt/raid6-storage/storage 38: option transport-type socket 39: option transport.address-family inet 40: option username affc2c53-f158-48fa-8191-2ae39ced889e 41: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 42: option transport.tcp-user-timeout 0 43: option transport.socket.keepalive-time 20 44: option transport.socket.keepalive-interval 2 45: option transport.socket.keepalive-count 9 46: option send-gids true 47: end-volume 48: 49: volume storage-client-2 50: type protocol/client 51: option opversion 50000 52: option clnt-lk-version 1 53: option volfile-checksum 0 54: option volfile-key /storage 55: option client-version 5.3 56: option process-name fuse 57: option process-uuid CTX_ID:d0875391-ca96-40d7-9350-63f487ef1210-GRAPH_ID:0-PID:23336-HOST:gluster07.pcic.uvic.ca-PC_NAME:storage-client-2-RECON_NO:-0 58: option fops-version 1298437 59: option ping-timeout 42 60: option remote-host 10.0.231.52 61: option remote-subvolume /mnt/raid6-storage/storage 62: option transport-type socket 63: option transport.address-family inet 64: option username affc2c53-f158-48fa-8191-2ae39ced889e 65: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 66: option transport.tcp-user-timeout 0 67: option transport.socket.keepalive-time 20 68: option transport.socket.keepalive-interval 2 69: option transport.socket.keepalive-count 9 70: option send-gids true 71: end-volume 72: 73: volume storage-client-3 74: type protocol/client 75: option opversion 50000 76: option clnt-lk-version 1 77: option volfile-checksum 0 78: option volfile-key /storage 79: option client-version 5.3 80: option process-name fuse 81: option process-uuid CTX_ID:d0875391-ca96-40d7-9350-63f487ef1210-GRAPH_ID:0-PID:23336-HOST:gluster07.pcic.uvic.ca-PC_NAME:storage-client-3-RECON_NO:-0 82: option fops-version 1298437 83: option ping-timeout 42 84: option remote-host 10.0.231.53 85: option remote-subvolume /mnt/raid6-storage/storage 86: option transport-type socket 87: option transport.address-family inet 88: option username affc2c53-f158-48fa-8191-2ae39ced889e 89: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 90: option transport.tcp-user-timeout 0 91: option transport.socket.keepalive-time 20 92: option transport.socket.keepalive-interval 2 93: option transport.socket.keepalive-count 9 94: option send-gids true 95: end-volume 96: 97: volume storage-client-4 98: type protocol/client 99: option opversion 50000 100: option clnt-lk-version 1 101: option volfile-checksum 0 102: option volfile-key /storage 103: option client-version 5.3 104: option process-name fuse 105: option process-uuid CTX_ID:d0875391-ca96-40d7-9350-63f487ef1210-GRAPH_ID:0-PID:23336-HOST:gluster07.pcic.uvic.ca-PC_NAME:storage-client-4-RECON_NO:-0 106: option fops-version 1298437 107: option ping-timeout 42 108: option remote-host 10.0.231.54 109: option remote-subvolume /mnt/raid6-storage/storage 110: option transport-type socket 111: option transport.address-family inet 112: option username affc2c53-f158-48fa-8191-2ae39ced889e 113: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 114: option transport.tcp-user-timeout 0 115: option transport.socket.keepalive-time 20 116: option transport.socket.keepalive-interval 2 117: option transport.socket.keepalive-count 9 118: option send-gids true 119: end-volume 120: 121: volume storage-client-5 122: type protocol/client 123: option ping-timeout 42 124: option remote-host 10.0.231.55 125: option remote-subvolume /mnt/raid6-storage/storage 126: option transport-type socket 127: option transport.address-family inet 128: option username affc2c53-f158-48fa-8191-2ae39ced889e 129: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 130: option transport.tcp-user-timeout 0 131: option transport.socket.keepalive-time 20 132: option transport.socket.keepalive-interval 2 133: option transport.socket.keepalive-count 9 134: option send-gids true 135: end-volume 136: 137: volume storage-client-6 138: type protocol/client 139: option ping-timeout 42 140: option remote-host 10.0.231.56 141: option remote-subvolume /mnt/raid6-storage/storage 142: option transport-type socket 143: option transport.address-family inet 144: option username affc2c53-f158-48fa-8191-2ae39ced889e 145: option password e4177b59-b486-4551-ba0c-d5768a3be3e0 146: option transport.tcp-user-timeout 0 147: option transport.socket.keepalive-time 20 148: option transport.socket.keepalive-interval 2 149: option transport.socket.keepalive-count 9 150: option send-gids true 151: end-volume 152: 153: volume storage-dht 154: type cluster/distribute 155: option lock-migration off 156: option force-migration off 157: subvolumes storage-client-0 storage-client-1 storage-client-2 storage-client-3 storage-client-4 storage-client-5 storage-client-6 158: end-volume 159: 160: volume storage-write-behind 161: type performance/write-behind 162: subvolumes storage-dht 163: end-volume 164: 165: volume storage-read-ahead 166: type performance/read-ahead 167: subvolumes storage-write-behind 168: end-volume 169: 170: volume storage-readdir-ahead 171: type performance/readdir-ahead 172: option parallel-readdir off 173: option rda-request-size 131072 174: option rda-cache-limit 10MB 175: subvolumes storage-read-ahead 176: end-volume 177: 178: volume storage-io-cache 179: type performance/io-cache 180: subvolumes storage-readdir-ahead 181: end-volume 182: 183: volume storage-quick-read 184: type performance/quick-read 185: subvolumes storage-io-cache 186: end-volume 187: 188: volume storage-open-behind 189: type performance/open-behind 190: subvolumes storage-quick-read 191: end-volume 192: 193: volume storage-md-cache 194: type performance/md-cache 195: subvolumes storage-open-behind 196: end-volume 197: 198: volume storage-io-threads 199: type performance/io-threads 200: subvolumes storage-md-cache 201: end-volume 202: 203: volume storage 204: type debug/io-stats 205: option log-level DEBUG 206: option latency-measurement off 207: option count-fop-hits off 208: subvolumes storage-io-threads 209: end-volume 210: 211: volume meta-autoload 212: type meta 213: subvolumes storage 214: end-volume 215: +------------------------------------------------------------------------------+ [2019-07-19 15:01:17.525571] D [glusterfsd-mgmt.c:2839:glusterfs_mgmt_pmap_signin] 0-fsd-mgmt: portmapper signin arguments not given [2019-07-19 15:01:17.525624] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-4: Connected to storage-client-4, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.525646] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-4: No fds to open - notifying all parents child up [2019-07-19 15:01:17.525724] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-6: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.525919] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.56:24007: ping timer event already removed [2019-07-19 15:01:17.525988] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-4': avail_percent is: 25.00 and avail_space is: 10120701075456 and avail_inodes is: 99.00 [2019-07-19 15:01:17.526024] D [MSGID: 0] [client-handshake.c:1394:server_has_portmap] 0-storage-client-6: detected portmapper on server [2019-07-19 15:01:17.526080] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-6: Ping latency is 0ms [2019-07-19 15:01:17.526127] I [rpc-clnt.c:2042:rpc_clnt_reconfig] 0-storage-client-6: changing port to 49154 (from 0) [2019-07-19 15:01:17.526162] D [socket.c:2927:socket_event_handler] 0-transport: EPOLLERR - disconnecting (sock:14) (non-SSL) [2019-07-19 15:01:17.526180] D [MSGID: 0] [client.c:2271:client_rpc_notify] 0-storage-client-6: got RPC_CLNT_DISCONNECT [2019-07-19 15:01:17.526194] D [MSGID: 0] [client.c:2312:client_rpc_notify] 0-storage-client-6: disconnected (skipped notify) [2019-07-19 15:01:17.527075] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.55 (port-24007) for hostname: 10.0.231.55 and port: 24007 [2019-07-19 15:01:17.527208] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.527287] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-5: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.527439] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.55:49155: ping timer event already removed [2019-07-19 15:01:17.527521] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-5: Ping latency is 0ms [2019-07-19 15:01:17.528010] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-5: Connected to storage-client-5, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.528024] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-5: No fds to open - notifying all parents child up [2019-07-19 15:01:17.528193] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-5': avail_percent is: 24.00 and avail_space is: 9646427553792 and avail_inodes is: 99.00 [2019-07-19 15:01:17.529409] D [MSGID: 0] [common-utils.c:535:gf_resolve_ip6] 0-resolver: returning ip-10.0.231.56 (port-24007) for hostname: 10.0.231.56 and port: 24007 [2019-07-19 15:01:17.529578] E [MSGID: 101191] [event-epoll.c:671:event_dispatch_epoll_worker] 0-epoll: Failed to dispatch handler [2019-07-19 15:01:17.529592] D [MSGID: 0] [client.c:2260:client_rpc_notify] 0-storage-client-6: got RPC_CLNT_CONNECT [2019-07-19 15:01:17.529785] D [rpc-clnt-ping.c:96:rpc_clnt_remove_ping_timer_locked] (--> /lib64/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa4e29eafbb] (--> /lib64/libgfrpc.so.0(+0x125cb)[0x7fa4e27b95cb] (--> /lib64/libgfrpc.so.0(+0x12d91)[0x7fa4e27b9d91] (--> /lib64/libgfrpc.so.0(rpc_clnt_submit+0x4bb)[0x7fa4e27b660b] (--> /usr/lib64/glusterfs/5.3/xlator/protocol/client.so(+0x13f99)[0x7fa4d4e5cf99] ))))) 0-: 10.0.231.56:49154: ping timer event already removed [2019-07-19 15:01:17.529851] D [rpc-clnt-ping.c:204:rpc_clnt_ping_cbk] 0-storage-client-6: Ping latency is 0ms [2019-07-19 15:01:17.530328] I [MSGID: 114046] [client-handshake.c:1107:client_setvolume_cbk] 0-storage-client-6: Connected to storage-client-6, attached to remote volume '/mnt/raid6-storage/storage'. [2019-07-19 15:01:17.530343] D [MSGID: 0] [client-handshake.c:946:client_post_handshake] 0-storage-client-6: No fds to open - notifying all parents child up [2019-07-19 15:01:17.530404] D [fuse-bridge.c:5332:notify] 0-fuse: got event 5 on graph 0 [2019-07-19 15:01:17.530559] D [MSGID: 0] [dht-diskusage.c:94:dht_du_info_cbk] 0-storage-dht: subvolume 'storage-client-6': avail_percent is: 23.00 and avail_space is: 9361142747136 and avail_inodes is: 99.00 [2019-07-19 15:01:17.532232] D [fuse-bridge.c:4919:fuse_get_mount_status] 0-fuse: mount status is 0 [2019-07-19 15:01:17.532356] D [fuse-bridge.c:4206:fuse_init] 0-glusterfs-fuse: Detected support for FUSE_AUTO_INVAL_DATA. Enabling fopen_keep_cache automatically. [2019-07-19 15:01:17.532384] I [fuse-bridge.c:4267:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.24 kernel 7.22 [2019-07-19 15:01:17.532397] I [fuse-bridge.c:4878:fuse_graph_sync] 0-fuse: switched to graph 0 [2019-07-19 15:01:17.532455] D [MSGID: 0] [io-threads.c:372:iot_schedule] 0-storage-io-threads: LOOKUP scheduled as fast fop [2019-07-19 15:01:17.532580] D [MSGID: 0] [dht-common.c:3454:dht_do_fresh_lookup] 0-storage-dht: Calling fresh lookup for / on storage-client-0 [2019-07-19 15:01:17.533478] D [MSGID: 0] [dht-common.c:3013:dht_lookup_cbk] 0-storage-dht: fresh_lookup returned for / with op_ret 0 [2019-07-19 15:01:17.534244] D [MSGID: 0] [dht-common.c:1354:dht_lookup_dir_cbk] 0-storage-dht: Internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.534805] D [logging.c:1998:_gf_msg_internal] 0-logging-infra: Buffer overflow of a buffer whose size limit is 5. About to flush least recently used log message to disk The message "D [MSGID: 0] [dht-common.c:1354:dht_lookup_dir_cbk] 0-storage-dht: Internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 " repeated 6 times between [2019-07-19 15:01:17.534244] and [2019-07-19 15:01:17.534563] [2019-07-19 15:01:17.534804] D [MSGID: 0] [io-threads.c:372:iot_schedule] 0-storage-io-threads: LOOKUP scheduled as fast fop [2019-07-19 15:01:17.535444] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535511] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.535552] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535580] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.535664] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535680] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.535697] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535714] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.535793] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535813] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.535834] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535846] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:17.535859] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:17.535875] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.371655] D [MSGID: 0] [io-threads.c:372:iot_schedule] 0-storage-io-threads: LOOKUP scheduled as fast fop [2019-07-19 15:01:19.372470] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372505] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.372528] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372576] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.372605] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372621] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.372701] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372719] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.372752] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372773] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.372828] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372841] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:19.372857] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:19.372876] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.898527] D [MSGID: 0] [io-threads.c:372:iot_schedule] 0-storage-io-threads: LOOKUP scheduled as fast fop [2019-07-19 15:01:20.899369] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899400] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.899448] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899488] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.899512] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899534] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.899562] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899596] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.899634] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899648] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.899753] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899768] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:20.899787] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:20.899809] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.358805] D [MSGID: 0] [io-threads.c:372:iot_schedule] 0-storage-io-threads: LOOKUP scheduled as fast fop [2019-07-19 15:01:27.359673] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.359705] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.359741] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.359771] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.359834] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.359852] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.359962] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.359991] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.360036] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.360051] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.360088] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.360102] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.361112] D [MSGID: 0] [dht-common.c:1559:dht_revalidate_cbk] 0-storage-dht: revalidate lookup of / returned with op_ret 0 [2019-07-19 15:01:27.361130] D [MSGID: 0] [dht-common.c:1651:dht_revalidate_cbk] 0-storage-dht: internal xattr trusted.glusterfs.dht.mds is not present on path / gfid is 00000000-0000-0000-0000-000000000001 [2019-07-19 15:01:27.361207] D [MSGID: 0] [io-threads.c:372:iot_schedule] 0-storage-io-threads: STATFS scheduled as fast fop [2019-07-19 15:01:27.455734] I [glusterfsd-mgmt.c:58:mgmt_cbk_spec] 0-mgmt: Volume file changed [2019-07-19 15:01:27.455790] D [rpc-clnt-ping.c:302:rpc_clnt_start_ping] 0-glusterfs: ping timeout is 0, returning [2019-07-19 15:01:27.542660] I [glusterfsd-mgmt.c:58:mgmt_cbk_spec] 0-mgmt: Volume file changed [2019-07-19 15:01:27.542721] D [rpc-clnt-ping.c:302:rpc_clnt_start_ping] 0-glusterfs: ping timeout is 0, returning [2019-07-19 15:01:27.550102] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/protocol/client.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550336] D [logging.c:1998:_gf_msg_internal] 0-logging-infra: Buffer overflow of a buffer whose size limit is 5. About to flush least recently used log message to disk The message "D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/protocol/client.so: undefined symbol: xlator_api. Fall back to old symbols" repeated 6 times between [2019-07-19 15:01:27.550102] and [2019-07-19 15:01:27.550306] [2019-07-19 15:01:27.550336] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/cluster/distribute.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550388] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/performance/write-behind.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550413] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/performance/read-ahead.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550441] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/performance/readdir-ahead.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550476] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/performance/io-cache.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550509] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/performance/open-behind.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550541] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/performance/io-threads.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550566] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/debug/io-stats.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550620] D [MSGID: 101097] [xlator.c:341:xlator_dynload_newway] 0-xlator: dlsym(xlator_api) on /usr/lib64/glusterfs/5.3/xlator/meta.so: undefined symbol: xlator_api. Fall back to old symbols [2019-07-19 15:01:27.550646] D [MSGID: 0] [graph.c:847:is_graph_topology_equal] 0-glusterfsd-mgmt: graphs are equal [2019-07-19 15:01:27.550655] D [MSGID: 0] [graph.c:897:glusterfs_volfile_reconfigure] 0-glusterfsd-mgmt: Only options have changed in the new graph [2019-07-19 15:01:27.550676] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-0: option ping-timeout using set value 42 [2019-07-19 15:01:27.550711] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-0: option send-gids using set value true [2019-07-19 15:01:27.550726] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-0: reconfigured [2019-07-19 15:01:27.550746] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-1: option ping-timeout using set value 42 [2019-07-19 15:01:27.550776] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-1: option send-gids using set value true [2019-07-19 15:01:27.550790] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-1: reconfigured [2019-07-19 15:01:27.550808] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-2: option ping-timeout using set value 42 [2019-07-19 15:01:27.550838] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-2: option send-gids using set value true [2019-07-19 15:01:27.550850] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-2: reconfigured [2019-07-19 15:01:27.550862] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-3: option ping-timeout using set value 42 [2019-07-19 15:01:27.550878] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-3: option send-gids using set value true [2019-07-19 15:01:27.550885] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-3: reconfigured [2019-07-19 15:01:27.550894] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-4: option ping-timeout using set value 42 [2019-07-19 15:01:27.550910] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-4: option send-gids using set value true [2019-07-19 15:01:27.550917] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-4: reconfigured [2019-07-19 15:01:27.550926] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-5: option ping-timeout using set value 42 [2019-07-19 15:01:27.550942] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-5: option send-gids using set value true [2019-07-19 15:01:27.550949] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-5: reconfigured [2019-07-19 15:01:27.550973] I [MSGID: 0] [options.c:1240:xlator_option_reconf_int32] 0-storage-client-6: option ping-timeout using set value 42 [2019-07-19 15:01:27.550996] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-client-6: option send-gids using set value true [2019-07-19 15:01:27.551003] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-client-6: reconfigured [2019-07-19 15:01:27.551039] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-dht: option lock-migration using set value off [2019-07-19 15:01:27.551053] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-dht: option force-migration using set value off [2019-07-19 15:01:27.551088] D [MSGID: 0] [dht-shared.c:356:dht_init_regex] 0-storage-dht: using regex rsync-hash-regex = ^\.(.+)\.[^.]+$ [2019-07-19 15:01:27.551103] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-dht: reconfigured [2019-07-19 15:01:27.551120] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-write-behind: reconfigured [2019-07-19 15:01:27.551133] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-read-ahead: reconfigured [2019-07-19 15:01:27.551142] I [MSGID: 0] [options.c:1242:xlator_option_reconf_size_uint64] 0-storage-readdir-ahead: option rda-request-size using set value 131072 [2019-07-19 15:01:27.551154] I [MSGID: 0] [options.c:1242:xlator_option_reconf_size_uint64] 0-storage-readdir-ahead: option rda-cache-limit using set value 10MB [2019-07-19 15:01:27.551163] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage-readdir-ahead: option parallel-readdir using set value off [2019-07-19 15:01:27.551172] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-readdir-ahead: reconfigured [2019-07-19 15:01:27.551245] D [logging.c:1998:_gf_msg_internal] 0-logging-infra: Buffer overflow of a buffer whose size limit is 5. About to flush least recently used log message to disk [2019-07-19 15:01:27.551243] D [MSGID: 0] [io-cache.c:1595:check_cache_size_ok] 0-storage-io-cache: Max cache size is 67271393280 [2019-07-19 15:01:27.551244] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-io-cache: reconfigured [2019-07-19 15:01:27.551313] D [logging.c:1998:_gf_msg_internal] 0-logging-infra: Buffer overflow of a buffer whose size limit is 5. About to flush least recently used log message to disk [2019-07-19 15:01:27.551312] D [MSGID: 0] [quick-read.c:1183:check_cache_size_ok] 0-storage-quick-read: Max cache size is 67271393280 [2019-07-19 15:01:27.551313] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-quick-read: reconfigured [2019-07-19 15:01:27.551342] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-open-behind: reconfigured [2019-07-19 15:01:27.551372] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-md-cache: reconfigured [2019-07-19 15:01:27.551394] D [MSGID: 0] [options.c:1115:xlator_reconfigure_rec] 0-storage-io-threads: reconfigured [2019-07-19 15:01:27.551405] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage: option count-fop-hits using set value off [2019-07-19 15:01:27.551415] I [MSGID: 0] [options.c:1245:xlator_option_reconf_bool] 0-storage: option latency-measurement using set value off [2019-07-19 15:01:27.551434] I [MSGID: 0] [options.c:1236:xlator_option_reconf_str] 0-storage: option log-level using set value INFO [2019-07-19 15:01:27.551475] I [glusterfsd-mgmt.c:2005:mgmt_getspec_cbk] 0-glusterfs: No change in volfile,continuing
_______________________________________________ Gluster-users mailing list Gluster-users@xxxxxxxxxxx https://lists.gluster.org/mailman/listinfo/gluster-users
