[2017-08-24 03:11:41.781948] I [fuse-bridge.c:5888:fini] 0-fuse: Closing fuse connection to '/mnt/gluster-brick'.
[2017-08-24 03:12:03.416432] I [MSGID: 100030] [glusterfsd.c:2476:main] 0-/usr/sbin/glusterfs: Started running /usr/sbin/glusterfs version 3.11.3 (args: /usr/sbin/glusterfs --direct-io-mode=disable --volfile-server=storage-1 --volfile-server=storage-2 --volfile-id=/gv0 /mnt/gluster-brick)
[2017-08-24 03:12:03.423025] W [MSGID: 101002] [options.c:954:xl_opt_validate] 0-glusterfs: option 'address-family' is deprecated, preferred is 'transport.address-family', continuing with correction
[2017-08-24 03:12:03.424050] I [MSGID: 101190] [event-epoll.c:602:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2017-08-24 03:12:03.428972] I [afr.c:94:fix_quorum_options] 0-gv0-replicate-0: reindeer: incoming qtype = none
[2017-08-24 03:12:03.428992] I [afr.c:116:fix_quorum_options] 0-gv0-replicate-0: reindeer: quorum_count = 0
[2017-08-24 03:12:03.430356] I [MSGID: 101190] [event-epoll.c:602:event_dispatch_epoll_worker] 0-epoll: Started thread with index 2
[2017-08-24 03:12:03.430766] W [MSGID: 101174] [graph.c:361:_log_if_unknown_option] 0-gv0-readdir-ahead: option 'parallel-readdir' is not recognized
[2017-08-24 03:12:03.431053] I [MSGID: 114020] [client.c:2360:notify] 0-gv0-client-0: parent translators are ready, attempting connect on transport
[2017-08-24 03:12:03.431516] I [MSGID: 114020] [client.c:2360:notify] 0-gv0-client-1: parent translators are ready, attempting connect on transport
[2017-08-24 03:12:03.431794] I [rpc-clnt.c:2000:rpc_clnt_reconfig] 0-gv0-client-0: changing port to 49152 (from 0)
Final graph:
+------------------------------------------------------------------------------+
  1: volume gv0-client-0
  2:     type protocol/client
  3:     option ping-timeout 42
  4:     option remote-host storage-1
  5:     option remote-subvolume /data/brick
  6:     option transport-type socket
  7:     option transport.address-family inet
  8:     option username 17aeb9b3-ff6e-45f9-bf02-64abeedaa52a
  9:     option password 0a1d62d6-522c-4c3b-b2fe-1143be28e5f2
 10:     option transport.tcp-user-timeout 0
 11:     option transport.socket.keepalive-time 20
 12:     option transport.socket.keepalive-interval 2
 13:     option transport.socket.keepalive-count 9
 14:     option send-gids true
 15: end-volume
 16:  
 17: volume gv0-client-1
 18:     type protocol/client
 19:     option ping-timeout 42
 20:     option remote-host storage-2
 21:     option remote-subvolume /data/brick
 22:     option transport-type socket
 23:     option transport.address-family inet
 24:     option username 17aeb9b3-ff6e-45f9-bf02-64abeedaa52a
 25:     option password 0a1d62d6-522c-4c3b-b2fe-1143be28e5f2
 26:     option transport.tcp-user-timeout 0
 27:     option transport.socket.keepalive-time 20
 28:     option transport.socket.keepalive-interval 2
 29:     option transport.socket.keepalive-count 9
 30:     option send-gids true
 31: end-volume
 32:  
 33: volume gv0-replicate-0
 34:     type cluster/replicate
 35:     option afr-pending-xattr gv0-client-0,gv0-client-1
 36:     option use-compound-fops off
 37:     subvolumes gv0-client-0 gv0-client-1
 38: end-volume
 39:  
 40: volume gv0-dht
 41:     type cluster/distribute
 42:     option lock-migration off
 43:     subvolumes gv0-replicate-0
 44: end-volume
 45:  
 46: volume gv0-write-behind
 47:     type performance/write-behind
 48:     subvolumes gv0-dht
 49: end-volume
 50:  
 51: volume gv0-read-ahead
 52:     type performance/read-ahead
 53:     subvolumes gv0-write-behind
 54: end-volume
 55:  
 56: volume gv0-readdir-ahead
 57:     type performance/readdir-ahead
 58:     option parallel-readdir off
 59:     option rda-request-size 131072
 60:     option rda-cache-limit 10MB
 61:     subvolumes gv0-read-ahead
 62: end-volume
 63:  
 64: volume gv0-io-cache
 65:     type performance/io-cache
 66:     subvolumes gv0-readdir-ahead
 67: end-volume
 68:  
 69: volume gv0-quick-read
 70:     type performance/quick-read
 71:     subvolumes gv0-io-cache
 72: end-volume
 73:  
 74: volume gv0-open-behind
 75:     type performance/open-behind
 76:     subvolumes gv0-quick-read
 77: end-volume
 78:  
 79: volume gv0-md-cache
 80:     type performance/md-cache
 81:     subvolumes gv0-open-behind
 82: end-volume
 83:  
 84: volume gv0
 85:     type debug/io-stats
 86:     option log-level INFO
 87:     option latency-measurement off
 88:     option count-fop-hits off
 89:     subvolumes gv0-md-cache
 90: end-volume
 91:  
 92: volume meta-autoload
[2017-08-24 03:12:03.432348] I [MSGID: 114057] [client-handshake.c:1451:select_server_supported_programs] 0-gv0-client-0: Using Program GlusterFS 3.3, Num (1298437), Version (330)
 93:     type meta
 94:     subvolumes gv0
 95: end-volume
 96:  
+------------------------------------------------------------------------------+
[2017-08-24 03:12:03.432755] I [rpc-clnt.c:2000:rpc_clnt_reconfig] 0-gv0-client-1: changing port to 49152 (from 0)
[2017-08-24 03:12:03.432897] I [MSGID: 114046] [client-handshake.c:1216:client_setvolume_cbk] 0-gv0-client-0: Connected to gv0-client-0, attached to remote volume '/data/brick'.
[2017-08-24 03:12:03.432920] I [MSGID: 114047] [client-handshake.c:1227:client_setvolume_cbk] 0-gv0-client-0: Server and Client lk-version numbers are not same, reopening the fds
[2017-08-24 03:12:03.433008] I [MSGID: 108005] [afr-common.c:4879:__afr_handle_child_up_event] 0-gv0-replicate-0: Subvolume 'gv0-client-0' came back up; going online.
[2017-08-24 03:12:03.433048] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-gv0-client-0: Server lk version = 1
[2017-08-24 03:12:03.433442] I [MSGID: 114057] [client-handshake.c:1451:select_server_supported_programs] 0-gv0-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2017-08-24 03:12:03.434060] I [MSGID: 114046] [client-handshake.c:1216:client_setvolume_cbk] 0-gv0-client-1: Connected to gv0-client-1, attached to remote volume '/data/brick'.
[2017-08-24 03:12:03.434081] I [MSGID: 114047] [client-handshake.c:1227:client_setvolume_cbk] 0-gv0-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2017-08-24 03:12:03.434310] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-gv0-client-1: Server lk version = 1
[2017-08-24 03:12:03.436045] I [fuse-bridge.c:4202:fuse_init] 0-glusterfs-fuse: FUSE inited with protocol versions: glusterfs 7.24 kernel 7.23
[2017-08-24 03:12:03.436087] I [fuse-bridge.c:4887:fuse_graph_sync] 0-fuse: switched to graph 0
[2017-08-24 03:12:03.437256] I [MSGID: 108031] [afr-common.c:2349:afr_local_discovery_cbk] 0-gv0-replicate-0: selecting local read_child gv0-client-0
[2017-08-24 03:13:28.428210] C [rpc-clnt-ping.c:166:rpc_clnt_ping_timer_expired] 0-gv0-client-1: server 192.168.212.12:49152 has not responded in the last 42 seconds, disconnecting.
[2017-08-24 03:13:28.428445] I [MSGID: 114018] [client.c:2285:client_rpc_notify] 0-gv0-client-1: disconnected from gv0-client-1. Client process will keep trying to connect to glusterd until brick's port is available
[2017-08-24 03:13:28.429066] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GlusterFS 3.3) op(WRITE(13)) called at 2017-08-24 03:12:45.477910 (xid=0x1f)
[2017-08-24 03:13:28.429108] W [MSGID: 114031] [client-rpc-fops.c:853:client3_3_writev_cbk] 0-gv0-client-1: remote operation failed [Transport endpoint is not connected]
[2017-08-24 03:13:28.429509] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GlusterFS 3.3) op(WRITE(13)) called at 2017-08-24 03:12:45.478015 (xid=0x20)
[2017-08-24 03:13:28.429546] W [MSGID: 114031] [client-rpc-fops.c:853:client3_3_writev_cbk] 0-gv0-client-1: remote operation failed [Transport endpoint is not connected]
[2017-08-24 03:13:28.429641] W [MSGID: 114061] [client-common.c:459:client_pre_fsync] 0-gv0-client-1:  (af82085b-2526-4c5f-925a-1c721c73b37a) remote_fd is -1. EBADFD [File descriptor in bad state]
[2017-08-24 03:13:28.429703] W [MSGID: 108035] [afr-transaction.c:2244:afr_changelog_fsync_cbk] 0-gv0-replicate-0: fsync(af82085b-2526-4c5f-925a-1c721c73b37a) failed on subvolume gv0-client-1. Transaction was WRITE [File descriptor in bad state]
[2017-08-24 03:13:28.429961] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GlusterFS 3.3) op(WRITE(13)) called at 2017-08-24 03:12:45.478264 (xid=0x21)
[2017-08-24 03:13:28.430000] W [MSGID: 114031] [client-rpc-fops.c:853:client3_3_writev_cbk] 0-gv0-client-1: remote operation failed [Transport endpoint is not connected]
[2017-08-24 03:13:28.430104] W [MSGID: 114061] [client-common.c:459:client_pre_fsync] 0-gv0-client-1:  (af82085b-2526-4c5f-925a-1c721c73b37a) remote_fd is -1. EBADFD [File descriptor in bad state]
[2017-08-24 03:13:28.430141] W [MSGID: 108035] [afr-transaction.c:2244:afr_changelog_fsync_cbk] 0-gv0-replicate-0: fsync(af82085b-2526-4c5f-925a-1c721c73b37a) failed on subvolume gv0-client-1. Transaction was WRITE [File descriptor in bad state]
[2017-08-24 03:13:28.430375] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GlusterFS 3.3) op(WRITE(13)) called at 2017-08-24 03:12:45.478718 (xid=0x22)
[2017-08-24 03:13:28.430407] W [MSGID: 114031] [client-rpc-fops.c:853:client3_3_writev_cbk] 0-gv0-client-1: remote operation failed [Transport endpoint is not connected]
[2017-08-24 03:13:28.430616] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GlusterFS 3.3) op(WRITE(13)) called at 2017-08-24 03:12:45.479297 (xid=0x23)
[2017-08-24 03:13:28.430830] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GlusterFS 3.3) op(WRITE(13)) called at 2017-08-24 03:12:45.479675 (xid=0x24)
[2017-08-24 03:13:28.431043] E [rpc-clnt.c:365:saved_frames_unwind] (--> /usr/lib/x86_64-linux-gnu/libglusterfs.so.0(_gf_log_callingfn+0x13b)[0x7fa42588ebcb] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_unwind+0x1d1)[0x7fa425656f91] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(saved_frames_destroy+0xe)[0x7fa4256570ae] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x94)[0x7fa425658734] (--> /usr/lib/x86_64-linux-gnu/libgfrpc.so.0(rpc_clnt_notify+0x288)[0x7fa425659218] ))))) 0-gv0-client-1: forced unwinding frame type(GF-DUMP) op(NULL(2)) called at 2017-08-24 03:12:46.423543 (xid=0x25)
[2017-08-24 03:13:28.431067] W [rpc-clnt-ping.c:223:rpc_clnt_ping_cbk] 0-gv0-client-1: socket disconnected
The message "W [MSGID: 114031] [client-rpc-fops.c:853:client3_3_writev_cbk] 0-gv0-client-1: remote operation failed [Transport endpoint is not connected]" repeated 2 times between [2017-08-24 03:13:28.430407] and [2017-08-24 03:13:28.430855]
[2017-08-24 03:13:28.462669] E [MSGID: 114031] [client-rpc-fops.c:1593:client3_3_finodelk_cbk] 0-gv0-client-1: remote operation failed [Transport endpoint is not connected]
[2017-08-24 03:13:39.431044] I [rpc-clnt.c:2000:rpc_clnt_reconfig] 0-gv0-client-1: changing port to 49152 (from 0)
[2017-08-24 03:13:39.432110] I [MSGID: 114057] [client-handshake.c:1451:select_server_supported_programs] 0-gv0-client-1: Using Program GlusterFS 3.3, Num (1298437), Version (330)
[2017-08-24 03:13:39.433268] I [MSGID: 114046] [client-handshake.c:1216:client_setvolume_cbk] 0-gv0-client-1: Connected to gv0-client-1, attached to remote volume '/data/brick'.
[2017-08-24 03:13:39.433311] I [MSGID: 114047] [client-handshake.c:1227:client_setvolume_cbk] 0-gv0-client-1: Server and Client lk-version numbers are not same, reopening the fds
[2017-08-24 03:13:39.433338] I [MSGID: 114042] [client-handshake.c:1047:client_post_handshake] 0-gv0-client-1: 1 fds open - Delaying child_up until they are re-opened
[2017-08-24 03:13:39.433933] I [MSGID: 114041] [client-handshake.c:678:client_child_up_reopen_done] 0-gv0-client-1: last fd open'd/lock-self-heal'd - notifying CHILD-UP
[2017-08-24 03:13:39.434249] I [MSGID: 114035] [client-handshake.c:202:client_set_lk_version_cbk] 0-gv0-client-1: Server lk version = 1