[2020-06-16 05:14:25.259400] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "17.18.11.14"
[2020-06-16 05:14:25.259457] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:ff0af68b-bc9a-4269-8589-f7ef60e27f5e-GRAPH_ID:0-PID:100753-HOST:node4-PC_NAME:glustervol-client-2-RECON_NO:-0 (version: 6.9) with subvol /data
[2020-06-16 05:16:06.512996] W [socket.c:775:__socket_rwv] 0-glusterfs: readv on 49.11.99.79:24007 failed (No data available)
[2020-06-16 05:16:06.513058] I [glusterfsd-mgmt.c:2719:mgmt_rpc_notify] 0-glusterfsd-mgmt: disconnected from remote-host: node3
[2020-06-16 05:16:06.513065] I [glusterfsd-mgmt.c:2739:mgmt_rpc_notify] 0-glusterfsd-mgmt: Exhausted all volfile servers
[2020-06-16 05:16:07.250816] W [glusterfsd.c:1596:cleanup_and_exit] (-->/lib64/libpthread.so.0(+0x7ea5) [0x7f86210a6ea5] -->/usr/sbin/glusterfsd(glusterfs_sigwaiter+0xe5) [0x55dd9c536625] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b) [0x55dd9c53648b] ) 0-: received signum (15), shutting down
[2020-06-16 05:16:07.291970] E [socket.c:3636:socket_connect] 0-glusterfs: connection attempt on 49.11.99.79:24007 failed, (Network is unreachable)
[2020-06-16 05:16:07.292123] W [rpc-clnt.c:1698:rpc_clnt_submit] 0-glusterfs: error returned while attempting to connect to host:(null), port:0
[2020-06-16 05:16:07.292362] I [timer.c:86:gf_timer_call_cancel] (-->/lib64/libgfrpc.so.0(+0xf4e8) [0x7f862200d4e8] -->/lib64/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x6e) [0x7f862200c8fe] -->/lib64/libglusterfs.so.0(gf_timer_call_cancel+0x149) [0x7f8622272f79] ) 0-timer: ctx cleanup started
[2020-06-16 05:16:07.292431] E [timer.c:34:gf_timer_call_after] (-->/lib64/libgfrpc.so.0(rpc_transport_notify+0x23) [0x7f8622009a93] -->/lib64/libgfrpc.so.0(+0xf512) [0x7f862200d512] -->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7f8622272cb9] ) 0-timer: Either ctx is NULL or ctx cleanup started [Invalid argument]
[2020-06-16 05:16:07.292448] W [rpc-clnt.c:850:rpc_clnt_handle_disconnect] 0-glusterfs: Cannot create rpc_clnt_reconnect timer
[2020-06-16 05:16:45.105033] I [MSGID: 100030] [glusterfsd.c:2867:main] 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 7.5 (args: /usr/sbin/glusterfsd -s node3 --volfile-id glustervol.node3.data -p /var/run/gluster/vols/glustervol/node3-data.pid -S /var/run/gluster/21fe7411872148f7.socket --brick-name /data -l /var/log/glusterfs/bricks/data.log --xlator-option *-posix.glusterd-uuid=ec137af6-4845-4ebb-955a-fac1d19b7b6c --process-name brick --brick-port 49152 --xlator-option glustervol-server.listen-port=49152)
[2020-06-16 05:16:45.106276] I [glusterfsd.c:2594:daemonize] 0-glusterfs: Pid of current running process is 2304
[2020-06-16 05:16:45.111777] I [socket.c:958:__socket_server_bind] 0-socket.glusterfsd: closing (AF_UNIX) reuse check socket 9
[2020-06-16 05:16:45.116766] I [MSGID: 101190] [event-epoll.c:682:event_dispatch_epoll_worker] 0-epoll: Started thread with index 0
[2020-06-16 05:16:45.116830] I [MSGID: 101190] [event-epoll.c:682:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2020-06-16 05:16:46.159277] I [rpcsvc.c:2690:rpcsvc_set_outstanding_rpc_limit] 0-rpc-service: Configured rpc.outstanding-rpc-limit with value 64
[2020-06-16 05:16:46.167295] I [socket.c:958:__socket_server_bind] 0-socket.glustervol-changelog: closing (AF_UNIX) reuse check socket 14
[2020-06-16 05:16:46.167589] I [trash.c:2450:init] 0-glustervol-trash: no option specified for 'eliminate', using NULL
Final graph:
+------------------------------------------------------------------------------+
  1: volume glustervol-posix
  2:     type storage/posix
  3:     option glusterd-uuid ec137af6-4845-4ebb-955a-fac1d19b7b6c
  4:     option directory /data
  5:     option volume-id 5422bb27-1863-47d5-b216-61751a01b759
  6:     option fips-mode-rchecksum on
  7:     option shared-brick-count 1
  8:     option reserve-size 0
  9: end-volume
 10:
 11: volume glustervol-trash
 12:     type features/trash
 13:     option trash-dir .trashcan
 14:     option brick-path /data
 15:     option trash-internal-op off
 16:     subvolumes glustervol-posix
 17: end-volume
 18:
 19: volume glustervol-changelog
 20:     type features/changelog
 21:     option changelog-brick /data
 22:     option changelog-dir /data/.glusterfs/changelogs
 23:     option changelog-barrier-timeout 120
 24:     subvolumes glustervol-trash
 25: end-volume
 26:
 27: volume glustervol-bitrot-stub
 28:     type features/bitrot-stub
 29:     option export /data
 30:     option bitrot disable
 31:     subvolumes glustervol-changelog
 32: end-volume
 33:
 34: volume glustervol-access-control
 35:     type features/access-control
 36:     subvolumes glustervol-bitrot-stub
 37: end-volume
 38:
 39: volume glustervol-locks
 40:     type features/locks
 41:     option enforce-mandatory-lock off
 42:     subvolumes glustervol-access-control
 43: end-volume
 44:
 45: volume glustervol-worm
 46:     type features/worm
 47:     option worm off
 48:     option worm-file-level off
 49:     option worm-files-deletable on
 50:     subvolumes glustervol-locks
 51: end-volume
 52:
 53: volume glustervol-read-only
 54:     type features/read-only
 55:     option read-only off
 56:     subvolumes glustervol-worm
 57: end-volume
 58:
 59: volume glustervol-leases
 60:     type features/leases
 61:     option leases off
 62:     subvolumes glustervol-read-only
 63: end-volume
 64:
 65: volume glustervol-upcall
 66:     type features/upcall
 67:     option cache-invalidation off
 68:     subvolumes glustervol-leases
 69: end-volume
 70:
 71: volume glustervol-io-threads
 72:     type performance/io-threads
 73:     subvolumes glustervol-upcall
 74: end-volume
 75:
 76: volume glustervol-selinux
 77:     type features/selinux
 78:     option selinux on
 79:     subvolumes glustervol-io-threads
 80: end-volume
 81:
 82: volume glustervol-marker
 83:     type features/marker
 84:     option volume-uuid 5422bb27-1863-47d5-b216-61751a01b759
 85:     option timestamp-file /var/lib/glusterd/vols/glustervol/marker.tstamp
 86:     option quota-version 0
 87:     option xtime off
 88:     option gsync-force-xtime off
 89:     option quota off
 90:     option inode-quota off
 91:     subvolumes glustervol-selinux
 92: end-volume
 93:
 94: volume glustervol-barrier
 95:     type features/barrier
 96:     option barrier disable
 97:     option barrier-timeout 120
 98:     subvolumes glustervol-marker
 99: end-volume
100:
101: volume glustervol-index
102:     type features/index
103:     option index-base /data/.glusterfs/indices
104:     option xattrop-dirty-watchlist trusted.afr.dirty
105:     option xattrop-pending-watchlist trusted.afr.glustervol-
106:     subvolumes glustervol-barrier
107: end-volume
108:
109: volume glustervol-quota
110:     type features/quota
111:     option volume-uuid glustervol
112:     option server-quota off
113:     option deem-statfs off
114:     subvolumes glustervol-index
115: end-volume
116:
117: volume /data
118:     type debug/io-stats
119:     option auth.addr./data.allow *
120:     option auth-path /data
121:     option auth.login.15e6dd85-fb90-4caf-971c-625ffec71229.password 6e2f0194-c002-439f-b6c8-6b1d71af66b8
122:     option auth.login./data.allow 15e6dd85-fb90-4caf-971c-625ffec71229
123:     option unique-id /data
124:     option log-level INFO
125:     option threads 16
126:     option latency-measurement off
127:     option count-fop-hits off
128:     option global-threading off
129:     subvolumes glustervol-quota
130: end-volume
131:
132: volume glustervol-server
133:     type protocol/server
134:     option transport.socket.listen-port 49152
135:     option rpc-auth.auth-glusterfs on
136:     option rpc-auth.auth-unix on
137:     option rpc-auth.auth-null on
138:     option rpc-auth-allow-insecure on
139:     option transport-type tcp
140:     option transport.address-family inet
141:     option auth.login./data.allow 15e6dd85-fb90-4caf-971c-625ffec71229
142:     option auth.login.15e6dd85-fb90-4caf-971c-625ffec71229.password 6e2f0194-c002-439f-b6c8-6b1d71af66b8
143:     option auth-path /data
144:     option auth.addr./data.allow *
145:     option transport.socket.keepalive 1
146:     option transport.socket.ssl-enabled off
147:     option transport.socket.keepalive-time 20
148:     option transport.socket.keepalive-interval 2
149:     option transport.socket.keepalive-count 9
150:     option transport.listen-backlog 1024
151:     subvolumes /data
152: end-volume
153:
+------------------------------------------------------------------------------+
[2020-06-16 05:16:47.112007] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "49.141.98.19"
[2020-06-16 05:16:47.112062] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 05:16:47.112077] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:8a2e528e-502d-4150-a994-1e1d44e17946-GRAPH_ID:0-PID:59760-HOST:node1-PC_NAME:glustervol-client-2-RECON_NO:-2 (version: 7.5) with subvol /data
[2020-06-16 05:16:47.114981] I [rpcsvc.c:866:rpcsvc_handle_rpc_call] 0-rpc-service: spawned a request handler thread for queue 0
[2020-06-16 05:16:47.116788] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "13.18.11.12"
[2020-06-16 05:16:47.116839] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 05:16:47.116850] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:5e6c8d94-95f8-4c61-a411-8649caa0f04c-GRAPH_ID:0-PID:3519-HOST:node2-PC_NAME:glustervol-client-2-RECON_NO:-2 (version: 7.5) with subvol /data
[2020-06-16 05:16:49.165088] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "49.11.99.79"
[2020-06-16 05:16:49.165137] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 05:16:49.165157] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:9628bd0e-07f7-4fba-abee-fe10bdd87944-GRAPH_ID:0-PID:2316-HOST:node3-PC_NAME:glustervol-client-2-RECON_NO:-0 (version: 7.5) with subvol /data
[2020-06-16 05:16:49.166590] I [rpcsvc.c:866:rpcsvc_handle_rpc_call] 0-rpc-service: spawned a request handler thread for queue 1
[2020-06-16 05:17:10.406294] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "17.18.11.14"
[2020-06-16 05:17:10.406348] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:ff0af68b-bc9a-4269-8589-f7ef60e27f5e-GRAPH_ID:0-PID:100753-HOST:node4-PC_NAME:glustervol-client-2-RECON_NO:-1 (version: 6.9) with subvol /data
[2020-06-16 06:17:08.352198] W [socket.c:775:__socket_rwv] 0-glusterfs: readv on 49.11.99.79:24007 failed (No data available)
[2020-06-16 06:17:08.352252] I [glusterfsd-mgmt.c:2719:mgmt_rpc_notify] 0-glusterfsd-mgmt: disconnected from remote-host: node3
[2020-06-16 06:17:08.352260] I [glusterfsd-mgmt.c:2739:mgmt_rpc_notify] 0-glusterfsd-mgmt: Exhausted all volfile servers
[2020-06-16 06:17:09.128929] W [glusterfsd.c:1596:cleanup_and_exit] (-->/lib64/libpthread.so.0(+0x7ea5) [0x7ff5125c0ea5] -->/usr/sbin/glusterfsd(glusterfs_sigwaiter+0xe5) [0x562d2c1f5625] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b) [0x562d2c1f548b] ) 0-: received signum (15), shutting down
[2020-06-16 06:17:09.177491] E [socket.c:3636:socket_connect] 0-glusterfs: connection attempt on 49.11.99.79:24007 failed, (Network is unreachable)
[2020-06-16 06:17:09.177602] W [rpc-clnt.c:1698:rpc_clnt_submit] 0-glusterfs: error returned while attempting to connect to host:(null), port:0
[2020-06-16 06:17:09.177743] I [timer.c:86:gf_timer_call_cancel] (-->/lib64/libgfrpc.so.0(+0xf4e8) [0x7ff5135274e8] -->/lib64/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x6e) [0x7ff5135268fe] -->/lib64/libglusterfs.so.0(gf_timer_call_cancel+0x149) [0x7ff51378cf79] ) 0-timer: ctx cleanup started
[2020-06-16 06:17:09.177789] E [timer.c:34:gf_timer_call_after] (-->/lib64/libgfrpc.so.0(rpc_transport_notify+0x23) [0x7ff513523a93] -->/lib64/libgfrpc.so.0(+0xf512) [0x7ff513527512] -->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7ff51378ccb9] ) 0-timer: Either ctx is NULL or ctx cleanup started [Invalid argument]
[2020-06-16 06:17:09.177800] W [rpc-clnt.c:850:rpc_clnt_handle_disconnect] 0-glusterfs: Cannot create rpc_clnt_reconnect timer
[2020-06-16 06:17:46.088080] I [MSGID: 100030] [glusterfsd.c:2867:main] 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 7.5 (args: /usr/sbin/glusterfsd -s node3 --volfile-id glustervol.node3.data -p /var/run/gluster/vols/glustervol/node3-data.pid -S /var/run/gluster/21fe7411872148f7.socket --brick-name /data -l /var/log/glusterfs/bricks/data.log --xlator-option *-posix.glusterd-uuid=ec137af6-4845-4ebb-955a-fac1d19b7b6c --process-name brick --brick-port 49152 --xlator-option glustervol-server.listen-port=49152)
[2020-06-16 06:17:46.089241] I [glusterfsd.c:2594:daemonize] 0-glusterfs: Pid of current running process is 1885
[2020-06-16 06:17:46.099072] I [socket.c:958:__socket_server_bind] 0-socket.glusterfsd: closing (AF_UNIX) reuse check socket 9
[2020-06-16 06:17:46.104753] I [MSGID: 101190] [event-epoll.c:682:event_dispatch_epoll_worker] 0-epoll: Started thread with index 0
[2020-06-16 06:17:46.104804] I [MSGID: 101190] [event-epoll.c:682:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2020-06-16 06:17:47.152209] I [rpcsvc.c:2690:rpcsvc_set_outstanding_rpc_limit] 0-rpc-service: Configured rpc.outstanding-rpc-limit with value 64
[2020-06-16 06:17:47.160983] I [socket.c:958:__socket_server_bind] 0-socket.glustervol-changelog: closing (AF_UNIX) reuse check socket 14
[2020-06-16 06:17:47.161238] I [trash.c:2450:init] 0-glustervol-trash: no option specified for 'eliminate', using NULL
Final graph:
+------------------------------------------------------------------------------+
  1: volume glustervol-posix
  2:     type storage/posix
  3:     option glusterd-uuid ec137af6-4845-4ebb-955a-fac1d19b7b6c
  4:     option directory /data
  5:     option volume-id 5422bb27-1863-47d5-b216-61751a01b759
  6:     option fips-mode-rchecksum on
  7:     option shared-brick-count 1
  8:     option reserve-size 0
  9: end-volume
 10:
 11: volume glustervol-trash
 12:     type features/trash
 13:     option trash-dir .trashcan
 14:     option brick-path /data
 15:     option trash-internal-op off
 16:     subvolumes glustervol-posix
 17: end-volume
 18:
 19: volume glustervol-changelog
 20:     type features/changelog
 21:     option changelog-brick /data
 22:     option changelog-dir /data/.glusterfs/changelogs
 23:     option changelog-barrier-timeout 120
 24:     subvolumes glustervol-trash
 25: end-volume
 26:
 27: volume glustervol-bitrot-stub
 28:     type features/bitrot-stub
 29:     option export /data
 30:     option bitrot disable
 31:     subvolumes glustervol-changelog
 32: end-volume
 33:
 34: volume glustervol-access-control
 35:     type features/access-control
 36:     subvolumes glustervol-bitrot-stub
 37: end-volume
 38:
 39: volume glustervol-locks
 40:     type features/locks
 41:     option enforce-mandatory-lock off
 42:     subvolumes glustervol-access-control
 43: end-volume
 44:
 45: volume glustervol-worm
 46:     type features/worm
 47:     option worm off
 48:     option worm-file-level off
 49:     option worm-files-deletable on
 50:     subvolumes glustervol-locks
 51: end-volume
 52:
 53: volume glustervol-read-only
 54:     type features/read-only
 55:     option read-only off
 56:     subvolumes glustervol-worm
 57: end-volume
 58:
 59: volume glustervol-leases
 60:     type features/leases
 61:     option leases off
 62:     subvolumes glustervol-read-only
 63: end-volume
 64:
 65: volume glustervol-upcall
 66:     type features/upcall
 67:     option cache-invalidation off
 68:     subvolumes glustervol-leases
 69: end-volume
 70:
 71: volume glustervol-io-threads
 72:     type performance/io-threads
 73:     subvolumes glustervol-upcall
 74: end-volume
 75:
 76: volume glustervol-selinux
 77:     type features/selinux
 78:     option selinux on
 79:     subvolumes glustervol-io-threads
 80: end-volume
 81:
 82: volume glustervol-marker
 83:     type features/marker
 84:     option volume-uuid 5422bb27-1863-47d5-b216-61751a01b759
 85:     option timestamp-file /var/lib/glusterd/vols/glustervol/marker.tstamp
 86:     option quota-version 0
 87:     option xtime off
 88:     option gsync-force-xtime off
 89:     option quota off
 90:     option inode-quota off
 91:     subvolumes glustervol-selinux
 92: end-volume
 93:
 94: volume glustervol-barrier
 95:     type features/barrier
 96:     option barrier disable
 97:     option barrier-timeout 120
 98:     subvolumes glustervol-marker
 99: end-volume
100:
101: volume glustervol-index
102:     type features/index
103:     option index-base /data/.glusterfs/indices
104:     option xattrop-dirty-watchlist trusted.afr.dirty
105:     option xattrop-pending-watchlist trusted.afr.glustervol-
106:     subvolumes glustervol-barrier
107: end-volume
108:
109: volume glustervol-quota
110:     type features/quota
111:     option volume-uuid glustervol
112:     option server-quota off
113:     option deem-statfs off
114:     subvolumes glustervol-index
115: end-volume
116:
117: volume /data
118:     type debug/io-stats
119:     option auth.addr./data.allow *
120:     option auth-path /data
121:     option auth.login.15e6dd85-fb90-4caf-971c-625ffec71229.password 6e2f0194-c002-439f-b6c8-6b1d71af66b8
122:     option auth.login./data.allow 15e6dd85-fb90-4caf-971c-625ffec71229
123:     option unique-id /data
124:     option log-level INFO
125:     option threads 16
126:     option latency-measurement off
127:     option count-fop-hits off
128:     option global-threading off
129:     subvolumes glustervol-quota
130: end-volume
131:
132: volume glustervol-server
133:     type protocol/server
134:     option transport.socket.listen-port 49152
135:     option rpc-auth.auth-glusterfs on
136:     option rpc-auth.auth-unix on
137:     option rpc-auth.auth-null on
138:     option rpc-auth-allow-insecure on
139:     option transport-type tcp
140:     option transport.address-family inet
141:     option auth.login./data.allow 15e6dd85-fb90-4caf-971c-625ffec71229
142:     option auth.login.15e6dd85-fb90-4caf-971c-625ffec71229.password 6e2f0194-c002-439f-b6c8-6b1d71af66b8
143:     option auth-path /data
144:     option auth.addr./data.allow *
145:     option transport.socket.keepalive 1
146:     option transport.socket.ssl-enabled off
147:     option transport.socket.keepalive-time 20
148:     option transport.socket.keepalive-interval 2
149:     option transport.socket.keepalive-count 9
150:     option transport.listen-backlog 1024
151:     subvolumes /data
152: end-volume
153:
+------------------------------------------------------------------------------+
[2020-06-16 06:17:49.097093] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "49.141.98.19"
[2020-06-16 06:17:49.097164] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 06:17:49.097191] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:8a2e528e-502d-4150-a994-1e1d44e17946-GRAPH_ID:0-PID:59760-HOST:node1-PC_NAME:glustervol-client-2-RECON_NO:-3 (version: 7.5) with subvol /data
[2020-06-16 06:17:49.100424] I [rpcsvc.c:866:rpcsvc_handle_rpc_call] 0-rpc-service: spawned a request handler thread for queue 0
[2020-06-16 06:17:49.233948] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "13.18.11.12"
[2020-06-16 06:17:49.234002] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 06:17:49.234014] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:5e6c8d94-95f8-4c61-a411-8649caa0f04c-GRAPH_ID:0-PID:3519-HOST:node2-PC_NAME:glustervol-client-2-RECON_NO:-3 (version: 7.5) with subvol /data
[2020-06-16 06:17:49.235248] I [rpcsvc.c:866:rpcsvc_handle_rpc_call] 0-rpc-service: spawned a request handler thread for queue 1
[2020-06-16 06:17:50.146556] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "49.11.99.79"
[2020-06-16 06:17:50.146584] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 06:17:50.146594] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:464d653b-1c3c-46e2-bfe0-d26c59860370-GRAPH_ID:0-PID:1894-HOST:node3-PC_NAME:glustervol-client-2-RECON_NO:-0 (version: 7.5) with subvol /data
[2020-06-16 06:18:04.226614] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "17.18.11.14"
[2020-06-16 06:18:04.226674] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:ff0af68b-bc9a-4269-8589-f7ef60e27f5e-GRAPH_ID:0-PID:100753-HOST:node4-PC_NAME:glustervol-client-2-RECON_NO:-2 (version: 6.9) with subvol /data
[2020-06-16 07:04:45.629024] W [socket.c:775:__socket_rwv] 0-glusterfs: readv on 49.11.99.79:24007 failed (No data available)
[2020-06-16 07:04:45.629121] I [glusterfsd-mgmt.c:2719:mgmt_rpc_notify] 0-glusterfsd-mgmt: disconnected from remote-host: node3
[2020-06-16 07:04:45.629129] I [glusterfsd-mgmt.c:2739:mgmt_rpc_notify] 0-glusterfsd-mgmt: Exhausted all volfile servers
[2020-06-16 07:04:46.396489] W [glusterfsd.c:1596:cleanup_and_exit] (-->/lib64/libpthread.so.0(+0x7ea5) [0x7fcd9d6afea5] -->/usr/sbin/glusterfsd(glusterfs_sigwaiter+0xe5) [0x56175628a625] -->/usr/sbin/glusterfsd(cleanup_and_exit+0x6b) [0x56175628a48b] ) 0-: received signum (15), shutting down
[2020-06-16 07:04:46.439188] E [socket.c:3636:socket_connect] 0-glusterfs: connection attempt on 49.11.99.79:24007 failed, (Network is unreachable)
[2020-06-16 07:04:46.439286] W [rpc-clnt.c:1698:rpc_clnt_submit] 0-glusterfs: error returned while attempting to connect to host:(null), port:0
[2020-06-16 07:04:46.439444] I [timer.c:86:gf_timer_call_cancel] (-->/lib64/libgfrpc.so.0(+0xf4e8) [0x7fcd9e6164e8] -->/lib64/libgfrpc.so.0(rpc_clnt_connection_cleanup+0x6e) [0x7fcd9e6158fe] -->/lib64/libglusterfs.so.0(gf_timer_call_cancel+0x149) [0x7fcd9e87bf79] ) 0-timer: ctx cleanup started
[2020-06-16 07:04:46.439489] E [timer.c:34:gf_timer_call_after] (-->/lib64/libgfrpc.so.0(rpc_transport_notify+0x23) [0x7fcd9e612a93] -->/lib64/libgfrpc.so.0(+0xf512) [0x7fcd9e616512] -->/lib64/libglusterfs.so.0(gf_timer_call_after+0x229) [0x7fcd9e87bcb9] ) 0-timer: Either ctx is NULL or ctx cleanup started [Invalid argument]
[2020-06-16 07:04:46.439500] W [rpc-clnt.c:850:rpc_clnt_handle_disconnect] 0-glusterfs: Cannot create rpc_clnt_reconnect timer
[2020-06-16 07:05:22.086336] I [MSGID: 100030] [glusterfsd.c:2867:main] 0-/usr/sbin/glusterfsd: Started running /usr/sbin/glusterfsd version 7.5 (args: /usr/sbin/glusterfsd -s node3 --volfile-id glustervol.node3.data -p /var/run/gluster/vols/glustervol/node3-data.pid -S /var/run/gluster/21fe7411872148f7.socket --brick-name /data -l /var/log/glusterfs/bricks/data.log --xlator-option *-posix.glusterd-uuid=ec137af6-4845-4ebb-955a-fac1d19b7b6c --process-name brick --brick-port 49152 --xlator-option glustervol-server.listen-port=49152)
[2020-06-16 07:05:22.087644] I [glusterfsd.c:2594:daemonize] 0-glusterfs: Pid of current running process is 1973
[2020-06-16 07:05:22.093003] I [socket.c:958:__socket_server_bind] 0-socket.glusterfsd: closing (AF_UNIX) reuse check socket 9
[2020-06-16 07:05:22.100197] I [MSGID: 101190] [event-epoll.c:682:event_dispatch_epoll_worker] 0-epoll: Started thread with index 0
[2020-06-16 07:05:22.100282] I [MSGID: 101190] [event-epoll.c:682:event_dispatch_epoll_worker] 0-epoll: Started thread with index 1
[2020-06-16 07:05:23.143298] I [rpcsvc.c:2690:rpcsvc_set_outstanding_rpc_limit] 0-rpc-service: Configured rpc.outstanding-rpc-limit with value 64
[2020-06-16 07:05:23.149723] I [socket.c:958:__socket_server_bind] 0-socket.glustervol-changelog: closing (AF_UNIX) reuse check socket 14
[2020-06-16 07:05:23.149998] I [trash.c:2450:init] 0-glustervol-trash: no option specified for 'eliminate', using NULL
Final graph:
+------------------------------------------------------------------------------+
  1: volume glustervol-posix
  2:     type storage/posix
  3:     option glusterd-uuid ec137af6-4845-4ebb-955a-fac1d19b7b6c
  4:     option directory /data
  5:     option volume-id 5422bb27-1863-47d5-b216-61751a01b759
  6:     option fips-mode-rchecksum on
  7:     option shared-brick-count 1
  8:     option reserve-size 0
  9: end-volume
 10:
 11: volume glustervol-trash
 12:     type features/trash
 13:     option trash-dir .trashcan
 14:     option brick-path /data
 15:     option trash-internal-op off
 16:     subvolumes glustervol-posix
 17: end-volume
 18:
 19: volume glustervol-changelog
 20:     type features/changelog
 21:     option changelog-brick /data
 22:     option changelog-dir /data/.glusterfs/changelogs
 23:     option changelog-barrier-timeout 120
 24:     subvolumes glustervol-trash
 25: end-volume
 26:
 27: volume glustervol-bitrot-stub
 28:     type features/bitrot-stub
 29:     option export /data
 30:     option bitrot disable
 31:     subvolumes glustervol-changelog
 32: end-volume
 33:
 34: volume glustervol-access-control
 35:     type features/access-control
 36:     subvolumes glustervol-bitrot-stub
 37: end-volume
 38:
 39: volume glustervol-locks
 40:     type features/locks
 41:     option enforce-mandatory-lock off
 42:     subvolumes glustervol-access-control
 43: end-volume
 44:
 45: volume glustervol-worm
 46:     type features/worm
 47:     option worm off
 48:     option worm-file-level off
 49:     option worm-files-deletable on
 50:     subvolumes glustervol-locks
 51: end-volume
 52:
 53: volume glustervol-read-only
 54:     type features/read-only
 55:     option read-only off
 56:     subvolumes glustervol-worm
 57: end-volume
 58:
 59: volume glustervol-leases
 60:     type features/leases
 61:     option leases off
 62:     subvolumes glustervol-read-only
 63: end-volume
 64:
 65: volume glustervol-upcall
 66:     type features/upcall
 67:     option cache-invalidation off
 68:     subvolumes glustervol-leases
 69: end-volume
 70:
 71: volume glustervol-io-threads
 72:     type performance/io-threads
 73:     subvolumes glustervol-upcall
 74: end-volume
 75:
 76: volume glustervol-selinux
 77:     type features/selinux
 78:     option selinux on
 79:     subvolumes glustervol-io-threads
 80: end-volume
 81:
 82: volume glustervol-marker
 83:     type features/marker
 84:     option volume-uuid 5422bb27-1863-47d5-b216-61751a01b759
 85:     option timestamp-file /var/lib/glusterd/vols/glustervol/marker.tstamp
 86:     option quota-version 0
 87:     option xtime off
 88:     option gsync-force-xtime off
 89:     option quota off
 90:     option inode-quota off
 91:     subvolumes glustervol-selinux
 92: end-volume
 93:
 94: volume glustervol-barrier
 95:     type features/barrier
 96:     option barrier disable
 97:     option barrier-timeout 120
 98:     subvolumes glustervol-marker
 99: end-volume
100:
101: volume glustervol-index
102:     type features/index
103:     option index-base /data/.glusterfs/indices
104:     option xattrop-dirty-watchlist trusted.afr.dirty
105:     option xattrop-pending-watchlist trusted.afr.glustervol-
106:     subvolumes glustervol-barrier
107: end-volume
108:
109: volume glustervol-quota
110:     type features/quota
111:     option volume-uuid glustervol
112:     option server-quota off
113:     option deem-statfs off
114:     subvolumes glustervol-index
115: end-volume
116:
117: volume /data
118:     type debug/io-stats
119:     option auth.addr./data.allow *
120:     option auth-path /data
121:     option auth.login.15e6dd85-fb90-4caf-971c-625ffec71229.password 6e2f0194-c002-439f-b6c8-6b1d71af66b8
122:     option auth.login./data.allow 15e6dd85-fb90-4caf-971c-625ffec71229
123:     option unique-id /data
124:     option log-level INFO
125:     option threads 16
126:     option latency-measurement off
127:     option count-fop-hits off
128:     option global-threading off
129:     subvolumes glustervol-quota
130: end-volume
131:
132: volume glustervol-server
133:     type protocol/server
134:     option transport.socket.listen-port 49152
135:     option rpc-auth.auth-glusterfs on
136:     option rpc-auth.auth-unix on
137:     option rpc-auth.auth-null on
138:     option rpc-auth-allow-insecure on
139:     option transport-type tcp
140:     option transport.address-family inet
141:     option auth.login./data.allow 15e6dd85-fb90-4caf-971c-625ffec71229
142:     option auth.login.15e6dd85-fb90-4caf-971c-625ffec71229.password 6e2f0194-c002-439f-b6c8-6b1d71af66b8
143:     option auth-path /data
144:     option auth.addr./data.allow *
145:     option transport.socket.keepalive 1
146:     option transport.socket.ssl-enabled off
147:     option transport.socket.keepalive-time 20
148:     option transport.socket.keepalive-interval 2
149:     option transport.socket.keepalive-count 9
150:     option transport.listen-backlog 1024
151:     subvolumes /data
152: end-volume
153:
+------------------------------------------------------------------------------+
[2020-06-16 07:05:23.817880] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "49.141.98.19"
[2020-06-16 07:05:23.817978] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 07:05:23.818006] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:8a2e528e-502d-4150-a994-1e1d44e17946-GRAPH_ID:0-PID:59760-HOST:node1-PC_NAME:glustervol-client-2-RECON_NO:-4 (version: 7.5) with subvol /data
[2020-06-16 07:05:23.821164] I [rpcsvc.c:866:rpcsvc_handle_rpc_call] 0-rpc-service: spawned a request handler thread for queue 0
[2020-06-16 07:05:23.950052] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "13.18.11.12"
[2020-06-16 07:05:23.950123] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 07:05:23.950143] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:5e6c8d94-95f8-4c61-a411-8649caa0f04c-GRAPH_ID:0-PID:3519-HOST:node2-PC_NAME:glustervol-client-2-RECON_NO:-4 (version: 7.5) with subvol /data
[2020-06-16 07:05:23.951663] I [rpcsvc.c:866:rpcsvc_handle_rpc_call] 0-rpc-service: spawned a request handler thread for queue 1
[2020-06-16 07:05:26.149895] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "49.11.99.79"
[2020-06-16 07:05:26.149932] I [login.c:110:gf_auth] 0-auth/login: allowed user names: 15e6dd85-fb90-4caf-971c-625ffec71229
[2020-06-16 07:05:26.149944] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:047b5329-a111-4e09-8fe2-ad632369e4db-GRAPH_ID:0-PID:1995-HOST:node3-PC_NAME:glustervol-client-2-RECON_NO:-0 (version: 7.5) with subvol /data
[2020-06-16 07:05:28.816629] I [addr.c:54:compare_addr_and_update] 0-/data: allowed = "*", received addr = "17.18.11.14"
[2020-06-16 07:05:28.816698] I [MSGID: 115029] [server-handshake.c:552:server_setvolume] 0-glustervol-server: accepted client from CTX_ID:ff0af68b-bc9a-4269-8589-f7ef60e27f5e-GRAPH_ID:0-PID:100753-HOST:node4-PC_NAME:glustervol-client-2-RECON_NO:-3 (version: 6.9) with subvol /data