[Gluster-devel] Question about epoll() based event loop

[Dmitry Antipov]

Is it expected that more than one socket_event_handler() operates on the same rpc_transport_t
instance in parallel? The most of the races reported by ThreadSanitizer comes from this area. Example:

WARNING: ThreadSanitizer: data race (pid=1366951)
   Write of size 8 at 0x7b40000307b0 by thread T8:
     #0 clock_gettime <null> (libtsan.so.0+0x396bf)
     #1 timespec_now glusterfs/libglusterfs/src/timespec.c:31 (libglusterfs.so.0+0x7cf83)
     #2 client4_0_setattr_cbk glusterfs/xlators/protocol/client/src/client-rpc-fops_v2.c:1969 (client.so+0x79d5b)
     #3 rpc_clnt_handle_reply glusterfs/rpc/rpc-lib/src/rpc-clnt.c:759 (libgfrpc.so.0+0x13c5a)
     #4 rpc_clnt_notify glusterfs/rpc/rpc-lib/src/rpc-clnt.c:926 (libgfrpc.so.0+0x1415e)
     #5 rpc_transport_notify glusterfs/rpc/rpc-lib/src/rpc-transport.c:520 (libgfrpc.so.0+0xf0a0)
     #6 socket_event_poll_in_async glusterfs/rpc/rpc-transport/socket/src/socket.c:2502 (socket.so+0x628d)
     #7 gf_async ../../../../libglusterfs/src/glusterfs/async.h:189 (socket.so+0xf3a5)
     #8 socket_event_poll_in glusterfs/rpc/rpc-transport/socket/src/socket.c:2543 (socket.so+0xf3a5)
     #9 socket_event_handler glusterfs/rpc/rpc-transport/socket/src/socket.c:2934 (socket.so+0xf3a5)
     #10 socket_event_handler glusterfs/rpc/rpc-transport/socket/src/socket.c:2854 (socket.so+0xf3a5)
     #11 event_dispatch_epoll_handler glusterfs/libglusterfs/src/event-epoll.c:640 (libglusterfs.so.0+0xcbab3)
     #12 event_dispatch_epoll_worker glusterfs/libglusterfs/src/event-epoll.c:751 (libglusterfs.so.0+0xcbab3)
     #13 <null> <null> (libtsan.so.0+0x2d33f)

   Previous write of size 8 at 0x7b40000307b0 by thread T7:
     #0 clock_gettime <null> (libtsan.so.0+0x396bf)
     #1 timespec_now glusterfs/libglusterfs/src/timespec.c:31 (libglusterfs.so.0+0x7cf83)
     #2 client4_0_setattr_cbk glusterfs/xlators/protocol/client/src/client-rpc-fops_v2.c:1969 (client.so+0x79d5b)
     #3 rpc_clnt_handle_reply glusterfs/rpc/rpc-lib/src/rpc-clnt.c:759 (libgfrpc.so.0+0x13c5a)
     #4 rpc_clnt_notify glusterfs/rpc/rpc-lib/src/rpc-clnt.c:926 (libgfrpc.so.0+0x1415e)
     #5 rpc_transport_notify glusterfs/rpc/rpc-lib/src/rpc-transport.c:520 (libgfrpc.so.0+0xf0a0)
     #6 socket_event_poll_in_async glusterfs/rpc/rpc-transport/socket/src/socket.c:2502 (socket.so+0x628d)
     #7 gf_async ../../../../libglusterfs/src/glusterfs/async.h:189 (socket.so+0xf3a5)
     #8 socket_event_poll_in glusterfs/rpc/rpc-transport/socket/src/socket.c:2543 (socket.so+0xf3a5)
     #9 socket_event_handler glusterfs/rpc/rpc-transport/socket/src/socket.c:2934 (socket.so+0xf3a5)
     #10 socket_event_handler glusterfs/rpc/rpc-transport/socket/src/socket.c:2854 (socket.so+0xf3a5)
     #11 event_dispatch_epoll_handler glusterfs/libglusterfs/src/event-epoll.c:640 (libglusterfs.so.0+0xcbab3)
     #12 event_dispatch_epoll_worker glusterfs/libglusterfs/src/event-epoll.c:751 (libglusterfs.so.0+0xcbab3)
     #13 <null> <null> (libtsan.so.0+0x2d33f)

   Location is heap block of size 256 at 0x7b4000030700 allocated by thread T9:
     #0 malloc <null> (libtsan.so.0+0x304d3)
     #1 mem_get_from_pool glusterfs/libglusterfs/src/mem-pool.c:802 (libglusterfs.so.0+0x70536)
     #2 mem_get glusterfs/libglusterfs/src/mem-pool.c:837 (libglusterfs.so.0+0x70536)
     #3 mem_get0 glusterfs/libglusterfs/src/mem-pool.c:705 (libglusterfs.so.0+0x7062a)
     #4 wb_lookup glusterfs/xlators/performance/write-behind/src/write-behind.c:2480 (write-behind.so+0x173ac)
     #5 default_lookup glusterfs/libglusterfs/src/defaults.c:3025 (libglusterfs.so.0+0x117155)
     #6 default_lookup glusterfs/libglusterfs/src/defaults.c:3025 (libglusterfs.so.0+0x117155)
     #7 qr_lookup glusterfs/xlators/performance/quick-read/src/quick-read.c:670 (quick-read.so+0x475c)
     #8 mdc_lookup glusterfs/xlators/performance/md-cache/src/md-cache.c:1323 (md-cache.so+0x260d8)
     #9 io_stats_lookup glusterfs/xlators/debug/io-stats/src/io-stats.c:2695 (io-stats.so+0x7314)
     #10 default_lookup glusterfs/libglusterfs/src/defaults.c:3025 (libglusterfs.so.0+0x117155)
     #11 meta_lookup glusterfs/xlators/meta/src/meta.c:43 (meta.so+0x40f9)
     #12 syncop_lookup glusterfs/libglusterfs/src/syncop.c:1309 (libglusterfs.so.0+0x92a1b)
     #13 fuse_first_lookup glusterfs/xlators/mount/fuse/src/fuse-bridge.c:5346 (fuse.so+0x37778)
     #14 fuse_graph_sync glusterfs/xlators/mount/fuse/src/fuse-bridge.c:5910 (fuse.so+0x39989)
     #15 fuse_thread_proc glusterfs/xlators/mount/fuse/src/fuse-bridge.c:6057 (fuse.so+0x3a127)
     #16 <null> <null> (libtsan.so.0+0x2d33f)

   Thread T8 'glfs_epoll001' (tid=1366962, running) created by main thread at:
     #0 pthread_create <null> (libtsan.so.0+0x5ec29)
     #1 gf_thread_vcreate glusterfs/libglusterfs/src/common-utils.c:4058 (libglusterfs.so.0+0x407ef)
     #2 gf_thread_create glusterfs/libglusterfs/src/common-utils.c:4081 (libglusterfs.so.0+0x40947)
     #3 event_dispatch_epoll glusterfs/libglusterfs/src/event-epoll.c:806 (libglusterfs.so.0+0xca23e)
     #4 gf_event_dispatch glusterfs/libglusterfs/src/event.c:115 (libglusterfs.so.0+0x6e3b2)
     #5 main glusterfs/glusterfsd/src/glusterfsd.c:2728 (glusterfsd+0x406c5e)

   Thread T7 'glfs_epoll000' (tid=1366961, running) created by main thread at:
     #0 pthread_create <null> (libtsan.so.0+0x5ec29)
     #1 gf_thread_vcreate glusterfs/libglusterfs/src/common-utils.c:4058 (libglusterfs.so.0+0x407ef)
     #2 gf_thread_create glusterfs/libglusterfs/src/common-utils.c:4081 (libglusterfs.so.0+0x40947)
     #3 event_dispatch_epoll glusterfs/libglusterfs/src/event-epoll.c:806 (libglusterfs.so.0+0xca23e)
     #4 gf_event_dispatch glusterfs/libglusterfs/src/event.c:115 (libglusterfs.so.0+0x6e3b2)
     #5 main glusterfs/glusterfsd/src/glusterfsd.c:2728 (glusterfsd+0x406c5e)

Dmitry