[Bugs] [Bug 1773532] Gluster brick randomly segfaults

bugzilla at redhat.com bugzilla at redhat.com
Tue Nov 26 08:25:22 UTC 2019


https://bugzilla.redhat.com/show_bug.cgi?id=1773532



--- Comment #18 from Dominik Drazyk <ddrazyk at gmail.com> ---
Ok I found it.

(gdb) bt
#0  0x00007f7d0c38564c in socket_event_handler () from
/usr/lib64/glusterfs/6.6/rpc-transport/socket.so
#1  0x00007f7d18028ae6 in event_dispatch_epoll_worker () from
/lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

(gdb) t a a bt

Thread 40 (Thread 0x7f7d184bf4c0 (LWP 138985)):
#0  0x00007f7d16e03fd7 in pthread_join () from /lib64/libpthread.so.0
#1  0x00007f7d18027cd8 in event_dispatch_epoll () from /lib64/libglusterfs.so.0
#2  0x000055f596714723 in main ()

Thread 39 (Thread 0x7f7d000ac700 (LWP 140549)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 38 (Thread 0x7f7cf46b3700 (LWP 140758)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 37 (Thread 0x7f7cf4431700 (LWP 140760)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 36 (Thread 0x7f7d0d59b700 (LWP 138991)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d18002ef0 in syncenv_task () from /lib64/libglusterfs.so.0
#2  0x00007f7d18003da0 in syncenv_processor () from /lib64/libglusterfs.so.0
#3  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 35 (Thread 0x7f7cf4472700 (LWP 140759)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 34 (Thread 0x7f7d0ed9e700 (LWP 138988)):
#0  0x00007f7d16e0a381 in sigwait () from /lib64/libpthread.so.0
#1  0x000055f5967181ab in glusterfs_sigwaiter ()
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 33 (Thread 0x7f7d0dd9c700 (LWP 138990)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d18002ef0 in syncenv_task () from /lib64/libglusterfs.so.0
#2  0x00007f7d18003da0 in syncenv_processor () from /lib64/libglusterfs.so.0
#3  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 32 (Thread 0x7f7d0cd9a700 (LWP 138992)):
#0  0x00007f7d166bf953 in select () from /lib64/libc.so.6
#1  0x00007f7d18043044 in runner () from /lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 31 (Thread 0x7f7d000ed700 (LWP 140548)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 30 (Thread 0x7f7d0016f700 (LWP 139188)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 29 (Thread 0x7f7d008f8700 (LWP 139011)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d02f69d24 in index_worker () from
/usr/lib64/glusterfs/6.6/xlator/features/index.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 28 (Thread 0x7f7d0e59d700 (LWP 138989)):
#0  0x00007f7d1668f80d in nanosleep () from /lib64/libc.so.6
#1  0x00007f7d1668f6a4 in sleep () from /lib64/libc.so.6
#2  0x00007f7d17fef678 in pool_sweeper () from /lib64/libglusterfs.so.0
#3  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 27 (Thread 0x7f7d0012e700 (LWP 139189)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 26 (Thread 0x7f7cedffb700 (LWP 139134)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d17d66bea in rpcsvc_request_handler () from /lib64/libgfrpc.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 25 (Thread 0x7f7d0f59f700 (LWP 138987)):
#0  0x00007f7d16e09e5d in nanosleep () from /lib64/libpthread.so.0
#1  0x00007f7d17fd2396 in gf_timer_proc () from /lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 24 (Thread 0x7f7cef7fe700 (LWP 139114)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d17d66bea in rpcsvc_request_handler () from /lib64/libgfrpc.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 23 (Thread 0x7f7d001f1700 (LWP 139118)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
---Type <return> to continue, or q <return> to quit---
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 22 (Thread 0x7f7cf46f4700 (LWP 140757)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 21 (Thread 0x7f7cf47b7700 (LWP 140552)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 20 (Thread 0x7f7cf4735700 (LWP 140756)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 19 (Thread 0x7f7cf4776700 (LWP 140755)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 18 (Thread 0x7f7cf47f8700 (LWP 140551)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 17 (Thread 0x7f7d0006b700 (LWP 140550)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 16 (Thread 0x7f7d001b0700 (LWP 139149)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 15 (Thread 0x7f7cee7fc700 (LWP 139126)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d17d66bea in rpcsvc_request_handler () from /lib64/libgfrpc.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 14 (Thread 0x7f7ceeffd700 (LWP 139117)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d17d66bea in rpcsvc_request_handler () from /lib64/libgfrpc.so.0
---Type <return> to continue, or q <return> to quit---
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 13 (Thread 0x7f7cf7fff700 (LWP 139013)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d08c17e23 in changelog_ev_connector () from
/usr/lib64/glusterfs/6.6/xlator/features/changelog.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 12 (Thread 0x7f7ceffff700 (LWP 139020)):
#0  0x00007f7d16e069f5 in pthread_cond_wait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d096812db in posix_fsyncer_pick () from
/usr/lib64/glusterfs/6.6/xlator/storage/posix.so
#2  0x00007f7d09681565 in posix_fsyncer () from
/usr/lib64/glusterfs/6.6/xlator/storage/posix.so
#3  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 11 (Thread 0x7f7cf4ff9700 (LWP 139019)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d0967b663 in posix_ctx_janitor_thread_proc () from
/usr/lib64/glusterfs/6.6/xlator/storage/posix.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 10 (Thread 0x7f7cf5ffb700 (LWP 139017)):
#0  0x00007f7d1668f80d in nanosleep () from /lib64/libc.so.6
#1  0x00007f7d1668f6a4 in sleep () from /lib64/libc.so.6
#2  0x00007f7d096810b0 in posix_disk_space_check_thread_proc () from
/usr/lib64/glusterfs/6.6/xlator/storage/posix.so
#3  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 9 (Thread 0x7f7d080f7700 (LWP 139012)):
#0  0x00007f7d16e06da2 in pthread_cond_timedwait@@GLIBC_2.3.2 () from
/lib64/libpthread.so.0
#1  0x00007f7d037b5c9d in iot_worker () from
/usr/lib64/glusterfs/6.6/xlator/performance/io-threads.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 8 (Thread 0x7f7cf67fc700 (LWP 139016)):
#0  0x00007f7d166bf953 in select () from /lib64/libc.so.6
#1  0x00007f7d08c1808a in changelog_ev_dispatch () from
/usr/lib64/glusterfs/6.6/xlator/features/changelog.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 7 (Thread 0x7f7cf57fa700 (LWP 139018)):
#0  0x00007f7d1668f80d in nanosleep () from /lib64/libc.so.6
#1  0x00007f7d1668f6a4 in sleep () from /lib64/libc.so.6
#2  0x00007f7d096808da in posix_health_check_thread_proc () from
/usr/lib64/glusterfs/6.6/xlator/storage/posix.so
#3  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#4  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 6 (Thread 0x7f7cf6ffd700 (LWP 139015)):
#0  0x00007f7d166bf953 in select () from /lib64/libc.so.6
#1  0x00007f7d08c1808a in changelog_ev_dispatch () from
/usr/lib64/glusterfs/6.6/xlator/features/changelog.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

---Type <return> to continue, or q <return> to quit---
Thread 5 (Thread 0x7f7cf77fe700 (LWP 139014)):
#0  0x00007f7d166bf953 in select () from /lib64/libc.so.6
#1  0x00007f7d08c1808a in changelog_ev_dispatch () from
/usr/lib64/glusterfs/6.6/xlator/features/changelog.so
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 4 (Thread 0x7f7d0a0c8700 (LWP 138998)):
#0  0x00007f7d166c8e63 in epoll_wait () from /lib64/libc.so.6
#1  0x00007f7d180288c0 in event_dispatch_epoll_worker () from
/lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 3 (Thread 0x7f7d01e01700 (LWP 139010)):
#0  0x00007f7d166c8e63 in epoll_wait () from /lib64/libc.so.6
#1  0x00007f7d180288c0 in event_dispatch_epoll_worker () from
/lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 2 (Thread 0x7f7d02602700 (LWP 139009)):
#0  0x00007f7d166c8e63 in epoll_wait () from /lib64/libc.so.6
#1  0x00007f7d180288c0 in event_dispatch_epoll_worker () from
/lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6

Thread 1 (Thread 0x7f7d0a8c9700 (LWP 138997)):
#0  0x00007f7d0c38564c in socket_event_handler () from
/usr/lib64/glusterfs/6.6/rpc-transport/socket.so
#1  0x00007f7d18028ae6 in event_dispatch_epoll_worker () from
/lib64/libglusterfs.so.0
#2  0x00007f7d16e02e65 in start_thread () from /lib64/libpthread.so.0
#3  0x00007f7d166c888d in clone () from /lib64/libc.so.6


I can share the core file as a link (it's more than 700MB). Is that fine with
bugzilla's policy?

-- 
You are receiving this mail because:
You are on the CC list for the bug.


More information about the Bugs mailing list