[Gluster-devel] crypt.t failed with core multiple times

Tue Jun 6 06:37:47 UTC 2017

fstat report:
https://fstat.gluster.org/summary?start_date=2017-05-25&end_date=2017-06-06

One of the regression regression link :
https://build.gluster.org/job/centos6-regression/4908/console

Who can help here?

(gdb) t a a bt

Thread 11 (LWP 25166):
#0  0x00007f1ead84500d in nanosleep () from ./lib64/libpthread.so.0
#1  0x00007f1eae576365 in gf_timer_proc (data=0xe34ea0)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/timer.c:164
#2  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#3  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 10 (LWP 25169):
#0  0x00007f1ead841a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from
./lib64/libpthread.so.0
#1  0x00007f1eae5b0f9c in syncenv_task (proc=0xe35690)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/syncop.c:603
#2  0x00007f1eae5b123e in syncenv_processor (thdata=0xe35690)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/syncop.c:695
#3  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#4  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 9 (LWP 25173):
#0  0x00007f1ead841a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from
./lib64/libpthread.so.0
#1  0x00007f1e9b9c5d7a in iot_worker (data=0x7f1e9c024ec0)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/performance/io-threads/src/io-threads.c:193
#2  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#3  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 8 (LWP 25170):
#0  0x00007f1ead841a5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from
./lib64/libpthread.so.0
#1  0x00007f1eae5b0f9c in syncenv_task (proc=0xe35a50)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/syncop.c:603
#2  0x00007f1eae5b123e in syncenv_processor (thdata=0xe35a50)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/syncop.c:695
#3  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#4  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 7 (LWP 25168):
#0  0x00007f1ead169c4d in nanosleep () from ./lib64/libc.so.6
#1  0x00007f1ead169ac0 in sleep () from ./lib64/libc.so.6
#2  0x00007f1eae59b946 in pool_sweeper (arg=0x0)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/mem-pool.c:465
#3  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#4  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 6 (LWP 25167):
#0  0x00007f1ead845585 in sigwait () from ./lib64/libpthread.so.0
#1  0x0000000000409f22 in glusterfs_sigwaiter (arg=0x7ffc45a51640)
    at
/home/jenkins/root/workspace/centos6-regression/glusterfsd/src/glusterfsd.c:2111
#2  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#3  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 5 (LWP 25165):
#0  0x00007f1ead83e2fd in pthread_join () from ./lib64/libpthread.so.0
#1  0x00007f1eae5d706f in event_dispatch_epoll (event_pool=0xe18050)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/event-epoll.c:732
#2  0x00007f1eae59a096 in event_dispatch (event_pool=0xe18050)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/event.c:124
#3  0x000000000040ac33 in main (argc=6, argv=0x7ffc45a528b8)
    at
/home/jenkins/root/workspace/centos6-regression/glusterfsd/src/glusterfsd.c:2522

Thread 4 (LWP 25177):
#0  0x00007f1ead84168c in pthread_cond_wait@@GLIBC_2.3.2 () from
./lib64/libpthread.so.0
#1  0x00007f1ea63110b6 in notify_kernel_loop (data=0xe1f910)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/mount/fuse/src/fuse-bridge.c:3997
#2  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
---Type <return> to continue, or q <return> to quit---
#3  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 3 (LWP 25176):
#0  0x00007f1ead19dbf7 in readv () from ./lib64/libc.so.6
#1  0x00007f1eae59d3dd in sys_readv (fd=8, iov=0x7f1e9a278e20, iovcnt=2)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/syscall.c:295
#2  0x00007f1ea6313a02 in fuse_thread_proc (data=0xe1f910)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/mount/fuse/src/fuse-bridge.c:4977
#3  0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#4  0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6

Thread 2 (LWP 25172):
#0  0x00007f1ead4d8f41 in ?? () from ./usr/lib64/libcrypto.so.10
#1  0x00007f1ead4d909b in AES_encrypt () from ./usr/lib64/libcrypto.so.10
#2  0x00007f1e9400ad3c in ?? ()
#3  0x00007f1e9400addc in ?? ()
#4  0x00007f1ea16a7290 in ?? ()
#5  0x00007f1ea16a7238 in ?? ()
#6  0x0000000000000000 in ?? ()

Thread 1 (LWP 25174):
#0  0x00007f1ea056393a in offset_at_tail (conf=0xc0, object=0x7f1e9400ac28)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/atom.c:96
#1  0x00007f1ea05639e4 in offset_at_data_tail (frame=0x7f1e94001320,
object=0x7f1e9400ac28)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/atom.c:110
#2  0x00007f1ea0564738 in rmw_partial_block (frame=0x7f1e94001320,
cookie=0x7f1e8c00c0b0, this=0x7f1e9c00b870,
    op_ret=4096, op_errno=0, vec=0x7f1e8c00bd10, count=1,
stbuf=0x7f1e8c017208, iobref=0x7f1e8c013900,
    atom=0x7f1ea07846a0 <atoms+64>)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/atom.c:523
#3  0x00007f1ea0565348 in rmw_data_tail (frame=0x7f1e94001320,
cookie=0x7f1e8c00c0b0, this=0x7f1e9c00b870,
    op_ret=4096, op_errno=0, vec=0x7f1e8c00bd10, count=1,
stbuf=0x7f1e8c017208, iobref=0x7f1e8c013900, xdata=0x0)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/atom.c:716
#4  0x00007f1ea0577699 in __crypt_readv_done (frame=0x7f1e8c00c0b0,
cookie=0x0, this=0x7f1e9c00b870, op_ret=0,
    op_errno=0, xdata=0x0)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/crypt.c:3460
#5  0x00007f1ea0577774 in crypt_readv_done (frame=0x7f1e8c00c0b0,
this=0x7f1e9c00b870)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/crypt.c:3487
#6  0x00007f1ea0577b3a in put_one_call_readv (frame=0x7f1e8c00c0b0,
this=0x7f1e9c00b870)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/crypt.c:3514
#7  0x00007f1ea0566885 in crypt_readv_cbk (frame=0x7f1e8c00c0b0,
cookie=0x7f1e8c00e8b0, this=0x7f1e9c00b870,
    op_ret=4096, op_errno=0, vec=0x7f1e9ac79880, count=1,
stbuf=0x7f1e9ac79810, iobref=0x7f1e8c013900, xdata=0x0)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/encryption/crypt/src/crypt.c:371
#8  0x00007f1ea0810b7f in dht_readv_cbk (frame=0x7f1e8c00e8b0,
cookie=0x7f1e8c003250, this=0x7f1e9c00a200,
    op_ret=4096, op_errno=0, vector=0x7f1e9ac79880, count=1,
stbuf=0x7f1e9ac79810, iobref=0x7f1e8c013900, xdata=0x0)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/cluster/dht/src/dht-inode-read.c:479
#9  0x00007f1ea0a73f83 in client3_3_readv_cbk (req=0x7f1e8c0034c0,
iov=0x7f1e8c003500, count=2,
    myframe=0x7f1e8c003250)
    at
/home/jenkins/root/workspace/centos6-regression/xlators/protocol/client/src/client-rpc-fops.c:2927
#10 0x00007f1eae32a84d in rpc_clnt_handle_reply (clnt=0x7f1e9c03ec30,
pollin=0x7f1e8c002570)
    at
/home/jenkins/root/workspace/centos6-regression/rpc/rpc-lib/src/rpc-clnt.c:778
#11 0x00007f1eae32ae17 in rpc_clnt_notify (trans=0x7f1e9c03ee00,
mydata=0x7f1e9c03ec60,
    event=RPC_TRANSPORT_MSG_RECEIVED, data=0x7f1e8c002570)
    at
/home/jenkins/root/workspace/centos6-regression/rpc/rpc-lib/src/rpc-clnt.c:971
#12 0x00007f1eae326dac in rpc_transport_notify (this=0x7f1e9c03ee00,
event=RPC_TRANSPORT_MSG_RECEIVED,
    data=0x7f1e8c002570) at
/home/jenkins/root/workspace/centos6-regression/rpc/rpc-lib/src/rpc-transport.c:538
#13 0x00007f1ea2ed858a in socket_event_poll_in (this=0x7f1e9c03ee00,
notify_handled=_gf_true)
    at
/home/jenkins/root/workspace/centos6-regression/rpc/rpc-transport/socket/src/socket.c:2315
#14 0x00007f1ea2ed8bd5 in socket_event_handler (fd=10, idx=2, gen=1,
data=0x7f1e9c03ee00, poll_in=1, poll_out=0,
    poll_err=0) at
/home/jenkins/root/workspace/centos6-regression/rpc/rpc-transport/socket/src/socket.c:2467
#15 0x00007f1eae5d6b42 in event_dispatch_epoll_handler
(event_pool=0xe18050, event=0x7f1e9ac79e70)
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/event-epoll.c:572
#16 0x00007f1eae5d6e44 in event_dispatch_epoll_worker (data=0x7f1e9c03a000)
---Type <return> to continue, or q <return> to quit---
    at
/home/jenkins/root/workspace/centos6-regression/libglusterfs/src/event-epoll.c:648
#17 0x00007f1ead83daa1 in start_thread () from ./lib64/libpthread.so.0
#18 0x00007f1ead1a5bcd in clone () from ./lib64/libc.so.6
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.gluster.org/pipermail/gluster-devel/attachments/20170606/a011ed87/attachment-0001.html>