[Gluster-devel] [master] double-free corruption

Wed May 9 08:05:49 UTC 2018

double-free corruption when running mutrace during a CREATE 5000 test run
using small-file test script on upstream master

following is the backtrace:

# mutrace --max=30 -d /usr/local/sbin/glusterfsd -s testsystem1
--volfile-id testvol.testsystem1.gluster-brick1-testvol -p
/var/run/gluster/vols/testvol/testsystem1-gluster-brick1-testvol.pid -S
/var/run/gluster/003b1b6d97b89ba7.socket --brick-name
/gluster/brick1/testvol -l
/var/log/glusterfs/bricks/gluster-brick1-testvol.log --xlator-option
*-posix.glusterd-uuid=94a150c5-d14f-4ecf-aa58-1caf45f40b90 --process-name
brick --brick-port 49152 --xlator-option testvol-server.listen-port=49152
--no-daemon
mutrace: 0.2 successfully initialized for process glusterfsd (PID: 7249).
BFD: /usr/local/lib/libglusterfs.so.0: warning: loop in section
dependencies detected
*** Error in `/usr/local/sbin/glusterfsd': double free or corruption
(!prev): 0x00007f30dc0ed1f0 ***
======= Backtrace: =========
/lib64/libc.so.6(+0x81429)[0x7f31429f6429]
/usr/local/lib/libmutrace-backtrace-symbols.so(+0x31fd8)[0x7f3144695fd8]
/usr/local/lib/libmutrace-backtrace-symbols.so(backtrace_symbols+0xda)[0x7f31446960da]
/usr/local/lib/libmutrace.so(backtrace_symbols+0x40)[0x7f3144991dc0]
/usr/local/lib/libglusterfs.so.0(+0x2aa87)[0x7f314435ba87]
/usr/local/lib/libglusterfs.so.0(_gf_msg+0x19d)[0x7f314435e044]
/usr/local/lib/libglusterfs.so.0(dict_get_bin+0x175)[0x7f3144351fdc]
/usr/local/lib/libglusterfs.so.0(gf_replace_old_iatt_in_dict+0x3b)[0x7f314436adb1]
/usr/local/lib/glusterfs/4.2dev/xlator/protocol/server.so(+0x3038d)[0x7f312f4f938d]
/usr/local/lib/glusterfs/4.2dev/xlator/debug/io-stats.so(+0x12254)[0x7f312f991254]
/usr/local/lib/glusterfs/4.2dev/xlator/features/marker.so(+0x15ab2)[0x7f313439cab2]
/usr/local/lib/glusterfs/4.2dev/xlator/features/selinux.so(+0x3868)[0x7f31345ba868]
/usr/local/lib/libglusterfs.so.0(default_setxattr_cbk+0x383)[0x7f31444148a2]
/usr/local/lib/glusterfs/4.2dev/xlator/features/upcall.so(+0x18055)[0x7f31349e5055]
/usr/local/lib/glusterfs/4.2dev/xlator/features/locks.so(+0x1c964)[0x7f313524e964]
/usr/local/lib/glusterfs/4.2dev/xlator/features/access-control.so(+0x127aa)[0x7f31354877aa]
/usr/local/lib/glusterfs/4.2dev/xlator/features/changelog.so(+0x10aa3)[0x7f31358c2aa3]
/usr/local/lib/glusterfs/4.2dev/xlator/features/changetimerecorder.so(+0xed2c)[0x7f3135fb8d2c]
/usr/local/lib/glusterfs/4.2dev/xlator/storage/posix.so(+0x3826f)[0x7f3136a3f26f]
/usr/local/lib/libglusterfs.so.0(default_setxattr+0x1f8)[0x7f314442ceb3]
/usr/local/lib/glusterfs/4.2dev/xlator/features/changetimerecorder.so(+0xf4e2)[0x7f3135fb94e2]
/usr/local/lib/glusterfs/4.2dev/xlator/features/changelog.so(+0x118f1)[0x7f31358c38f1]
/usr/local/lib/glusterfs/4.2dev/xlator/features/bitrot-stub.so(+0xc314)[0x7f313569c314]
/usr/local/lib/glusterfs/4.2dev/xlator/features/access-control.so(+0x12c0c)[0x7f3135487c0c]
/usr/local/lib/glusterfs/4.2dev/xlator/features/locks.so(+0x1d51e)[0x7f313524f51e]
/usr/local/lib/libglusterfs.so.0(default_setxattr+0x1f8)[0x7f314442ceb3]
/usr/local/lib/glusterfs/4.2dev/xlator/features/read-only.so(+0x8a4a)[0x7f3134e19a4a]
/usr/local/lib/libglusterfs.so.0(default_setxattr+0x1f8)[0x7f314442ceb3]
/usr/local/lib/glusterfs/4.2dev/xlator/features/upcall.so(+0x184fa)[0x7f31349e54fa]
/usr/local/lib/libglusterfs.so.0(default_setxattr_resume+0x3d0)[0x7f3144420c51]
/usr/local/lib/libglusterfs.so.0(call_resume_wind+0x637)[0x7f314437bcea]
/usr/local/lib/libglusterfs.so.0(call_resume+0xc7)[0x7f314438dc6e]
/usr/local/lib/glusterfs/4.2dev/xlator/performance/io-threads.so(+0x58bb)[0x7f31347c28bb]
/lib64/libpthread.so.0(+0x7dd5)[0x7f31431aadd5]
/lib64/libc.so.6(clone+0x6d)[0x7f3142a73b3d]
======= Memory map: ========
00400000-00418000 r-xp 00000000 fd:00 68968720
/usr/local/sbin/glusterfsd
00617000-00618000 r--p 00017000 fd:00 68968720
/usr/local/sbin/glusterfsd
00618000-0061a000 rw-p 00018000 fd:00 68968720
/usr/local/sbin/glusterfsd
00fad000-01362000 rw-p 00000000 00:00 0
[heap]
7f30d4000000-7f30d4068000 rw-p 00000000 00:00 0
7f30d4068000-7f30d8000000 ---p 00000000 00:00 0
7f30dc000000-7f30dc307000 rw-p 00000000 00:00 0
7f30dc307000-7f30e0000000 ---p 00000000 00:00 0
7f30e0000000-7f30e029b000 rw-p 00000000 00:00 0
7f30e029b000-7f30e4000000 ---p 00000000 00:00 0
7f30e4000000-7f30e42ec000 rw-p 00000000 00:00 0
7f30e42ec000-7f30e8000000 ---p 00000000 00:00 0
7f30e8000000-7f30e82dd000 rw-p 00000000 00:00 0
7f30e82dd000-7f30ec000000 ---p 00000000 00:00 0
7f30ec000000-7f30ec021000 rw-p 00000000 00:00 0
7f30ec021000-7f30f0000000 ---p 00000000 00:00 0
7f30f0000000-7f30f0331000 rw-p 00000000 00:00 0
7f30f0331000-7f30f4000000 ---p 00000000 00:00 0
7f30f4000000-7f30f4021000 rw-p 00000000 00:00 0
7f30f4021000-7f30f8000000 ---p 00000000 00:00 0
7f30f8000000-7f30f8021000 rw-p 00000000 00:00 0
7f30f8021000-7f30fc000000 ---p 00000000 00:00 0
7f30fc000000-7f30fc021000 rw-p 00000000 00:00 0
7f30fc021000-7f3100000000 ---p 00000000 00:00 0
7f3100000000-7f3100021000 rw-p 00000000 00:00 0
7f3100021000-7f3104000000 ---p 00000000 00:00 0
7f3104000000-7f3104021000 rw-p 00000000 00:00 0
7f3104021000-7f3108000000 ---p 00000000 00:00 0
7f3108000000-7f3108021000 rw-p 00000000 00:00 0
7f3108021000-7f310c000000 ---p 00000000 00:00 0
7f310e6fc000-7f310e7fd000 rw-p 00000000 00:00 0
7f310e7fd000-7f310e7fe000 ---p 00000000 00:00 0
7f310e7fe000-7f310effe000 rw-p 00000000 00:00 0
7f310effe000-7f310efff000 ---p 00000000 00:00 0
7f310efff000-7f310f7ff000 rw-p 00000000 00:00 0
7f310f7ff000-7f310f800000 ---p 00000000 00:00 0
7f310f800000-7f3110000000 rw-p 00000000 00:00 0
7f3110000000-7f3110308000 rw-p 00000000 00:00 0
7f3110308000-7f3114000000 ---p 00000000 00:00 0
7f3114000000-7f3114021000 rw-p 00000000 00:00 0
7f3114021000-7f3118000000 ---p 00000000 00:00 0
7f3118000000-7f3118217000 rw-p 00000000 00:00 0
7f3118217000-7f311c000000 ---p 00000000 00:00 0
7f311c072000-7f311c073000 ---p 00000000 00:00 0
7f311c073000-7f311c0b3000 rw-p 00000000 00:00 0
7f311c0b3000-7f311c0b4000 ---p 00000000 00:00 0
7f311c0b4000-7f311c0f4000 rw-p 00000000 00:00 0
7f311c0f4000-7f311c0f5000 ---p 00000000 00:00 0
7f311c0f5000-7f311c9f6000 rw-p 00000000 00:00 0
7f311c9f6000-7f311c9f7000 ---p 00000000 00:00 0
7f311c9f7000-7f311d1f7000 rw-p 00000000 00:00 0
7f311d1f7000-7f311d1f8000 ---p 00000000 00:00 0
7f311d1f8000-7f311d9f8000 rw-p 00000000 00:00 0
7f311d9f8000-7f311d9f9000 ---p 00000000 00:00 0
7f311d9f9000-7f311e1f9000 rw-p 00000000 00:00 0
7f311e1f9000-7f311e1fa000 ---p 00000000 00:00 0
7f311e1fa000-7f311edfe000 rw-p 00000000 00:00 0
7f311edfe000-7f311edff000 ---p 00000000 00:00 0
7f311edff000-7f311eeff000 rw-p 00000000 00:00 0
7f311eeff000-7f311ef00000 ---p 00000000 00:00 0
7f311ef00000-7f3120000000 rw-p 00000000 00:00 0
7f3120000000-7f3120127000 rw-p 00000000 00:00 0
7f3120127000-7f3124000000 ---p 00000000 00:00 0
7f3124000000-7f312417d000 rw-p 00000000 00:00 0
7f312417d000-7f3128000000 ---p 00000000 00:00 0
7f3128000000-7f31280f1000 rw-p 00000000 00:00 0
7f31280f1000-7f312c000000 ---p 00000000 00:00 0
7f312c039000-7f312c03a000 ---p 00000000 00:00 0
7f312c03a000-7f312c07a000 rw-p 00000000 00:00 0
7f312c07a000-7f312c07b000 ---p 00000000 00:00 0
7f312c07b000-7f312c0bb000 rw-p 00000000 00:00 0
7f312c0bb000-7f312c0bc000 ---p 00000000 00:00 0
7f312c0bc000-7f312c8bc000 rw-p 00000000 00:00 0
7f312c8bc000-7f312c8bd000 ---p 00000000 00:00 0
7f312c8bd000-7f312d0bd000 rw-p 00000000 00:00 0
7f312d0bd000-7f312d0be000 ---p 00000000 00:00 0
7f312d0be000-7f312d8be000 rw-p 00000000 00:00 0
7f312d8be000-7f312d8c0000 r-xp 00000000 fd:00 68836409
/usr/local/lib/glusterfs/4.2dev/auth/login.so
7f312d8c0000-7f312dabf000 ---p 00002000 fd:00 68836409
/usr/local/lib/glusterfs/4.2dev/auth/login.so
7f312dabf000-7f312dac0000 r--p 00001000 fd:00 68836409
/usr/local/lib/glusterfs/4.2dev/auth/login.so
7f312dac0000-7f312dac1000 rw-p 00002000 fd:00 68836409
/usr/local/lib/glusterfs/4.2dev/auth/login.so
7f312dac1000-7f312dac3000 r-xp 00000000 fd:00 68836403
/usr/local/lib/glusterfs/4.2dev/auth/addr.so
7f312dac3000-7f312dcc3000 ---p 00002000 fd:00 68836403
/usr/local/lib/glusterfs/4.2dev/auth/addr.so
7f312dcc3000-7f312dcc4000 r--p 00002000 fd:00 68836403
/usr/local/lib/glusterfs/4.2dev/auth/addr.so
7f312dcc4000-7f312dcc6000 rw-p 00003000 fd:00 68836403
/usr/local/lib/glusterfs/4.2dev/auth/addr.so
7f312dcc6000-7f312dcc7000 ---p 00000000 00:00 0
7f312dcc7000-7f312e4c7000 rw-p 00000000 00:00 0
7f312e4c7000-7f312e4c8000 ---p 00000000 00:00 0
7f312e4c8000-7f312ecc8000 rw-p 00000000 00:00 0
7f312ecc8000-7f312ecc9000 ---p 00000000 00:00 0
7f312ecc9000-7f312f4c9000 rw-p 00000000 00:00 0
7f312f4c9000-7f312f547000 r-xp 00000000 fd:00 33598413
/usr/local/lib/glusterfs/4.2dev/xlator/protocol/server.so
7f312f547000-7f312f747000 ---p 0007e000 fd:00 33598413
/usr/local/lib/glusterfs/4.2dev/xlator/protocol/server.so
7f312f747000-7f312f748000 r--p 0007e000 fd:00 33598413
/usr/local/lib/glusterfs/4.2dev/xlator/protocol/server.so
7f312f748000-7f312f752000 rw-p 0007f000 fd:00 33598413
/usr/local/lib/glusterfs/4.2dev/xlator/protocol/server.so
7f312f752000-7f312f77e000 r-xp 00000000 fd:00 858608
/usr/local/lib/glusterfs/4.2dev/xlator/performance/decompounder.so
7f312f77e000-7f312f97d000 ---p 0002c000 fd:00 858608
/usr/local/lib/glusterfs/4.2dev/xlator/performance/decompounder.so
7f312f97d000-7f312f97e000 r--p 0002b000 fd:00 858608
/usr/local/lib/glusterfs/4.2dev/xlator/performance/decompounder.so
7f312f97e000-7f312f97f000 rw-p 0002c000 fd:00 858608
/usr/local/lib/glusterfs/4.2dev/xlator/performance/decompounder.so
7f312f97f000-7f312f9ae000 r-xp 00000000 fd:00 68836488
/usr/local/lib/glusterfs/4.2dev/xlator/debug/io-stats.so
7f312f9ae000-7f312fbad000 ---p 0002f000 fd:00 68836488
/usr/local/lib/glusterfs/4.2dev/xlator/debug/io-stats.so
7f312fbad000-7f312fbae000 r--p 0002e000 fd:00 68836488
/usr/local/lib/glusterfs/4.2dev/xlator/debug/io-stats.so
7f312fbae000-7f312fbb6000 rw-p 0002f000 fd:00 68836488
/usr/local/lib/glusterfs/4.2dev/xlator/debug/io-stats.so
7f312fbb6000-7f312fbe7000 r-xp 00000000 fd:00 858632
/usr/local/lib/glusterfs/4.2dev/xlator/features/quota.so
7f312fbe7000-7f312fde6000 ---p 00031000 fd:00 858632
/usr/local/lib/glusterfs/4.2dev/xlator/features/quota.so
7f312fde6000-7f312fde7000 r--p 00030000 fd:00 858632
/usr/local/lib/glusterfs/4.2dev/xlator/features/quota.so
7f312fde7000-7f312fdea000 rw-p 00031000 fd:00 858632
/usr/local/lib/glusterfs/4.2dev/xlator/features/quota.so
7f312fdea000-7f312fdfe000 r-xp 00000000 fd:00 858655
/usr/local/lib/glusterfs/4.2dev/xlator/features/index.so
7f312fdfe000-7f312fffd000 ---p 00014000 fd:00 858655
/usr/local/lib/glusterfs/4.2dev/xlator/features/index.so
7f312fffd000-7f312fffe000 r--p 00013000 fd:00 858655
/usr/local/lib/glusterfs/4.2dev/xlator/features/index.so
7f312fffe000-7f3130000000 rw-p 00014000 fd:00 858655
/usr/local/lib/glusterfs/4.2dev/xlator/features/index.so
7f3130000000-7f3130345000 rw-p 00000000 00:00 0
7f3130345000-7f3134000000 ---p 00000000 00:00 0
7f313402f000-7f3134030000 ---p 00000000 00:00 0
7f3134030000-7f3134070000 rw-p 00000000 00:00 0
7f3134070000-7f3134071000 ---p 00000000 00:00 0
7f3134071000-7f3134078000 rw-p 00000000 00:00 0
7f3134078000-7f3134079000 ---p 00000000 00:00 0
7f3134079000-7f3134179000 rw-p 00000000 00:00 0
7f3134179000-7f3134185000 r-xp 00000000 fd:00 858473
/usr/local/lib/glusterfs/4.2dev/xlator/features/barrier.so
7f3134185000-7f3134384000 ---p 0000c000 fd:00 858473
/usr/local/lib/glusterfs/4.2dev/xlator/features/barrier.so
7f3134384000-7f3134385000 r--p 0000b000 fd:00 858473
/usr/local/lib/glusterfs/4.2dev/xlator/features/barrier.so
7f3134385000-7f3134387000 rw-p 0000c000 fd:00 858473
/usr/local/lib/glusterfs/4.2dev/xlator/features/barrier.so
7f3134387000-7f31343b4000 r-xp 00000000 fd:00 858653
/usr/local/lib/glusterfs/4.2dev/xlator/features/marker.so
7f31343b4000-7f31345b3000 ---p 0002d000 fd:00 858653
/usr/local/lib/glusterfs/4.2dev/xlator/features/marker.so
7f31345b3000-7f31345b4000 r--p 0002c000 fd:00 858653
/usr/local/lib/glusterfs/4.2dev/xlator/features/marker.so
7f31345b4000-7f31345b7000 rw-p 0002d000 fd:00 858653
/usr/local/lib/glusterfs/4.2dev/xlator/features/marker.so
7f31345b7000-7f31345bc000 r-xp 00000000 fd:00 858484
/usr/local/lib/glusterfs/4.2dev/xlator/features/selinux.so
7f31345bc000-7f31347bb000 ---p 00005000 fd:00 858484
/usr/local/lib/glusterfs/4.2dev/xlator/features/selinux.so
7f31347bb000-7f31347bc000 r--p 00004000 fd:00 858484
/usr/local/lib/glusterfs/4.2dev/xlator/features/selinux.so
7f31347bc000-7f31347bd000 rw-p 00005000 fd:00 858484
/usr/local/lib/glusterfs/4.2dev/xlator/features/selinux.so
7f31347bd000-7f31347c9000 r-xp 00000000 fd:00 858706
/usr/local/lib/glusterfs/4.2dev/xlator/performance/io-threads.so
7f31347c9000-7f31349c8000 ---p 0000c000 fd:00 858706
/usr/local/lib/glusterfs/4.2dev/xlator/performance/io-threads.so
7f31349c8000-7f31349c9000 r--p 0000b000 fd:00 858706
/usr/local/lib/glusterfs/4.2dev/xlator/performance/io-threads.so
7f31349c9000-7f31349cd000 rw-p 0000c000 fd:00 858706
/usr/local/lib/glusterfs/4.2dev/xlator/performance/io-threads.so
7f31349cd000-7f31349f1000 r-xp 00000000 fd:00 858491
/usr/local/lib/glusterfs/4.2dev/xlator/features/upcall.so
7f31349f1000-7f3134bf0000 ---p 00024000 fd:00 858491
/usr/local/lib/glusterfs/4.2dev/xlator/features/upcall.so
7f3134bf0000-7f3134bf1000 r--p 00023000 fd:00 858491
/usr/local/lib/glusterfs/4.2dev/xlator/features/upcall.so
7f3134bf1000-7f3134bf3000 rw-p 00024000 fd:00 858491
/usr/local/lib/glusterfs/4.2dev/xlator/features/upcall.so
7f3134bf3000-7f3134c0f000 r-xp 00000000 fd:00 858494
/usr/local/lib/glusterfs/4.2dev/xlator/features/leases.so
7f3134c0f000-7f3134e0e000 ---p 0001c000 fd:00 858494
/usr/local/lib/glusterfs/4.2dev/xlator/features/leases.so
7f3134e0e000-7f3134e0f000 r--p 0001b000 fd:00 858494
/usr/local/lib/glusterfs/4.2dev/xlator/features/leases.so
7f3134e0f000-7f3134e11000 rw-p 0001c000 fd:00 858494
/usr/local/lib/glusterfs/4.2dev/xlator/features/leases.so
7f3134e11000-7f3134e1b000 r-xp 00000000 fd:00 858637
/usr/local/lib/glusterfs/4.2dev/xlator/features/read-only.so
7f3134e1b000-7f313501b000 ---p 0000a000 fd:00 858637
/usr/local/lib/glusterfs/4.2dev/xlator/features/read-only.so
7f313501b000-7f313501c000 r--p 0000a000 fd:00 858637
/usr/local/lib/glusterfs/4.2dev/xlator/features/read-only.so
7f313501c000-7f313501d000 rw-p 0000b000 fd:00 858637
/usr/local/lib/glusterfs/4.2dev/xlator/features/read-only.so
7f313501d000-7f313502f000 r-xp 00000000 fd:00 858639
/usr/local/lib/glusterfs/4.2dev/xlator/features/worm.so
7f313502f000-7f313522e000 ---p 00012000 fd:00 858639
/usr/local/lib/glusterfs/4.2dev/xlator/features/worm.so
7f313522e000-7f313522f000 r--p 00011000 fd:00 858639
/usr/local/lib/glusterfs/4.2dev/xlator/features/worm.so
7f313522f000-7f3135232000 rw-p 00012000 fd:00 858639
/usr/local/lib/glusterfs/4.2dev/xlator/features/worm.so
7f3135232000-7f3135271000 r-xp 00000000 fd:00 858629
/usr/local/lib/glusterfs/4.2dev/xlator/features/locks.so
7f3135271000-7f3135470000 ---p 0003f000 fd:00 858629
/usr/local/lib/glusterfs/4.2dev/xlator/features/locks.so
7f3135470000-7f3135471000 r--p 0003e000 fd:00 858629
/usr/local/lib/glusterfs/4.2dev/xlator/features/locks.so
7f3135471000-7f3135475000 rw-p 0003f000 fd:00 858629
/usr/local/lib/glusterfs/4.2dev/xlator/features/locks.so
7f3135475000-7f313548e000 r-xp 00000000 fd:00 868968
/usr/local/lib/glusterfs/4.2dev/xlator/system/posix-acl.so
7f313548e000-7f313568e000 ---p 00019000 fd:00 868968
/usr/local/lib/glusterfs/4.2dev/xlator/system/posix-acl.so
7f313568e000-7f313568f000 r--p 00019000 fd:00 868968
/usr/local/lib/glusterfs/4.2dev/xlator/system/posix-acl.so
7f313568f000-7f3135690000 rw-p 0001a000 fd:00 868968
/usr/local/lib/glusterfs/4.2dev/xlator/system/posix-acl.so
7f3135690000-7f31356b0000 r-xp 00000000 fd:00 868841
/usr/local/lib/glusterfs/4.2dev/xlator/features/bitrot-stub.so
7f31356b0000-7f31358af000 ---p 00020000 fd:00 868841
/usr/local/lib/glusterfs/4.2dev/xlator/features/bitrot-stub.soAborted (core
dumped)


-- 
Milind
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.gluster.org/pipermail/gluster-devel/attachments/20180509/4b4af763/attachment-0001.html>