[Gluster-users] 2.0.6
Stephan von Krawczynski
skraw at ithnet.com
Wed Aug 19 12:29:15 UTC 2009
On Tue, 18 Aug 2009 22:41:05 +0530
Vijay Bellur <vijay at gluster.com> wrote:
> Stephan von Krawczynski wrote:
> > On Tue, 18 Aug 2009 15:01:46 +0200
> > And, I forgot to mention: it took me around 1 hour of bonnie to crash a server
> >
> > in a classical distribute setup...
> >
> This is new to us. We have done quite a bit of testing with bonnie++ and
> did not encounter issues. Can you please point us to the core/logs and
> help us by filing a bug report in bugzilla (http://bugs.gluster.com) ?
Hello Vijay,
we tried to reproduce the freeze, but instead a classical hang showed up. This one is nice as everything seems fine with the boxes, only glusterfs mountpoints hang on both clients. There are no unusual outputs in the standard glusterfs logfiles, so we can only show you this:
These are strace output from the hanging processes, I don't know if this is really useful for you.
client 1:
Process 4460 attached - interrupt to quit
epoll_wait(3, {{EPOLLIN, {u32=10, u64=12884901898}}}, 260, 4294967295) = 1
readv(10, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(10, [{"\0\0\0\0\0\1\313r\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=10, u64=12884901898}}}, 260, 4294967295) = 1
readv(10, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(10, [{"\0\0\0\0\0\1\313s\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, ^C <unfinished ...>
Process 4460 detached
client 2:
Process 4109 attached - interrupt to quit
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\276\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\277\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\300\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\301\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\302\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\303\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\304\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\305\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\306\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\307\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\310\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\311\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\312\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\313\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, {{EPOLLIN, {u32=9, u64=8589934601}}}, 260, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\314\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
epoll_wait(3, ^C <unfinished ...>
Process 4109 detached
server 1:
Process 3430 attached - interrupt to quit
epoll_wait(3, {{EPOLLIN, {u32=8, u64=8589934600}}}, 257, 4294967295) = 1
readv(8, [{":O\0\0\0\0*\0\0\0\0*", 12}], 1) = 12
readv(8, [{"\0\0\0\0\0\1_\4\0\0\0\1\0\0\0\17\0\0\0\n\0\0\27\302\0\0\0\0\0\0\0\0"..., 42}], 1) = 42
futex(0x61058c, FUTEX_CMP_REQUEUE_PRIVATE, 1, 2147483647, 0x6105b8, 43038) = 1
futex(0x6105b8, FUTEX_WAKE_PRIVATE, 1) = 1
epoll_wait(3, {{EPOLLIN, {u32=8, u64=8589934600}}}, 257, 4294967295) = 1
readv(8, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(8, [{"\0\0\0\0\0\1_\5\0\0\0\2\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
writev(8, [{":O\0\0\0\0 \0\0\0\0*", 12}, {"\0\0\0\0\0\1_\5\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 2)= 44
epoll_wait(3, ^C <unfinished ...>
Process 3430 detached
server 2:
epoll_wait(3, {{EPOLLIN, {u32=9, u64=12884901897}}}, 257, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\242\0\0\0\2\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
writev(9, [{":O\0\0\0\0 \0\0\0\0*", 12}, {"\0\0\0\0\0&\311\242\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 2) = 44
epoll_wait(3, {{EPOLLIN, {u32=8, u64=8589934600}}}, 257, 4294967295) = 1
readv(8, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(8, [{"\0\0\0\0\0\1\313s\0\0\0\2\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
writev(8, [{":O\0\0\0\0 \0\0\0\0*", 12}, {"\0\0\0\0\0\1\313s\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 2) = 44
epoll_wait(3, {{EPOLLIN, {u32=9, u64=12884901897}}}, 257, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\243\0\0\0\2\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
writev(9, [{":O\0\0\0\0 \0\0\0\0*", 12}, {"\0\0\0\0\0&\311\243\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 2) = 44
epoll_wait(3, {{EPOLLIN, {u32=8, u64=8589934600}}}, 257, 4294967295) = 1
readv(8, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(8, [{"\0\0\0\0\0\1\313t\0\0\0\2\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
writev(8, [{":O\0\0\0\0 \0\0\0\0*", 12}, {"\0\0\0\0\0\1\313t\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 2) = 44
epoll_wait(3, {{EPOLLIN, {u32=9, u64=12884901897}}}, 257, 4294967295) = 1
readv(9, [{":O\0\0\0\0 \0\0\0\0*", 12}], 1) = 12
readv(9, [{"\0\0\0\0\0&\311\244\0\0\0\2\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 1) = 32
writev(9, [{":O\0\0\0\0 \0\0\0\0*", 12}, {"\0\0\0\0\0&\311\244\0\0\0\5\0\0\0\5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", 32}], 2) = 44
epoll_wait(3, ^C <unfinished ...>
Process 3768 detached
Server 2 warnings:
Aug 19 14:10:10 linux-tnpx kernel: INFO: task pdflush:288 blocked for more than 120 seconds.
Aug 19 14:10:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:10:10 linux-tnpx kernel: ffff8801279f8110 0000000000000046 ffff8801110584a8 ffff880127132a20
Aug 19 14:10:10 linux-tnpx kernel: 0000000027123998 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:10:10 linux-tnpx kernel: 0000000000000000 0000000000000000 ffff8801279f8398 00000002271239f0
Aug 19 14:10:10 linux-tnpx kernel: Call Trace:
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8031dc53>] ? __make_request+0x2e1/0x42f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469a85>] ? __wait_on_bit+0x41/0x70
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469b1f>] ? out_of_line_wait_on_bit+0x6b/0x77
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802e63ea>] ? write_ordered_buffers+0x224/0x2bf
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802e6830>] ? flush_commit_list+0x1cc/0x65d
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802e8b51>] ? do_journal_end+0xb03/0xcff
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802292ef>] ? set_next_entity+0x34/0x56
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802db717>] ? reiserfs_sync_fs+0x38/0x5c
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80288b2e>] ? sync_supers+0x60/0xa5
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8026596e>] ? wb_kupdate+0x2f/0x121
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802665f9>] ? pdflush+0x14a/0x21e
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8026593f>] ? wb_kupdate+0x0/0x121
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802664af>] ? pdflush+0x0/0x21e
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Aug 19 14:10:10 linux-tnpx kernel: INFO: task reiserfs/2:1042 blocked for more than 120 seconds.
Aug 19 14:10:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:10:10 linux-tnpx kernel: ffff880127a3b910 0000000000000046 0000000000000000 ffff880028061f18
Aug 19 14:10:10 linux-tnpx kernel: ffff88002805e840 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:10:10 linux-tnpx kernel: 000000000000000e 0000000000000002 ffff880127a3bb98 00000002ffffffff
Aug 19 14:10:10 linux-tnpx kernel: Call Trace:
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8032df06>] ? vsnprintf+0x75e/0x7b4
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80228d43>] ? update_curr+0x52/0xe4
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469c6e>] ? __mutex_lock_slowpath+0xd8/0x123
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469fe4>] ? mutex_lock+0x1a/0x31
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802e67aa>] ? flush_commit_list+0x146/0x65d
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802e6d99>] ? flush_async_commits+0x42/0x4b
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802e6d57>] ? flush_async_commits+0x0/0x4b
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80240d4c>] ? worker_thread+0x11f/0x1b0
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80244348>] ? autoremove_wake_function+0x0/0x2e
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80240c2d>] ? worker_thread+0x0/0x1b0
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Aug 19 14:10:10 linux-tnpx kernel: INFO: task glusterfsd:3775 blocked for more than 120 seconds.
Aug 19 14:10:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:10:10 linux-tnpx kernel: ffff88012735b2a0 0000000000000086 0000000000000000 0000000000000000
Aug 19 14:10:10 linux-tnpx kernel: 0000000000000000 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:10:10 linux-tnpx kernel: ffff88000d981708 0000000000000005 ffff88012735b528 000000020000003c
Aug 19 14:10:10 linux-tnpx kernel: Call Trace:
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8026363a>] ? free_pages_bulk+0x16b/0x29b
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469988>] ? __wait_on_bit_lock+0x3f/0x84
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80469a38>] ? out_of_line_wait_on_bit_lock+0x6b/0x77
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802d2e1b>] ? reiserfs_invalidatepage+0x9d/0x196
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80267abf>] ? truncate_complete_page+0x1d/0x59
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80267bc5>] ? truncate_inode_pages_range+0xca/0x31d
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802d6eb3>] ? reiserfs_delete_inode+0x0/0xb9
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802d6ecc>] ? reiserfs_delete_inode+0x19/0xb9
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802ab4c1>] ? dnotify_parent+0x25/0x71
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80327e87>] ? _atomic_dec_and_lock+0x33/0x50
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff802d6eb3>] ? reiserfs_delete_inode+0x0/0xb9
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80297dfb>] ? generic_delete_inode+0x8e/0x10c
Aug 19 14:10:10 linux-tnpx kernel: [<ffffffff80290b44>] ? do_unlinkat+0xe6/0x13f
Aug 19 14:10:11 linux-tnpx kernel: [<ffffffff80287776>] ? sys_writev+0x60/0x6e
Aug 19 14:10:11 linux-tnpx kernel: [<ffffffff8020ae6b>] ? system_call_fastpath+0x16/0x1b
Aug 19 14:12:10 linux-tnpx kernel: INFO: task pdflush:288 blocked for more than 120 seconds.
Aug 19 14:12:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:12:10 linux-tnpx kernel: ffff8801279f8110 0000000000000046 ffff8801110584a8 ffff880127132a20
Aug 19 14:12:10 linux-tnpx kernel: 0000000027123998 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:12:10 linux-tnpx kernel: 0000000000000000 0000000000000000 ffff8801279f8398 00000002271239f0
Aug 19 14:12:10 linux-tnpx kernel: Call Trace:
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8031dc53>] ? __make_request+0x2e1/0x42f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469a85>] ? __wait_on_bit+0x41/0x70
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469b1f>] ? out_of_line_wait_on_bit+0x6b/0x77
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802e63ea>] ? write_ordered_buffers+0x224/0x2bf
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802e6830>] ? flush_commit_list+0x1cc/0x65d
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802e8b51>] ? do_journal_end+0xb03/0xcff
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802292ef>] ? set_next_entity+0x34/0x56
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802db717>] ? reiserfs_sync_fs+0x38/0x5c
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80288b2e>] ? sync_supers+0x60/0xa5
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8026596e>] ? wb_kupdate+0x2f/0x121
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802665f9>] ? pdflush+0x14a/0x21e
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8026593f>] ? wb_kupdate+0x0/0x121
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802664af>] ? pdflush+0x0/0x21e
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Aug 19 14:12:10 linux-tnpx kernel: INFO: task reiserfs/2:1042 blocked for more than 120 seconds.
Aug 19 14:12:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:12:10 linux-tnpx kernel: ffff880127a3b910 0000000000000046 0000000000000000 ffff880028061f18
Aug 19 14:12:10 linux-tnpx kernel: ffff88002805e840 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:12:10 linux-tnpx kernel: 000000000000000e 0000000000000002 ffff880127a3bb98 00000002ffffffff
Aug 19 14:12:10 linux-tnpx kernel: Call Trace:
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8032df06>] ? vsnprintf+0x75e/0x7b4
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80228d43>] ? update_curr+0x52/0xe4
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469c6e>] ? __mutex_lock_slowpath+0xd8/0x123
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469fe4>] ? mutex_lock+0x1a/0x31
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802e67aa>] ? flush_commit_list+0x146/0x65d
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802e6d99>] ? flush_async_commits+0x42/0x4b
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802e6d57>] ? flush_async_commits+0x0/0x4b
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80240d4c>] ? worker_thread+0x11f/0x1b0
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80244348>] ? autoremove_wake_function+0x0/0x2e
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80240c2d>] ? worker_thread+0x0/0x1b0
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Aug 19 14:12:10 linux-tnpx kernel: INFO: task glusterfsd:3775 blocked for more than 120 seconds.
Aug 19 14:12:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:12:10 linux-tnpx kernel: ffff88012735b2a0 0000000000000086 0000000000000000 0000000000000000
Aug 19 14:12:10 linux-tnpx kernel: 0000000000000000 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:12:10 linux-tnpx kernel: ffff88000d981708 0000000000000005 ffff88012735b528 000000020000003c
Aug 19 14:12:10 linux-tnpx kernel: Call Trace:
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8026363a>] ? free_pages_bulk+0x16b/0x29b
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469988>] ? __wait_on_bit_lock+0x3f/0x84
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80469a38>] ? out_of_line_wait_on_bit_lock+0x6b/0x77
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802d2e1b>] ? reiserfs_invalidatepage+0x9d/0x196
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80267abf>] ? truncate_complete_page+0x1d/0x59
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80267bc5>] ? truncate_inode_pages_range+0xca/0x31d
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802d6eb3>] ? reiserfs_delete_inode+0x0/0xb9
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802d6ecc>] ? reiserfs_delete_inode+0x19/0xb9
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802ab4c1>] ? dnotify_parent+0x25/0x71
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80327e87>] ? _atomic_dec_and_lock+0x33/0x50
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff802d6eb3>] ? reiserfs_delete_inode+0x0/0xb9
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80297dfb>] ? generic_delete_inode+0x8e/0x10c
Aug 19 14:12:10 linux-tnpx kernel: [<ffffffff80290b44>] ? do_unlinkat+0xe6/0x13f
Aug 19 14:12:11 linux-tnpx kernel: [<ffffffff80287776>] ? sys_writev+0x60/0x6e
Aug 19 14:12:11 linux-tnpx kernel: [<ffffffff8020ae6b>] ? system_call_fastpath+0x16/0x1b
Aug 19 14:14:10 linux-tnpx kernel: INFO: task pdflush:288 blocked for more than 120 seconds.
Aug 19 14:14:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:14:10 linux-tnpx kernel: ffff8801279f8110 0000000000000046 ffff8801110584a8 ffff880127132a20
Aug 19 14:14:10 linux-tnpx kernel: 0000000027123998 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:14:10 linux-tnpx kernel: 0000000000000000 0000000000000000 ffff8801279f8398 00000002271239f0
Aug 19 14:14:10 linux-tnpx kernel: Call Trace:
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8031dc53>] ? __make_request+0x2e1/0x42f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469a85>] ? __wait_on_bit+0x41/0x70
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469b1f>] ? out_of_line_wait_on_bit+0x6b/0x77
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802e63ea>] ? write_ordered_buffers+0x224/0x2bf
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802e6830>] ? flush_commit_list+0x1cc/0x65d
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802e8b51>] ? do_journal_end+0xb03/0xcff
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802292ef>] ? set_next_entity+0x34/0x56
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802db717>] ? reiserfs_sync_fs+0x38/0x5c
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80288b2e>] ? sync_supers+0x60/0xa5
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8026596e>] ? wb_kupdate+0x2f/0x121
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802665f9>] ? pdflush+0x14a/0x21e
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8026593f>] ? wb_kupdate+0x0/0x121
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802664af>] ? pdflush+0x0/0x21e
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Aug 19 14:14:10 linux-tnpx kernel: INFO: task reiserfs/2:1042 blocked for more than 120 seconds.
Aug 19 14:14:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:14:10 linux-tnpx kernel: ffff880127a3b910 0000000000000046 0000000000000000 ffff880028061f18
Aug 19 14:14:10 linux-tnpx kernel: ffff88002805e840 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:14:10 linux-tnpx kernel: 000000000000000e 0000000000000002 ffff880127a3bb98 00000002ffffffff
Aug 19 14:14:10 linux-tnpx kernel: Call Trace:
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8032df06>] ? vsnprintf+0x75e/0x7b4
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80228d43>] ? update_curr+0x52/0xe4
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469c6e>] ? __mutex_lock_slowpath+0xd8/0x123
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469fe4>] ? mutex_lock+0x1a/0x31
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802e67aa>] ? flush_commit_list+0x146/0x65d
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff803250ce>] ? cfq_may_queue+0x0/0xd9
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802e6d99>] ? flush_async_commits+0x42/0x4b
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802e6d57>] ? flush_async_commits+0x0/0x4b
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80240d4c>] ? worker_thread+0x11f/0x1b0
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80244348>] ? autoremove_wake_function+0x0/0x2e
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80240c2d>] ? worker_thread+0x0/0x1b0
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Aug 19 14:14:10 linux-tnpx kernel: INFO: task glusterfsd:3775 blocked for more than 120 seconds.
Aug 19 14:14:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:14:10 linux-tnpx kernel: ffff88012735b2a0 0000000000000086 0000000000000000 0000000000000000
Aug 19 14:14:10 linux-tnpx kernel: 0000000000000000 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:14:10 linux-tnpx kernel: ffff88000d981708 0000000000000005 ffff88012735b528 000000020000003c
Aug 19 14:14:10 linux-tnpx kernel: Call Trace:
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8026363a>] ? free_pages_bulk+0x16b/0x29b
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469988>] ? __wait_on_bit_lock+0x3f/0x84
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80469a38>] ? out_of_line_wait_on_bit_lock+0x6b/0x77
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802d2e1b>] ? reiserfs_invalidatepage+0x9d/0x196
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80267abf>] ? truncate_complete_page+0x1d/0x59
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80267bc5>] ? truncate_inode_pages_range+0xca/0x31d
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802d6eb3>] ? reiserfs_delete_inode+0x0/0xb9
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802d6ecc>] ? reiserfs_delete_inode+0x19/0xb9
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802ab4c1>] ? dnotify_parent+0x25/0x71
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80327e87>] ? _atomic_dec_and_lock+0x33/0x50
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff802d6eb3>] ? reiserfs_delete_inode+0x0/0xb9
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80297dfb>] ? generic_delete_inode+0x8e/0x10c
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80290b44>] ? do_unlinkat+0xe6/0x13f
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff80287776>] ? sys_writev+0x60/0x6e
Aug 19 14:14:10 linux-tnpx kernel: [<ffffffff8020ae6b>] ? system_call_fastpath+0x16/0x1b
Aug 19 14:16:10 linux-tnpx kernel: INFO: task pdflush:288 blocked for more than 120 seconds.
Aug 19 14:16:10 linux-tnpx kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Aug 19 14:16:10 linux-tnpx kernel: ffff8801279f8110 0000000000000046 ffff8801110584a8 ffff880127132a20
Aug 19 14:16:10 linux-tnpx kernel: 0000000027123998 0000000000004000 000000000000fec0 000000000000c9b8
Aug 19 14:16:10 linux-tnpx kernel: 0000000000000000 0000000000000000 ffff8801279f8398 00000002271239f0
Aug 19 14:16:10 linux-tnpx kernel: Call Trace:
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff8031dc53>] ? __make_request+0x2e1/0x42f
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff8046950e>] ? schedule+0x9/0x1e
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80469550>] ? io_schedule+0x2d/0x3a
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802a4162>] ? sync_buffer+0x3b/0x3f
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80469a85>] ? __wait_on_bit+0x41/0x70
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802a4127>] ? sync_buffer+0x0/0x3f
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80469b1f>] ? out_of_line_wait_on_bit+0x6b/0x77
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80244376>] ? wake_bit_function+0x0/0x23
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802e63ea>] ? write_ordered_buffers+0x224/0x2bf
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802e6830>] ? flush_commit_list+0x1cc/0x65d
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802e8b51>] ? do_journal_end+0xb03/0xcff
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802292ef>] ? set_next_entity+0x34/0x56
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802db717>] ? reiserfs_sync_fs+0x38/0x5c
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80469fd7>] ? mutex_lock+0xd/0x31
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80288b2e>] ? sync_supers+0x60/0xa5
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff8026596e>] ? wb_kupdate+0x2f/0x121
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802665f9>] ? pdflush+0x14a/0x21e
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff8026593f>] ? wb_kupdate+0x0/0x121
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff802664af>] ? pdflush+0x0/0x21e
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80243ff8>] ? kthread+0x54/0x80
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff8020bdba>] ? child_rip+0xa/0x20
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff80243fa4>] ? kthread+0x0/0x80
Aug 19 14:16:10 linux-tnpx kernel: [<ffffffff8020bdb0>] ? child_rip+0x0/0x20
Regards,
Stephan
More information about the Gluster-users
mailing list