[Gluster-devel] io-cache exceeding cache-size value
Gordan Bobic
gordan at bobich.net
Wed Feb 25 02:50:20 UTC 2009
Just after mounting:
top:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
4605 root 20 0 51348 2368 1296 S 0 0.1 0:00.08 glusterfs
# kill -s SIGUSR1 4605
Arena 0:
system bytes = 479232
in use bytes = 258832
Total (incl. mmap):
system bytes = 1466368
in use bytes = 1245968
max mmap regions = 5
max mmap bytes = 1122304
After compiling the kernel:
# make clean; make dep; make -j32 all
top:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
4605 root 20 0 202m 97m 1316 S 0 2.5 2:42.09 glusterfs
# kill -s SIGUSR1 4605
Arena 0:
system bytes = 26423296
in use bytes = 25972704
Arena 1:
system bytes = 74113024
in use bytes = 73516560
Total (incl. mmap):
system bytes = 101523456
in use bytes = 100476400
max mmap regions = 5
max mmap bytes = 1122304
# ./pmap 4605
glusterfs(4605)
00400000 (24 KB) r-xp (09:02 442904105) /usr/sbin/glusterfsd
00606000 (4 KB) rw-p (09:02 442904105) /usr/sbin/glusterfsd
00607000 (25804 KB) rw-p (00:00 0) [heap]
40000000 (4 KB) ---p (00:00 0)
40001000 (8192 KB) rw-p (00:00 0)
40801000 (4 KB) ---p (00:00 0)
40802000 (8192 KB) rw-p (00:00 0)
3a38600000 (176 KB) r-xp (09:02 442904212)
/usr/lib64/libglusterfs.so.0.0.0
3a3862c000 (2048 KB) ---p (09:02 442904212)
/usr/lib64/libglusterfs.so.0.0.0
3a3882c000 (4 KB) rw-p (09:02 442904212)
/usr/lib64/libglusterfs.so.0.0.0
3a3882d000 (72 KB) rw-p (00:00 0)
3d4c800000 (104 KB) r-xp (09:02 1194229765) /lib64/ld-2.5.so
3d4ca1a000 (4 KB) r--p (09:02 1194229765) /lib64/ld-2.5.so
3d4ca1b000 (4 KB) rw-p (09:02 1194229765) /lib64/ld-2.5.so
3d4cc00000 (1320 KB) r-xp (09:02 1194229772) /lib64/libc-2.5.so
3d4cd4a000 (2048 KB) ---p (09:02 1194229772) /lib64/libc-2.5.so
3d4cf4a000 (16 KB) r--p (09:02 1194229772) /lib64/libc-2.5.so
3d4cf4e000 (4 KB) rw-p (09:02 1194229772) /lib64/libc-2.5.so
3d4cf4f000 (20 KB) rw-p (00:00 0)
3d4d000000 (8 KB) r-xp (09:02 1194229776) /lib64/libdl-2.5.so
3d4d002000 (2048 KB) ---p (09:02 1194229776) /lib64/libdl-2.5.so
3d4d202000 (4 KB) r--p (09:02 1194229776) /lib64/libdl-2.5.so
3d4d203000 (4 KB) rw-p (09:02 1194229776) /lib64/libdl-2.5.so
3d4dc00000 (84 KB) r-xp (09:02 1194229786) /lib64/libpthread-2.5.so
3d4dc15000 (2044 KB) ---p (09:02 1194229786) /lib64/libpthread-2.5.so
3d4de14000 (4 KB) r--p (09:02 1194229786) /lib64/libpthread-2.5.so
3d4de15000 (4 KB) rw-p (09:02 1194229786) /lib64/libpthread-2.5.so
3d4de16000 (16 KB) rw-p (00:00 0)
3d4e400000 (28 KB) r-xp (09:02 1194229788) /lib64/librt-2.5.so
3d4e407000 (2048 KB) ---p (09:02 1194229788) /lib64/librt-2.5.so
3d4e607000 (4 KB) r--p (09:02 1194229788) /lib64/librt-2.5.so
3d4e608000 (4 KB) rw-p (09:02 1194229788) /lib64/librt-2.5.so
2aaaaaaab000 (28 KB) r-xp (09:02 443222345)
/usr/lib64/glusterfs/2.0.0rc1/transport/socket.so.0.0.0
2aaaaaab2000 (2044 KB) ---p (09:02 443222345)
/usr/lib64/glusterfs/2.0.0rc1/transport/socket.so.0.0.0
2aaaaacb1000 (8 KB) rw-p (09:02 443222345)
/usr/lib64/glusterfs/2.0.0rc1/transport/socket.so.0.0.0
2aaaaacb3000 (576 KB) rw-p (00:00 0)
2aaaaad43000 (8 KB) r-xp (09:02 443222303)
/usr/lib64/glusterfs/2.0.0rc1/auth/addr.so.0.0.0
2aaaaad45000 (2044 KB) ---p (09:02 443222303)
/usr/lib64/glusterfs/2.0.0rc1/auth/addr.so.0.0.0
2aaaaaf44000 (4 KB) rw-p (09:02 443222303)
/usr/lib64/glusterfs/2.0.0rc1/auth/addr.so.0.0.0
2aaaaaf4e000 (40 KB) r-xp (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaaaf58000 (2044 KB) ---p (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaab157000 (4 KB) r--p (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaab158000 (4 KB) rw-p (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaac000000 (72380 KB) rw-p (00:00 0)
2aaab06af000 (58692 KB) ---p (00:00 0)
2b4ac185b000 (8 KB) rw-p (00:00 0)
2b4ac1866000 (404 KB) rw-p (00:00 0)
2b4ac18cb000 (104 KB) r-xp (09:02 443252818)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/client.so.0.0.0
2b4ac18e5000 (2048 KB) ---p (09:02 443252818)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/client.so.0.0.0
2b4ac1ae5000 (8 KB) rw-p (09:02 443252818)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/client.so.0.0.0
2b4ac1ae7000 (64 KB) r-xp (09:02 443252834)
/usr/lib64/glusterfs/2.0.0rc1/xlator/storage/posix.so.0.0.0
2b4ac1af7000 (2044 KB) ---p (09:02 443252834)
/usr/lib64/glusterfs/2.0.0rc1/xlator/storage/posix.so.0.0.0
2b4ac1cf6000 (8 KB) rw-p (09:02 443252834)
/usr/lib64/glusterfs/2.0.0rc1/xlator/storage/posix.so.0.0.0
2b4ac1cf8000 (28 KB) r-xp (09:02 443252764)
/usr/lib64/glusterfs/2.0.0rc1/xlator/features/locks.so.0.0.0
2b4ac1cff000 (2048 KB) ---p (09:02 443252764)
/usr/lib64/glusterfs/2.0.0rc1/xlator/features/locks.so.0.0.0
2b4ac1eff000 (4 KB) rw-p (09:02 443252764)
/usr/lib64/glusterfs/2.0.0rc1/xlator/features/locks.so.0.0.0
2b4ac1f00000 (108 KB) r-xp (09:02 443252823)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/server.so.0.0.0
2b4ac1f1b000 (2044 KB) ---p (09:02 443252823)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/server.so.0.0.0
2b4ac211a000 (8 KB) rw-p (09:02 443252823)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/server.so.0.0.0
2b4ac211c000 (176 KB) r-xp (09:02 443222352)
/usr/lib64/glusterfs/2.0.0rc1/xlator/cluster/afr.so.0.0.0
2b4ac2148000 (2048 KB) ---p (09:02 443222352)
/usr/lib64/glusterfs/2.0.0rc1/xlator/cluster/afr.so.0.0.0
2b4ac2348000 (12 KB) rw-p (09:02 443222352)
/usr/lib64/glusterfs/2.0.0rc1/xlator/cluster/afr.so.0.0.0
2b4ac234b000 (56 KB) r-xp (09:02 443252786)
/usr/lib64/glusterfs/2.0.0rc1/xlator/mount/fuse.so.0.0.0
2b4ac2359000 (2044 KB) ---p (09:02 443252786)
/usr/lib64/glusterfs/2.0.0rc1/xlator/mount/fuse.so.0.0.0
2b4ac2558000 (8 KB) rw-p (09:02 443252786)
/usr/lib64/glusterfs/2.0.0rc1/xlator/mount/fuse.so.0.0.0
2b4ac255a000 (116 KB) r-xp (09:02 442904974) /usr/lib64/libfuse.so.2.7.4
2b4ac2577000 (2048 KB) ---p (09:02 442904974) /usr/lib64/libfuse.so.2.7.4
2b4ac2777000 (8 KB) rw-p (09:02 442904974) /usr/lib64/libfuse.so.2.7.4
7fffe923a000 (84 KB) rw-p (00:00 0) [stack]
7fffe93fe000 (8 KB) r-xp (00:00 0) [vdso]
ffffffffff600000 (4 KB) r-xp (00:00 0) [vsyscall]
mapped: 207760 KB writable/private: 115848 KB shared: 0 KB
And again:
# make clean; make dep; make -j32 all
top:
PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+ COMMAND
4605 root 20 0 266m 192m 1316 S 0 4.9 5:26.57 glusterfs
# kill -s SIGUSR1 4605
Arena 0:
system bytes = 26423296
in use bytes = 25972784
Arena 1:
system bytes = 173174784
in use bytes = 172023232
Total (incl. mmap):
system bytes = 200585216
in use bytes = 198983152
max mmap regions = 5
max mmap bytes = 1122304
# ./pmap 4605
glusterfs(4605)
00400000 (24 KB) r-xp (09:02 442904105) /usr/sbin/glusterfsd
00606000 (4 KB) rw-p (09:02 442904105) /usr/sbin/glusterfsd
00607000 (25804 KB) rw-p (00:00 0) [heap]
40000000 (4 KB) ---p (00:00 0)
40001000 (8192 KB) rw-p (00:00 0)
40801000 (4 KB) ---p (00:00 0)
40802000 (8192 KB) rw-p (00:00 0)
3a38600000 (176 KB) r-xp (09:02 442904212)
/usr/lib64/libglusterfs.so.0.0.0
3a3862c000 (2048 KB) ---p (09:02 442904212)
/usr/lib64/libglusterfs.so.0.0.0
3a3882c000 (4 KB) rw-p (09:02 442904212)
/usr/lib64/libglusterfs.so.0.0.0
3a3882d000 (72 KB) rw-p (00:00 0)
3d4c800000 (104 KB) r-xp (09:02 1194229765) /lib64/ld-2.5.so
3d4ca1a000 (4 KB) r--p (09:02 1194229765) /lib64/ld-2.5.so
3d4ca1b000 (4 KB) rw-p (09:02 1194229765) /lib64/ld-2.5.so
3d4cc00000 (1320 KB) r-xp (09:02 1194229772) /lib64/libc-2.5.so
3d4cd4a000 (2048 KB) ---p (09:02 1194229772) /lib64/libc-2.5.so
3d4cf4a000 (16 KB) r--p (09:02 1194229772) /lib64/libc-2.5.so
3d4cf4e000 (4 KB) rw-p (09:02 1194229772) /lib64/libc-2.5.so
3d4cf4f000 (20 KB) rw-p (00:00 0)
3d4d000000 (8 KB) r-xp (09:02 1194229776) /lib64/libdl-2.5.so
3d4d002000 (2048 KB) ---p (09:02 1194229776) /lib64/libdl-2.5.so
3d4d202000 (4 KB) r--p (09:02 1194229776) /lib64/libdl-2.5.so
3d4d203000 (4 KB) rw-p (09:02 1194229776) /lib64/libdl-2.5.so
3d4dc00000 (84 KB) r-xp (09:02 1194229786) /lib64/libpthread-2.5.so
3d4dc15000 (2044 KB) ---p (09:02 1194229786) /lib64/libpthread-2.5.so
3d4de14000 (4 KB) r--p (09:02 1194229786) /lib64/libpthread-2.5.so
3d4de15000 (4 KB) rw-p (09:02 1194229786) /lib64/libpthread-2.5.so
3d4de16000 (16 KB) rw-p (00:00 0)
3d4e400000 (28 KB) r-xp (09:02 1194229788) /lib64/librt-2.5.so
3d4e407000 (2048 KB) ---p (09:02 1194229788) /lib64/librt-2.5.so
3d4e607000 (4 KB) r--p (09:02 1194229788) /lib64/librt-2.5.so
3d4e608000 (4 KB) rw-p (09:02 1194229788) /lib64/librt-2.5.so
2aaaaaaab000 (28 KB) r-xp (09:02 443222345)
/usr/lib64/glusterfs/2.0.0rc1/transport/socket.so.0.0.0
2aaaaaab2000 (2044 KB) ---p (09:02 443222345)
/usr/lib64/glusterfs/2.0.0rc1/transport/socket.so.0.0.0
2aaaaacb1000 (8 KB) rw-p (09:02 443222345)
/usr/lib64/glusterfs/2.0.0rc1/transport/socket.so.0.0.0
2aaaaacb3000 (576 KB) rw-p (00:00 0)
2aaaaad43000 (8 KB) r-xp (09:02 443222303)
/usr/lib64/glusterfs/2.0.0rc1/auth/addr.so.0.0.0
2aaaaad45000 (2044 KB) ---p (09:02 443222303)
/usr/lib64/glusterfs/2.0.0rc1/auth/addr.so.0.0.0
2aaaaaf44000 (4 KB) rw-p (09:02 443222303)
/usr/lib64/glusterfs/2.0.0rc1/auth/addr.so.0.0.0
2aaaaaf4e000 (40 KB) r-xp (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaaaf58000 (2044 KB) ---p (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaab157000 (4 KB) r--p (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaab158000 (4 KB) rw-p (09:02 1194229787)
/lib64/libnss_files-2.5.so
2aaaac000000 (169120 KB) rw-p (00:00 0)
2aaab6528000 (27488 KB) ---p (00:00 0)
2b4ac185b000 (8 KB) rw-p (00:00 0)
2b4ac1866000 (404 KB) rw-p (00:00 0)
2b4ac18cb000 (104 KB) r-xp (09:02 443252818)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/client.so.0.0.0
2b4ac18e5000 (2048 KB) ---p (09:02 443252818)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/client.so.0.0.0
2b4ac1ae5000 (8 KB) rw-p (09:02 443252818)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/client.so.0.0.0
2b4ac1ae7000 (64 KB) r-xp (09:02 443252834)
/usr/lib64/glusterfs/2.0.0rc1/xlator/storage/posix.so.0.0.0
2b4ac1af7000 (2044 KB) ---p (09:02 443252834)
/usr/lib64/glusterfs/2.0.0rc1/xlator/storage/posix.so.0.0.0
2b4ac1cf6000 (8 KB) rw-p (09:02 443252834)
/usr/lib64/glusterfs/2.0.0rc1/xlator/storage/posix.so.0.0.0
2b4ac1cf8000 (28 KB) r-xp (09:02 443252764)
/usr/lib64/glusterfs/2.0.0rc1/xlator/features/locks.so.0.0.0
2b4ac1cff000 (2048 KB) ---p (09:02 443252764)
/usr/lib64/glusterfs/2.0.0rc1/xlator/features/locks.so.0.0.0
2b4ac1eff000 (4 KB) rw-p (09:02 443252764)
/usr/lib64/glusterfs/2.0.0rc1/xlator/features/locks.so.0.0.0
2b4ac1f00000 (108 KB) r-xp (09:02 443252823)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/server.so.0.0.0
2b4ac1f1b000 (2044 KB) ---p (09:02 443252823)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/server.so.0.0.0
2b4ac211a000 (8 KB) rw-p (09:02 443252823)
/usr/lib64/glusterfs/2.0.0rc1/xlator/protocol/server.so.0.0.0
2b4ac211c000 (176 KB) r-xp (09:02 443222352)
/usr/lib64/glusterfs/2.0.0rc1/xlator/cluster/afr.so.0.0.0
2b4ac2148000 (2048 KB) ---p (09:02 443222352)
/usr/lib64/glusterfs/2.0.0rc1/xlator/cluster/afr.so.0.0.0
2b4ac2348000 (12 KB) rw-p (09:02 443222352)
/usr/lib64/glusterfs/2.0.0rc1/xlator/cluster/afr.so.0.0.0
2b4ac234b000 (56 KB) r-xp (09:02 443252786)
/usr/lib64/glusterfs/2.0.0rc1/xlator/mount/fuse.so.0.0.0
2b4ac2359000 (2044 KB) ---p (09:02 443252786)
/usr/lib64/glusterfs/2.0.0rc1/xlator/mount/fuse.so.0.0.0
2b4ac2558000 (8 KB) rw-p (09:02 443252786)
/usr/lib64/glusterfs/2.0.0rc1/xlator/mount/fuse.so.0.0.0
2b4ac255a000 (116 KB) r-xp (09:02 442904974) /usr/lib64/libfuse.so.2.7.4
2b4ac2577000 (2048 KB) ---p (09:02 442904974) /usr/lib64/libfuse.so.2.7.4
2b4ac2777000 (8 KB) rw-p (09:02 442904974) /usr/lib64/libfuse.so.2.7.4
7fffe923a000 (84 KB) rw-p (00:00 0) [stack]
7fffe93fe000 (8 KB) r-xp (00:00 0) [vdso]
ffffffffff600000 (4 KB) r-xp (00:00 0) [vsyscall]
mapped: 273296 KB writable/private: 212588 KB shared: 0 KB
I hope this is useful.
Gordan
Amar Tumballi (bulde) wrote:
> Hi Gordan and Dan,
>
> It would help me a lot if its possible for you to get the info as
> described below,
>
> compile glusterfs like
>
> bash# make clean > /dev/null
> bash# make CFLAGS="-g -O0 -DDEBUG" > /dev/null
> bash# make install
>
> run the process which consumes memory (mostly client process) like below:
>
> bash# glusterfs <any argument you give generally> -N
> <this process will run in foreground now>
>
> Open another terminal
>
> bash# ps aux | grep glusterfs
> bash# kill -s SIGUSR1 <pid of glusterfs -N process>
> <Check in other terminal for memory usage stats>
>
> bash# <run your application over glusterfs as you do till you get high
> memory usage of glusterfs.. >
> bash# kill -s SIGUSR1 <pid of glusterfs -N process>
> <Check the stat in another terminal>
>
> bash# echo 3 > /proc/sys/vm/drop_caches
> bash# kill -s SIGUSR1 <pid of glusterfs -N process>
> <Check the stat in another terminal>
>
> Even after dropping caches, if 'in use bytes =' in malloc stats shows a
> high value, then it is a leak. If its showing less, but just 'system
> bytes = ' is a high value, this means glusterfs is not really consuming
> high memory, but the problem is really in the memory allocation segments.
>
> Regards,
> Amar
>
> NOTE: 'malloc_stats' will be printed to 'stdout' if we enable -DDEBUG
> while compiling glusterfs, as it hits performance badly otherwise.
>
>
> 2009/2/23 Gordan Bobic <gordan at bobich.net <mailto:gordan at bobich.net>>
>
> Dan Parsons wrote:
>
> I'm having an issue with glusterfs exceeding its cache-size
> value. Right now I have it set to 4000MB and I've seen it climb
> as high as 4800MB. If I set it to 5000, I've seen it go as high
> as 6000MB. This is a problem because it causes me to set the
> value very low so that my apps don't get pushed into swap. Is
> there any way to fix this? To get it to stick to the limit I set
> and not exceed?
>
>
> It's possible you are running into the same memory leak that I'm
> seeing, and I'm not using io-cache or any other performance
> translators at all. With rootfs on Gluster, doing a kernel compile
> (kernel source tree being on NFS, so this won't be contributing to
> the bloat, hopefully) makes the glusterfsd bloat by about 80MB per
> pass, and never frees it.
>
> Gordan
>
>
> _______________________________________________
> Gluster-devel mailing list
> Gluster-devel at nongnu.org <mailto:Gluster-devel at nongnu.org>
> http://lists.nongnu.org/mailman/listinfo/gluster-devel
>
>
>
>
> --
> Amar Tumballi
> Gluster/GlusterFS Hacker
> [bulde on #gluster/irc.gnu.org <http://irc.gnu.org>]
> http://www.zresearch.com - Commoditizing Super Storage!
More information about the Gluster-devel
mailing list